diff --git a/Makefile b/Makefile index 250ac8ca..885c4a61 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ lint: FORCE test: FORCE bun test packages/ apps/ bin/ tests/agent/ tests/agent-audit-log/ tests/agent-blob-spill/ tests/agent-common/ tests/agent-multi-provider/ tests/agent-quickstart/ tests/agent-resume/ tests/agent-rewind/ tests/agent-rich-tool/ tests/agent-structured-payload/ tests/coding-agent/ tests/hub-agent/lib/ tests/inference-testing/ tests/tool-packaging/ tests/workflow/ - bun test --timeout 120000 tests/hub-agent/deploy-flow.test.ts tests/workflow-deploy/trivial-roundtrip.test.ts tests/workflow-deploy/multistep-signal.test.ts tests/workflow-deploy/single-step-real-agent.test.ts tests/workflow-deploy/single-step-message-input.test.ts tests/workflow-deploy/single-step-posix-tool.test.ts tests/workflow-deploy/single-step-grants-bridge.test.ts tests/workflow-deploy/single-step-event-threading.test.ts tests/workflow-deploy/single-step-conversation-durability.test.ts tests/workflow-deploy/single-step-full-lifecycle.test.ts tests/workflow-deploy/cross-process-custom-adapter.test.ts tests/workflow-deploy/conversation-state-wal.test.ts tests/workflow-deploy/drain-roundtrip.test.ts tests/workflow-deploy/child-workflow-roundtrip.test.ts tests/workflow-deploy/unresolvable-director.test.ts tests/workflow-deploy/fifo-mail.test.ts tests/workflow-deploy/mail-edge-cases.test.ts tests/inference/ tests/skill-attachment-flow.test.ts tests/hub-api/ tests/db/ + bun test --timeout 120000 tests/workflow-deploy/multistep-signal.test.ts tests/workflow-deploy/single-step-real-agent.test.ts tests/workflow-deploy/instance-reroute-real-agent.test.ts tests/workflow-deploy/instance-failover-real-agent.test.ts tests/workflow-deploy/multistep-signed-send.test.ts tests/workflow-deploy/single-step-message-input.test.ts tests/workflow-deploy/single-step-posix-tool.test.ts tests/workflow-deploy/single-step-grants-bridge.test.ts tests/workflow-deploy/single-step-event-threading.test.ts tests/workflow-deploy/single-step-conversation-durability.test.ts tests/workflow-deploy/single-step-full-lifecycle.test.ts tests/workflow-deploy/cross-process-custom-adapter.test.ts tests/workflow-deploy/conversation-state-wal.test.ts tests/workflow-deploy/drain-roundtrip.test.ts tests/workflow-deploy/child-workflow-roundtrip.test.ts tests/workflow-deploy/unresolvable-director.test.ts tests/workflow-deploy/fifo-mail.test.ts tests/workflow-deploy/mail-edge-cases.test.ts tests/inference/ tests/skill-attachment-flow.test.ts tests/hub-api/ tests/db/ test-load: FORCE bun test --timeout 300000 tests/workflow-deploy/fifo-mail-load.test.ts diff --git a/apps/hub/src/index.ts b/apps/hub/src/index.ts index 3feda98f..bedc5d5d 100644 --- a/apps/hub/src/index.ts +++ b/apps/hub/src/index.ts @@ -1,4 +1,4 @@ -import { createDB, createGrantStore } from "@intx/db"; +import { createDB } from "@intx/db"; import { createApp, createAuth } from "@intx/hub-api"; import { createAgentRepoStore, @@ -35,7 +35,6 @@ const { db } = createDB({ }); const auth = createAuth(db); -const grantStore = createGrantStore(db); const hubDataDir = process.env["HUB_DATA_DIR"]; if (!hubDataDir) { @@ -141,7 +140,6 @@ createHubSessionOrchestrator({ router: sidecarRouter, db, eventCollectors, - grantStore, agentRepoStore, }); diff --git a/apps/sidecar/src/agent-key-registration-lifecycle.test.ts b/apps/sidecar/src/agent-key-registration-lifecycle.test.ts index 7c715cb6..4cab2388 100644 --- a/apps/sidecar/src/agent-key-registration-lifecycle.test.ts +++ b/apps/sidecar/src/agent-key-registration-lifecycle.test.ts @@ -29,7 +29,13 @@ import { } from "@intx/workflow-host"; import type { AgentDeployFrame } from "@intx/types/sidecar"; -import { createSidecarDeployRouter } from "./workflow-host-wiring"; +import { type } from "arktype"; + +import { + createSidecarDeployRouter, + deriveDeploymentId, +} from "./workflow-host-wiring"; +import { WorkflowDeploymentRecord } from "./workflow-deployment-record"; import { createMultistepDrainRouter, createMultistepMailRouter, @@ -219,7 +225,7 @@ describe("agent signing-key registration lifecycle on the host transport", () => const repoStore = createSpawnTestRepoStore(tempBase); const router = createSidecarDeployRouter({ - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- multi-step branch never invokes sessions + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the single-step branch invokes only initRepo (head deploy-tree repo); provisionAgent/persistHubPublicKey stay unused (the supervised child mints its own key and persists no hub-agent config) sessions: { provisionAgent: async () => { throw new Error("must not invoke provisionAgent"); @@ -227,14 +233,13 @@ describe("agent signing-key registration lifecycle on the host transport", () => persistHubPublicKey: async () => { throw new Error("must not invoke persistHubPublicKey"); }, + initRepo: async () => undefined, } as unknown as Parameters< typeof createSidecarDeployRouter >[0]["sessions"], - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- only loadOrGenerateKey is exercised (the single-step branch registers the agent's signing key on the transport before spawn) + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the single-step branch registers the agent's signing key (loadOrGenerateKey) and records the hub key (recordHubKey) at the head before spawn keyStore: { - recordHubKey: () => { - throw new Error("must not invoke recordHubKey"); - }, + recordHubKey: () => undefined, loadOrGenerateKey: async () => ({ keyPair: await generateKeyPair(), isNew: false, @@ -242,11 +247,11 @@ describe("agent signing-key registration lifecycle on the host transport", () => } as unknown as Parameters< typeof createSidecarDeployRouter >[0]["keyStore"], - onAgentEvent: () => () => undefined, transport, repoStore, signingKeySeed: keyPair.privateKey, createAgentCrypto: createEd25519Crypto, + assertSourceBuildable: () => undefined, registerDeployment: () => undefined, unregisterDeployment: () => undefined, multistepSubprocessSpawner: spawner, @@ -271,13 +276,15 @@ describe("agent signing-key registration lifecycle on the host transport", () => steps: { "step-1": { kind: "step" } }, }, sources: { - "step-1": { - id: "step-1", - provider: "anthropic", - baseURL: "https://api.anthropic.com", - apiKey: "sk-step-1", - model: "claude-3-5", - }, + "step-1": [ + { + id: "step-1", + provider: "anthropic", + baseURL: "https://api.anthropic.com", + apiKey: "sk-step-1", + model: "claude-3-5", + }, + ], }, }, }; @@ -316,6 +323,26 @@ describe("agent signing-key registration lifecycle on the host transport", () => // (b) Registered after deploy -- the supervisor can now sign agent mail. expect(isRegistered(transport)).toBe(true); + // (b') The deploy persisted a schema-valid restore record for the + // deployment, carrying the head address so a boot-time restore can + // re-establish it. + const deploymentId = deriveDeploymentId(AGENT_ADDRESS); + const recordFile = path.join( + dataDir, + "workflow-runs", + deploymentId, + "deployment.json", + ); + const parsedRecord = WorkflowDeploymentRecord( + JSON.parse(await fs.readFile(recordFile, "utf8")), + ); + if (parsedRecord instanceof type.errors) { + throw new Error( + `deployment record failed validation: ${parsedRecord.summary}`, + ); + } + expect(parsedRecord.agentAddress).toBe(AGENT_ADDRESS); + const undeploy = router.undeploy; if (undeploy === undefined) throw new Error("router.undeploy undefined"); await undeploy({ @@ -327,6 +354,15 @@ describe("agent signing-key registration lifecycle on the host transport", () => // (c) Unregistered after undeploy -- no leaked key for a torn-down agent. expect(isRegistered(transport)).toBe(false); + // (c') Undeploy dropped the restore record so a boot-time restore will + // not re-spawn the torn-down deployment. + expect( + await fs.access(recordFile).then( + () => true, + () => false, + ), + ).toBe(false); + await fs.rm(tempBase, { recursive: true, force: true }); await fs.rm(dataDir, { recursive: true, force: true }); }); diff --git a/apps/sidecar/src/atomic-write.test.ts b/apps/sidecar/src/atomic-write.test.ts new file mode 100644 index 00000000..7b81961a --- /dev/null +++ b/apps/sidecar/src/atomic-write.test.ts @@ -0,0 +1,102 @@ +import { describe, test, expect } from "bun:test"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import { writeFileAtomicDurable } from "./atomic-write"; + +async function makeDir(): Promise { + return fs.mkdtemp(path.join(os.tmpdir(), "atomic-write-")); +} + +async function listNames(dir: string): Promise { + return (await fs.readdir(dir)).sort(); +} + +describe("writeFileAtomicDurable", () => { + test("round-trips contents to the target path", async () => { + const dir = await makeDir(); + const file = path.join(dir, "record.json"); + + await writeFileAtomicDurable(file, '{"a":1}', { mode: 0o600 }); + expect(await fs.readFile(file, "utf8")).toBe('{"a":1}'); + + await fs.rm(dir, { recursive: true, force: true }); + }); + + test("applies mode on the created file", async () => { + const dir = await makeDir(); + const file = path.join(dir, "record.json"); + + await writeFileAtomicDurable(file, "x", { mode: 0o600 }); + const stat = await fs.stat(file); + expect(stat.mode & 0o777).toBe(0o600); + + await fs.rm(dir, { recursive: true, force: true }); + }); + + test("re-applies mode when overwriting an existing file", async () => { + const dir = await makeDir(); + const file = path.join(dir, "record.json"); + + // Seed a pre-existing file with a laxer mode. A plain in-place + // overwrite keeps the original mode; the temp+rename path creates a + // fresh file every write, so the requested mode actually lands. + await fs.writeFile(file, "old", { mode: 0o644 }); + await writeFileAtomicDurable(file, "new", { mode: 0o600 }); + + expect(await fs.readFile(file, "utf8")).toBe("new"); + const stat = await fs.stat(file); + expect(stat.mode & 0o777).toBe(0o600); + + await fs.rm(dir, { recursive: true, force: true }); + }); + + test("leaves no temp orphan after a successful write", async () => { + const dir = await makeDir(); + const file = path.join(dir, "record.json"); + + await writeFileAtomicDurable(file, "x", { mode: 0o600 }); + expect(await listNames(dir)).toEqual(["record.json"]); + + await fs.rm(dir, { recursive: true, force: true }); + }); + + test("a write that fails before staging leaves the prior file intact", async () => { + const dir = await makeDir(); + const file = path.join(dir, "record.json"); + await fs.writeFile(file, "old", { mode: 0o600 }); + + // Deny writes to the directory so the temp-file creation fails + // outright. The prior complete record must survive untouched -- an + // interrupted rotation must never corrupt the sole restore source. + await fs.chmod(dir, 0o500); + try { + await expect( + writeFileAtomicDurable(file, "new", { mode: 0o600 }), + ).rejects.toThrow(); + expect(await fs.readFile(file, "utf8")).toBe("old"); + expect(await listNames(dir)).toEqual(["record.json"]); + } finally { + await fs.chmod(dir, 0o700); + await fs.rm(dir, { recursive: true, force: true }); + } + }); + + test("a write that fails at the rename unlinks its staged temp", async () => { + const dir = await makeDir(); + const file = path.join(dir, "record.json"); + + // Occupy the target path with a directory so the rename fails after + // the temp file has already been staged and fsynced. This exercises + // the error-path unlink: the staged temp must be removed rather than + // stranded as an orphan. + await fs.mkdir(file); + await expect( + writeFileAtomicDurable(file, "new", { mode: 0o600 }), + ).rejects.toThrow(); + expect(await listNames(dir)).toEqual(["record.json"]); + + await fs.rm(dir, { recursive: true, force: true }); + }); +}); diff --git a/apps/sidecar/src/atomic-write.ts b/apps/sidecar/src/atomic-write.ts new file mode 100644 index 00000000..48b116e3 --- /dev/null +++ b/apps/sidecar/src/atomic-write.ts @@ -0,0 +1,79 @@ +// Atomic, durable file replacement for the sidecar's non-rebuildable +// on-disk records. Distinct from the cache's rebuildable temp+rename +// (no fsync, a lost write just forces a re-fetch) and from +// `fsyncWriteFile`'s in-place fsync write (no atomicity, a torn write +// leaves a half-file): this is the tier for a sole restore source that +// must survive both a process kill and a power loss without ever +// exposing a torn record. + +import { open, rename, unlink } from "node:fs/promises"; +import { dirname } from "node:path"; + +import { getLogger } from "@intx/log"; +import { hexEncode } from "@intx/types"; + +const logger = getLogger(["interchange", "sidecar", "atomic-write"]); + +export interface AtomicWriteOptions { + /** Permission mode applied when the temp file is created. */ + mode: number; +} + +/** + * Replace `path` with `contents` atomically and durably. The bytes land + * in a fresh per-write temp file that is fsynced and then `rename`d over + * `path`; because rename is atomic within a directory, a reader only + * ever observes the prior complete file or the new complete file, never + * a torn one. The fsync before the rename is what extends that + * guarantee past process death to OS crash / power loss: without it, the + * ext4 delayed-allocation window can surface the renamed path as a + * zero-length file after a power loss. + * + * The parent directory is fsynced after the rename so the new link is + * itself durable, but a filesystem that rejects directory fsync + * (FAT/exFAT, some network mounts) only degrades durability -- the file + * is already renamed and fsynced -- so that failure is logged, not + * thrown. + * + * `mode` is applied on the temp file's creation, so it takes effect on + * every write. A plain in-place overwrite of an existing file would + * silently keep the original file's mode instead. + * + * The temp file follows `createTarballCache`'s `.tmp..` + * naming convention for consistency across the sidecar's staged writes. + */ +export async function writeFileAtomicDurable( + path: string, + contents: string, + options: AtomicWriteOptions, +): Promise { + const tmp = `${path}.tmp.${String(process.pid)}.${hexEncode(crypto.getRandomValues(new Uint8Array(8)))}`; + try { + const handle = await open(tmp, "w", options.mode); + try { + await handle.writeFile(contents); + await handle.sync(); + } finally { + await handle.close(); + } + await rename(tmp, path); + } catch (cause) { + // The write failed and is about to rethrow; unlink the temp so a + // failed write leaves no orphan. Best-effort: the temp may never + // have been created, and a second failure here must not mask the + // original cause. + await unlink(tmp).catch(() => undefined); + throw cause; + } + + try { + const dirHandle = await open(dirname(path), "r"); + try { + await dirHandle.sync(); + } finally { + await dirHandle.close(); + } + } catch (err) { + logger.warn`parent-dir fsync failed for ${path}; durability is degraded but the file is renamed and fsynced — ${err instanceof Error ? err.message : String(err)}`; + } +} diff --git a/apps/sidecar/src/config.ts b/apps/sidecar/src/config.ts index a9223a6f..21425e95 100644 --- a/apps/sidecar/src/config.ts +++ b/apps/sidecar/src/config.ts @@ -9,7 +9,6 @@ // re-reading env, so the boundary stays at the boot edge. import { AdapterManifest } from "@intx/inference"; -import type { GCPolicy, RetentionPolicy } from "@intx/storage-isogit"; const DEFAULT_CACHE_MAX_BYTES = 10 * 1024 * 1024 * 1024; @@ -69,55 +68,3 @@ export function readAdapterManifest(): AdapterManifest { } return AdapterManifest.assert(parsed); } - -// Write-path GC for the sidecar's deployed-agent repos. The reactor -// commits loose objects every cycle, so these repos accumulate loose -// objects far faster than packs; the loose threshold is the dominant -// trigger here. Retention defaults to tip-only: the sidecar treats the -// repo as the agent's current state, not a long-term archive, so dropping -// commit history keeps the repo small for reactor read/commit latency. An -// operator that needs the history preserved sets -// SIDECAR_AGENT_GC_RETENTION=keep-history. -const DEFAULT_SIDECAR_AGENT_GC_PACK_THRESHOLD = 16; -const DEFAULT_SIDECAR_AGENT_GC_LOOSE_THRESHOLD = 512; -const DEFAULT_SIDECAR_AGENT_GC_WARN_BYTES = 128 * 1024 * 1024; - -function readPositiveIntEnv(name: string, fallback: number): number { - const raw = process.env[name]; - if (raw === undefined || raw.trim() === "") return fallback; - const n = Number(raw); - if (!Number.isInteger(n) || n <= 0) { - throw new Error(`${name} must be a positive integer; got ${raw}`); - } - return n; -} - -function readRetentionEnv( - name: string, - fallback: RetentionPolicy, -): RetentionPolicy { - const raw = process.env[name]; - if (raw === undefined || raw.trim() === "") return fallback; - if (raw === "tip-only" || raw === "keep-history") return raw; - throw new Error( - `${name} must be "tip-only" or "keep-history"; got ${JSON.stringify(raw)}`, - ); -} - -export function readAgentGCPolicy(): GCPolicy { - return { - packThreshold: readPositiveIntEnv( - "SIDECAR_AGENT_GC_PACK_THRESHOLD", - DEFAULT_SIDECAR_AGENT_GC_PACK_THRESHOLD, - ), - looseThreshold: readPositiveIntEnv( - "SIDECAR_AGENT_GC_LOOSE_THRESHOLD", - DEFAULT_SIDECAR_AGENT_GC_LOOSE_THRESHOLD, - ), - warnBytes: readPositiveIntEnv( - "SIDECAR_AGENT_GC_WARN_BYTES", - DEFAULT_SIDECAR_AGENT_GC_WARN_BYTES, - ), - retention: readRetentionEnv("SIDECAR_AGENT_GC_RETENTION", "tip-only"), - }; -} diff --git a/apps/sidecar/src/conversation-state.ts b/apps/sidecar/src/conversation-state.ts index 97200b25..0448c82d 100644 --- a/apps/sidecar/src/conversation-state.ts +++ b/apps/sidecar/src/conversation-state.ts @@ -347,6 +347,40 @@ export async function createDurableConversationStore( // checkpointBoundarySeq) and when to compact. let checkpointBoundarySeq = 0; + // Serialize the shared-counter critical section. Both `mirrorToSubstrate` + // and `restoreFromSubstrate` read and advance the three counts above, and + // either can re-enter the other: `connectorRouter.restore()` can fire + // `onStateChanged` synchronously, which enqueues a mirror. A single + // per-instance tail runs every such op one-at-a-time regardless of entry + // point, so two overlapping runs cannot read the same boundary seq and + // emit two WAL entries at it. Modeled on `runRepoOp` in the hub-agent + // session manager: the stored tail swallows rejections so one failed op + // does not poison the chain, while each caller still observes its own op's + // result (or rejection) through the returned promise. + // + // This serializes mirror-vs-mirror and mirror-vs-restore only. It does NOT + // address the reactor-vs-mirror peek-snapshot window documented on + // `runMirror` below (nothing must append to the reactor's turn array + // between its last writeTurns and the mirror's peekTurns) -- that is a + // different concurrency axis and is out of scope here. + let stateOpTail: Promise = Promise.resolve(); + function serializeStateOp(op: () => Promise): Promise { + const result = stateOpTail.then(op, op); + stateOpTail = result.then( + () => undefined, + () => undefined, + ); + return result; + } + + function restoreFromSubstrate(): Promise { + return serializeStateOp(runRestore); + } + + function mirrorToSubstrate(): Promise { + return serializeStateOp(runMirror); + } + function substrateAgentStateFsDir(): string { const repoDir = opts.substrate.getRepoDir(opts.workflowRunRepoId); return path.join( @@ -376,7 +410,7 @@ export async function createDurableConversationStore( return `${agentStatePrefix}${CHECKPOINT_META_FILE}`; } - async function restoreFromSubstrate(): Promise { + async function runRestore(): Promise { const reconstructed = await reconstructDurableConversation( substrateAgentStateFsDir(), opts.agentKey, @@ -397,6 +431,17 @@ export async function createDurableConversationStore( // so a future change-driven mirror carries the right base. await baseStorage.writeTurns(reconstructed.turns); baseStorage.setConnectorState(reconstructed.connectorState); + // Establish the committed counts BEFORE restoring the connector state. + // `connectorRouter.restore()` can fire `onStateChanged` synchronously + // (when the restored state differs from current), which enqueues a + // mirror. Serialization already chains that mirror behind this restore, + // but setting the counts first keeps them correct even if that ordering + // guarantee is ever weakened. The counts reflect the substrate state + // `reconstructed` was read from, which is durable independent of the + // local-store commit below. + mirroredBoundaryCount = reconstructed.boundaryCount; + mirroredTurnCount = reconstructed.totalTurns; + checkpointBoundarySeq = reconstructed.checkpointBoundarySeq; connectorRouter.restore(reconstructed.connectorState); await baseStorage.writeMetadata({ pendingOperations: reconstructed.pendingOperations, @@ -405,9 +450,6 @@ export async function createDurableConversationStore( await baseStorage.commit({ message: `restore conversation for ${opts.agentKey} from substrate`, }); - mirroredBoundaryCount = reconstructed.boundaryCount; - mirroredTurnCount = reconstructed.totalTurns; - checkpointBoundarySeq = reconstructed.checkpointBoundarySeq; return true; } @@ -501,13 +543,21 @@ export async function createDurableConversationStore( ); } - async function mirrorToSubstrate(): Promise { - const loaded = await baseStorage.load(); - const metadata = { - pendingOperations: loaded.pendingOperations, - tokenUsage: loaded.tokenUsage, - connectorState: loaded.connectorState, - }; + async function runMirror(): Promise { + // Slice the new turns from the reactor's in-memory array (retained + // by the local single-writer store at the last writeTurns) instead + // of re-reading and re-parsing the whole turns.jsonl every + // boundary; only the bounded metadata.json is read from disk. This + // rests on a sequencing invariant the store cannot enforce: nothing + // mutates the reactor's turn array between its last writeTurns and + // this read. The mirror runs at onRunBoundary after send() settles, + // and the reactor only appends inside a cycle (each ending in + // writeTurns), so peekTurns() equals the on-disk state here. + // Serializing the mirror entry points keeps two mirrors from + // overlapping this read, but the reactor must still not append + // between its writeTurns and this peek -- that axis is not serialized. + const turns = baseStorage.peekTurns(); + const metadata = await baseStorage.loadMetadata(); // First mirror in this store's lifetime that did not run through // `restoreFromSubstrate` (which sets the counts): learn the durable @@ -532,11 +582,16 @@ export async function createDurableConversationStore( // turn DELTA plus bounded metadata -- never the whole conversation, so // the O(N^2) growth stays gone. This is the single synchronous write // per boundary on the reply path. - const newTurns = loaded.turns.slice(mirroredTurnCount); + const newTurns = turns.slice(mirroredTurnCount); const boundarySeq = mirroredBoundaryCount; await appendWalEntry(boundarySeq, newTurns, metadata); mirroredBoundaryCount = boundarySeq + 1; - mirroredTurnCount = loaded.turns.length; + // Advance by the count actually persisted -- newTurns is a pre-await + // snapshot -- not by turns.length. `turns` is the reactor's live array + // by reference; reading its length after the await would count any turn + // appended during appendWalEntry as mirrored, so the next mirror would + // slice past it and drop it from the WAL permanently. + mirroredTurnCount = mirroredTurnCount + newTurns.length; // Compact once the live WAL reaches the interval (measured in mirror // boundaries = WAL entries, which bounds both the bucket fan-out and the @@ -546,7 +601,7 @@ export async function createDurableConversationStore( if (mirroredBoundaryCount - checkpointBoundarySeq >= CHECKPOINT_INTERVAL) { await writeCheckpoint( mirroredBoundaryCount, - loaded.turns.slice(0, mirroredTurnCount), + turns.slice(0, mirroredTurnCount), metadata, ); checkpointBoundarySeq = mirroredBoundaryCount; diff --git a/apps/sidecar/src/default-harness.ts b/apps/sidecar/src/default-harness.ts index 3679c308..dac66522 100644 --- a/apps/sidecar/src/default-harness.ts +++ b/apps/sidecar/src/default-harness.ts @@ -1,73 +1,21 @@ // Default HarnessBuilder for the sidecar app. // -// Implements the HarnessBuilder seam declared by @intx/hub-agent using -// the concrete plugins the sidecar app ships with: posix + LSP tools, -// the mail tools package, the isogit-backed context and mail audit -// stores, the authz engine, and the inference runtime. Any host that -// wants a different mix of plugins ships its own builder; the package -// never sees these concrete dependencies. +// Implements the HarnessBuilder source-admission seam declared by +// @intx/hub-agent using the sidecar app's adapter registry. The package +// never sees the concrete inference dependencies the check consults. -import fs from "node:fs"; -import path from "node:path"; -import { evaluateGrants } from "@intx/authz"; -import { - type BaseEnv, - createDefaultDirectorRegistry, - defineAgent, -} from "@intx/agent"; -import { createHarness, type MailEnv } from "@intx/harness"; -import { - readDeployTree, - type HarnessBuilder, - type HarnessBundle, -} from "@intx/hub-agent"; -import { createDependencies, type AdapterRegistry } from "@intx/inference"; -import { getLogger } from "@intx/log"; -import { - createIsogitStore, - createMailAuditStore, - type GCPolicy, -} from "@intx/storage-isogit"; +import { type AdapterRegistry } from "@intx/inference"; +import type { HarnessBuilder } from "@intx/hub-agent"; import type { InferenceSource } from "@intx/types/runtime"; -import { materializeToolPackages } from "./tool-materialization"; - -const logger = getLogger(["sidecar", "harness-builder"]); - export interface DefaultHarnessBuilderConfig { - /** - * Root directory of the content-addressable tarball cache. The - * loader instantiated on each apply opens a fresh `TarballCache` - * against this path. - */ - readonly cacheRoot: string; - /** - * Cache size cap, resolved at the boot edge so the per-apply - * `TarballCache` receives a concrete value rather than re-reading - * env at a non-boundary call site. - */ - readonly cacheMaxBytes: number; - /** - * Per-tarball cap on the loader's HTTP-registry fetcher. Resolved - * at the boot edge from `SIDECAR_REGISTRY_MAX_TARBALL_BYTES` so the - * per-apply loader receives a concrete value. - */ - readonly registryMaxTarballBytes: number; /** * Adapter registry resolved once at the boot edge (built-ins merged - * with any operator-configured custom adapters). It backs both the - * `canBuildSource` membership check and the per-agent - * `env.deps.adapters` used to resolve inference adapters at run time, - * so the single-agent in-process path resolves the same provider set - * the operator configured. + * with any operator-configured custom adapters). It backs the + * `canBuildSource` membership check the deploy router calls to admit a + * step's pinned inference source before spawning. */ readonly adapters: AdapterRegistry; - /** - * Write-path GC policy for the per-agent context repo. Resolved at the - * boot edge and handed to `createIsogitStore` so the reactor's commits - * reclaim the repo once it crosses the policy's thresholds. - */ - readonly gcPolicy: GCPolicy; } export function createDefaultHarnessBuilder( @@ -81,277 +29,5 @@ export function createDefaultHarnessBuilder( ); } }, - - async build({ - agentAddress, - agentConfig, - sources, - defaultSource, - storeDir, - agentTransport, - crypto, - onEvent, - onConnectorStateChanged, - emitDeployApplyError, - }): Promise { - const signer = (payload: string) => crypto.signSSH(payload); - - const storage = await createIsogitStore( - storeDir, - signer, - config.gcPolicy, - ); - const mailStore = await createMailAuditStore(storeDir, signer); - - const deployTree = await readDeployTree(storeDir); - const systemPrompt = deployTree.systemPrompt ?? agentConfig.systemPrompt; - - // Materialize tool-package manifest (if present) via the loader. - // Failures emit a deploy.apply.error frame back to the hub and - // abort harness construction; the prior deploy remains active. - const { factories: loadedToolFactories, pluginFactories: loadedPlugins } = - await materializeToolPackages({ - rawManifestBytes: deployTree.toolPackageManifestRaw, - assetMounts: deployTree.assetMounts, - storeDir, - agentAddress, - cacheRoot: config.cacheRoot, - cacheMaxBytes: config.cacheMaxBytes, - registryMaxTarballBytes: config.registryMaxTarballBytes, - emitDeployApplyError, - }); - - const grantsRef = { current: agentConfig.grants }; - const { principalId, tenantId } = agentConfig; - const authorize = async ( - resource: string, - action: string, - _context: unknown, - ) => - evaluateGrants(grantsRef.current, resource, action, { - principalId, - tenantId, - }); - - const workDir = path.join(storeDir, "workspace"); - await fs.promises.mkdir(workDir, { recursive: true }); - - // Per the @intx/agent ToolBundle contract, the agent does not - // invoke `bundle.dispose` on normal shutdown — only on - // construction rollback. The caller of `createHarness` is - // responsible for disposing the bundle resources at session - // end. Wrap each loaded factory so its bundle's `dispose` - // (when present) lands in `capturedDisposers`; the harness's - // `disposers` array below replays them on session teardown. - // Dedupe by disposer-function identity: a factory whose bundle - // returns the same `dispose` closure on every invocation would - // otherwise re-append it on agent rebuild / hot-reapply paths - // and the teardown loop would call it once per push. A `Set` - // keyed on the closure reference collapses identical pushes - // while still capturing distinct disposers from genuinely - // distinct bundles. - const capturedDisposers = new Set<() => unknown>(); - const factoriesWithCapture = loadedToolFactories.map((f) => { - const wrapped = (env: BaseEnv) => { - const bundle = f(env); - if (bundle.dispose !== undefined) - capturedDisposers.add(bundle.dispose); - return bundle; - }; - // Freeze the harness wrapper so downstream consumers cannot - // mutate `wrapped.id` / `wrapped.requires`. The loader already - // froze the source factory's wrapper, but `Object.assign` here - // produces a fresh outer object whose own `id`/`requires` - // would be writable without this freeze — defeating the - // invariant the loader's wrapper documents. Object.freeze on - // the outer reference is enough; the inner `requires` array - // is already frozen at the loader layer. - return Object.freeze( - Object.assign(wrapped, { id: f.id, requires: f.requires }), - ); - }); - - const def = defineAgent({ - id: agentAddress, - systemPrompt, - // Mail, posix, and LSP travel through the loader path as - // ordinary tool packages — the agent definition pins them and - // the materialization above resolved their factories. The - // sidecar harness no longer constructs them directly. - tools: factoriesWithCapture, - capabilities: [], - inference: { - sources: sources.map((s) => ({ - provider: s.provider, - model: s.model, - })), - }, - }); - - const baseEnv: MailEnv = { - sources, - defaultSource, - storage, - workdir: workDir, - audit: storage, - authorize, - directors: createDefaultDirectorRegistry(), - // Resolve inference adapters through the boot-edge registry - // (built-ins + operator custom adapters). Without this the agent - // would fall back to the built-ins-only default and a - // custom-provider source would fail to resolve at run time even - // though `canBuildSource` admitted it. - deps: createDependencies(config.adapters), - transport: agentTransport, - address: agentAddress, - onConnectorStateChanged, - }; - - // Instantiate plugin factories one at a time so each successive - // factory sees the prior plugins' instances on `env.plugins`. - // The chain is rebuilt incrementally rather than handing every - // factory the same baseEnv (which would force a plugin that - // wanted to consume other plugins to receive an undefined slot). - // Posix's bundle reads `env.plugins` and threads ToolPlugin- - // shaped values into `createPosixTools`; LSP's plugin factory - // is what populates them. - // - // If a midway factory throws, every plugin instance already - // constructed has to release whatever it acquired (LSP starts a - // subprocess) before the construction error propagates. - // Otherwise a partial-success chain leaks resources the - // harness never owned. - const pluginInstances: unknown[] = []; - let chainEnv: MailEnv = baseEnv; - try { - for (const factory of loadedPlugins) { - const instance = factory(chainEnv); - pluginInstances.push(instance); - // Each iteration constructs a fresh `chainEnv` object so a - // factory observing `env.plugins` sees the prior chain - // entries. The non-`plugins` fields (storage, transport, - // address, etc.) are shared by reference across every - // chainEnv produced here, so factories must not compare - // envs by identity — equality checks across iterations - // would always return false even though the underlying - // substrate is the same object graph. - chainEnv = { - ...baseEnv, - plugins: [...pluginInstances], - }; - } - } catch (err) { - for (const instance of pluginInstances) { - if (instance === null || typeof instance !== "object") continue; - if (!("dispose" in instance)) continue; - const dispose: unknown = (instance as { dispose: unknown }).dispose; - if (typeof dispose !== "function") continue; - try { - // `await` accepts non-promise values verbatim, so this - // works whether the plugin's disposer is sync or async - // without the Promise.resolve indirection that would - // silently flatten a thrown-then-rejected pair. - const result: unknown = dispose.call(instance); - await result; - } catch (disposeErr) { - // A disposer that throws during harness construction - // rollback leaks whatever the plugin acquired (LSP starts - // a subprocess; a leak survives the harness teardown). At - // this point the tool-package apply itself already - // succeeded — materializeToolPackages returned — so the - // failure is not attributable to any specific apply - // attempt. Emitting a deploy.apply.error frame with a - // freshly-minted attemptId and a `none` previousDeployId - // would land in the hub's audit trail uncorrelated with - // any real apply record, so log the leak and rely on the - // sidecar log line; the hub's session-launch failure - // (raised by the outer throw below) already informs the - // operator that the harness did not come up. - logger.error`plugin dispose failed during harness construction rollback: ${disposeErr instanceof Error ? disposeErr.message : String(disposeErr)}`; - } - } - throw err; - } - const env: MailEnv = chainEnv; - - const harness = await createHarness(def, env); - - // The sidecar's event channel expects onEvent calls. The - // harness exposes the underlying agent's event stream; forward - // every event onto the legacy callback EXCEPT `message.received`, - // which is the single intentional exclusion. - // - // Why `message.received` is filtered: - // - It is an assembly-internal signal. The reactor emits it - // when an inbound mail dequeues, but the hub-facing audit - // chain expresses per-message work as the - // `message.run.started` / `message.run.ended` bracket pair, - // minted around the same dequeue. Forwarding both would - // double-count per-message work in downstream consumers - // (SessionManager, workflow-runtime translation) and would - // leak the dequeue-vs-bracket distinction into a layer that - // has no use for it. - // - The raw inbound mail bytes are already authoritative in - // the mail-audit store; the bracket events carry messageId - // plus the reactor-minted messageRunId, which is what the - // audit chain needs to correlate. - // - // The filter is an allowlist-of-everything-except, so new - // InferenceEvent members flow through by default. In particular, - // `message.run.started` and `message.run.ended` are forwarded - // unchanged; widening this filter to exclude other event types - // would silently drop hub-facing audit data and must be done - // deliberately. - // - // `harness.stream()` is invoked synchronously here so the - // underlying StreamConsumer is registered before any microtask - // runs. The IIFE below iterates the returned AsyncIterable; - // events emitted in the window between createHarness() - // resolving and the for-await loop starting are buffered by the - // consumer and delivered on the first iteration. - const events = harness.stream(); - let stopForward = false; - const forwardDone = (async () => { - try { - for await (const event of events) { - if (stopForward) break; - if (event.type === "message.received") continue; - onEvent(event); - } - } catch (cause) { - logger.warn`Event forwarder terminated: ${cause}`; - } - })(); - - return { - harness, - mailStore, - updateGrants(grants) { - grantsRef.current = grants; - }, - // The event-forwarder shutdown runs first so no more events - // reach a downstream that is about to release resources. Then - // every captured `ToolBundle.dispose` runs in registration - // order — the agent built the bundles in factory order, so - // disposing in the same order matches that build sequence. - // Each dispose is wrapped in its own try so one runner's - // failure does not stop the others from running. - disposers: [ - async () => { - stopForward = true; - await forwardDone; - }, - async () => { - for (const dispose of capturedDisposers) { - try { - await dispose(); - } catch (disposeErr) { - logger.warn`tool bundle dispose failed during session teardown: ${disposeErr instanceof Error ? disposeErr.message : String(disposeErr)}`; - } - } - }, - ], - }; - }, }; } diff --git a/apps/sidecar/src/index.ts b/apps/sidecar/src/index.ts index 22762c19..3c0ba1c6 100644 --- a/apps/sidecar/src/index.ts +++ b/apps/sidecar/src/index.ts @@ -17,7 +17,6 @@ import { loadAdapterRegistry } from "@intx/inference/providers"; import { readAdapterManifest, - readAgentGCPolicy, readCacheMaxBytes, readRegistryMaxTarballBytes, } from "./config"; @@ -26,20 +25,21 @@ import { createDefaultHarnessBuilder } from "./default-harness"; // graph so the supervisor surface is reachable from this binary. // `createSidecarDeployRouter` is the production routing the // orchestrator hands to the link's `agent.deploy` handler; every -// inbound frame flows through a freshly-constructed workflow-host -// supervisor whose trivial branch calls back into the sidecar's -// existing single-agent provisioning surface. +// inbound frame stages through the workflow-run substrate, spawning a +// supervised workflow-process child for a workflow deploy. import type { DispatchTimingMark } from "@intx/workflow-host"; import { createSidecarDeployRouter, createSidecarWorkflowSupervisor, + type SidecarDeployRouter, } from "./workflow-host-wiring"; import { createDeploymentAddressRegistry, createMultistepDrainRouter, createMultistepMailRouter, createMultistepSignalRouter, + createMultistepSourcesRouter, createWorkflowRunPackClient, createWorkflowRunPackPushingRepoStore, } from "./workflow-run-pack-client"; @@ -57,9 +57,9 @@ function requireEnv(name: string): string { const dataDir = requireEnv("SIDECAR_DATA_DIR"); -// Resolve cache configuration at the boot edge so the per-apply loader -// inside the harness builder receives a concrete path and cap rather -// than re-reading env at non-boundary call sites. +// Resolve cache configuration at the boot edge so the workflow-child's +// per-apply loader receives a concrete path and cap through its spawn +// env rather than re-reading env at non-boundary call sites. const sidecarCacheDir = process.env["SIDECAR_CACHE_DIR"]; const cacheRoot = sidecarCacheDir !== undefined && sidecarCacheDir.trim() !== "" @@ -67,15 +67,14 @@ const cacheRoot = : path.join(dataDir, "cache", "tarballs"); const cacheMaxBytes = readCacheMaxBytes(); const registryMaxTarballBytes = readRegistryMaxTarballBytes(); -const agentGCPolicy = readAgentGCPolicy(); // Operator-configured custom inference adapters, resolved once at the // boot edge. `loadAdapterRegistry` merges the statically-linked // built-ins with any custom adapters the manifest names, importing each // custom module eagerly here so a bad specifier fails the sidecar at -// boot rather than at first inference. The SAME registry is threaded -// into the in-process single-agent harness builder below; the workflow -// child cannot receive this object across the fork, so the validated +// boot rather than at first inference. The SAME registry backs the +// deploy router's source-admission check below; the workflow child +// cannot receive this object across the fork, so the validated // manifest is serialized into the child's spawn env (see // `multistepSubstrateEnv`) and the child rebuilds an equivalent // registry from it. Specifiers are operator-config-only — the agent @@ -165,6 +164,23 @@ if (consumedRetentionRaw !== undefined && consumedRetentionRaw.trim() !== "") { consumedRetentionMs = parsed; } +// Bound on the child's spawn-time `ready` handshake. Threaded to every +// per-deployment supervisor; on expiry the supervisor kills the child and +// rejects the spawn, so a child that spawns but never signals ready fails +// the deploy (or is skipped by boot-time restore) instead of hanging it. +// Absent, the supervisor applies its 30s default. +const readyTimeoutRaw = process.env["CHILD_READY_TIMEOUT_MS"]; +let readyTimeoutMs: number | undefined; +if (readyTimeoutRaw !== undefined && readyTimeoutRaw.trim() !== "") { + const parsed = Number.parseInt(readyTimeoutRaw, 10); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error( + `CHILD_READY_TIMEOUT_MS must be a positive integer (milliseconds), got ${readyTimeoutRaw}`, + ); + } + readyTimeoutMs = parsed; +} + // Sweep any tmp staging directories left behind by a `put` or // `extractTarball` that crashed between staging and the final rename // on a previous boot. Running here, before the orchestrator starts @@ -186,11 +202,11 @@ const SIDECAR_SIGNING_DIR = path.join(dataDir, ".sidecar-signing"); const sidecarSigningKey = await loadOrMintSidecarKeypair(SIDECAR_SIGNING_DIR); // Construct the substrate-backed RepoStore at the boot edge. The -// supervisor consumes this through the deploy router; the trivial -// branch's `recordRunEvent` reaches `writeTreePreservingPrefix` on -// this store. The boot-edge facade below wraps the store so a -// successful workflow-run write fires the pack push hook before its -// Promise resolves. +// supervisor consumes this through the deploy router; the supervisor's +// conversation-state writes reach `writeTreePreservingPrefix` on this +// store. The boot-edge facade below wraps the store so a successful +// workflow-run write fires the pack push hook before its Promise +// resolves. const agentRepoStore = createAgentRepoStore({ dataDir, signingKey: sidecarSigningKey, @@ -202,15 +218,13 @@ const agentRepoStore = createAgentRepoStore({ // agentAddress for hub-side routing. const deploymentAddressRegistry = createDeploymentAddressRegistry(); -// Per-deployment-address mail handler registry the hub-link consults -// before falling back to the legacy `transport.deliver` / -// `sessions.commitInboundMail` path. The deploy router's multi-step +// Per-deployment-address mail handler registry the hub-link consults on +// every inbound `mail.inbound` frame. The deploy router's multi-step // branch registers `wired.routeInbound` against the deployment's mail // address once its supervisor spawns; the hub-link's `mail.inbound` -// handler calls `tryRoute` first so an inbound deployment-address -// message lands on the supervisor's mail-bus subscription instead of -// the legacy session path (which has no `transport` registration and -// no `sessions` entry for the deployment address). +// handler calls `tryRoute` so an inbound deployment-address message +// lands on the supervisor's mail-bus subscription. Mail for an address +// with no registered handler has no receiver and is logged-and-dropped. const multistepMailRouter = createMultistepMailRouter(); // Per-deployment-address signal handler registry the hub-link consults @@ -235,6 +249,15 @@ const multistepSignalRouter = createMultistepSignalRouter(); // workflow-run repo when the deadline expires. const multistepDrainRouter = createMultistepDrainRouter(); +// Per-deployment-address sources-rotation handler registry. Only a +// single-step warm deployment registers a handler once its supervisor +// spawns; the handler forwards the rotated list into the supervisor's +// `deliverSources`, which sends a `sources-updated` control IPC frame to +// the child, where the warm agent's live sources are swapped in place. A +// multi-step deployment registers none, so a rotation resolved against +// its address is unrouted. +const multistepSourcesRouter = createMultistepSourcesRouter(); + const transport = createInMemoryTransport(); // The pack-push client closes over the substrate (for `createPack`) @@ -296,9 +319,9 @@ const multistepSubstrateEnv: Record = { SIDECAR_TOKEN: sidecarToken, PATH: requireEnv("PATH"), // Tool-loader caps the child's per-step tool materialization uses. - // Resolved once at the boot edge (same values the in-process harness - // builder receives) and threaded into the child through the substrate - // config so the child does not re-read env at a non-boundary site. + // Resolved once at the boot edge and threaded into the child through + // the substrate config so the child does not re-read env at a + // non-boundary site. SIDECAR_CACHE_MAX_BYTES: String(cacheMaxBytes), SIDECAR_REGISTRY_MAX_TARBALL_BYTES: String(registryMaxTarballBytes), // Serialize the parent's ALREADY-VALIDATED manifest (the object @@ -320,20 +343,22 @@ if (hostTmpdir !== undefined) { multistepSubstrateEnv["TMPDIR"] = hostTmpdir; } +// The deploy router's source-admission gate reuses this exact +// `canBuildSource` predicate, against the one adapter registry, rather +// than a second copy of the check. +const buildHarness = createDefaultHarnessBuilder({ adapters }); + +// Set by the `createDeployRouter` callback below (invoked synchronously +// during construction) so the boot edge can drive the router's restore pass +// before `orchestrator.start()` connects to the hub. +let sidecarDeployRouter: SidecarDeployRouter | undefined; + const orchestrator = createSidecarOrchestrator({ hubURL: hubWsUrl, sidecarId, token: sidecarToken, dataDir, transport, - buildHarness: createDefaultHarnessBuilder({ - cacheRoot, - cacheMaxBytes, - registryMaxTarballBytes, - adapters, - gcPolicy: agentGCPolicy, - }), - createAgentCrypto: createEd25519Crypto, cryptoOps: { generateKeyPair, signEd25519, @@ -342,20 +367,34 @@ const orchestrator = createSidecarOrchestrator({ mailInboundRouter: multistepMailRouter, signalInboundRouter: multistepSignalRouter, drainInboundRouter: multistepDrainRouter, + sourcesInboundRouter: multistepSourcesRouter, + // The hub link calls this on every (re)connect to announce the workflow + // deployments this sidecar hosts so the hub re-registers their routes. + // `createDeployRouter` runs synchronously during construction (below), so + // the router is captured before the link ever connects; assert rather than + // optional-chain so a wiring regression fails loud instead of silently + // announcing no deployments. + getWorkflowAddresses: () => { + if (sidecarDeployRouter === undefined) { + throw new Error( + "sidecar boot: deploy router was not constructed before the hub link requested workflow addresses", + ); + } + return sidecarDeployRouter.activeAddresses(); + }, createDeployRouter: ({ sessions, keyStore, - onAgentEvent, publishWorkflowInferenceEvent, - }) => - createSidecarDeployRouter({ + }) => { + const router = createSidecarDeployRouter({ sessions, keyStore, - onAgentEvent, transport, repoStore: wrappedRepoStore, signingKeySeed: sidecarSigningKey.privateKey, createAgentCrypto: createEd25519Crypto, + assertSourceBuildable: buildHarness.canBuildSource, registerDeployment: ({ deploymentId, agentAddress }) => { deploymentAddressRegistry.record(deploymentId, agentAddress); }, @@ -365,16 +404,39 @@ const orchestrator = createSidecarOrchestrator({ multistepMailRouter, multistepSignalRouter, multistepDrainRouter, + multistepSourcesRouter, multistepSubstrateEnv, publishWorkflowInferenceEvent, ...(onDispatchTiming !== undefined ? { onDispatchTiming } : {}), ...(repackEveryMessages !== undefined ? { repackEveryMessages } : {}), ...(consumedRetentionMs !== undefined ? { consumedRetentionMs } : {}), - }), + ...(readyTimeoutMs !== undefined ? { readyTimeoutMs } : {}), + }); + // Capture the router so the boot edge can drive its restore pass before + // connecting. `createDeployRouter` runs synchronously during + // `createSidecarOrchestrator` construction (exactly once, before the + // handle returns), so `sidecarDeployRouter` is populated by the time the + // restore call below runs. + sidecarDeployRouter = router; + return router; + }, }); resolvedHubLink = orchestrator.hubLink; +// Re-establish the workflow deployments a prior sidecar process persisted, +// BEFORE opening the hub connection: each single-step head must have its +// mailbox/transport registration live before the hub can route to it. +// Assert the router was captured rather than optional-chaining it, so a +// future refactor that made `createDeployRouter` fire lazily would fail loud +// here instead of silently skipping restore. +if (sidecarDeployRouter === undefined) { + throw new Error( + "sidecar boot: deploy router was not constructed before workflow-deployment restore", + ); +} +await sidecarDeployRouter.restoreWorkflowDeployments(); + orchestrator.start(); // Keep `createSidecarWorkflowSupervisor` reachable from this entry diff --git a/apps/sidecar/src/step-agent-tools.ts b/apps/sidecar/src/step-agent-tools.ts index ef7d5934..d1d5e4ed 100644 --- a/apps/sidecar/src/step-agent-tools.ts +++ b/apps/sidecar/src/step-agent-tools.ts @@ -37,14 +37,10 @@ import { type BaseEnv, type ToolBundle, } from "@intx/agent"; -import { - readDeployTree, - sanitizeAddress, - type DeployApplyErrorEmitter, -} from "@intx/hub-agent/paths"; +import { readDeployTree, sanitizeAddress } from "@intx/hub-agent/paths"; import { getLogger } from "@intx/log"; import type { LoadedToolFactory } from "@intx/tool-packaging"; -import { deriveStepAddress } from "@intx/workflow-deploy"; +import { resolveStepAddress } from "@intx/workflow-deploy"; import { parseAgentAddress } from "@intx/types"; import { materializeToolPackages } from "./tool-materialization"; @@ -57,8 +53,8 @@ const INSTANCE_PREFIX = "ins_"; * Cache and registry caps the per-step tool loader needs. Resolved at * the sidecar boot edge from the existing `SIDECAR_CACHE_*` / * `SIDECAR_REGISTRY_*` config keys and threaded into the child through - * the substrate config, so the child's per-step materialization uses - * the same caps the in-process harness builder does. + * the substrate config, so the child's per-step materialization is + * bounded by those boot-edge-resolved caps. */ export interface StepToolCacheConfig { readonly cacheMaxBytes: number; @@ -133,18 +129,23 @@ function isStepToolMaterialization( * substrate's `agent-state/` layout -- the multi-step deploy path * never pushes step `agent-state` packs to the child's substrate. * - * The step's mail address is the orchestrator's - * `deriveStepAddress(deploymentId, stepId, deploymentDomain)`. The - * orchestrator's `deploymentId`/`deploymentDomain` are recovered from - * the deployment mailbox address the supervisor threaded into the + * The step's mail address is `resolveStepAddress(...)`, the single owner + * of the head/step collapse: for a single-step deployment the lone step + * IS the head (the deployment mailbox itself), so the tree is read at the + * head; for multi-step it is `deriveStepAddress(deploymentId, stepId, + * deploymentDomain)`. The `deploymentId`/`deploymentDomain` are recovered + * from the deployment mailbox address the supervisor threaded into the * child as `MAILBOX_ADDRESS` (`ins_@`): the - * instance-id local part minus the `ins_` prefix is the deploymentId, - * and the address domain is the deploymentDomain. + * instance-id local part minus the `ins_` prefix is the deploymentId, and + * the address domain is the deploymentDomain. `stepCount` is sourced from + * the host (via `substrateEnv`) so producer and consumer never derive + * divergent addresses. */ export function stepDeployTreeDir(args: { dataDir: string; mailboxAddress: string; stepId: string; + stepCount: number; }): string { const parsed = parseAgentAddress(args.mailboxAddress); if (parsed === null) { @@ -158,10 +159,11 @@ export function stepDeployTreeDir(args: { ); } const deploymentId = parsed.instanceId.slice(INSTANCE_PREFIX.length); - const stepAddress = deriveStepAddress({ + const stepAddress = resolveStepAddress({ deploymentId, stepId: args.stepId, deploymentDomain: parsed.domain, + stepCount: args.stepCount, }); return path.join(args.dataDir, sanitizeAddress(stepAddress)); } @@ -176,22 +178,23 @@ export function stepDeployTreeDir(args: { * A deploy with no tool-package manifest yields empty factories -- the * legitimate `rawManifestBytes === undefined` case. A manifest that is * present but fails to load surfaces loudly through - * `materializeToolPackages` (the deploy.apply.error / throw path), - * never a silent empty-tools fallback that would mask a broken deploy. + * `materializeToolPackages` (the throw path), never a silent + * empty-tools fallback that would mask a broken deploy. */ export async function materializeStepTools(args: { dataDir: string; mailboxAddress: string; stepId: string; + stepCount: number; /** Per-step state root; cache + instance dir + workspace live under it. */ storeDir: string; cache: StepToolCacheConfig; - emitDeployApplyError?: DeployApplyErrorEmitter; }): Promise { const deployTreeDir = stepDeployTreeDir({ dataDir: args.dataDir, mailboxAddress: args.mailboxAddress, stepId: args.stepId, + stepCount: args.stepCount, }); const deployTree = await readDeployTree(deployTreeDir); @@ -219,9 +222,6 @@ export async function materializeStepTools(args: { cacheRoot, cacheMaxBytes: args.cache.cacheMaxBytes, registryMaxTarballBytes: args.cache.registryMaxTarballBytes, - ...(args.emitDeployApplyError !== undefined - ? { emitDeployApplyError: args.emitDeployApplyError } - : { emitDeployApplyError: undefined }), }); return { factories: materialized.factories, diff --git a/apps/sidecar/src/tool-materialization.test.ts b/apps/sidecar/src/tool-materialization.test.ts index 46d13c3a..6139f61a 100644 --- a/apps/sidecar/src/tool-materialization.test.ts +++ b/apps/sidecar/src/tool-materialization.test.ts @@ -27,31 +27,9 @@ afterEach(async () => { ); }); -interface CapturedDeployApplyError { - attemptId: string; - previousDeployId: string; - category: string; - message: string; - occurredAt: string; -} - -function captureEmitter(): { - calls: CapturedDeployApplyError[]; - emit: (e: CapturedDeployApplyError) => void; -} { - const calls: CapturedDeployApplyError[] = []; - return { - calls, - emit: (e) => { - calls.push(e); - }, - }; -} - describe("materializeToolPackages — manifest.invalid gate", () => { test("returns empty factories when no manifest bytes supplied", async () => { const storeDir = await tempDir(); - const { calls, emit } = captureEmitter(); const result = await materializeToolPackages({ assetMounts: new Map(), rawManifestBytes: undefined, @@ -60,16 +38,13 @@ describe("materializeToolPackages — manifest.invalid gate", () => { cacheRoot: "/unused-by-manifest-invalid-gate", cacheMaxBytes: 1024 * 1024, registryMaxTarballBytes: 10 * 1024 * 1024, - emitDeployApplyError: emit, }); expect(result.factories).toEqual([]); expect(result.pluginFactories).toEqual([]); - expect(calls).toEqual([]); }); - test("JSON.parse failure persists the raw bytes verbatim and emits manifest.invalid", async () => { + test("JSON.parse failure throws and persists the raw bytes verbatim", async () => { const storeDir = await tempDir(); - const { calls, emit } = captureEmitter(); const corrupt = "{this is not valid json"; let caught: unknown; try { @@ -81,7 +56,6 @@ describe("materializeToolPackages — manifest.invalid gate", () => { cacheRoot: "/unused-by-manifest-invalid-gate", cacheMaxBytes: 1024 * 1024, registryMaxTarballBytes: 10 * 1024 * 1024, - emitDeployApplyError: emit, }); } catch (err) { caught = err; @@ -90,12 +64,6 @@ describe("materializeToolPackages — manifest.invalid gate", () => { expect(String(caught)).toMatch(/manifest\.invalid/); expect(String(caught)).toMatch(/JSON\.parse failed/); - expect(calls).toHaveLength(1); - const call = calls[0]; - if (call === undefined) throw new Error("emitter was not called"); - expect(call.category).toBe("manifest.invalid"); - expect(call.previousDeployId).toBe("none"); - const auditRoot = path.join(storeDir, "audit", "rejected-applies"); const attemptDirs = await fs.promises.readdir(auditRoot); expect(attemptDirs).toHaveLength(1); @@ -121,12 +89,13 @@ describe("materializeToolPackages — manifest.invalid gate", () => { ); expect(errorJson.category).toBe("manifest.invalid"); expect(errorJson.message).toMatch(/JSON\.parse failed/); - expect(errorJson.attemptId).toBe(call.attemptId); + expect(errorJson.previousDeployId).toBe("none"); + expect(typeof errorJson.attemptId).toBe("string"); + expect(errorJson.attemptId.length).toBeGreaterThan(0); }); - test("schema failure persists the raw bytes verbatim and emits manifest.invalid", async () => { + test("schema failure throws and persists the raw bytes verbatim", async () => { const storeDir = await tempDir(); - const { calls, emit } = captureEmitter(); // Valid JSON but wrong shape — arktype rejects it. const wrongShape = JSON.stringify({ schemaVersion: 99 }); let caught: unknown; @@ -139,7 +108,6 @@ describe("materializeToolPackages — manifest.invalid gate", () => { cacheRoot: "/unused-by-manifest-invalid-gate", cacheMaxBytes: 1024 * 1024, registryMaxTarballBytes: 10 * 1024 * 1024, - emitDeployApplyError: emit, }); } catch (err) { caught = err; @@ -148,11 +116,6 @@ describe("materializeToolPackages — manifest.invalid gate", () => { expect(String(caught)).toMatch(/manifest\.invalid/); expect(String(caught)).toMatch(/schema validation failed/); - expect(calls).toHaveLength(1); - const call = calls[0]; - if (call === undefined) throw new Error("emitter was not called"); - expect(call.category).toBe("manifest.invalid"); - const auditRoot = path.join(storeDir, "audit", "rejected-applies"); const attemptDirs = await fs.promises.readdir(auditRoot); expect(attemptDirs).toHaveLength(1); @@ -170,6 +133,11 @@ describe("materializeToolPackages — manifest.invalid gate", () => { "utf-8", ); expect(persisted).toBe(wrongShape); + + const errorJson = JSON.parse( + await fs.promises.readFile(path.join(attemptDir, "error.json"), "utf-8"), + ); + expect(errorJson.category).toBe("manifest.invalid"); }); describe("parseActiveDeployId", () => { @@ -290,12 +258,12 @@ describe("materializeToolPackages — manifest.invalid gate", () => { }); }); - test("boot reconciliation reads the dirty marker as previousDeployId", async () => { + test("boot reconciliation records the dirty marker as previousDeployId", async () => { // Seed an instance where the prior apply could not durably record // the committed deploy id and a dirty marker carries the truth. // The next apply (any apply — here, a manifest-invalid one) must - // surface the marker's id as `previousDeployId` in the failure - // frame, not the stale recorded id and not "none". + // surface the marker's id as `previousDeployId` in the audit + // record, not the stale recorded id and not "none". const storeDir = await tempDir(); const instanceDir = path.join(storeDir, "tool-packages"); await fs.promises.mkdir(instanceDir, { recursive: true }); @@ -308,7 +276,6 @@ describe("materializeToolPackages — manifest.invalid gate", () => { "v1:truth-from-marker", ); - const { calls, emit } = captureEmitter(); let caught: unknown; try { await materializeToolPackages({ @@ -319,21 +286,29 @@ describe("materializeToolPackages — manifest.invalid gate", () => { cacheRoot: "/unused-by-manifest-invalid-gate", cacheMaxBytes: 1024 * 1024, registryMaxTarballBytes: 10 * 1024 * 1024, - emitDeployApplyError: emit, }); } catch (err) { caught = err; } expect(caught).toBeInstanceOf(Error); - expect(calls).toHaveLength(1); - const call = calls[0]; - if (call === undefined) throw new Error("emitter was not called"); - expect(call.previousDeployId).toBe("truth-from-marker"); + + const auditRoot = path.join(storeDir, "audit", "rejected-applies"); + const attemptDirs = await fs.promises.readdir(auditRoot); + expect(attemptDirs).toHaveLength(1); + const attemptDirName = attemptDirs[0]; + if (attemptDirName === undefined) { + throw new Error("no attempt dir was created"); + } + const errorJson = JSON.parse( + await fs.promises.readFile( + path.join(auditRoot, attemptDirName, "error.json"), + "utf-8", + ), + ); + expect(errorJson.previousDeployId).toBe("truth-from-marker"); }); - test("manifest.invalid still throws when no emitDeployApplyError is wired", async () => { - // The hub-side emitter is optional (e.g., for tests or offline - // boots). The gate must still reject and surface to the caller. + test("manifest.invalid throws and writes an audit entry", async () => { const storeDir = await tempDir(); let caught: unknown; try { @@ -341,11 +316,10 @@ describe("materializeToolPackages — manifest.invalid gate", () => { assetMounts: new Map(), rawManifestBytes: "{ not json", storeDir, - agentAddress: "agent-no-emitter", + agentAddress: "agent-throws", cacheRoot: "/unused-by-manifest-invalid-gate", cacheMaxBytes: 1024 * 1024, registryMaxTarballBytes: 10 * 1024 * 1024, - emitDeployApplyError: undefined, }); } catch (err) { caught = err; diff --git a/apps/sidecar/src/tool-materialization.ts b/apps/sidecar/src/tool-materialization.ts index d7fb24d8..af623345 100644 --- a/apps/sidecar/src/tool-materialization.ts +++ b/apps/sidecar/src/tool-materialization.ts @@ -2,24 +2,23 @@ // // The deploy-tree reader, the `@intx/tool-packaging` loader // construction, `applyAtomic`, and the active-deploy-id persistence -// ladder live here so both the in-process harness builder -// (`default-harness.ts`) and the workflow-process child's substrate -// factory (`workflow-substrate-factory.ts`) reach the same -// implementation. Keeping this in `apps/sidecar` (and out of -// `@intx/workflow-host`) preserves the portable orchestration -// package's host-agnostic layering: the child IS the sidecar binary, -// so the sidecar's tool runtime is already present in the child's -// address space without the portable package depending on it. +// ladder live here so the workflow-process child's substrate factory +// (`workflow-substrate-factory.ts`) reaches this implementation without +// the portable orchestration package depending on the sidecar's tool +// runtime. Keeping this in `apps/sidecar` (and out of +// `@intx/workflow-host`) preserves that package's host-agnostic +// layering: the child IS the sidecar binary, so the sidecar's tool +// runtime is already present in the child's address space. // // The module is deliberately free of any `@intx/harness` reactor -// dependency so the workflow-process child does not drag the -// in-process harness's transport/reactor ownership into its graph. +// dependency so the workflow-process child's boot graph does not pull +// in `@intx/harness`'s transport/reactor stack -- a forbidden-import +// guard enforces that boundary. import fs from "node:fs"; import path from "node:path"; import { type } from "arktype"; import { type AnnotatedPluginFactory } from "@intx/agent"; -import { type DeployApplyErrorEmitter } from "@intx/hub-agent/paths"; import { getLogger } from "@intx/log"; import { type LoadedToolFactory, @@ -172,10 +171,9 @@ interface MaterializedToolPackages { } // Exported for direct unit testing of the manifest-invalid gate -// (JSON.parse failure + arktype schema failure). The in-process -// harness builder reaches it through `createDefaultHarnessBuilder`; -// the workflow-process child reaches it through the substrate -// factory's per-step agent builder. +// (JSON.parse failure + arktype schema failure). The workflow-process +// child reaches it through the substrate factory's per-step agent +// builder. export async function materializeToolPackages(args: { rawManifestBytes: string | undefined; /** @@ -188,9 +186,8 @@ export async function materializeToolPackages(args: { /** * Workspace root the loader resolves `kind: "asset"` tarball mounts * against (`//...`). Defaults to - * `/workspace` -- the in-process harness builder's layout, - * where the deploy flow stages asset packs into the same dir the - * agent runs in. + * `/workspace` -- the default workspace layout, where the + * deploy flow stages asset packs into the same dir the agent runs in. * * The workflow-process child overrides this: it stages assets in the * step's LEGACY agent dir workspace (where the hub's asset-pack push @@ -203,7 +200,6 @@ export async function materializeToolPackages(args: { cacheRoot: string; cacheMaxBytes: number; registryMaxTarballBytes: number; - emitDeployApplyError: DeployApplyErrorEmitter | undefined; }): Promise { if (args.rawManifestBytes === undefined) { return { factories: [], pluginFactories: [] }; @@ -264,8 +260,8 @@ export async function materializeToolPackages(args: { // failures and arktype schema failures route through the same // `manifest.invalid` category so the hub sees a single failure // shape for malformed manifests regardless of the specific defect. - // The helper persists the rejected bytes + the failure payload and - // emits the WS frame, then returns the Error for the caller to throw. + // The helper persists the rejected bytes + the failure payload to + // the durable audit, then returns the Error for the caller to throw. // Returning instead of throwing lets the caller use `throw await ...`, // which TypeScript narrows control flow against without any // dead-code suffix at the call site. @@ -285,15 +281,6 @@ export async function materializeToolPackages(args: { occurredAt, }, }); - if (args.emitDeployApplyError !== undefined) { - args.emitDeployApplyError({ - attemptId, - previousDeployId, - category: "manifest.invalid", - message, - occurredAt, - }); - } return new Error( `tool-package apply rejected (manifest.invalid): ${reason}`, ); @@ -356,16 +343,6 @@ export async function materializeToolPackages(args: { manifestBytes: rawManifestBytes, failure: result, }); - if (args.emitDeployApplyError !== undefined) { - args.emitDeployApplyError({ - attemptId: result.attemptId, - previousDeployId: result.previousDeployId, - category: result.category, - message: result.message, - ...(result.package !== undefined ? { package: result.package } : {}), - occurredAt: result.occurredAt, - }); - } throw new Error( `tool-package apply rejected (${result.category}): ${result.message}`, ); @@ -379,13 +356,12 @@ export async function materializeToolPackages(args: { // `persistActiveDeployIdWithFallback` has already written the new id // through its no-fsync / dirty-marker degradation ladder, so the new // deploy is logically committed, but the id was not durably flushed. - // Route this through `apply.previous-rotation.failed` — the wire - // contract for that category says `previousDeployId` carries the - // **new** deploy id (the one now live) so the hub records the on-disk - // truth. Emit the failure frame and the audit entry, then throw so - // the harness tears down: the durability gap means the next apply - // cannot trust `previousDeployId` until the next boot reconciles via - // the dirty marker. + // Record this as `apply.previous-rotation.failed` — for that category + // `previousDeployId` carries the **new** deploy id (the one now live) + // so the durable audit records the on-disk truth. Write the audit + // entry, then throw so the harness tears down: the durability gap + // means the next apply cannot trust `previousDeployId` until the next + // boot reconciles via the dirty marker. const persistOutcome = await persistActiveDeployIdWithFallback( instanceDir, activeIdFile, @@ -408,15 +384,6 @@ export async function materializeToolPackages(args: { occurredAt, }, }); - if (args.emitDeployApplyError !== undefined) { - args.emitDeployApplyError({ - attemptId, - previousDeployId: result.activeDeployId, - category: "apply.previous-rotation.failed", - message, - occurredAt, - }); - } throw new Error( `tool-package apply rejected (apply.previous-rotation.failed): ${message}`, { cause: err }, @@ -560,11 +527,11 @@ export async function clearDirtyMarker( * * Returns `{ degraded: false }` on full success. Returns * `{ degraded: true, error }` when the primary persist failed; the - * caller routes the existing `apply.previous-rotation.failed` frame on - * the back of it. + * caller records an `apply.previous-rotation.failed` audit entry and + * throws on the back of it. * * If the fallback persist also fails, on-disk state will diverge from - * the failure frame for one boot cycle. Boot-time reconciliation reads + * the recorded id for one boot cycle. Boot-time reconciliation reads * the dirty marker and prefers it. * * Exported for direct unit testing of the dirty-marker contract. @@ -592,7 +559,7 @@ export async function persistActiveDeployIdWithFallback( logger.warn`active-deploy-id dirty marker written to ${dirtyPath}; next boot will prefer it over the stale recorded id`; return { degraded: true, error: primary }; } catch (marker) { - logger.error`active-deploy-id dirty marker write also failed for ${activeIdFile}.dirty: ${marker instanceof Error ? marker.message : String(marker)}; on-disk state will diverge from the failure frame for one boot cycle`; + logger.error`active-deploy-id dirty marker write also failed for ${activeIdFile}.dirty: ${marker instanceof Error ? marker.message : String(marker)}; on-disk state will diverge from the recorded id for one boot cycle`; return { degraded: true, error: primary }; } } @@ -645,14 +612,14 @@ async function writeRejectedApplyAudit(args: { ); await fs.promises.mkdir(dir, { recursive: true }); // fsync the files and their parent directory before returning so the - // WS frame emitted by the caller is the second event in the durable - // sequence, not the first. Otherwise, a crash after the frame leaves - // the wire but before the audit bytes hit disk would have the hub - // know about a rejection the sidecar cannot prove the manifest for - // on replay. fsync failures are downgraded to warnings: the bytes - // are written either way, and a refusing fsync (rare networked-FS - // failure mode) should not turn into a second cascading failure on - // an already-failing apply. + // rejection's evidence is durable before the caller throws and tears + // the apply down. Otherwise, a crash after the throw begins + // propagating but before the audit bytes hit disk would leave a + // rejected apply the sidecar cannot prove the manifest for on replay. + // fsync failures are downgraded to warnings: the bytes are written + // either way, and a refusing fsync (rare networked-FS failure mode) + // should not turn into a second cascading failure on an already- + // failing apply. await fsyncWriteFile(path.join(dir, "manifest.json"), args.manifestBytes); await fsyncWriteFile( path.join(dir, "error.json"), diff --git a/apps/sidecar/src/workflow-deployment-record.test.ts b/apps/sidecar/src/workflow-deployment-record.test.ts new file mode 100644 index 00000000..8b3c8426 --- /dev/null +++ b/apps/sidecar/src/workflow-deployment-record.test.ts @@ -0,0 +1,204 @@ +import { describe, test, expect } from "bun:test"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import { type } from "arktype"; + +import { + WorkflowDeploymentRecord, + writeWorkflowDeploymentRecord, + deleteWorkflowDeploymentRecord, + scanWorkflowDeploymentRecords, +} from "./workflow-deployment-record"; + +async function makeDataDir(): Promise { + return fs.mkdtemp(path.join(os.tmpdir(), "wdr-")); +} + +function recordPath(dataDir: string, deploymentId: string): string { + return path.join(dataDir, "workflow-runs", deploymentId, "deployment.json"); +} + +async function fileExists(p: string): Promise { + try { + await fs.access(p); + return true; + } catch { + return false; + } +} + +const SINGLE_STEP: WorkflowDeploymentRecord = { + version: 1, + agentAddress: "ins_abc123@tenant.example", + definitionId: "wf_abc123", + sources: { + "step-1": [ + { + id: "anthropic:mock", + provider: "anthropic", + baseURL: "https://api.example/anthropic", + apiKey: "sk-x", + model: "claude-mock", + }, + ], + }, + sessionId: "ses_1", + hubPublicKey: "deadbeef", +}; + +// A multi-step deployment records no head hub key and may carry no session +// id -- both optional fields absent. +const MULTI_STEP: WorkflowDeploymentRecord = { + version: 1, + agentAddress: "ins_dep_xyz@tenant.example", + definitionId: "wf_xyz", + sources: { + plan: [ + { + id: "anthropic:mock", + provider: "anthropic", + baseURL: "https://api.example/anthropic", + apiKey: "sk-y", + model: "claude-mock", + }, + ], + execute: [ + { + id: "openai:mock", + provider: "openai", + baseURL: "https://api.example/openai", + apiKey: "sk-z", + model: "gpt-mock", + }, + ], + }, +}; + +describe("workflow deployment record store", () => { + test("round-trips a schema-valid record through disk (single-step)", async () => { + const dataDir = await makeDataDir(); + const deploymentId = "abc123-tenant-example"; + await writeWorkflowDeploymentRecord(dataDir, deploymentId, SINGLE_STEP); + + // The record embeds source apiKeys, so it must not be group/world + // readable on a shared host. + const stat = await fs.stat(recordPath(dataDir, deploymentId)); + expect(stat.mode & 0o077).toBe(0); + + const raw = await fs.readFile(recordPath(dataDir, deploymentId), "utf8"); + const parsed = WorkflowDeploymentRecord(JSON.parse(raw)); + if (parsed instanceof type.errors) { + throw new Error(`record failed validation: ${parsed.summary}`); + } + expect(parsed).toEqual(SINGLE_STEP); + + await fs.rm(dataDir, { recursive: true, force: true }); + }); + + test("round-trips a record with the optional fields absent (multi-step)", async () => { + const dataDir = await makeDataDir(); + const deploymentId = "dep_xyz-tenant-example"; + await writeWorkflowDeploymentRecord(dataDir, deploymentId, MULTI_STEP); + + const raw = await fs.readFile(recordPath(dataDir, deploymentId), "utf8"); + const parsed = WorkflowDeploymentRecord(JSON.parse(raw)); + if (parsed instanceof type.errors) { + throw new Error(`record failed validation: ${parsed.summary}`); + } + expect(parsed).toEqual(MULTI_STEP); + expect("hubPublicKey" in parsed).toBe(false); + expect("sessionId" in parsed).toBe(false); + + await fs.rm(dataDir, { recursive: true, force: true }); + }); + + test("overwriting a record leaves the new one and no temp orphan", async () => { + const dataDir = await makeDataDir(); + const deploymentId = "rotated-1"; + + // A source rotation overwrites the existing record in place. The + // atomic write must replace it cleanly, leaving only the record and + // no `.tmp` staging file behind. + await writeWorkflowDeploymentRecord(dataDir, deploymentId, SINGLE_STEP); + await writeWorkflowDeploymentRecord(dataDir, deploymentId, MULTI_STEP); + + const dir = path.join(dataDir, "workflow-runs", deploymentId); + expect(await fs.readdir(dir)).toEqual(["deployment.json"]); + + const scanned = await scanWorkflowDeploymentRecords(dataDir); + expect(scanned.map((s) => s.record)).toEqual([MULTI_STEP]); + + await fs.rm(dataDir, { recursive: true, force: true }); + }); + + test("delete removes the record and is a no-op when absent", async () => { + const dataDir = await makeDataDir(); + const deploymentId = "gone-1"; + + // No-op when the record was never written. + await deleteWorkflowDeploymentRecord(dataDir, deploymentId); + + await writeWorkflowDeploymentRecord(dataDir, deploymentId, SINGLE_STEP); + expect(await fileExists(recordPath(dataDir, deploymentId))).toBe(true); + + await deleteWorkflowDeploymentRecord(dataDir, deploymentId); + expect(await fileExists(recordPath(dataDir, deploymentId))).toBe(false); + + await fs.rm(dataDir, { recursive: true, force: true }); + }); +}); + +describe("scanWorkflowDeploymentRecords", () => { + test("returns an empty list when the workflow-runs directory is absent", async () => { + const dataDir = await makeDataDir(); + // First boot: nothing has been deployed, so `workflow-runs/` does not + // exist. That is the legitimate empty case, not an error. + expect(await scanWorkflowDeploymentRecords(dataDir)).toEqual([]); + await fs.rm(dataDir, { recursive: true, force: true }); + }); + + test("returns every schema-valid record keyed by its directory name", async () => { + const dataDir = await makeDataDir(); + await writeWorkflowDeploymentRecord(dataDir, "dep-a", SINGLE_STEP); + await writeWorkflowDeploymentRecord(dataDir, "dep-b", MULTI_STEP); + + const scanned = await scanWorkflowDeploymentRecords(dataDir); + const byId = new Map(scanned.map((s) => [s.deploymentId, s.record])); + expect(byId.size).toBe(2); + expect(byId.get("dep-a")).toEqual(SINGLE_STEP); + expect(byId.get("dep-b")).toEqual(MULTI_STEP); + + await fs.rm(dataDir, { recursive: true, force: true }); + }); + + test("soft-fails a corrupt or schema-invalid record while returning the valid ones", async () => { + const dataDir = await makeDataDir(); + await writeWorkflowDeploymentRecord(dataDir, "dep-valid", SINGLE_STEP); + + // A directory whose record is not valid JSON. + const corruptDir = path.join(dataDir, "workflow-runs", "dep-corrupt"); + await fs.mkdir(corruptDir, { recursive: true }); + await fs.writeFile(path.join(corruptDir, "deployment.json"), "{ not json"); + + // A directory whose record parses but fails the schema (missing fields). + const invalidDir = path.join(dataDir, "workflow-runs", "dep-invalid"); + await fs.mkdir(invalidDir, { recursive: true }); + await fs.writeFile( + path.join(invalidDir, "deployment.json"), + JSON.stringify({ version: 1 }), + ); + + // A bare run directory with no record at all. + await fs.mkdir(path.join(dataDir, "workflow-runs", "dep-empty"), { + recursive: true, + }); + + const scanned = await scanWorkflowDeploymentRecords(dataDir); + expect(scanned.map((s) => s.deploymentId)).toEqual(["dep-valid"]); + expect(scanned[0]?.record).toEqual(SINGLE_STEP); + + await fs.rm(dataDir, { recursive: true, force: true }); + }); +}); diff --git a/apps/sidecar/src/workflow-deployment-record.ts b/apps/sidecar/src/workflow-deployment-record.ts new file mode 100644 index 00000000..01aaaf14 --- /dev/null +++ b/apps/sidecar/src/workflow-deployment-record.ts @@ -0,0 +1,171 @@ +// Sidecar-local persistence of the per-deployment record needed to +// re-establish a workflow deployment across a sidecar PROCESS restart. The +// record is co-located with the deployment's workflow-run substrate at +// `${dataDir}/workflow-runs//deployment.json`, so a single +// teardown reclaims both and a boot scan can enumerate the active +// deployments beside the run state they resume. +// +// It carries only the inputs that are otherwise frame/in-memory only: +// `sources` (each step's ordered inference-source failover chain, threaded to +// the child via the spawn env and durable nowhere else), `sessionId` +// (inference-event +// correlation), and `hubPublicKey` (the head's deploy-pack / inbound +// verification key, recorded only in memory today). The definition itself +// lives in `assets/workflow//workflow.json`, referenced by +// `definitionId`, and each step's grants live in its agent-state repo, so +// neither is duplicated here. + +import { mkdir, readdir, readFile, rm } from "node:fs/promises"; +import { dirname, join as pathJoin } from "node:path"; + +import { type } from "arktype"; + +import { getLogger } from "@intx/log"; +import { InferenceSource } from "@intx/types/runtime"; + +import { writeFileAtomicDurable } from "./atomic-write"; + +const logger = getLogger([ + "interchange", + "sidecar", + "workflow-deployment-record", +]); + +const RECORD_FILENAME = "deployment.json"; + +/** True for a `node:fs` rejection whose `code` is `ENOENT`. */ +function isENOENT(cause: unknown): boolean { + return ( + cause instanceof Error && + "code" in cause && + (cause as { code: unknown }).code === "ENOENT" + ); +} + +/** + * The on-disk deployment record. `version` guards the schema shape so a + * future reader can reject or migrate a stale record rather than parse it + * blindly. Validated at read time (the boot scan) at the trust boundary. + */ +export const WorkflowDeploymentRecord = type({ + version: "1", + agentAddress: "string > 0", + definitionId: "string > 0", + sources: { + "[string]": InferenceSource.array().atLeastLength(1), + }, + "sessionId?": "string > 0", + "hubPublicKey?": "string > 0", +}); +export type WorkflowDeploymentRecord = typeof WorkflowDeploymentRecord.infer; + +function recordPath(dataDir: string, deploymentId: string): string { + return pathJoin(dataDir, "workflow-runs", deploymentId, RECORD_FILENAME); +} + +/** + * Persist a deployment record. Written after the deployment's slug is + * claimed and before the child is spawned, so a crash mid-spawn leaves a + * record the boot scan re-drives. Idempotent: it overwrites any existing + * record for the same deployment. + */ +export async function writeWorkflowDeploymentRecord( + dataDir: string, + deploymentId: string, + record: WorkflowDeploymentRecord, +): Promise { + const path = recordPath(dataDir, deploymentId); + await mkdir(dirname(path), { recursive: true }); + // Atomic + durable: this is the sole restore source for the + // deployment's `sources`/`hubPublicKey`, and a rotation overwrites the + // existing record in place, so an interrupted write must never expose a + // torn record the boot scan would then skip. Owner-only (0o600): the + // record embeds each source's `apiKey`, so it must not be world-readable + // on a shared host, matching the private-key writes elsewhere on the + // sidecar. + await writeFileAtomicDurable(path, JSON.stringify(record, null, 2), { + mode: 0o600, + }); +} + +/** + * Remove a deployment record. Called on undeploy and on a soft-failed + * deploy so a torn-down or never-completed deployment is not restored on + * the next boot. A missing record is not an error (`force`). + */ +export async function deleteWorkflowDeploymentRecord( + dataDir: string, + deploymentId: string, +): Promise { + await rm(recordPath(dataDir, deploymentId), { force: true }); +} + +/** A restorable deployment: its directory-derived id plus the validated record. */ +export interface ScannedWorkflowDeployment { + /** The `workflow-runs/` directory name the record was found under. */ + deploymentId: string; + record: WorkflowDeploymentRecord; +} + +/** + * Enumerate the persisted deployment records under `workflow-runs/` so a + * boot-time restore can re-establish each deployment. Soft-fails per record: + * a missing `deployment.json`, unparseable JSON, or a record that fails schema + * validation is logged and skipped rather than wedging the whole boot -- one + * corrupt record must not strand every other deployment. An absent + * `workflow-runs/` directory is the legitimate first-boot case and yields an + * empty list, not an error. + * + * The returned `deploymentId` is the directory name; the caller cross-checks it + * against the record's own address before trusting it. + */ +export async function scanWorkflowDeploymentRecords( + dataDir: string, +): Promise { + const runsDir = pathJoin(dataDir, "workflow-runs"); + let entries; + try { + entries = await readdir(runsDir, { withFileTypes: true }); + } catch (cause) { + if (isENOENT(cause)) return []; + throw cause; + } + + const scanned: ScannedWorkflowDeployment[] = []; + for (const entry of entries) { + if (!entry.isDirectory()) continue; + const deploymentId = entry.name; + const path = recordPath(dataDir, deploymentId); + + let raw: string; + try { + raw = await readFile(path, "utf8"); + } catch (cause) { + // A run directory with no record: a crash between mkdir and the record + // write, or a run whose record was already reclaimed. Nothing to + // restore from -- skip. + if (isENOENT(cause)) { + logger.warn`skipping workflow-runs/${deploymentId}: no ${RECORD_FILENAME} to restore from`; + continue; + } + throw cause; + } + + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (cause) { + const reason = cause instanceof Error ? cause.message : String(cause); + logger.warn`skipping workflow-runs/${deploymentId}: ${RECORD_FILENAME} is not valid JSON: ${reason}`; + continue; + } + + const record = WorkflowDeploymentRecord(parsed); + if (record instanceof type.errors) { + logger.warn`skipping workflow-runs/${deploymentId}: ${RECORD_FILENAME} failed validation: ${record.summary}`; + continue; + } + scanned.push({ deploymentId, record }); + } + return scanned; +} diff --git a/apps/sidecar/src/workflow-host-wiring-deploy-failure.test.ts b/apps/sidecar/src/workflow-host-wiring-deploy-failure.test.ts index 3414031a..b2fd9220 100644 --- a/apps/sidecar/src/workflow-host-wiring-deploy-failure.test.ts +++ b/apps/sidecar/src/workflow-host-wiring-deploy-failure.test.ts @@ -1,19 +1,16 @@ // Pins H-A1: a deploy-router rejection must not leave the -// `DeploymentAddressRegistry` populated. The deploy router used to -// call `registerDeployment` BEFORE every step that can throw (the -// trivial branch's `provisionAgent`, the multi-step branch's asset -// materialization and `supervisor.spawn`). The link's -// `handleAgentDeploy` catches the rejection and sends `agent.error` -// without invoking `deployRouter.undeploy(frame)`, so the registry -// retained the `(deploymentId -> agentAddress)` mapping for a -// deployment that does not exist. The fix defers registration until -// every throwy step has succeeded; both branches must leave the -// registry clean on deploy failure. +// `DeploymentAddressRegistry` populated. The multi-step branch defers +// `registerDeployment` until every step that can throw (asset +// materialization, `supervisor.spawn`) has succeeded. The link's +// `handleAgentDeploy` catches a rejection and sends `agent.error` +// without invoking `deployRouter.undeploy(frame)`, so a premature +// registration would retain a `(deploymentId -> agentAddress)` mapping +// for a deployment that does not exist. The multi-step test drives that +// failure through a subprocess spawner that throws synchronously. // -// Both branches drive their failure path through this test: -// - trivial: `sessions.provisionAgent` throws. -// - multi-step: the subprocess spawner throws synchronously so -// `supervisor.spawn` rejects. +// The first test pins the router's frame-shape guard: a frame carrying +// neither `provisionStep` nor a workflow definition is rejected before +// any deploy work, so a malformed frame cannot leak registry state. import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; @@ -47,9 +44,6 @@ function stubKeyStore(): Parameters< // exercises. return { keyPair: await generateKeyPair(), isNew: false }; }, - async scanKeys() { - return []; - }, signChallenge() { return null; }, @@ -86,24 +80,25 @@ function makeRouterDeps() { } describe("deploy-failure registry leak", () => { - test("trivial deploy: provisionAgent throws and registry stays clean", async () => { + test("a frame carrying neither provisionStep nor a workflow is rejected before any deploy work", async () => { const { registry, mailRouter, signalRouter, drainRouter, transport } = makeRouterDeps(); - const sessions = stubFailingSessions(); - const keyStore = stubKeyStore(); - + let spawnerInvoked = false; const router = createSidecarDeployRouter({ - sessions, - keyStore, - onAgentEvent: () => () => undefined, + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- an unsupported frame throws before sessions is touched + sessions: {} as Parameters< + typeof createSidecarDeployRouter + >[0]["sessions"], + keyStore: stubKeyStore(), transport, - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- trivial branch reaches provisionAgent before any repoStore usage + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- an unsupported frame throws before any repoStore usage repoStore: {} as Parameters< typeof createSidecarDeployRouter >[0]["repoStore"], signingKeySeed: new Uint8Array(32), createAgentCrypto: createEd25519Crypto, + assertSourceBuildable: () => undefined, registerDeployment: ({ deploymentId, agentAddress }) => { registry.record(deploymentId, agentAddress); }, @@ -113,35 +108,31 @@ describe("deploy-failure registry leak", () => { multistepMailRouter: mailRouter, multistepSignalRouter: signalRouter, multistepDrainRouter: drainRouter, + multistepSubprocessSpawner: () => { + spawnerInvoked = true; + throw new Error("spawner must not run for an unsupported frame"); + }, }); + // Neither `provisionStep` nor a workflow definition: the router has + // no path to stage this frame through the substrate, so it rejects on + // shape before reaching any deploy work. const frame: AgentDeployFrame = { type: "agent.deploy", - agentAddress: "agent-fail@x.example", - agentId: "agent-fail", + agentAddress: "agent-unsupported@x.example", + agentId: "agent-unsupported", hubPublicKey: "00".repeat(32), - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- trivialLaunch surfaces config to the failing provisionAgent stub - config: { - agentAddress: "agent-fail@x.example", - agentId: "agent-fail", - sessionId: "session-fail", - sources: [], - defaultSource: "primary", - grants: [], - } as unknown as AgentDeployFrame["config"], + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- config is irrelevant; the frame is rejected on shape before config is read + config: {} as unknown as AgentDeployFrame["config"], }; - let threw = false; - try { - await router.deploy(frame); - } catch { - threw = true; - } - expect(threw).toBe(true); - - // `deriveTrivialDeploymentId` replaces every char outside - // `[a-zA-Z0-9_-]` with `-`, so `@` and `.` both become `-`. - const slug = "agent-fail-x-example"; + await expect(router.deploy(frame)).rejects.toThrow( + /unsupported deploy frame/, + ); + // The throw fires before any deploy work: no spawn, and the registry + // is never touched. + expect(spawnerInvoked).toBe(false); + const slug = "agent-unsupported-x-example"; expect(registry.resolve(slug)).toBeNull(); }); @@ -182,12 +173,12 @@ describe("deploy-failure registry leak", () => { const router = createSidecarDeployRouter({ sessions, keyStore, - onAgentEvent: () => () => undefined, transport, // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub: only getRepoDir + writeTree are exercised before the spawn-time failure repoStore: repoStoreStub as RepoStore, signingKeySeed: new Uint8Array(32), createAgentCrypto: createEd25519Crypto, + assertSourceBuildable: () => undefined, registerDeployment: ({ deploymentId, agentAddress }) => { registry.record(deploymentId, agentAddress); }, @@ -234,13 +225,15 @@ describe("deploy-failure registry leak", () => { steps: { s1: { kind: "step" } }, }, sources: { - s1: { - id: "primary", - provider: "anthropic", - baseURL: "https://api.anthropic.com", - apiKey: "sk-x", - model: "claude-3-5", - }, + s1: [ + { + id: "primary", + provider: "anthropic", + baseURL: "https://api.anthropic.com", + apiKey: "sk-x", + model: "claude-3-5", + }, + ], }, }, }; @@ -256,4 +249,150 @@ describe("deploy-failure registry leak", () => { const slug = "ins_mstep-x-example"; expect(registry.resolve(slug)).toBeNull(); }); + + test("single-step spawn failure reverses the recorded hub key but preserves the head repo", async () => { + const { registry, mailRouter, signalRouter, drainRouter, transport } = + makeRouterDeps(); + + const initedRepos: string[] = []; + const removedRepos: string[] = []; + const recordedHubKeys: string[] = []; + const forgottenAgents: string[] = []; + + // A sessions stub whose `initRepo` succeeds (so the single-step head + // proceeds to record the hub key and then fail at spawn) and whose + // `deleteAgentDir` is spied: the unwind must NOT call it, because the + // head repo directory also holds the durable identity keypair. + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub: only initRepo runs on the happy path; deleteAgentDir is spied to prove it is NOT called + const sessions = { + initRepo: async (address: string) => { + initedRepos.push(address); + }, + deleteAgentDir: async (address: string) => { + removedRepos.push(address); + }, + } as unknown as Parameters[0]["sessions"]; + + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub + const keyStore = { + async loadOrGenerateKey() { + return { keyPair: await generateKeyPair(), isNew: false }; + }, + signChallenge() { + return null; + }, + recordHubKey(address: string) { + recordedHubKeys.push(address); + }, + verifyDeployCommit() { + return true; + }, + forgetAgent(address: string) { + forgottenAgents.push(address); + }, + } as unknown as Parameters[0]["keyStore"]; + + const failingSpawner: SubprocessSpawner = () => { + throw new Error("spawner forced failure"); + }; + + const tmpDir = mkdtempSync(pathJoin(tmpdir(), "single-step-unwind-")); + + const repoStoreStub: Partial = { + getRepoDir(repoId: RepoId): string { + return pathJoin(tmpDir, repoId.kind, repoId.id); + }, + writeTree(_p, repoId, _ref, content) { + const dir = pathJoin(tmpDir, repoId.kind, repoId.id); + for (const [relPath, contents] of Object.entries(content.files)) { + const full = pathJoin(dir, relPath); + mkdirSync(dirname(full), { recursive: true }); + writeFileSync(full, contents); + } + return Promise.resolve({ + commitSha: "stub-sha", + newlyTerminalRuns: [], + }); + }, + }; + + const router = createSidecarDeployRouter({ + sessions, + keyStore, + transport, + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub: only getRepoDir + writeTree are exercised before the spawn-time failure + repoStore: repoStoreStub as RepoStore, + signingKeySeed: new Uint8Array(32), + createAgentCrypto: createEd25519Crypto, + assertSourceBuildable: () => undefined, + registerDeployment: ({ deploymentId, agentAddress }) => { + registry.record(deploymentId, agentAddress); + }, + unregisterDeployment: ({ deploymentId }) => { + registry.unregister(deploymentId); + }, + multistepMailRouter: mailRouter, + multistepSignalRouter: signalRouter, + multistepDrainRouter: drainRouter, + multistepSubstrateEnv: { + SIDECAR_DATA_DIR: tmpDir, + SIDECAR_SIGNING_PUBLIC_KEY: "00".repeat(32), + SIDECAR_SIGNING_PRIVATE_KEY: "00".repeat(32), + HUB_WS_URL: "ws://test", + SIDECAR_ID: "sc", + SIDECAR_TOKEN: "tok", + PATH: "/usr/bin", + }, + multistepSubprocessSpawner: failingSpawner, + }); + + const frame: AgentDeployFrame = { + type: "agent.deploy", + agentAddress: "ins_single@x.example", + agentId: "single", + hubPublicKey: "00".repeat(32), + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- single-step branch does not consult config before failing at spawn + config: { + agentAddress: "ins_single@x.example", + agentId: "single", + sessionId: "s", + sources: [], + defaultSource: "primary", + grants: [], + } as unknown as AgentDeployFrame["config"], + workflow: { + definition: { + id: "wf-single", + triggers: [{ type: "manual" }], + stepOrder: ["s1"], + steps: { s1: { kind: "step" } }, + }, + sources: { + s1: [ + { + id: "primary", + provider: "anthropic", + baseURL: "https://api.anthropic.com", + apiKey: "sk-x", + model: "claude-3-5", + }, + ], + }, + }, + }; + + await expect(router.deploy(frame)).rejects.toThrow(); + + // The single-step head initialized its repo and recorded the hub key + // before the spawn failed. + expect(initedRepos).toEqual(["ins_single@x.example"]); + expect(recordedHubKeys).toEqual(["ins_single@x.example"]); + // The unwind reversed the in-memory hub key... + expect(forgottenAgents).toEqual(["ins_single@x.example"]); + // ...but did NOT remove the head repo directory (it holds the durable + // identity keypair a rerouted head must keep across a failed redeploy). + expect(removedRepos).toEqual([]); + // Registry untouched. + expect(registry.resolve("ins_single-x-example")).toBeNull(); + }); }); diff --git a/apps/sidecar/src/workflow-host-wiring-undeploy-supervisor.test.ts b/apps/sidecar/src/workflow-host-wiring-undeploy-supervisor.test.ts index 031db701..fddbb51b 100644 --- a/apps/sidecar/src/workflow-host-wiring-undeploy-supervisor.test.ts +++ b/apps/sidecar/src/workflow-host-wiring-undeploy-supervisor.test.ts @@ -31,7 +31,7 @@ import type { AgentDeployFrame } from "@intx/types/sidecar"; import { createSidecarDeployRouter, - deriveTrivialDeploymentId, + deriveDeploymentId, } from "./workflow-host-wiring"; import { createMultistepDrainRouter, @@ -234,24 +234,23 @@ describe("createSidecarDeployRouter multi-step undeploy shuts the supervisor dow const drainRouter = createMultistepDrainRouter(); const router = createSidecarDeployRouter({ - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- multi-step branch never invokes sessions + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the single-step branch invokes only initRepo (head deploy-tree repo); provisionAgent/persistHubPublicKey stay unused (the supervised child mints its own key and persists no hub-agent config) sessions: { provisionAgent: async () => { - throw new Error("multi-step branch must not invoke provisionAgent"); + throw new Error("single-step branch must not invoke provisionAgent"); }, persistHubPublicKey: async () => { throw new Error( - "multi-step branch must not invoke persistHubPublicKey", + "single-step branch must not invoke persistHubPublicKey", ); }, + initRepo: async () => undefined, } as unknown as Parameters< typeof createSidecarDeployRouter >[0]["sessions"], - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- only loadOrGenerateKey is exercised (the single-step branch registers the agent's signing key on the transport before spawn) + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the single-step branch registers the agent's signing key (loadOrGenerateKey) and records the hub key (recordHubKey) at the head before spawn keyStore: { - recordHubKey: () => { - throw new Error("multi-step branch must not invoke recordHubKey"); - }, + recordHubKey: () => undefined, loadOrGenerateKey: async () => ({ keyPair: await generateKeyPair(), isNew: false, @@ -259,13 +258,11 @@ describe("createSidecarDeployRouter multi-step undeploy shuts the supervisor dow } as unknown as Parameters< typeof createSidecarDeployRouter >[0]["keyStore"], - onAgentEvent: () => () => { - /* unused */ - }, transport, repoStore, signingKeySeed: keyPair.privateKey, createAgentCrypto: createEd25519Crypto, + assertSourceBuildable: () => undefined, registerDeployment: () => { /* no-op */ }, @@ -299,13 +296,15 @@ describe("createSidecarDeployRouter multi-step undeploy shuts the supervisor dow steps: { "step-1": { kind: "step" } }, }, sources: { - "step-1": { - id: "step-1", - provider: "anthropic", - baseURL: "https://api.anthropic.com", - apiKey: "sk-step-1", - model: "claude-3-5", - }, + "step-1": [ + { + id: "step-1", + provider: "anthropic", + baseURL: "https://api.anthropic.com", + apiKey: "sk-step-1", + model: "claude-3-5", + }, + ], }, }, }; @@ -358,7 +357,7 @@ describe("createSidecarDeployRouter multi-step undeploy shuts the supervisor dow // one-per-message); a stale cold `runs//` subtree models a // multi-step leftover the per-run cleanup did not drop. An unrelated // deployment's step-state subtree must survive the undeploy sweep. - const deploymentId = deriveTrivialDeploymentId(frame.agentAddress); + const deploymentId = deriveDeploymentId(frame.agentAddress); const stepStateRoot = path.join(dataDir, "workflow-step-state"); const warmWorkspaceFile = path.join( stepStateRoot, diff --git a/apps/sidecar/src/workflow-host-wiring.test.ts b/apps/sidecar/src/workflow-host-wiring.test.ts index 021c8093..ec917564 100644 --- a/apps/sidecar/src/workflow-host-wiring.test.ts +++ b/apps/sidecar/src/workflow-host-wiring.test.ts @@ -9,31 +9,34 @@ import { createInMemoryTransport } from "@intx/mail-memory"; import type { RepoId, RepoStore } from "@intx/hub-sessions"; import { createControlChannelSender, - type CommitRunEventResult, type EventPayload, type FrameReader, type NdjsonReader, type NdjsonWriter, type SubprocessHandle, type SubprocessSpawner, - type SupervisorRunEvent, } from "@intx/workflow-host"; -import type { InferenceEvent } from "@intx/types/runtime"; import type { AgentDeployFrame } from "@intx/types/sidecar"; import { computeWireDefinitionHash, createSidecarDeployRouter, createSidecarWorkflowSupervisor, - driveTrivialRunChain, + deriveDeploymentId, STEP_INFERENCE_SOURCES_ENV_KEY, validateWorkflowProjection, - type TrivialRunCell, } from "./workflow-host-wiring"; import { createMultistepMailRouter, + createMultistepSourcesRouter, type MultistepMailRouter, + type MultistepSourcesRouter, } from "./workflow-run-pack-client"; +import { + scanWorkflowDeploymentRecords, + writeWorkflowDeploymentRecord, + type WorkflowDeploymentRecord, +} from "./workflow-deployment-record"; function createMinimalStubRepoStore(): RepoStore { const stub: Partial = { @@ -79,12 +82,13 @@ describe("createSidecarWorkflowSupervisor", () => { workflowRunRepoId: { kind: "workflow-run", id: "wire-test" }, workflowRunRef: "refs/heads/main", deploymentId: "wire-test", + stepCount: 1, deploymentMailAddress: "wire-test@example.com", deriveStepAddress: ({ deploymentId, stepId }) => `${deploymentId}-${stepId}@example.com`, substrateEnv: { DATA_DIR: "/tmp/wire" }, + dynamicSpawnEnv: () => ({}), subprocessSpawner: spawner, - trivialLaunch: () => Promise.resolve(), }); expect(typeof wired.supervisor.spawn).toBe("function"); @@ -114,14 +118,15 @@ describe("createSidecarWorkflowSupervisor", () => { workflowRunRepoId: { kind: "workflow-run", id: "inbound" }, workflowRunRef: "refs/heads/main", deploymentId: "inbound", + stepCount: 1, deploymentMailAddress: "inbound@example.com", deriveStepAddress: ({ deploymentId, stepId }) => `${deploymentId}-${stepId}@example.com`, substrateEnv: {}, + dynamicSpawnEnv: () => ({}), subprocessSpawner: () => { throw new Error("spawner not invoked in this test"); }, - trivialLaunch: () => Promise.resolve(), }); // Without a subscriber, routeInbound is a no-op rather than a // throw -- the wiring's mail bus map is a per-address Set that @@ -132,303 +137,77 @@ describe("createSidecarWorkflowSupervisor", () => { }); }); -describe("driveTrivialRunChain projects reactor events onto the workflow-run chain", () => { - function makeRecorder(): { - calls: SupervisorRunEvent[]; - record: (e: SupervisorRunEvent) => Promise; - } { - const calls: SupervisorRunEvent[] = []; - return { - calls, - record: async (e) => { - calls.push(e); - return { - commitSha: "stub", - seq: calls.length - 1, - signature: { sig: new Uint8Array(64), principalKind: "supervisor" }, - }; - }, - }; - } - - test("completed run drives RunStarted, StepStarted, StepCompleted, RunCompleted", async () => { - const cell: TrivialRunCell = { runId: null, stepStarted: false }; - const { calls, record } = makeRecorder(); - - const started: InferenceEvent = { - type: "message.run.started", - seq: 0, - data: { messageId: "m-1", messageRunId: "r-1", receivedAt: 1 }, - }; - const inferStart: InferenceEvent = { - type: "inference.start", - seq: 1, - data: { model: "test" }, - }; - const ended: InferenceEvent = { - type: "message.run.ended", - seq: 2, - data: { messageRunId: "r-1", messageId: "m-1", status: "completed" }, - }; - - await driveTrivialRunChain(started, record, cell); - await driveTrivialRunChain(inferStart, record, cell); - // A second inference.start within the same bracket does NOT mint a - // duplicate StepStarted. - await driveTrivialRunChain(inferStart, record, cell); - await driveTrivialRunChain(ended, record, cell); - - const kinds = calls.map((c) => c.kind); - expect(kinds).toEqual([ - "RunStarted", - "StepStarted", - "StepCompleted", - "RunCompleted", - ]); - for (const call of calls) { - expect(call.runId).toBe("r-1"); - } - const runStarted = calls[0]; - if (runStarted?.kind !== "RunStarted") { - throw new Error("unreachable"); - } - expect(runStarted.consumedMessageId).toBe("m-1"); - expect(cell.runId).toBeNull(); - expect(cell.stepStarted).toBe(false); - }); - - test("failed run emits StepCompleted but not RunCompleted", async () => { - const cell: TrivialRunCell = { runId: null, stepStarted: false }; - const { calls, record } = makeRecorder(); - - await driveTrivialRunChain( - { - type: "message.run.started", - seq: 0, - data: { messageId: "m-2", messageRunId: "r-2", receivedAt: 1 }, - }, - record, - cell, - ); - await driveTrivialRunChain( - { type: "inference.start", seq: 1, data: { model: "test" } }, - record, - cell, - ); - await driveTrivialRunChain( - { - type: "message.run.ended", - seq: 2, - data: { - messageRunId: "r-2", - messageId: "m-2", - status: "failed", - error: { message: "boom" }, - }, - }, - record, - cell, - ); - - expect(calls.map((c) => c.kind)).toEqual([ - "RunStarted", - "StepStarted", - "StepCompleted", - ]); - }); - - test("inference.start without a live bracket is ignored", async () => { - const cell: TrivialRunCell = { runId: null, stepStarted: false }; - const { calls, record } = makeRecorder(); - - await driveTrivialRunChain( - { type: "inference.start", seq: 0, data: { model: "test" } }, - record, - cell, - ); - - expect(calls).toEqual([]); - expect(cell.runId).toBeNull(); - }); -}); - -describe("createSidecarDeployRouter wires the InferenceEvent subscription to recordRunEvent", () => { - test("the trivial-launch closure brackets a real mail trigger via onAgentEvent", async () => { +describe("createSidecarDeployRouter provision-step (no-spawn) mode", () => { + test("a provisionStep frame inits the repo and records the hub key without spawning", async () => { const transport = createInMemoryTransport(); const keyPair = await generateKeyPair(); - // Capture every `merge` payload the supervisor commits through the - // substrate. Each successful `recordRunEvent` call drives one - // writeTreePreservingPrefix invocation that runs `merge` against - // an empty existing-tree map; we read the files the merge wrote - // back out and snapshot their `type` discriminator. - const writtenEvents: string[] = []; - const repoStore: RepoStore = ((): RepoStore => { - // Mirror the real store's per-repo serialization (withRepoLock) so - // concurrent recordRunEvent writes land in invocation order even - // though the per-event signature is async. - let writeTail: Promise = Promise.resolve(); - const stub: Partial = { - getRepoDir(_repoId: RepoId): string { - return "/tmp/unused"; - }, - writeTreePreservingPrefix(_p, _id, _ref, args) { - const previous = writeTail; - let release: () => void = () => undefined; - writeTail = new Promise((resolve) => { - release = resolve; - }); - return (async () => { - await previous; - try { - const files = await args.merge(new Map()); - for (const value of Object.values(files)) { - const text = - value instanceof Uint8Array - ? new TextDecoder().decode(value) - : value; - const parsed: unknown = JSON.parse(text); - if ( - typeof parsed === "object" && - parsed !== null && - "type" in parsed && - typeof parsed.type === "string" - ) { - writtenEvents.push(parsed.type); - } - } - return { - commitSha: `c-${String(writtenEvents.length)}`, - newlyTerminalRuns: [], - }; - } finally { - release(); - } - })(); - }, - }; - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- in-test stub; the unused RepoStore methods are guarded by the Proxy below - return new Proxy(stub as RepoStore, { - get(target, prop, receiver) { - const value = Reflect.get(target, prop, receiver); - if (value !== undefined) return value; - return () => { - throw new Error(`stub RepoStore: ${String(prop)} not implemented`); - }; - }, - }); - })(); + const initRepoCalls: string[] = []; + const recordHubKeyCalls: { address: string; hubKey: string }[] = []; - // Capture the per-agent InferenceEvent listener the trivial-launch - // closure registers so the test can fire events at it directly. - type CapturedListener = { - address: string; - listener: (e: InferenceEvent) => void; - }; - const captured: CapturedListener[] = []; - const onAgentEvent = ( - address: string, - listener: (e: InferenceEvent) => void, - ): (() => void) => { - captured.push({ address, listener }); - return () => { - const idx = captured.findIndex((c) => c.listener === listener); - if (idx >= 0) captured.splice(idx, 1); - }; - }; + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- provisionStep touches no RepoStore method; the Proxy throws if it ever does + const repoStore = new Proxy({} as RepoStore, { + get(_target, prop) { + return () => { + throw new Error(`stub RepoStore: ${String(prop)} not implemented`); + }; + }, + }); const router = createSidecarDeployRouter({ - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the deploy-router test exercises only provisionAgent + persistHubPublicKey + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- provisionStep exercises only initRepo sessions: { - provisionAgent: async (_config: unknown) => ({ - publicKey: "pk-trivial", - keyPair: { - publicKey: new Uint8Array(32), - privateKey: new Uint8Array(32), - }, - }), - persistHubPublicKey: async (_a: string, _h: string) => { - /* no-op */ + initRepo: async (a: string) => { + initRepoCalls.push(a); }, } as unknown as Parameters< typeof createSidecarDeployRouter >[0]["sessions"], - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the deploy-router test exercises only recordHubKey + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- provisionStep exercises only recordHubKey keyStore: { - recordHubKey: (_a: string, _h: string) => { - /* no-op */ + recordHubKey: (a: string, h: string) => { + recordHubKeyCalls.push({ address: a, hubKey: h }); }, - loadOrGenerateKey: async () => ({ - keyPair: await generateKeyPair(), - isNew: false, - }), } as unknown as Parameters< typeof createSidecarDeployRouter >[0]["keyStore"], - onAgentEvent, transport, repoStore, signingKeySeed: keyPair.privateKey, createAgentCrypto: createEd25519Crypto, - registerDeployment: () => { - /* the in-test repoStore is a stub; the pack-push facade is exercised separately */ - }, - unregisterDeployment: () => { - /* no-op for parity with registerDeployment in this stub */ - }, + assertSourceBuildable: () => undefined, + registerDeployment: () => undefined, + unregisterDeployment: () => undefined, }); + const STEP_ADDR = "ins_dep_abc-step1@example.com"; + const HUB_KEY = "aa".repeat(32); const result = await router.deploy({ type: "agent.deploy", - agentAddress: "trivial@example.com", - agentId: "trivial-agent", - hubPublicKey: "hub-pk", - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the trivial-launch closure passes config to the SessionManager mock above without inspecting it + agentAddress: STEP_ADDR, + agentId: "ins_dep_abc-step1", + hubPublicKey: HUB_KEY, + provisionStep: true, + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- provisionStep never inspects config config: {} as unknown as Parameters< ReturnType["deploy"] >[0]["config"], }); - expect(result.publicKey).toBe("pk-trivial"); - if (captured.length !== 1) { - throw new Error( - `expected exactly one onAgentEvent registration, got ${String(captured.length)}`, - ); - } - const entry = captured[0]; - if (entry === undefined) throw new Error("unreachable"); - // The slug derivation strips the `@example.com` suffix; the - // listener is registered against the original frame address. - expect(entry.address).toBe("trivial@example.com"); - - entry.listener({ - type: "message.run.started", - seq: 0, - data: { messageId: "m-1", messageRunId: "r-1", receivedAt: 1 }, - }); - entry.listener({ - type: "inference.start", - seq: 1, - data: { model: "test" }, - }); - entry.listener({ - type: "message.run.ended", - seq: 2, - data: { messageRunId: "r-1", messageId: "m-1", status: "completed" }, - }); + // The step's agent-state repo is initialized and the hub key recorded, + // so the follow-up full-closure deploy pack applies into a repo and + // verifies against the recorded key. + expect(initRepoCalls).toEqual([STEP_ADDR]); + expect(recordHubKeyCalls).toEqual([ + { address: STEP_ADDR, hubKey: HUB_KEY }, + ]); - // recordRunEvent fires are sequenced through Promises and the - // per-event signature is async; poll until all four events land. - for (let i = 0; i < 200 && writtenEvents.length < 4; i++) { - await new Promise((r) => setTimeout(r, 1)); - } + // The ack carries the sidecar principal key (the hub discards it for a + // workflow-derived per-step address). + expect(result.publicKey).toMatch(/^[0-9a-f]{64}$/); - expect(writtenEvents).toEqual([ - "RunStarted", - "StepStarted", - "StepCompleted", - "RunCompleted", - ]); + // Nothing spawned: no supervisor, so no active address. + expect(router.activeAddresses()).toEqual([]); }); }); @@ -580,10 +359,12 @@ function createSpawnTestRepoStore(tempBase: string): RepoStore { } type WorkflowProjection = NonNullable; -type InferenceSourceFixture = WorkflowProjection["sources"][string]; +// A single source: each step's `sources` value is an ordered failover chain, +// so the fixture element type is the chain's member. +type InferenceSourceFixture = WorkflowProjection["sources"][string][number]; type MultistepDeployArgs = { - sources: Record; + sources: WorkflowProjection["sources"]; definition: { id: string; triggers: unknown[]; @@ -618,11 +399,12 @@ function makeMultistepFrame(args: MultistepDeployArgs): AgentDeployFrame { agentAddress: args.agentAddress ?? "multi@example.com", agentId: "multi-agent", hubPublicKey: "hub-pk", - // The wire-side HarnessConfig has many required fields. The router - // never inspects `config` on the multi-step branch (only the - // trivial branch hands it to provisionAgent), so an opaque - // placeholder satisfies the surface contract for these tests. - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the multi-step branch does not read config + // The wire-side HarnessConfig has many required fields. On the + // workflow deploy path the router reads only `config.sessionId` + // and `config.grants`, both of which tolerate the empty + // placeholder (they resolve to `undefined`), so an opaque `{}` + // satisfies the surface contract for these tests. + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the workflow path reads only config.sessionId/config.grants, which tolerate undefined config: {} as AgentDeployFrame["config"], workflow: { definition: args.definition, @@ -631,10 +413,10 @@ function makeMultistepFrame(args: MultistepDeployArgs): AgentDeployFrame { }; } -function defaultMultistepSources(): Record { +function defaultMultistepSources(): WorkflowProjection["sources"] { return { - "step-1": makeInferenceSource("step-1"), - "step-2": makeInferenceSource("step-2"), + "step-1": [makeInferenceSource("step-1")], + "step-2": [makeInferenceSource("step-2")], }; } @@ -674,6 +456,32 @@ describe("validateWorkflowProjection", () => { ).toThrow(/sources is missing entry/); }); + test("rejects an empty sources chain for a stepOrder id", () => { + expect(() => + validateWorkflowProjection({ + definition: { + id: "w-1", + stepOrder: ["step-1"], + steps: { "step-1": {} }, + }, + sources: { "step-1": [] }, + }), + ).toThrow(/must be a non-empty array/); + }); + + test("rejects a non-array sources entry for a stepOrder id", () => { + expect(() => + validateWorkflowProjection({ + definition: { + id: "w-1", + stepOrder: ["step-1"], + steps: { "step-1": {} }, + }, + sources: { "step-1": {} }, + }), + ).toThrow(/must be a non-empty array/); + }); + test("accepts a well-formed projection", () => { expect(() => validateWorkflowProjection({ @@ -682,7 +490,7 @@ describe("validateWorkflowProjection", () => { stepOrder: ["step-1", "step-2"], steps: { "step-1": {}, "step-2": {} }, }, - sources: { "step-1": {}, "step-2": {} }, + sources: { "step-1": [{}], "step-2": [{}] }, }), ).not.toThrow(); }); @@ -706,548 +514,195 @@ describe("computeWireDefinitionHash", () => { }); }); -describe("createSidecarDeployRouter trivial-frame regression", () => { - test("a frame without `workflow` still drives the trivial provisioning path", async () => { - const transport = createInMemoryTransport(); +describe("createSidecarDeployRouter multi-step branch", () => { + async function buildMultistepFixture(opts: { + spawner: SubprocessSpawner; + publishWorkflowInferenceEvent?: ( + address: string, + event: EventPayload, + sessionId: string | undefined, + ) => void; + multistepBinaryPath?: string; + multistepSubstrateEnv?: Record; + multistepMailRouter?: MultistepMailRouter; + multistepSourcesRouter?: MultistepSourcesRouter; + registerDeployment?: (args: { + deploymentId: string; + agentAddress: string; + }) => void; + assertSourceBuildable?: Parameters< + typeof createSidecarDeployRouter + >[0]["assertSourceBuildable"]; + /** + * Reuse an existing transport instead of a fresh one. The restore + * tests deploy through one fixture, then build a SECOND fixture over + * the same on-disk data dir with a FRESH transport to model a sidecar + * process restart (the in-memory transport is process-local, so a + * restart starts with an empty registration table). + */ + transport?: ReturnType; + /** + * Spawn ready-handshake timeout (ms) threaded to every supervisor the + * router constructs. The ready-timeout test uses a small value with a + * spawner that never drives `ready`, asserting the deploy rejects with + * the threaded value echoed in the message. + */ + readyTimeoutMs?: number; + /** + * Fixed keypair the keyStore's `loadOrGenerateKey` returns for the head. + * The B-key test pins the single-step deploy ack to this agent key; when + * omitted a fresh keypair is minted per call as before. + */ + headKeyPair?: Awaited>; + /** + * Injectable deployment-record writer. The rotation-interleave tests + * pass a blockable/failing stub so a recycle can be driven into the + * rotation's persist window; omitted, the router uses the real writer. + */ + writeWorkflowDeploymentRecord?: typeof writeWorkflowDeploymentRecord; + }) { + const transport = opts.transport ?? createInMemoryTransport(); const keyPair = await generateKeyPair(); - const repoStore = createMinimalStubRepoStore(); - - let provisionAgentCalled = false; - let spawnerInvoked = false; - + const tempBase = await createTempBaseDir("sidecar-multistep-"); + const repoStore = createSpawnTestRepoStore(tempBase); + // The deploy router's multi-step branch materializes + // `workflow.json` under `${SIDECAR_DATA_DIR}/assets/workflow//` + // before invoking the spawner. The test fixture defaults the data + // dir to a per-test mkdtemp so the wiring tests do not have to + // touch a real /tmp path; callers can override + // `SIDECAR_DATA_DIR` (and any other key) by passing + // `multistepSubstrateEnv`. + const defaultSubstrateEnv: Record = { + SIDECAR_DATA_DIR: await createTempBaseDir("sidecar-multistep-data-"), + }; + const mergedSubstrateEnv: Record = { + ...defaultSubstrateEnv, + ...(opts.multistepSubstrateEnv ?? {}), + }; const router = createSidecarDeployRouter({ - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub; trivial branch exercises only provisionAgent + persistHubPublicKey + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the workflow path never invokes provisionAgent/persistHubPublicKey (single-step uses the narrow initRepo; the child mints its own key); the stubs throw if it does. initRepo is a no-op for the single-step head repo. sessions: { - provisionAgent: async (_config: unknown) => { - provisionAgentCalled = true; - return { - publicKey: "pk-trivial-regression", - keyPair: { - publicKey: new Uint8Array(32), - privateKey: new Uint8Array(32), - }, - }; + provisionAgent: async () => { + throw new Error("workflow branch must not invoke provisionAgent"); }, - persistHubPublicKey: async (_a: string, _h: string) => { - /* no-op */ + persistHubPublicKey: async () => { + throw new Error( + "workflow branch must not invoke persistHubPublicKey", + ); }, + initRepo: async () => undefined, } as unknown as Parameters< typeof createSidecarDeployRouter >[0]["sessions"], - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub; the single-step head deploy records the hub key for pack verification keyStore: { - recordHubKey: (_a: string, _h: string) => { - /* no-op */ - }, + recordHubKey: () => undefined, loadOrGenerateKey: async () => ({ - keyPair: await generateKeyPair(), + keyPair: opts.headKeyPair ?? (await generateKeyPair()), isNew: false, }), + // A single-step spawn failure unwinds the recorded hub key via + // forgetAgent; the fixture exercises that unwind, so the stub must + // honor the call. + forgetAgent: () => undefined, } as unknown as Parameters< typeof createSidecarDeployRouter >[0]["keyStore"], - onAgentEvent: () => () => { - /* unused in this test */ - }, transport, repoStore, signingKeySeed: keyPair.privateKey, createAgentCrypto: createEd25519Crypto, - registerDeployment: () => { - /* no-op */ - }, + assertSourceBuildable: opts.assertSourceBuildable ?? (() => undefined), + registerDeployment: opts.registerDeployment ?? (() => undefined), unregisterDeployment: () => { /* no-op */ }, - multistepSubprocessSpawner: () => { - spawnerInvoked = true; - throw new Error("the trivial branch must not invoke the spawner"); - }, + multistepSubprocessSpawner: opts.spawner, + ...(opts.multistepBinaryPath !== undefined + ? { multistepBinaryPath: opts.multistepBinaryPath } + : {}), + multistepSubstrateEnv: mergedSubstrateEnv, + ...(opts.publishWorkflowInferenceEvent !== undefined + ? { + publishWorkflowInferenceEvent: opts.publishWorkflowInferenceEvent, + } + : {}), + ...(opts.multistepMailRouter !== undefined + ? { multistepMailRouter: opts.multistepMailRouter } + : {}), + ...(opts.multistepSourcesRouter !== undefined + ? { multistepSourcesRouter: opts.multistepSourcesRouter } + : {}), + ...(opts.readyTimeoutMs !== undefined + ? { readyTimeoutMs: opts.readyTimeoutMs } + : {}), + ...(opts.writeWorkflowDeploymentRecord !== undefined + ? { + writeWorkflowDeploymentRecord: opts.writeWorkflowDeploymentRecord, + } + : {}), }); + return { + router, + tempBase, + keyPair, + substrateEnv: mergedSubstrateEnv, + transport, + }; + } - const result = await router.deploy({ - type: "agent.deploy", - agentAddress: "trivial-regression@example.com", - agentId: "trivial-agent", - hubPublicKey: "hub-pk", - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the trivial-launch closure passes config to the SessionManager mock above without inspecting it - config: {} as unknown as Parameters< - ReturnType["deploy"] - >[0]["config"], + test("validates the projection, constructs SpawnOpts from the frame, drives spawn, and acks the deployment address's public key", async () => { + const supervisorIpcKeyPair = await generateKeyPair(); + const childIpcKeyPair = await generateKeyPair(); + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const eventChildToSupervisor = createMemoryFrameStream(); + let resolveExit: ((code: number) => void) | undefined; + const exited = new Promise((resolve) => { + resolveExit = resolve; }); - - expect(provisionAgentCalled).toBe(true); - expect(spawnerInvoked).toBe(false); - expect(result.publicKey).toBe("pk-trivial-regression"); - }); - - test("two distinct agent addresses whose deriveTrivialDeploymentId slugs collide are rejected at the second deploy", async () => { - // `deriveTrivialDeploymentId` substitutes every disallowed - // character with `-`, so two agent addresses that differ only in - // disallowed characters collapse to the same slug. The slug IS - // the workflow-run repoId, so a silent collision would let the - // second deploy overwrite the first deploy's repo state. The - // slug-claims map rejects the second deploy at the router edge. - const transport = createInMemoryTransport(); - const keyPair = await generateKeyPair(); - const repoStore = createMinimalStubRepoStore(); - const router = createSidecarDeployRouter({ - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub; trivial branch exercises only provisionAgent + persistHubPublicKey - sessions: { - provisionAgent: async (_config: unknown) => ({ - publicKey: "pk-slug-collision", - keyPair: { - publicKey: new Uint8Array(32), - privateKey: new Uint8Array(32), - }, - }), - persistHubPublicKey: async (_a: string, _h: string) => { - /* no-op */ - }, - } as unknown as Parameters< - typeof createSidecarDeployRouter - >[0]["sessions"], - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub - keyStore: { - recordHubKey: (_a: string, _h: string) => { - /* no-op */ + let observedBinary: string | undefined; + let observedEnv: Record | undefined; + const spawner: SubprocessSpawner = ({ binaryPath, env }) => { + observedBinary = binaryPath; + observedEnv = env; + const handle: SubprocessHandle = { + pid: 7321, + controlWriter: supervisorToChild.writer, + controlReader: childToSupervisor.reader, + eventReader: eventChildToSupervisor.reader, + kill: () => { + childToSupervisor.close(); + eventChildToSupervisor.close(); + resolveExit?.(0); }, - loadOrGenerateKey: async () => ({ - keyPair: await generateKeyPair(), - isNew: false, - }), - } as unknown as Parameters< - typeof createSidecarDeployRouter - >[0]["keyStore"], - onAgentEvent: () => () => { - /* unused in this test */ - }, - transport, - repoStore, - signingKeySeed: keyPair.privateKey, - createAgentCrypto: createEd25519Crypto, - registerDeployment: () => { - /* no-op */ - }, - unregisterDeployment: () => { - /* no-op */ - }, - multistepSubprocessSpawner: () => { - throw new Error("the trivial branch must not invoke the spawner"); - }, - }); + exited, + }; + return handle; + }; - // `agent@a.b.com` and `agent!a!b!com` both project to - // `agent-a-b-com` under the slug derivation. - const first = await router.deploy({ - type: "agent.deploy", - agentAddress: "agent@a.b.com", - agentId: "agent-id-1", - hubPublicKey: "hub-pk", - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the trivial-launch closure forwards config opaquely - config: {} as unknown as Parameters< - ReturnType["deploy"] - >[0]["config"], + const multiDataDir = await createTempBaseDir("sidecar-multi-data-"); + // The deployment address's own key -- what `loadOrGenerateKey` mints and + // `signChallenge` signs reconnect challenges with. Pin it so the ack + // assertion below is deterministic. + const deploymentKeyPair = await generateKeyPair(); + const { router } = await buildMultistepFixture({ + spawner, + headKeyPair: deploymentKeyPair, + multistepBinaryPath: "/fake/bin/multistep-workflow-child", + multistepSubstrateEnv: { + SIDECAR_DATA_DIR: multiDataDir, + }, }); - expect(first.publicKey).toBe("pk-slug-collision"); - let caught: unknown; - try { - await router.deploy({ - type: "agent.deploy", - agentAddress: "agent!a!b!com", - agentId: "agent-id-2", - hubPublicKey: "hub-pk", - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- forwarded opaquely - config: {} as unknown as Parameters< - ReturnType["deploy"] - >[0]["config"], - }); - } catch (err) { - caught = err; - } - expect(caught).toBeInstanceOf(Error); - expect(caught instanceof Error && caught.message).toMatch( - /deriveTrivialDeploymentId collision/, - ); - expect(caught instanceof Error && caught.message).toMatch(/agent-a-b-com/); - }); - - test("re-deploying the same address is a no-op claim and succeeds", async () => { - // The slug-claims map records the FIRST claimer's agent - // address; a second claim from the SAME address must not - // throw. Otherwise idempotent re-deploys would be rejected as - // self-collisions. - const transport = createInMemoryTransport(); - const keyPair = await generateKeyPair(); - const repoStore = createMinimalStubRepoStore(); - let provisionCount = 0; - const router = createSidecarDeployRouter({ - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub - sessions: { - provisionAgent: async (_config: unknown) => { - provisionCount += 1; - return { - publicKey: `pk-redeploy-${String(provisionCount)}`, - keyPair: { - publicKey: new Uint8Array(32), - privateKey: new Uint8Array(32), - }, - }; - }, - persistHubPublicKey: async (_a: string, _h: string) => undefined, - } as unknown as Parameters< - typeof createSidecarDeployRouter - >[0]["sessions"], - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub - keyStore: { - recordHubKey: (_a: string, _h: string) => undefined, - loadOrGenerateKey: async () => ({ - keyPair: await generateKeyPair(), - isNew: false, - }), - } as unknown as Parameters< - typeof createSidecarDeployRouter - >[0]["keyStore"], - onAgentEvent: () => () => undefined, - transport, - repoStore, - signingKeySeed: keyPair.privateKey, - createAgentCrypto: createEd25519Crypto, - registerDeployment: () => undefined, - unregisterDeployment: () => undefined, - multistepSubprocessSpawner: () => { - throw new Error("trivial branch must not invoke the spawner"); - }, - }); - - const first = await router.deploy({ - type: "agent.deploy", - agentAddress: "redeploy@example.com", - agentId: "redeploy-1", - hubPublicKey: "hub-pk", - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- forwarded opaquely - config: {} as unknown as Parameters< - ReturnType["deploy"] - >[0]["config"], - }); - const second = await router.deploy({ - type: "agent.deploy", - agentAddress: "redeploy@example.com", - agentId: "redeploy-2", - hubPublicKey: "hub-pk", - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- forwarded opaquely - config: {} as unknown as Parameters< - ReturnType["deploy"] - >[0]["config"], - }); - expect(first.publicKey).toBe("pk-redeploy-1"); - expect(second.publicKey).toBe("pk-redeploy-2"); - }); - - test("a failed deploy releases the slug so a subsequent deploy on the same address succeeds", async () => { - // Without the release-on-failure guard, the first deploy's - // `claimSlug` would leak after `provisionAgent` throws, and - // every subsequent retry on the same address would be rejected - // as a phantom collision. - const transport = createInMemoryTransport(); - const keyPair = await generateKeyPair(); - const repoStore = createMinimalStubRepoStore(); - let provisionCount = 0; - const router = createSidecarDeployRouter({ - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub - sessions: { - provisionAgent: async (_config: unknown) => { - provisionCount += 1; - if (provisionCount === 1) { - throw new Error("provision failed (synthetic)"); - } - return { - publicKey: "pk-after-retry", - keyPair: { - publicKey: new Uint8Array(32), - privateKey: new Uint8Array(32), - }, - }; - }, - persistHubPublicKey: async (_a: string, _h: string) => undefined, - } as unknown as Parameters< - typeof createSidecarDeployRouter - >[0]["sessions"], - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub - keyStore: { - recordHubKey: (_a: string, _h: string) => undefined, - loadOrGenerateKey: async () => ({ - keyPair: await generateKeyPair(), - isNew: false, - }), - } as unknown as Parameters< - typeof createSidecarDeployRouter - >[0]["keyStore"], - onAgentEvent: () => () => undefined, - transport, - repoStore, - signingKeySeed: keyPair.privateKey, - createAgentCrypto: createEd25519Crypto, - registerDeployment: () => undefined, - unregisterDeployment: () => undefined, - multistepSubprocessSpawner: () => { - throw new Error("trivial branch must not invoke the spawner"); - }, - }); - - let firstCaught: unknown; - try { - await router.deploy({ - type: "agent.deploy", - agentAddress: "retry@example.com", - agentId: "retry-1", - hubPublicKey: "hub-pk", - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- forwarded opaquely - config: {} as unknown as Parameters< - ReturnType["deploy"] - >[0]["config"], - }); - } catch (err) { - firstCaught = err; - } - expect(firstCaught).toBeInstanceOf(Error); - expect(firstCaught instanceof Error && firstCaught.message).toMatch( - /provision failed \(synthetic\)/, - ); - - // Retry on the SAME address must succeed -- if the slug were - // leaked, this would throw `deriveTrivialDeploymentId collision`. - const retry = await router.deploy({ - type: "agent.deploy", - agentAddress: "retry@example.com", - agentId: "retry-2", - hubPublicKey: "hub-pk", - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- forwarded opaquely - config: {} as unknown as Parameters< - ReturnType["deploy"] - >[0]["config"], - }); - expect(retry.publicKey).toBe("pk-after-retry"); - }); - - test("undeploy releases the slug so a different-address deploy on the same slug succeeds", async () => { - // After deploy -> undeploy on `release@a.b.com` (slug - // `release-a-b-com`), a fresh deploy on `release!a!b!com` - // (same slug) must be accepted. Without `releaseSlug` running - // on undeploy, the slug would stay claimed and the second - // deploy would surface a phantom collision. - const transport = createInMemoryTransport(); - const keyPair = await generateKeyPair(); - const repoStore = createMinimalStubRepoStore(); - let provisionCount = 0; - const router = createSidecarDeployRouter({ - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub - sessions: { - provisionAgent: async (_config: unknown) => { - provisionCount += 1; - return { - publicKey: `pk-release-${String(provisionCount)}`, - keyPair: { - publicKey: new Uint8Array(32), - privateKey: new Uint8Array(32), - }, - }; - }, - persistHubPublicKey: async (_a: string, _h: string) => undefined, - destroySession: async (_a: string, _r: string) => undefined, - } as unknown as Parameters< - typeof createSidecarDeployRouter - >[0]["sessions"], - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub - keyStore: { - recordHubKey: (_a: string, _h: string) => undefined, - loadOrGenerateKey: async () => ({ - keyPair: await generateKeyPair(), - isNew: false, - }), - } as unknown as Parameters< - typeof createSidecarDeployRouter - >[0]["keyStore"], - onAgentEvent: () => () => undefined, - transport, - repoStore, - signingKeySeed: keyPair.privateKey, - createAgentCrypto: createEd25519Crypto, - registerDeployment: () => undefined, - unregisterDeployment: () => undefined, - multistepSubprocessSpawner: () => { - throw new Error("trivial branch must not invoke the spawner"); - }, - }); - - await router.deploy({ - type: "agent.deploy", - agentAddress: "release@a.b.com", - agentId: "release-1", - hubPublicKey: "hub-pk", - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- forwarded opaquely - config: {} as unknown as Parameters< - ReturnType["deploy"] - >[0]["config"], - }); - if (router.undeploy === undefined) { - throw new Error("router.undeploy is required for this test"); - } - await router.undeploy({ - type: "agent.undeploy", - agentAddress: "release@a.b.com", - reason: "test", - }); - const reclaimed = await router.deploy({ - type: "agent.deploy", - agentAddress: "release!a!b!com", - agentId: "release-2", - hubPublicKey: "hub-pk", - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- forwarded opaquely - config: {} as unknown as Parameters< - ReturnType["deploy"] - >[0]["config"], - }); - expect(reclaimed.publicKey).toBe("pk-release-2"); - }); -}); - -describe("createSidecarDeployRouter multi-step branch", () => { - async function buildMultistepFixture(opts: { - spawner: SubprocessSpawner; - publishWorkflowInferenceEvent?: ( - address: string, - event: EventPayload, - sessionId: string | undefined, - ) => void; - multistepBinaryPath?: string; - multistepSubstrateEnv?: Record; - multistepMailRouter?: MultistepMailRouter; - registerDeployment?: (args: { - deploymentId: string; - agentAddress: string; - }) => void; - }) { - const transport = createInMemoryTransport(); - const keyPair = await generateKeyPair(); - const tempBase = await createTempBaseDir("sidecar-multistep-"); - const repoStore = createSpawnTestRepoStore(tempBase); - // The deploy router's multi-step branch materializes - // `workflow.json` under `${SIDECAR_DATA_DIR}/assets/workflow//` - // before invoking the spawner. The test fixture defaults the data - // dir to a per-test mkdtemp so the wiring tests do not have to - // touch a real /tmp path; callers can override - // `SIDECAR_DATA_DIR` (and any other key) by passing - // `multistepSubstrateEnv`. - const defaultSubstrateEnv: Record = { - SIDECAR_DATA_DIR: await createTempBaseDir("sidecar-multistep-data-"), - }; - const mergedSubstrateEnv: Record = { - ...defaultSubstrateEnv, - ...(opts.multistepSubstrateEnv ?? {}), - }; - const router = createSidecarDeployRouter({ - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the multi-step branch never invokes provisionAgent; the stub throws if it does - sessions: { - provisionAgent: async () => { - throw new Error("multi-step branch must not invoke provisionAgent"); - }, - persistHubPublicKey: async () => { - throw new Error( - "multi-step branch must not invoke persistHubPublicKey", - ); - }, - } as unknown as Parameters< - typeof createSidecarDeployRouter - >[0]["sessions"], - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub - keyStore: { - recordHubKey: () => { - throw new Error("multi-step branch must not invoke recordHubKey"); - }, - loadOrGenerateKey: async () => ({ - keyPair: await generateKeyPair(), - isNew: false, - }), - } as unknown as Parameters< - typeof createSidecarDeployRouter - >[0]["keyStore"], - onAgentEvent: () => () => { - /* unused in multi-step branch */ - }, - transport, - repoStore, - signingKeySeed: keyPair.privateKey, - createAgentCrypto: createEd25519Crypto, - registerDeployment: opts.registerDeployment ?? (() => undefined), - unregisterDeployment: () => { - /* no-op */ - }, - multistepSubprocessSpawner: opts.spawner, - ...(opts.multistepBinaryPath !== undefined - ? { multistepBinaryPath: opts.multistepBinaryPath } - : {}), - multistepSubstrateEnv: mergedSubstrateEnv, - ...(opts.publishWorkflowInferenceEvent !== undefined - ? { - publishWorkflowInferenceEvent: opts.publishWorkflowInferenceEvent, - } - : {}), - ...(opts.multistepMailRouter !== undefined - ? { multistepMailRouter: opts.multistepMailRouter } - : {}), - }); - return { router, tempBase, keyPair, substrateEnv: mergedSubstrateEnv }; - } - - test("validates the projection, constructs SpawnOpts from the frame, drives spawn, and surfaces the supervisor's principal pubkey", async () => { - const supervisorIpcKeyPair = await generateKeyPair(); - const childIpcKeyPair = await generateKeyPair(); - const supervisorToChild = createMemoryNdjsonStream(); - const childToSupervisor = createMemoryNdjsonStream(); - const eventChildToSupervisor = createMemoryFrameStream(); - let resolveExit: ((code: number) => void) | undefined; - const exited = new Promise((resolve) => { - resolveExit = resolve; - }); - let observedBinary: string | undefined; - let observedEnv: Record | undefined; - const spawner: SubprocessSpawner = ({ binaryPath, env }) => { - observedBinary = binaryPath; - observedEnv = env; - const handle: SubprocessHandle = { - pid: 7321, - controlWriter: supervisorToChild.writer, - controlReader: childToSupervisor.reader, - eventReader: eventChildToSupervisor.reader, - kill: () => { - childToSupervisor.close(); - eventChildToSupervisor.close(); - resolveExit?.(0); - }, - exited, - }; - return handle; - }; - - const multiDataDir = await createTempBaseDir("sidecar-multi-data-"); - const { router, keyPair } = await buildMultistepFixture({ - spawner, - multistepBinaryPath: "/fake/bin/multistep-workflow-child", - multistepSubstrateEnv: { - SIDECAR_DATA_DIR: multiDataDir, - }, - }); - - // Hijack the supervisor's ipc keypair factory by routing through - // the test-construction surface: the router constructs the - // supervisor via createSidecarWorkflowSupervisor which does not - // expose ipcKeyPairFactory. The supervisor's default keypair is - // generated with generateKeyPair; the test signs the `ready` frame - // with whatever channelId the spawn-time env carries plus the - // child's keypair, and the supervisor accepts a bootstrap - // signature from any childPublicKey carried in the `ready` payload. + // Hijack the supervisor's ipc keypair factory by routing through + // the test-construction surface: the router constructs the + // supervisor via createSidecarWorkflowSupervisor which does not + // expose ipcKeyPairFactory. The supervisor's default keypair is + // generated with generateKeyPair; the test signs the `ready` frame + // with whatever channelId the spawn-time env carries plus the + // child's keypair, and the supervisor accepts a bootstrap + // signature from any childPublicKey carried in the `ready` payload. const sources = defaultMultistepSources(); const definition = { @@ -1302,14 +757,13 @@ describe("createSidecarDeployRouter multi-step branch", () => { }); const result = await deployPromise; - // The publicKey is the sidecar's principal public key (hex). The - // router derives it from the signing seed; the resulting hex is a - // 64-character lowercase string. + // Every deployment -- single- or multi-step -- acks the deployment + // address's own public key, the one `signChallenge` signs reconnect + // challenges with, so the hub can verify ownership on reconnect. A + // multi-step deployment previously acked the supervisor principal key, + // which the hub discarded. The hex is a 64-character lowercase string. expect(result.publicKey).toMatch(/^[0-9a-f]{64}$/); - - // Sanity check: re-deriving the public key from the keypair lines - // up with the router's returned value. - expect(result.publicKey).toBe(hexEncode(keyPair.publicKey)); + expect(result.publicKey).toBe(hexEncode(deploymentKeyPair.publicKey)); // Teardown: kill the child so the spawn-time pumps unwind. // Use unused supervisorToChild to silence the linter. @@ -1317,14 +771,128 @@ describe("createSidecarDeployRouter multi-step branch", () => { void supervisorIpcKeyPair; }); - test("registers a multistepMailRouter handler against the deployment address once spawn succeeds", async () => { - // Drives the spawn handshake the same way the first multi-step - // test does, but injects a `multistepMailRouter` and asserts the - // deploy router registered a handler against the deployment's - // mail address by the time `deploy(frame)` resolves. The handler - // is what the sidecar hub-link's `mail.inbound` path dispatches - // through; without this registration, an inbound mail aimed at - // the deployment address falls into the legacy session path, + test("a second same-address deploy is rejected mid-spawn and never deletes the live deployment record", async () => { + // Pins the synchronous single-flight reservation guard. The first + // deploy runs its durable writes and then suspends inside + // supervisor.spawn awaiting the child's `ready` handshake -- the window + // in which its reservation is held but `activeSupervisors` is not yet + // populated. A second same-address frame arriving in that window must be + // rejected at the reservation guard (its own message, distinct from the + // spawn-core backstop) before it touches durable state, so it cannot + // delete the first deploy's live record via the soft-fail catch. + const childIpcKeyPair = await generateKeyPair(); + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const eventChildToSupervisor = createMemoryFrameStream(); + let resolveExit: ((code: number) => void) | undefined; + const exited = new Promise((resolve) => { + resolveExit = resolve; + }); + let spawnCount = 0; + let observedEnv: Record | undefined; + const spawner: SubprocessSpawner = ({ env }) => { + spawnCount += 1; + observedEnv = env; + const handle: SubprocessHandle = { + pid: 4242, + controlWriter: supervisorToChild.writer, + controlReader: childToSupervisor.reader, + eventReader: eventChildToSupervisor.reader, + kill: () => { + childToSupervisor.close(); + eventChildToSupervisor.close(); + resolveExit?.(0); + }, + exited, + }; + return handle; + }; + + const multiDataDir = await createTempBaseDir("sidecar-concurrent-deploy-"); + const registered: string[] = []; + const { router } = await buildMultistepFixture({ + spawner, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: multiDataDir }, + registerDeployment: ({ agentAddress }) => { + registered.push(agentAddress); + }, + }); + + const sources = defaultMultistepSources(); + const definition = { + id: "wf-concurrent", + triggers: [{ type: "manual" }], + stepOrder: ["step-1", "step-2"], + steps: { "step-1": { kind: "step" }, "step-2": { kind: "step" } }, + }; + const frame = makeMultistepFrame({ definition, sources }); + const deploymentId = deriveDeploymentId(frame.agentAddress); + const recordFile = path.join( + multiDataDir, + "workflow-runs", + deploymentId, + "deployment.json", + ); + + const firstDeploy = router.deploy(frame); + // Wait until the first deploy has spawned: its record is on disk and it + // is now suspended in the ready handshake with the reservation held. + while (observedEnv === undefined) { + await new Promise((r) => setTimeout(r, 1)); + } + const recordBefore = await fs.readFile(recordFile, "utf8"); + expect(recordBefore.length).toBeGreaterThan(0); + + // The loser is rejected at the reservation guard, not the spawn-core + // backstop -- the guard's message is the one asserted here. + await expect(router.deploy(frame)).rejects.toThrow( + /is already deployed; undeploy it before redeploying/, + ); + // It never reached the spawner and never deleted the live record. + expect(spawnCount).toBe(1); + const recordAfter = await fs.readFile(recordFile, "utf8"); + expect(recordAfter).toBe(recordBefore); + + // Drive the first deploy's ready handshake so it completes, then confirm + // the winner is the live, registered deployment. + const channelId = observedEnv.IPC_CHANNEL_ID; + if (channelId === undefined) { + throw new Error("IPC_CHANNEL_ID not set in spawn-time env"); + } + const childSender = createControlChannelSender({ + privateKeySeed: childIpcKeyPair.privateKey, + channelId, + writer: { + write(line: string) { + childToSupervisor.inject(line); + return Promise.resolve(); + }, + }, + }); + await childSender.send({ + type: "ready", + data: { + childPid: 4242, + childPublicKey: Buffer.from(childIpcKeyPair.publicKey).toString("hex"), + }, + }); + + const result = await firstDeploy; + expect(result.publicKey).toMatch(/^[0-9a-f]{64}$/); + expect(registered).toEqual([frame.agentAddress]); + expect(router.activeAddresses()).toEqual([frame.agentAddress]); + + void supervisorToChild; + }); + + test("registers a multistepMailRouter handler against the deployment address once spawn succeeds", async () => { + // Drives the spawn handshake the same way the first multi-step + // test does, but injects a `multistepMailRouter` and asserts the + // deploy router registered a handler against the deployment's + // mail address by the time `deploy(frame)` resolves. The handler + // is what the sidecar hub-link's `mail.inbound` path dispatches + // through; without this registration, an inbound mail aimed at + // the deployment address falls into the legacy session path, // which has no transport entry and no `sessions` row for the // deployment address. const childIpcKeyPair = await generateKeyPair(); @@ -1428,7 +996,7 @@ describe("createSidecarDeployRouter multi-step branch", () => { stepOrder: ["step-1"], steps: { "step-1": { kind: "step" } }, }, - sources: { "step-1": makeInferenceSource("step-1") }, + sources: { "step-1": [makeInferenceSource("step-1")] }, }); await expect(router.deploy(frame)).rejects.toThrow( @@ -1440,6 +1008,94 @@ describe("createSidecarDeployRouter multi-step branch", () => { ); }); + test("a soft-failed deploy (spawn rejects) leaves no restore record", async () => { + const crashSpawner: SubprocessSpawner = () => { + throw new Error("ENOENT: binary missing"); + }; + const { router, substrateEnv } = await buildMultistepFixture({ + spawner: crashSpawner, + }); + const agentAddress = "ins_softfail@example.com"; + const frame = makeMultistepFrame({ + agentAddress, + definition: { + id: "wf-softfail", + triggers: [{ type: "manual" }], + stepOrder: ["step-1"], + steps: { "step-1": { kind: "step" } }, + }, + sources: { "step-1": [makeInferenceSource("step-1")] }, + }); + + await expect(router.deploy(frame)).rejects.toThrow(/ENOENT/); + + // The record is written before the spawn, so the soft-failure catch must + // delete it -- a boot-time restore must not re-spawn a deploy that never + // completed. (A hard crash mid-spawn, by contrast, deliberately leaves + // the record for the restore to re-drive.) + const dataDir = substrateEnv.SIDECAR_DATA_DIR; + if (dataDir === undefined) + throw new Error("fixture SIDECAR_DATA_DIR unset"); + const recordFile = path.join( + dataDir, + "workflow-runs", + deriveDeploymentId(agentAddress), + "deployment.json", + ); + expect( + await fs.access(recordFile).then( + () => true, + () => false, + ), + ).toBe(false); + }); + + test("rejects a deploy whose step pins an unbuildable provider before spawning", async () => { + // The source-admission gate runs before any state is claimed or the + // child is spawned. A step whose pinned source names a provider the + // sidecar cannot build must reject the whole deploy synchronously -- + // the admission control property -- rather than spawning a child that + // fails when the step's inference first resolves. + let spawnCount = 0; + const trackingSpawner: SubprocessSpawner = () => { + spawnCount++; + throw new Error("spawn must not be reached for an inadmissible source"); + }; + const { router } = await buildMultistepFixture({ + spawner: trackingSpawner, + assertSourceBuildable: (source) => { + if (source.provider === "ghost-provider") { + throw new Error( + `Source provider "${source.provider}" is not registered`, + ); + } + }, + }); + + const frame = makeMultistepFrame({ + agentAddress: "ins_unbuildable@example.com", + definition: { + id: "wf-unbuildable", + triggers: [{ type: "manual" }], + stepOrder: ["step-1"], + steps: { "step-1": { kind: "step" } }, + }, + sources: { + "step-1": [ + { + ...makeInferenceSource("step-1"), + provider: "ghost-provider", + }, + ], + }, + }); + + await expect(router.deploy(frame)).rejects.toThrow( + /ghost-provider.*not registered/, + ); + expect(spawnCount).toBe(0); + }); + test("a spawner that throws synchronously surfaces a structured rejection rather than hanging in starting", async () => { // Simulates `Bun.spawn` failing to launch (binary missing, // permissions error). The router must surface the rejection @@ -1459,7 +1115,7 @@ describe("createSidecarDeployRouter multi-step branch", () => { steps: { "step-1": { kind: "step" } }, }, sources: { - "step-1": makeInferenceSource("step-1"), + "step-1": [makeInferenceSource("step-1")], }, }); @@ -1486,7 +1142,7 @@ describe("createSidecarDeployRouter multi-step branch", () => { steps: { "step-1": { kind: "step" } }, }, sources: { - "step-1": makeInferenceSource("step-1"), + "step-1": [makeInferenceSource("step-1")], }, }); @@ -1870,4 +1526,898 @@ describe("createSidecarDeployRouter multi-step branch", () => { expect(secondResult.publicKey).toMatch(/^[0-9a-f]{64}$/); expect(registerCallCount).toBe(2); }); + + // ------------------------------------------------------------------ + // Boot-time restore of persisted workflow deployments + // ------------------------------------------------------------------ + + // A mock spawner that serves a fresh control/event channel per spawn and + // lets the test complete each child's `ready` handshake. Both `deploy` and + // `restoreWorkflowDeployments` block on `supervisor.spawn` until `ready` + // lands, so every spawned child needs its handshake driven. + function makeReadyDrivingSpawner(pidBase: number) { + type Spawn = { + env: Record; + childToSupervisor: ReturnType; + eventChildToSupervisor: ReturnType; + childSender?: ReturnType; + }; + const spawns: Spawn[] = []; + const spawner: SubprocessSpawner = ({ env }) => { + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const eventChildToSupervisor = createMemoryFrameStream(); + let resolveExit: ((code: number) => void) | undefined; + const exited = new Promise((resolve) => { + resolveExit = resolve; + }); + spawns.push({ env, childToSupervisor, eventChildToSupervisor }); + const handle: SubprocessHandle = { + pid: pidBase + spawns.length, + controlWriter: supervisorToChild.writer, + controlReader: childToSupervisor.reader, + eventReader: eventChildToSupervisor.reader, + kill: () => { + childToSupervisor.close(); + eventChildToSupervisor.close(); + resolveExit?.(0); + }, + exited, + }; + return handle; + }; + async function driveReadyFor( + index: number, + opts?: { sendRecycleRequest?: boolean }, + ): Promise { + while (spawns.length <= index) { + await new Promise((r) => setTimeout(r, 1)); + } + const spawn = spawns[index]; + if (spawn === undefined) { + throw new Error(`spawn ${String(index)} missing`); + } + const channelId = spawn.env.IPC_CHANNEL_ID; + if (channelId === undefined) { + throw new Error("IPC_CHANNEL_ID missing in spawn env"); + } + const childIpcKeyPair = await generateKeyPair(); + const childSender = createControlChannelSender({ + privateKeySeed: childIpcKeyPair.privateKey, + channelId, + writer: { + write(line: string) { + spawn.childToSupervisor.inject(line); + return Promise.resolve(); + }, + }, + }); + // Retain the sender so a later recycle.request (recycleRequestFor) + // signs with the SAME keypair the supervisor pinned from this ready. + spawn.childSender = childSender; + await childSender.send({ + type: "ready", + data: { + childPid: pidBase + index, + childPublicKey: Buffer.from(childIpcKeyPair.publicKey).toString( + "hex", + ), + }, + }); + if (opts?.sendRecycleRequest === true) { + // Model the child asking to recycle; the supervisor's upstream pump + // consumes it and respawns. + await childSender.send({ + type: "recycle.request", + data: { reason: "sources-rotation-recycle" }, + }); + } + } + return { + spawner, + driveReadyFor, + spawnCount: () => spawns.length, + envFor: (index: number): Record | undefined => + spawns[index]?.env, + async recycleRequestFor(index: number): Promise { + const sender = spawns[index]?.childSender; + if (sender === undefined) { + throw new Error( + `spawn ${String(index)} has no sender; drive its ready first`, + ); + } + await sender.send({ + type: "recycle.request", + data: { reason: "sources-rotation-recycle" }, + }); + }, + }; + } + + function isRegistered( + transport: ReturnType, + address: string, + ): boolean { + try { + transport.getTransportFor(address); + return true; + } catch { + return false; + } + } + + function recordExists( + dataDir: string, + deploymentId: string, + ): Promise { + return fs + .access( + path.join(dataDir, "workflow-runs", deploymentId, "deployment.json"), + ) + .then( + () => true, + () => false, + ); + } + + function singleStepFrame( + agentAddress: string, + definitionId: string, + ): AgentDeployFrame { + return makeMultistepFrame({ + agentAddress, + definition: { + id: definitionId, + triggers: [{ type: "manual" }], + stepOrder: ["step-1"], + steps: { "step-1": { kind: "step" } }, + }, + sources: { "step-1": [makeInferenceSource("step-1")] }, + }); + } + + test("restore re-spawns a persisted single-step deployment and re-registers its head on a fresh transport", async () => { + const dataDir = await createTempBaseDir("sidecar-restore-restart-data-"); + const head = "ins_restart@example.com"; + + // First process: deploy a single-step workflow. The deploy persists a + // restore record under `dataDir` and materializes its `workflow.json`. + const first = makeReadyDrivingSpawner(9100); + const { router: routerA } = await buildMultistepFixture({ + spawner: first.spawner, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + const deployPromise = routerA.deploy(singleStepFrame(head, "wf-restart")); + await first.driveReadyFor(0); + await deployPromise; + + // Second process (simulated restart): a FRESH transport (empty + // registration table) and fresh in-memory router state over the SAME + // on-disk data dir. + const second = makeReadyDrivingSpawner(9200); + const freshTransport = createInMemoryTransport(); + const { router: routerB } = await buildMultistepFixture({ + spawner: second.spawner, + transport: freshTransport, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + + // Nothing is registered before restore -- the restart started clean. + expect(isRegistered(freshTransport, head)).toBe(false); + + const restorePromise = routerB.restoreWorkflowDeployments(); + await second.driveReadyFor(0); + await restorePromise; + + // The deployment was re-spawned exactly once and its head is live again. + expect(second.spawnCount()).toBe(1); + expect(isRegistered(freshTransport, head)).toBe(true); + }); + + test("restore soft-fails a record whose workflow.json is missing and restores the rest", async () => { + const dataDir = await createTempBaseDir("sidecar-restore-softfail-data-"); + const goodHead = "ins_good@example.com"; + const badHead = "ins_bad@example.com"; + + const first = makeReadyDrivingSpawner(9300); + const { router: routerA } = await buildMultistepFixture({ + spawner: first.spawner, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + const deployGood = routerA.deploy(singleStepFrame(goodHead, "wf-good")); + await first.driveReadyFor(0); + await deployGood; + const deployBad = routerA.deploy(singleStepFrame(badHead, "wf-bad")); + await first.driveReadyFor(1); + await deployBad; + + // Remove the bad deployment's definition so its restore read faults. + await fs.rm( + path.join(dataDir, "assets", "workflow", "wf-bad", "workflow.json"), + ); + + const second = makeReadyDrivingSpawner(9400); + const freshTransport = createInMemoryTransport(); + const { router: routerB } = await buildMultistepFixture({ + spawner: second.spawner, + transport: freshTransport, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + + // The good deployment re-spawns (exactly one handshake to drive); + // scan order is filesystem-dependent, but only the good record spawns. + const restorePromise = routerB.restoreWorkflowDeployments(); + await second.driveReadyFor(0); + await restorePromise; + + expect(second.spawnCount()).toBe(1); + expect(isRegistered(freshTransport, goodHead)).toBe(true); + expect(isRegistered(freshTransport, badHead)).toBe(false); + // The failed record is KEPT on disk -- never deleted, unlike a + // soft-failed deploy -- so a later boot can retry it. + expect(await recordExists(dataDir, deriveDeploymentId(badHead))).toBe(true); + }); + + test("restore applies validateWorkflowProjection: a stepOrder entry with no matching steps is skipped", async () => { + const dataDir = await createTempBaseDir("sidecar-restore-validator-data-"); + const head = "ins_validator@example.com"; + const deploymentId = deriveDeploymentId(head); + + // Hand-write a record plus a workflow.json whose `stepOrder` names a step + // `steps` does not define. This clears the wire arktype + // (`AgentDeployWorkflow` only checks that `sources` cover `stepOrder`) but + // MUST be rejected by `validateWorkflowProjection`, the second gate the + // deploy path applies. If restore ran only the arktype it would spawn a + // child for a structurally invalid definition. + const record: WorkflowDeploymentRecord = { + version: 1, + agentAddress: head, + definitionId: "wf-missing-step", + sources: { "step-1": [makeInferenceSource("step-1")] }, + hubPublicKey: "hub-pk", + }; + await writeWorkflowDeploymentRecord(dataDir, deploymentId, record); + const workflowJsonPath = path.join( + dataDir, + "assets", + "workflow", + "wf-missing-step", + "workflow.json", + ); + await fs.mkdir(path.dirname(workflowJsonPath), { recursive: true }); + await fs.writeFile( + workflowJsonPath, + JSON.stringify({ + id: "wf-missing-step", + triggers: [{ type: "manual" }], + stepOrder: ["step-1"], + steps: {}, + }), + "utf8", + ); + + const spawner = makeReadyDrivingSpawner(9500); + const freshTransport = createInMemoryTransport(); + const { router } = await buildMultistepFixture({ + spawner: spawner.spawner, + transport: freshTransport, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + + await router.restoreWorkflowDeployments(); + + expect(spawner.spawnCount()).toBe(0); + expect(isRegistered(freshTransport, head)).toBe(false); + }); + + test("restore is a no-op for a deployment already live in this process", async () => { + const dataDir = await createTempBaseDir("sidecar-restore-guard-data-"); + const head = "ins_guard@example.com"; + + const spawner = makeReadyDrivingSpawner(9600); + const { router, transport } = await buildMultistepFixture({ + spawner: spawner.spawner, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + + const deployPromise = router.deploy(singleStepFrame(head, "wf-guard")); + await spawner.driveReadyFor(0); + await deployPromise; + expect(spawner.spawnCount()).toBe(1); + + // The record is on disk and the address is live in this same process. A + // restore pass must NOT spawn a second child for an address the core's + // double-spawn guard already owns (the transition guard the B-reroute + // follow-up leans on). + await router.restoreWorkflowDeployments(); + + expect(spawner.spawnCount()).toBe(1); + expect(isRegistered(transport, head)).toBe(true); + }); + + test("a second deploy for a live address is rejected without orphaning its restore record", async () => { + const dataDir = await createTempBaseDir("sidecar-restore-dup-data-"); + const head = "ins_dup@example.com"; + const deploymentId = deriveDeploymentId(head); + + const spawner = makeReadyDrivingSpawner(9700); + const { router, transport } = await buildMultistepFixture({ + spawner: spawner.spawner, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + + const deployPromise = router.deploy(singleStepFrame(head, "wf-dup")); + await spawner.driveReadyFor(0); + await deployPromise; + expect(await recordExists(dataDir, deploymentId)).toBe(true); + + // A second deploy for the already-live address must be rejected WITHOUT + // touching the running deployment's durable state. The reject fires + // before any overwrite; without it, deployMultiStep's catch would delete + // the live deployment's record and release its slug, silently breaking + // the next restart for a still-running agent. + await expect( + router.deploy(singleStepFrame(head, "wf-dup")), + ).rejects.toThrow(/already deployed/); + expect(spawner.spawnCount()).toBe(1); + expect(await recordExists(dataDir, deploymentId)).toBe(true); + expect(isRegistered(transport, head)).toBe(true); + }); + + test("restore skips a record whose address does not derive its directory name", async () => { + const dataDir = await createTempBaseDir("sidecar-restore-mismatch-data-"); + const head = "ins_mismatch@example.com"; + // A record filed under a directory that is NOT its own derived slug -- + // a corrupt or misplaced record that must not be restored under the + // wrong slug. + const wrongDir = "not-the-right-slug"; + const record: WorkflowDeploymentRecord = { + version: 1, + agentAddress: head, + definitionId: "wf-mismatch", + sources: { "step-1": [makeInferenceSource("step-1")] }, + hubPublicKey: "hub-pk", + }; + await writeWorkflowDeploymentRecord(dataDir, wrongDir, record); + + const spawner = makeReadyDrivingSpawner(9800); + const freshTransport = createInMemoryTransport(); + const { router } = await buildMultistepFixture({ + spawner: spawner.spawner, + transport: freshTransport, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + + await router.restoreWorkflowDeployments(); + + expect(spawner.spawnCount()).toBe(0); + expect(isRegistered(freshTransport, head)).toBe(false); + // The record is kept on a skip, not deleted. + expect(await recordExists(dataDir, wrongDir)).toBe(true); + }); + + test("restore soft-fails and keeps the record when the pinned source is no longer buildable", async () => { + const dataDir = await createTempBaseDir( + "sidecar-restore-unbuildable-data-", + ); + const head = "ins_unbuildable_restore@example.com"; + const deploymentId = deriveDeploymentId(head); + + // First process: a permissive gate lets the deploy through, persisting + // the record and its workflow.json. + const first = makeReadyDrivingSpawner(9900); + const { router: routerA } = await buildMultistepFixture({ + spawner: first.spawner, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + const deployPromise = routerA.deploy( + singleStepFrame(head, "wf-unbuildable-restore"), + ); + await first.driveReadyFor(0); + await deployPromise; + + // Restart with a gate that now rejects the pinned provider. + const second = makeReadyDrivingSpawner(10000); + const freshTransport = createInMemoryTransport(); + const { router: routerB } = await buildMultistepFixture({ + spawner: second.spawner, + transport: freshTransport, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + assertSourceBuildable: (source) => { + throw new Error( + `Source provider "${source.provider}" is not registered`, + ); + }, + }); + + await routerB.restoreWorkflowDeployments(); + + expect(second.spawnCount()).toBe(0); + expect(isRegistered(freshTransport, head)).toBe(false); + // The record survives so a later boot, once the provider is buildable + // again, can retry it. + expect(await recordExists(dataDir, deploymentId)).toBe(true); + }); + + test("a deploy whose child never signals ready times out and rejects", async () => { + const dataDir = await createTempBaseDir("sidecar-ready-timeout-data-"); + const head = "ins_readytimeout@example.com"; + const deploymentId = deriveDeploymentId(head); + + // A spawner whose child is created but never driven through the `ready` + // handshake. With a small threaded readyTimeoutMs the supervisor times + // out, kills the child, and rejects the spawn. The message echoes the + // threaded value, so this also proves readyTimeoutMs reaches the + // supervisor across the router's forwarding. + const spawner = makeReadyDrivingSpawner(10100); + const { router } = await buildMultistepFixture({ + spawner: spawner.spawner, + readyTimeoutMs: 40, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + + await expect( + router.deploy(singleStepFrame(head, "wf-readytimeout")), + ).rejects.toThrow(/did not emit ready within 40ms/); + + // The deploy soft-failed, so its restore record was cleaned up -- a + // wedged deploy leaves nothing for a later boot to re-spawn. + expect(await recordExists(dataDir, deploymentId)).toBe(false); + }); + + test("a single-step deploy acks the agent key, not the supervisor key", async () => { + // The single-step head IS an agent identity: it signs its own reconnect + // challenges with the agent key, and the hub records the ack's key into + // agent_instance.publicKey and verifies the challenge against it. So the + // ack must surface the AGENT key, not the supervisor principal key -- + // otherwise a rerouted instance's reconnect signature never matches. + const headKeyPair = await generateKeyPair(); + const spawner = makeReadyDrivingSpawner(10300); + const { router, keyPair: fixtureKeyPair } = await buildMultistepFixture({ + spawner: spawner.spawner, + headKeyPair, + multistepSubstrateEnv: { + SIDECAR_DATA_DIR: await createTempBaseDir("sidecar-bkey-data-"), + }, + }); + + const deployPromise = router.deploy( + singleStepFrame("ins_bkey@example.com", "wf-bkey"), + ); + await spawner.driveReadyFor(0); + const result = await deployPromise; + + // The supervisor principal key is derived from the fixture's signing seed + // (fixtureKeyPair); the head's agent key is the distinct headKeyPair. + expect(result.publicKey).toBe( + Buffer.from(headKeyPair.publicKey).toString("hex"), + ); + expect(result.publicKey).not.toBe( + Buffer.from(fixtureKeyPair.publicKey).toString("hex"), + ); + }); + + test("registers a sources-rotation handler for a single-step deployment", async () => { + const sourcesRouter = createMultistepSourcesRouter(); + const spawner = makeReadyDrivingSpawner(10600); + const { router } = await buildMultistepFixture({ + spawner: spawner.spawner, + multistepSourcesRouter: sourcesRouter, + multistepSubstrateEnv: { + SIDECAR_DATA_DIR: await createTempBaseDir("sidecar-sources-single-"), + }, + }); + + const deployPromise = router.deploy( + singleStepFrame("ins_srcsingle@example.com", "wf-srcsingle"), + ); + await spawner.driveReadyFor(0); + await deployPromise; + + // The single-step deploy registered a rotation handler, so an inbound + // sources.update for its address routes. + expect( + await sourcesRouter.tryRoute({ + type: "sources.update", + agentAddress: "ins_srcsingle@example.com", + sources: [makeInferenceSource("primary")], + defaultSource: "primary", + }), + ).toBe(true); + }); + + test("does not register a sources-rotation handler for a multi-step deployment", async () => { + const sourcesRouter = createMultistepSourcesRouter(); + const spawner = makeReadyDrivingSpawner(10700); + const { router } = await buildMultistepFixture({ + spawner: spawner.spawner, + multistepSourcesRouter: sourcesRouter, + multistepSubstrateEnv: { + SIDECAR_DATA_DIR: await createTempBaseDir("sidecar-sources-multi-"), + }, + }); + + const frame = makeMultistepFrame({ + definition: { + id: "wf-srcmulti", + triggers: [{ type: "manual" }], + stepOrder: ["step-1", "step-2"], + steps: { "step-1": { kind: "step" }, "step-2": { kind: "step" } }, + }, + sources: { + "step-1": [makeInferenceSource("step-1")], + "step-2": [makeInferenceSource("step-2")], + }, + }); + const deployPromise = router.deploy(frame); + await spawner.driveReadyFor(0); + await deployPromise; + + // A multi-step deployment has no single warm agent to rotate, so no + // handler is registered and the inbound rotation is unrouted. + expect( + await sourcesRouter.tryRoute({ + type: "sources.update", + agentAddress: frame.agentAddress, + sources: [makeInferenceSource("primary")], + defaultSource: "primary", + }), + ).toBe(false); + }); + + test("a source rotation survives a recycle respawn", async () => { + // End-to-end guard for the rotation-survives-recycle fix: the single-step + // rotation handler mutates `currentSources`, `dynamicSpawnEnv` + // re-serializes it, and the recycle respawn's STEP_INFERENCE_SOURCES + // carries the ROTATED table -- not the frozen deploy-time one. + const sourcesRouter = createMultistepSourcesRouter(); + const spawner = makeReadyDrivingSpawner(10800); + const { router } = await buildMultistepFixture({ + spawner: spawner.spawner, + multistepSourcesRouter: sourcesRouter, + multistepSubstrateEnv: { + SIDECAR_DATA_DIR: await createTempBaseDir("sidecar-rot-survive-"), + }, + }); + + const addr = "ins_rotsurvive@example.com"; + const deployPromise = router.deploy(singleStepFrame(addr, "wf-rotsurvive")); + await spawner.driveReadyFor(0); + await deployPromise; + + // The initial spawn carries the deploy-time source table. + const initialEnv = spawner.envFor(0); + if (initialEnv === undefined) throw new Error("initial spawn env missing"); + expect( + JSON.parse(initialEnv[STEP_INFERENCE_SOURCES_ENV_KEY] ?? "null"), + ).toEqual({ "step-1": [makeInferenceSource("step-1")] }); + + // Rotate the single-step deployment's sources in place. + const rotated = makeInferenceSource("rotated"); + expect( + await sourcesRouter.tryRoute({ + type: "sources.update", + agentAddress: addr, + sources: [rotated], + defaultSource: "rotated", + }), + ).toBe(true); + + // The child asks to recycle; the supervisor respawns. + await spawner.recycleRequestFor(0); + while (spawner.spawnCount() < 2) { + await new Promise((r) => setTimeout(r, 1)); + } + await spawner.driveReadyFor(1); + + // The recycle respawn's sources are the ROTATED table, proving the + // rotation survived the recycle (before the fix it reverted to the + // deploy-time list frozen in substrateEnv). + const respawnEnv = spawner.envFor(1); + if (respawnEnv === undefined) throw new Error("respawn env missing"); + expect( + JSON.parse(respawnEnv[STEP_INFERENCE_SOURCES_ENV_KEY] ?? "null"), + ).toEqual({ "step-1": [rotated] }); + }); + + test("a source rotation survives a full sidecar restart", async () => { + // Restart-durability: a rotation is persisted into the deployment record, + // so a fresh sidecar process (a restore over the same data dir) respawns + // the deployment on the ROTATED sources, not the deploy-time ones. + const dataDir = await createTempBaseDir("sidecar-rot-restart-"); + const addr = "ins_rotrestart@example.com"; + + // First process: deploy a single-step deployment and rotate its sources. + const sourcesRouter = createMultistepSourcesRouter(); + const first = makeReadyDrivingSpawner(10900); + const { router: routerA } = await buildMultistepFixture({ + spawner: first.spawner, + multistepSourcesRouter: sourcesRouter, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + const deployPromise = routerA.deploy( + singleStepFrame(addr, "wf-rotrestart"), + ); + await first.driveReadyFor(0); + await deployPromise; + + const rotated = makeInferenceSource("rotated"); + expect( + await sourcesRouter.tryRoute({ + type: "sources.update", + agentAddress: addr, + sources: [rotated], + defaultSource: "rotated", + }), + ).toBe(true); + + // Second process (simulated restart): a fresh router over the SAME data + // dir restores the deployment from its durable record. + const second = makeReadyDrivingSpawner(11000); + const { router: routerB } = await buildMultistepFixture({ + spawner: second.spawner, + transport: createInMemoryTransport(), + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + const restorePromise = routerB.restoreWorkflowDeployments(); + await second.driveReadyFor(0); + await restorePromise; + + // The restored spawn carries the ROTATED sources, read back from the + // durable record -- not the deploy-time list. + const restoredEnv = second.envFor(0); + if (restoredEnv === undefined) { + throw new Error("restored spawn env missing"); + } + expect( + JSON.parse(restoredEnv[STEP_INFERENCE_SOURCES_ENV_KEY] ?? "null"), + ).toEqual({ "step-1": [rotated] }); + }); + + test("a rotation whose persist fails leaves no partial effect", async () => { + // Atomicity: if the durable write rejects, the rotation takes NO net + // effect. The handler swaps currentSources synchronously, then rolls it + // back on a failed persist, so once the handler throws currentSources is + // on the deploy-time table (a recycle respawn AFTER the failure is + // unrotated) and deliverSources is never reached. Rolling the in-memory + // hint back keeps currentSources and the record in agreement in this + // no-interleaved-recycle failure case. + const dataDir = await createTempBaseDir("sidecar-rot-failatomic-"); + const addr = "ins_rotfailatomic@example.com"; + const sourcesRouter = createMultistepSourcesRouter(); + const spawner = makeReadyDrivingSpawner(11100); + const { router } = await buildMultistepFixture({ + spawner: spawner.spawner, + multistepSourcesRouter: sourcesRouter, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + + const deployPromise = router.deploy(singleStepFrame(addr, "wf-rotfail")); + await spawner.driveReadyFor(0); + await deployPromise; + + // Replace the deployment record file with a directory so the rotation's + // writeWorkflowDeploymentRecord rejects (EISDIR) -- a deterministic, + // root-immune write fault (a chmod guard would be bypassed under root). + const recordFile = path.join( + dataDir, + "workflow-runs", + deriveDeploymentId(addr), + "deployment.json", + ); + await fs.rm(recordFile); + await fs.mkdir(recordFile); + + // The persist rejects, so the route rejects and nothing after the write + // runs. + await expect( + sourcesRouter.tryRoute({ + type: "sources.update", + agentAddress: addr, + sources: [makeInferenceSource("rotated")], + defaultSource: "rotated", + }), + ).rejects.toThrow(); + + // currentSources was rolled back: a recycle respawn AFTER the failed + // rotation carries the DEPLOY-TIME sources, proving the failed rotation + // left no net in-memory effect. + await spawner.recycleRequestFor(0); + while (spawner.spawnCount() < 2) { + await new Promise((r) => setTimeout(r, 1)); + } + await spawner.driveReadyFor(1); + const respawnEnv = spawner.envFor(1); + if (respawnEnv === undefined) throw new Error("respawn env missing"); + expect( + JSON.parse(respawnEnv[STEP_INFERENCE_SOURCES_ENV_KEY] ?? "null"), + ).toEqual({ "step-1": [makeInferenceSource("step-1")] }); + }); + + test("a recycle interleaving the rotation persist respawns on the rotated sources", async () => { + // Interleave guard: the rotation swaps currentSources synchronously + // BEFORE the durable persist, so a recycle that lands inside the persist + // window respawns the child on the ROTATED sources -- consistent with + // what is being persisted -- rather than the stale deploy-time table. + const dataDir = await createTempBaseDir("sidecar-rot-interleave-ok-"); + const addr = "ins_rotinterok@example.com"; + const sourcesRouter = createMultistepSourcesRouter(); + const spawner = makeReadyDrivingSpawner(11200); + + // A persist that blocks on a test-controlled gate once armed, so a recycle + // can be driven into the rotation's persist window. The deploy's own + // persist (before the gate is armed) passes straight through to the real + // writer. + let gate: Promise | null = null; + let release: () => void = () => undefined; + const persist: typeof writeWorkflowDeploymentRecord = async ( + d, + id, + rec, + ) => { + if (gate !== null) await gate; + await writeWorkflowDeploymentRecord(d, id, rec); + }; + + const { router } = await buildMultistepFixture({ + spawner: spawner.spawner, + multistepSourcesRouter: sourcesRouter, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + writeWorkflowDeploymentRecord: persist, + }); + + const deployPromise = router.deploy(singleStepFrame(addr, "wf-rotinterok")); + await spawner.driveReadyFor(0); + await deployPromise; + + // Arm the gate so the rotation's persist blocks mid-window. + gate = new Promise((resolve) => { + release = resolve; + }); + + // Start the rotation but do not await it: the handler swaps currentSources + // synchronously, then parks on the blocked persist before deliverSources. + const rotated = makeInferenceSource("rotated"); + const rotatePromise = sourcesRouter.tryRoute({ + type: "sources.update", + agentAddress: addr, + sources: [rotated], + defaultSource: "rotated", + }); + + // Drive a recycle while the persist is blocked. The respawn env must carry + // the ROTATED sources, because the synchronous swap already ran. + await spawner.recycleRequestFor(0); + while (spawner.spawnCount() < 2) { + await new Promise((r) => setTimeout(r, 1)); + } + await spawner.driveReadyFor(1); + const respawnEnv = spawner.envFor(1); + if (respawnEnv === undefined) throw new Error("respawn env missing"); + expect( + JSON.parse(respawnEnv[STEP_INFERENCE_SOURCES_ENV_KEY] ?? "null"), + ).toEqual({ "step-1": [rotated] }); + + // Release the persist; the rotation completes without throwing and the + // durable record converges on the rotated sources. + release(); + await rotatePromise; + const scanned = await scanWorkflowDeploymentRecords(dataDir); + const record = scanned.find( + (s) => s.deploymentId === deriveDeploymentId(addr), + ); + expect(record?.record.sources).toEqual({ "step-1": [rotated] }); + }); + + test("a recycle interleaving a failed rotation persist respawns on new sources then rolls back", async () => { + // The benign residual, pinned: a persist that fails WHILE a recycle + // interleaves leaves the just-respawned child transiently ahead on the + // rotated sources (the intended, self-healing direction), while + // currentSources rolls back to the deploy-time table so the NEXT recycle + // reverts the child to durable truth. + const dataDir = await createTempBaseDir("sidecar-rot-interleave-fail-"); + const addr = "ins_rotinterfail@example.com"; + const sourcesRouter = createMultistepSourcesRouter(); + const spawner = makeReadyDrivingSpawner(11300); + + let gate: Promise | null = null; + let release: () => void = () => undefined; + const persist: typeof writeWorkflowDeploymentRecord = async ( + d, + id, + rec, + ) => { + if (gate !== null) { + await gate; + throw new Error("rotation persist boom"); + } + await writeWorkflowDeploymentRecord(d, id, rec); + }; + + const { router } = await buildMultistepFixture({ + spawner: spawner.spawner, + multistepSourcesRouter: sourcesRouter, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + writeWorkflowDeploymentRecord: persist, + }); + + const deployPromise = router.deploy( + singleStepFrame(addr, "wf-rotinterfail"), + ); + await spawner.driveReadyFor(0); + await deployPromise; + + gate = new Promise((resolve) => { + release = resolve; + }); + + const rotated = makeInferenceSource("rotated"); + const rotatePromise = sourcesRouter.tryRoute({ + type: "sources.update", + agentAddress: addr, + sources: [rotated], + defaultSource: "rotated", + }); + + // A recycle interleaves the about-to-fail persist: the respawn still + // carries the ROTATED sources, because the swap already ran. + await spawner.recycleRequestFor(0); + while (spawner.spawnCount() < 2) { + await new Promise((r) => setTimeout(r, 1)); + } + await spawner.driveReadyFor(1); + const respawnEnv = spawner.envFor(1); + if (respawnEnv === undefined) throw new Error("respawn env missing"); + expect( + JSON.parse(respawnEnv[STEP_INFERENCE_SOURCES_ENV_KEY] ?? "null"), + ).toEqual({ "step-1": [rotated] }); + + // Release the persist so it rejects; the rotation rolls currentSources + // back and rethrows. + release(); + await expect(rotatePromise).rejects.toThrow(/rotation persist boom/); + + // A SECOND recycle now respawns on the ROLLED-BACK deploy-time sources, + // healing the transient down to durable truth. + await spawner.recycleRequestFor(1); + while (spawner.spawnCount() < 3) { + await new Promise((r) => setTimeout(r, 1)); + } + await spawner.driveReadyFor(2); + const secondRespawnEnv = spawner.envFor(2); + if (secondRespawnEnv === undefined) { + throw new Error("second respawn env missing"); + } + expect( + JSON.parse(secondRespawnEnv[STEP_INFERENCE_SOURCES_ENV_KEY] ?? "null"), + ).toEqual({ "step-1": [makeInferenceSource("step-1")] }); + }); + + test("two addresses whose deriveDeploymentId slugs collide are rejected at the second deploy", async () => { + // deriveDeploymentId substitutes every disallowed character with + // `-`, so two distinct addresses can collapse to the same slug. The slug + // IS the workflow-run repoId, so a silent collision would let the second + // deploy overwrite the first deploy's repo state. claimSlug rejects the + // second deploy at the router edge, before any spawn or repo write. + const spawner = makeReadyDrivingSpawner(10400); + const { router } = await buildMultistepFixture({ + spawner: spawner.spawner, + multistepSubstrateEnv: { + SIDECAR_DATA_DIR: await createTempBaseDir("sidecar-collision-data-"), + }, + }); + + // `ins_col.a@example.com` and `ins_col-a@example.com` both project to + // `ins_col-a-example-com` under the slug derivation. + const deployPromise = router.deploy( + singleStepFrame("ins_col.a@example.com", "wf-collide"), + ); + await spawner.driveReadyFor(0); + await deployPromise; + + await expect( + router.deploy(singleStepFrame("ins_col-a@example.com", "wf-collide")), + ).rejects.toThrow(/deriveDeploymentId collision/); + expect(spawner.spawnCount()).toBe(1); + }); }); diff --git a/apps/sidecar/src/workflow-host-wiring.ts b/apps/sidecar/src/workflow-host-wiring.ts index e61d00a0..704f82cb 100644 --- a/apps/sidecar/src/workflow-host-wiring.ts +++ b/apps/sidecar/src/workflow-host-wiring.ts @@ -1,10 +1,8 @@ // Thin wiring module that constructs `createWorkflowSupervisor` with // this sidecar's host-specific bindings: the existing mail-bus // instance, the sidecar's Ed25519 signing keypair, the substrate -// RepoStore handle, `Bun.spawn` as the subprocess spawner, and a -// host-injected `trivialLaunch` callback that drives the legacy -// single-agent provisioning surface for trivial (1-step) deploys. -// Any logic that would benefit a future alternative-sidecar +// RepoStore handle, and `Bun.spawn` as the subprocess spawner. Any +// logic that would benefit a future alternative-sidecar // implementation lives inside `@intx/workflow-host`, not here. import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; @@ -42,12 +40,9 @@ import { type HubTransportMailBusAdapter, type NdjsonReader, type NdjsonWriter, - type RecordRunEvent, type SpawnOpts, type SubprocessHandle, type SubprocessSpawner, - type SupervisorRunEvent, - type TrivialLaunch, type WorkflowSupervisor, } from "@intx/workflow-host"; import { hexEncode } from "@intx/types"; @@ -55,9 +50,13 @@ import { parseInferenceEvent, type CryptoProvider, type InferenceEvent, + type InferenceSource, type KeyPair, } from "@intx/types/runtime"; -import type { AgentDeployFrame } from "@intx/types/sidecar"; +import { + AgentDeployWorkflow, + type AgentDeployFrame, +} from "@intx/types/sidecar"; import { STEP_ID_PATTERN } from "@intx/workflow"; import { deriveWorkflowRunRepoId } from "@intx/workflow-deploy"; @@ -65,7 +64,14 @@ import type { MultistepDrainRouter, MultistepMailRouter, MultistepSignalRouter, + MultistepSourcesRouter, } from "./workflow-run-pack-client"; +import { + deleteWorkflowDeploymentRecord, + scanWorkflowDeploymentRecords, + writeWorkflowDeploymentRecord, + type WorkflowDeploymentRecord, +} from "./workflow-deployment-record"; const logger = getLogger(["interchange", "sidecar", "workflow-host-wiring"]); @@ -79,7 +85,7 @@ const logger = getLogger(["interchange", "sidecar", "workflow-host-wiring"]); * the sidecar's call sites readable while the rationale and the * substrate `SAFE_REPO_ID` contract live with the shared function. */ -export function deriveTrivialDeploymentId(agentAddress: string): string { +export function deriveDeploymentId(agentAddress: string): string { return deriveWorkflowRunRepoId(agentAddress); } @@ -402,6 +408,12 @@ export type CreateSidecarWorkflowSupervisorOpts = { workflowRunRef: string; /** Deployment id baked into principal claims and address derivation. */ deploymentId: string; + /** + * Step count of the deployed `WorkflowDefinition` (`stepOrder.length`). + * Threaded into the child's spawn-time env so its deploy-tree read + * collapses onto the head for a single-step deployment. + */ + stepCount: number; /** Deployment's mail address. */ deploymentMailAddress: string; /** Per-step mail-address derivation. */ @@ -419,12 +431,11 @@ export type CreateSidecarWorkflowSupervisorOpts = { /** Substrate-config keys propagated to the child via spawn-time env. */ substrateEnv: Record; /** - * Host-injected callback the supervisor's trivial branch invokes. - * For the production sidecar this closes over the - * `SessionManager.provisionAgent` flow plus the hub-pairing-key - * recording the legacy `agent.deploy` handler performed inline. + * Dynamic spawn-env fragment the supervisor recomputes on every spawn and + * recycle respawn (e.g. a live-rotated inference-source list). Its keys + * layer over `substrateEnv`. See the `dynamicSpawnEnv` supervisor binding. */ - trivialLaunch: TrivialLaunch; + dynamicSpawnEnv: () => Record; /** * Override the subprocess spawner. Tests inject a deterministic * mock; production defaults to the `Bun.spawn`-backed @@ -455,6 +466,13 @@ export type CreateSidecarWorkflowSupervisorOpts = { * supervisor applies `DEFAULT_CONSUMED_RETENTION_MS` (24h). */ consumedRetentionMs?: number; + /** + * Spawn ready-handshake timeout (ms), forwarded to the supervisor's + * `readyTimeoutMs` binding. The boot edge resolves the operator's + * `CHILD_READY_TIMEOUT_MS` config; absent, the supervisor applies + * `DEFAULT_READY_TIMEOUT_MS` (30s). + */ + readyTimeoutMs?: number; }; export type SidecarWorkflowSupervisor = { @@ -466,38 +484,12 @@ export type SidecarWorkflowSupervisor = { }; /** - * Construct the sidecar's `DeployRouter`. The router holds the - * `sessions` + `keyStore` closures the workflow-host supervisor's - * `trivialLaunch` needs and constructs a fresh per-deployment - * supervisor on every inbound `agent.deploy` frame. The trivial - * branch then calls back into `sessions.provisionAgent` plus the - * pre-existing hub-pairing-key recording so the bytes flowing - * through the deploy-flow gate test path stay bit-identical to the - * pre-supervisor path. - * - * Multi-step deploys (frames carrying a workflow definition with - * `steps.length >= 2`) route through `supervisor.deploy`'s - * multi-step branch -- the IPC channel, child spawn, and - * `credentialsSnapshot` assembly. The routing seam exists here so - * the frame-format extension that carries the workflow definition is - * a pure data-shape change. - */ -/** - * Stable step identifier the trivial workflow uses for its single - * step. The on-disk `StepStarted` / `StepCompleted` envelopes carry - * this value verbatim; it is opaque to the supervisor and the - * substrate, but downstream audit-log consumers join run events on - * it. The trivial workflow has exactly one step per run, so the id - * is a constant rather than a per-deploy mint. - */ -const TRIVIAL_STEP_ID = "trivial"; - -/** - * Env key the multi-step branch uses to carry per-step inference source - * pins from `frame.workflow.sources` down to the workflow-process child. - * The substrate factory's `buildEnv` reads this and resolves a source - * per step at step invocation; the supervisor itself is opaque to the - * value (it is plumbed through `bindings.substrateEnv` verbatim). + * Env key the multi-step branch uses to carry each step's ordered + * inference-source failover chain from `frame.workflow.sources` down to + * the workflow-process child. The substrate factory's `buildEnv` reads + * this and resolves a step's chain at step invocation, feeding it to the + * reactor for forward-only failover; the supervisor itself is opaque to + * the value (it is plumbed through `bindings.substrateEnv` verbatim). * * Listed here so the router and the future substrate-factory consumer * spell the key the same way without a magic-string trip hazard. @@ -527,11 +519,14 @@ export const STEP_INFERENCE_SOURCES_ENV_KEY = "STEP_INFERENCE_SOURCES"; * entry be absent; presence is required so the workflow-process * child can resolve each step's primitive at run time. * - Every `stepOrder` entry has a corresponding `sources[id]` - * entry. The arktype narrow already enforces this; the re-check - * here surfaces a structured router-side error instead of an - * arktype validation failure at the wire boundary, which keeps + * entry, and that entry is a non-empty array (the step's ordered + * failover chain). The arktype narrow already enforces both; the + * re-check here surfaces a structured router-side error instead of + * an arktype validation failure at the wire boundary, which keeps * the failure shape consistent with the rest of the validations - * this function owns. + * this function owns. An empty chain would leave the reactor with + * no initial source, so it is rejected here rather than deferred to + * a deep-stack child failure. * * A rejection here surfaces as a thrown `Error` the link's deploy * frame caller converts into a structured failure reply. @@ -584,6 +579,13 @@ export function validateWorkflowProjection(projection: { `sidecar deploy router: workflow.sources is missing entry for stepId ${JSON.stringify(stepId)}`, ); } + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- sources is checked to be a non-null object above; this reads a value to re-check its array shape + const stepSources = (sources as Record)[stepId]; + if (!Array.isArray(stepSources) || stepSources.length === 0) { + throw new Error( + `sidecar deploy router: workflow.sources[${JSON.stringify(stepId)}] must be a non-empty array (the step's ordered inference-source failover chain)`, + ); + } } } @@ -647,10 +649,40 @@ async function derivePrincipalPublicKeyHex( return hexEncode(await derivePublicKeyBytes(signingKeySeed)); } +/** + * The sidecar's `DeployRouter` plus the boot-time restore driver. The link + * routes `agent.deploy`/`agent.undeploy` through the `DeployRouter` surface; + * the sidecar boot edge additionally calls `restoreWorkflowDeployments` once, + * before connecting to the hub, to re-establish the deployments a prior + * process persisted. The extra method is sidecar-app-only, so it rides on the + * concrete router type rather than the shared `DeployRouter` contract. + */ +export interface SidecarDeployRouter extends DeployRouter { + /** + * Re-establish every persisted workflow deployment on this sidecar's local + * substrate. Runs once at boot, before `hubLink.connect()`, so a single-step + * head's mailbox/transport registration is live before the hub routes to it. + * Soft-fails per deployment: a record that cannot be restored (unbuildable + * provider, corrupt `workflow.json`, spawn failure) is logged and left on + * disk for a later boot to retry -- it is never deleted here. + */ + restoreWorkflowDeployments(): Promise; + /** + * The workflow-substrate deployment addresses (`ins_dep_...`) this router + * currently hosts a live supervisor for -- the set of addresses this + * sidecar can route mail to. The boot edge announces these to the hub on + * (re)connect so the hub re-registers them for routing: they are hub-minted + * and carry no per-address key, so unlike single-agent sessions they are + * not re-established by the challenge flow, and without this announcement + * the hub drops their route on a WS reconnect. Reflects `deploy`/`undeploy` + * and boot-time restore live, so a caller re-reads it per connect. + */ + activeAddresses(): string[]; +} + export function createSidecarDeployRouter(deps: { sessions: SessionManager; keyStore: AgentKeyStore; - onAgentEvent: SessionManager["onAgentEvent"]; transport: HubTransport; repoStore: RepoStore; signingKeySeed: Uint8Array; @@ -663,18 +695,33 @@ export function createSidecarDeployRouter(deps: { * mail path (`MailBusBindings.sendOutbound`) signs the agent's replies * with the AGENT's identity -- the OUTBOUND half of mailbox ownership * (§3a). Without this registration the spawned agent's address has no - * `CryptoProvider` on the transport (no `startSession` runs for it), - * and an outbound send would throw "address is not registered" rather - * than emit unsigned mail. + * `CryptoProvider` on the transport (nothing else registers one for + * it), and an outbound send would throw "address is not registered" + * rather than emit unsigned mail. */ createAgentCrypto: (keyPair: KeyPair) => CryptoProvider; + /** + * Source-admission gate: throws if a step's pinned inference source + * names a provider this sidecar cannot build. The buildable-provider + * set is sidecar config (the boot edge's adapter registry), so this + * admission control lives at the sidecar -- the hub is a different + * process and cannot know a given sidecar's providers. Production wires + * the default harness builder's `canBuildSource` verbatim, so a rejected + * provider carries the same `"... is not registered"` message. + * + * Distinct from the orchestrator's operator-approval check + * (`pickStepInferenceSource`): that gates on whether the operator + * approved a `provider:model`; this gates on whether the provider is + * buildable at all. A source can be approved yet unbuildable. + */ + assertSourceBuildable: (source: InferenceSource) => void; /** * Record a `(deploymentId -> agentAddress)` mapping the boot edge's * workflow-run pack push facade consults when it must address an * outbound pack frame. Fires once per inbound `agent.deploy` frame - * before the supervisor's `deploy()` call so the first `recordRunEvent` - * commit (which triggers the push hook) sees the mapping. Tests that - * do not exercise the pack push path may pass a no-op. + * before the deployment's supervisor spawns, so the first pack push + * the child triggers sees the mapping. Tests that do not exercise + * the pack push path may pass a no-op. */ registerDeployment: (entry: { deploymentId: string; @@ -699,16 +746,14 @@ export function createSidecarDeployRouter(deps: { * the workflow-process child's spawn-time env (see * `SIDECAR_SUBSTRATE_CONFIG_KEYS` in `workflow-substrate-factory.ts`). * The router merges `STEP_INFERENCE_SOURCES` on top per multi-step - * frame. Defaults to an empty record so trivial-only deployments do - * not require the boot edge to thread substrate config through the - * router. + * frame. Defaults to an empty record so a router built without + * substrate config (e.g. a test) needs no boot-edge threading. */ multistepSubstrateEnv?: Record; /** * Subprocess spawner the multi-step branch hands to the supervisor. * Defaults to the production `Bun.spawn`-backed * `defaultSubprocessSpawner`; tests inject a deterministic mock. - * The trivial branch never invokes the spawner. */ multistepSubprocessSpawner?: SubprocessSpawner; /** @@ -750,13 +795,12 @@ export function createSidecarDeployRouter(deps: { * `wired.routeInbound` against the deployment's mail address once * `supervisor.spawn` succeeds so inbound mail aimed at the * deployment address flows into the supervisor's mail-bus - * subscription. The trivial branch never touches this registry -- - * its mail path is the legacy session surface. + * subscription. * - * Optional so tests that exercise the trivial branch (or the - * multi-step branch without an end-to-end mail loop) can omit the - * binding; an absent registry simply means multi-step inbound mail - * cannot route through the hub-link until the wiring is plumbed. + * Optional so tests that exercise the multi-step branch without an + * end-to-end mail loop can omit the binding; an absent registry + * simply means multi-step inbound mail cannot route through the + * hub-link until the wiring is plumbed. */ multistepMailRouter?: MultistepMailRouter; /** @@ -770,10 +814,10 @@ export function createSidecarDeployRouter(deps: { * preserving the workflow-run repo's single-writer invariant on the * sidecar side. * - * Optional so tests that exercise the trivial branch (or the - * multi-step branch without an end-to-end signal loop) can omit the - * binding; an absent registry means hub-side signals cannot route - * through the hub-link until the wiring is plumbed. + * Optional so tests that exercise the multi-step branch without an + * end-to-end signal loop can omit the binding; an absent registry + * means hub-side signals cannot route through the hub-link until the + * wiring is plumbed. */ multistepSignalRouter?: MultistepSignalRouter; /** @@ -788,12 +832,26 @@ export function createSidecarDeployRouter(deps: { * signed `CancelRequested{origin: "supervisor-drain"}` against the * workflow-run repo when the deadline expires. * - * Optional so tests that exercise the trivial branch (or the - * multi-step branch without an end-to-end drain loop) can omit the - * binding; an absent registry means hub-side drain frames cannot - * route through the hub-link until the wiring is plumbed. + * Optional so tests that exercise the multi-step branch without an + * end-to-end drain loop can omit the binding; an absent registry + * means hub-side drain frames cannot route through the hub-link until + * the wiring is plumbed. */ multistepDrainRouter?: MultistepDrainRouter; + /** + * Per-deployment-address sources-rotation handler registry. Only a + * single-step warm deployment registers a handler (against the + * deployment's mail address once `supervisor.spawn` succeeds) so a + * rotation resolved for its address flows into + * `wired.supervisor.deliverSources` and on to the child's warm agent. A + * multi-step deployment registers none -- it has no single warm agent to + * rotate -- so `tryRoute` reports its address as unrouted. + * + * Optional so tests that exercise deploys without a rotation loop can + * omit the binding; an absent registry means no rotation handler is + * installed for any deployment. + */ + multistepSourcesRouter?: MultistepSourcesRouter; /** * Optional per-message dispatch-timing observer the multi-step branch * forwards to each supervisor it constructs. Resolved at the sidecar @@ -818,7 +876,24 @@ export function createSidecarDeployRouter(deps: { * handler for the operator-owned horizon invariant. */ consumedRetentionMs?: number; -}): DeployRouter { + /** + * Spawn ready-handshake timeout (ms) forwarded to every supervisor the + * router constructs. The sidecar boot edge resolves the operator's + * `CHILD_READY_TIMEOUT_MS` config; absent, the supervisor applies + * `DEFAULT_READY_TIMEOUT_MS` (30s). A child that spawns but never + * signals ready is killed and its spawn rejected rather than hanging + * the deploy or boot-time restore. + */ + readyTimeoutMs?: number; + /** + * Deployment-record writer, injectable so a test can block or fail the + * persist at a controlled point -- the natural seam for exercising a + * recycle that interleaves the source-rotation persist window. Defaults + * to the real `writeWorkflowDeploymentRecord`; production never overrides + * it. + */ + writeWorkflowDeploymentRecord?: typeof writeWorkflowDeploymentRecord; +}): SidecarDeployRouter { // Validate the signing seed at construction so a malformed key fails // sidecar boot rather than the first multi-step deploy, where the // public key is derived from it (`derivePrincipalPublicKeyHex`). The @@ -843,10 +918,12 @@ export function createSidecarDeployRouter(deps: { // (`/workflow-step-state//...`). Resolved once // from the boot-edge substrate env so the undeploy hook can reclaim // the whole subtree. Absent only when the router is wired without - // substrate config (trivial-only paths / tests that never spawn a - // child), in which case no child ever rooted scratch and the - // undeploy reclaim is correctly skipped. + // substrate config (a test that never spawns a child), in which case + // no child ever rooted scratch and the undeploy reclaim is correctly + // skipped. const stepStateDataDir = multistepSubstrateEnv.SIDECAR_DATA_DIR; + const persistDeploymentRecord = + deps.writeWorkflowDeploymentRecord ?? writeWorkflowDeploymentRecord; const multistepSpawner = deps.multistepSubprocessSpawner ?? defaultSubprocessSpawner; const multistepDeriveStepAddress: DeriveStepAddress = @@ -858,13 +935,23 @@ export function createSidecarDeployRouter(deps: { // frame; the supervisor owns the workflow-process child, its IPC // pipes, and its event-channel fd. The undeploy hook consults this // map to call `supervisor.shutdown()` so the child's lifetime ends - // with the deployment. The trivial branch never spawns a child -- - // its supervisor is constructed and immediately dropped after the - // deploy callback runs -- so no entry is recorded for trivial - // deploys and the undeploy hook's lookup is a no-op for them. + // with the deployment. const activeSupervisors = new Map(); - // Slug-collision tracking. `deriveTrivialDeploymentId` substitutes + // Synchronous single-flight guard for the deploy path. The real supervisor + // does not exist until inside `spawnWorkflowDeployment`, so `deployMultiStep` + // cannot reserve its `activeSupervisors` slot up front; instead it records + // the address here synchronously, before its first await, and clears it in a + // finally once the deploy settles. `activeSupervisors` is populated only + // after `spawn` succeeds, so the has-check alone leaves a window in which two + // same-address frames both pass and the loser's unwind deletes the winner's + // live deployment record. This set closes that window: a second frame that + // arrives while the first is mid-deploy is rejected before it touches any + // durable state. Only the live deploy path reserves; the boot restore path + // is serial and relies on the `activeSupervisors` backstop instead. + const reservingDeployAddresses = new Set(); + + // Slug-collision tracking. `deriveDeploymentId` substitutes // disallowed characters with `-`, which is deterministic but lossy: // two distinct agent addresses can collapse to the same slug, and // a collision would let the second deploy silently overwrite the @@ -878,9 +965,13 @@ export function createSidecarDeployRouter(deps: { const existing = slugClaims.get(deploymentId); if (existing !== undefined && existing !== agentAddress) { throw new Error( - `deriveTrivialDeploymentId collision: agent addresses ${JSON.stringify(existing)} and ${JSON.stringify(agentAddress)} both project to deploymentId ${JSON.stringify(deploymentId)}`, + `deriveDeploymentId collision: agent addresses ${JSON.stringify(existing)} and ${JSON.stringify(agentAddress)} both project to deploymentId ${JSON.stringify(deploymentId)}`, ); } + // A same-address re-claim is a defensive no-op: the `activeSupervisors` + // guard rejects a live re-deploy before claimSlug is re-invoked, and a + // failed or undeployed deploy releases the slug first, so in practice + // `existing` is only ever undefined or a different address here. slugClaims.set(deploymentId, agentAddress); } @@ -889,167 +980,211 @@ export function createSidecarDeployRouter(deps: { if (existing === agentAddress) slugClaims.delete(deploymentId); } - async function deployMultiStep( - frame: AgentDeployFrame, - projection: NonNullable, - ): Promise { - // Boundary validation: a malformed projection is rejected at the - // router edge before the supervisor is constructed so the link - // surfaces a structured failure rather than a hung `starting` - // supervisor. - validateWorkflowProjection(projection); + /** + * Materialize the workflow definition on the sidecar's local substrate so + * the workflow-process child's `loadWorkflowDefinition` can read + * `workflow.json` out of the workflow-asset repo's working tree. The + * destination mirrors the bare RepoStore's `getRepoDir` for + * `{ kind: "workflow", id }`: + * `${SIDECAR_DATA_DIR}/assets/workflow//workflow.json`. The child reads + * via `fs.readFile`, so writing the bytes outside git suffices. This is + * deploy-only durable state; the restore path finds it already on disk. + */ + async function materializeWorkflowJson( + sidecarDataDir: string | undefined, + definition: NonNullable["definition"], + ): Promise { + if (typeof sidecarDataDir !== "string" || sidecarDataDir.length === 0) { + throw new Error( + "sidecar deploy router: SIDECAR_DATA_DIR must be present in the multi-step substrate env; the workflow-process child resolves the workflow-asset repo dir against this data dir", + ); + } + const workflowAssetPath = pathJoin( + sidecarDataDir, + "assets", + "workflow", + definition.id, + "workflow.json", + ); + const workflowAssetBytes = JSON.stringify(definition, null, 2); + try { + await mkdir(dirname(workflowAssetPath), { recursive: true }); + // Idempotent: only rewrite when the on-disk content differs. Treats a + // missing file as different. + let existing: string | null = null; + try { + existing = await readFile(workflowAssetPath, "utf8"); + } catch (cause) { + if ( + !( + cause instanceof Error && + "code" in cause && + (cause as { code: unknown }).code === "ENOENT" + ) + ) { + throw cause; + } + } + if (existing !== workflowAssetBytes) { + await writeFile(workflowAssetPath, workflowAssetBytes, "utf8"); + } + } catch (cause) { + const reason = cause instanceof Error ? cause.message : String(cause); + throw new Error( + `sidecar deploy router: failed to materialize workflow.json at ${workflowAssetPath}: ${reason}`, + { cause }, + ); + } + } - const deploymentId = deriveTrivialDeploymentId(frame.agentAddress); + /** + * Read a workflow definition back off the sidecar's local substrate for a + * boot-time restore. Mirrors `materializeWorkflowJson`'s path derivation + * (`${dataDir}/assets/workflow//workflow.json`). Returns the + * parsed-but-unvalidated JSON: the on-disk file is untrusted at restore + * (partial write, corruption, tamper), so the caller re-validates it through + * the same wire + structural gates the deploy path applies. A missing file + * or unparseable JSON throws; the restore loop's per-record catch converts + * that into a warn-and-skip. + */ + async function readWorkflowJson( + sidecarDataDir: string, + definitionId: string, + ): Promise { + const workflowAssetPath = pathJoin( + sidecarDataDir, + "assets", + "workflow", + definitionId, + "workflow.json", + ); + const raw = await readFile(workflowAssetPath, "utf8"); + return JSON.parse(raw); + } - // Single-step launched-agent deploy vs. derived multi-step deploy. - // - // A one-step projection is the agent-launch identity path: the sole - // step keeps the deployment's own (legacy) mail address, and its - // grants live in the legacy agent-state repo keyed by the legacy - // instance id (`parseAgentId(frame.agentAddress)`). This preserves - // the identity the legacy agent-deploy path established -- the - // workflow-run repo stays keyed by `deriveWorkflowRunRepoId(legacy)` - // and `agent_instance.address` remains the `ins_` legacy shape. - // - // A multi-step projection derives `-` per step - // for both the mail address and the agent-state repo id, isolating - // each step's grants in its own repo. + /** + * The per-deployment inputs the shared spawn core needs to stand up a + * workflow deployment, independent of the live deploy frame. The live + * deploy path builds this from `frame`/`projection`; a boot-time restore + * path builds the same shape from the persisted deployment record. + */ + interface WorkflowDeploySpec { + agentAddress: string; + definition: NonNullable["definition"]; + sources: NonNullable["sources"]; + /** Correlates the child's inference events to the deploy's session. */ + sessionId: string | undefined; + /** + * Hub public key recorded at the head for deploy-pack verification and + * inbound hub-frame verification. Required for a single-step + * deployment (whose head IS the agent identity); undefined for a + * genuine multi-step deployment, which derives per-step addresses and + * records no head key. + */ + hubPublicKey: string | undefined; + } + + /** + * Build the durable deployment record from a spec and a source table. The + * table is a parameter (not `spec.sources`) so the deploy path writes the + * deploy-time sources while the rotation handler writes the live-rotated + * ones -- both through one shape, so a rotation persists the same record a + * boot-time restore reseeds from. + */ + function buildDeploymentRecord( + spec: WorkflowDeploySpec, + sources: WorkflowDeploymentRecord["sources"], + ): WorkflowDeploymentRecord { + return { + version: 1, + agentAddress: spec.agentAddress, + definitionId: spec.definition.id, + sources, + ...(spec.sessionId !== undefined ? { sessionId: spec.sessionId } : {}), + ...(spec.hubPublicKey !== undefined + ? { hubPublicKey: spec.hubPublicKey } + : {}), + }; + } + + /** + * The single owner of the workflow-deployment spawn sequence: construct + * the supervisor, register the single-step agent's outbound key + head + * repo + hub key, spawn the workflow-process child, then register the + * live deployment (supervisor, mail/signal/drain routers, address + * mapping). Its `try/finally` unwinds every piece of partial state if any + * step throws, so a failed spawn leaks nothing. Both the live deploy path + * and the boot-time restore path route through here so the two can never + * diverge on how a deployment is stood up. Callers materialize the + * deploy-only durable state (`workflow.json`, step grants) before calling. + */ + async function spawnWorkflowDeployment( + spec: WorkflowDeploySpec, + ): Promise { + // Fail loud if this address already has a live supervisor. Both single- + // and multi-step now register on the transport, so both carry the + // `transport.register` duplicate-throw backstop; this `has()` check is the + // primary early guard that gives a clean error before that lower-level + // throw and before the `activeSupervisors.set` below could clobber the + // running deployment's handle. Both the deploy path and the boot restore + // path route through here, so this is the single transition guard against + // a double-spawn -- notably a boot restore racing a legacy restore for the + // same address (the B-reroute follow-up relies on it). + if (activeSupervisors.has(spec.agentAddress)) { + throw new Error( + `sidecar deploy router: a supervisor is already active for ${spec.agentAddress}; refusing to spawn a second`, + ); + } + const deploymentId = deriveDeploymentId(spec.agentAddress); + + // Single-step launched-agent deploy vs. derived multi-step deploy. A + // one-step deployment keeps the deployment's own (legacy) mail address + // and its grants in the legacy agent-state repo keyed by the legacy + // instance id. A multi-step deployment derives `-` + // per step for both the mail address and the agent-state repo id. const stepStrategy = createStepStrategy({ - legacyAddress: frame.agentAddress, - stepOrder: projection.definition.stepOrder, + legacyAddress: spec.agentAddress, + stepOrder: spec.definition.stepOrder, multistepDeriveStepAddress, }); - claimSlug(deploymentId, frame.agentAddress); - // Release every piece of partial state if any step between here - // and `registerDeployment` throws. The undeploy hook -- the only - // other release caller -- never fires for failed deploys, so - // without this finally the slug, the freshly-spawned - // workflow-process child, the `activeSupervisors` entry, and the - // three multistep router registrations would all leak. The + // Unwind every piece of spawn state if any step between here and + // `registerDeployment` throws, so a failed spawn leaks no + // freshly-spawned workflow-process child, `activeSupervisors` entry, + // transport registration, or multistep router registration. The // ordering inside the finally is the reverse of the success-path - // registration order so each release runs against state the - // success path has already established. - let claimedSlugSucceeded = false; + // registration order. The caller owns the deployment slug: it must + // claim the collision guard before any durable write and release it on + // failure, so the slug is not touched here. + let succeeded = false; let wiredForUnwind: SidecarWorkflowSupervisor | undefined; let supervisorRegistered = false; let routersRegistered = false; let agentTransportRegistered = false; + let hubKeyRecorded = false; try { - const definitionHash = await computeWireDefinitionHash( - projection.definition, - ); + const definitionHash = await computeWireDefinitionHash(spec.definition); // Per-deployment substrate-config keys the workflow-substrate-factory - // validator requires (`SIDECAR_SUBSTRATE_CONFIG_KEYS` / - // `SubstrateConfig` in `workflow-substrate-factory.ts`). The boot - // edge's `multistepSubstrateEnv` only carries the boot-edge constants - // (`SIDECAR_DATA_DIR`, signing keys, hub link anchors); the four - // workflow-definition / workflow-run identity keys must be derived - // per-deploy here so the child's substrate-config validator passes - // at startup. - // - // `WORKFLOW_RUN_REPO_ID` mirrors `workflowRunRepoId.id` (the - // substrate-safe slug of the deployment address) and - // `WORKFLOW_RUN_REF` mirrors `workflowRunRef`, so the child resolves - // the same workflow-run repo the supervisor is writing into. - // - // `WORKFLOW_DEFINITION_REPO_ID` is set to `projection.definition.id` - // (the workflow asset's repo id; see the orchestrator's - // `WorkflowRepoWriter.writeWorkflowRepo` call, which writes the - // asset repo keyed by `workflow.id`). The child's - // `loadWorkflowDefinition` in - // `packages/workflow-host/src/child/run-child.ts` reads - // `workflow.json` out of this repo's working tree. The current - // Phase I multi-step test path does not yet stage the workflow - // asset on the sidecar's substrate (the orchestrator writes the - // asset to the hub repo store, not the sidecar's data dir), so the - // child's `loadWorkflowDefinition` will not actually find a - // `workflow.json` until the workflow-asset deploy lands on the - // sidecar. The value is structurally correct -- a consistent, - // deterministic id derived from the definition -- so the substrate- - // config validator passes and the next gap (if any) surfaces - // structurally rather than at the env-keys boundary. - // - // `WORKFLOW_DEFINITION_REF` is `"refs/heads/main"` to mirror the - // hub's `DEFAULT_ASSET_REF` and the workflow-run ref the supervisor - // uses. + // validator requires. The boot edge's `multistepSubstrateEnv` carries + // the boot-edge constants; the four workflow-definition / workflow-run + // identity keys are derived per-deploy here. const substrateEnv: Record = { ...multistepSubstrateEnv, - WORKFLOW_DEFINITION_REPO_ID: projection.definition.id, + WORKFLOW_DEFINITION_REPO_ID: spec.definition.id, WORKFLOW_DEFINITION_REF: "refs/heads/main", WORKFLOW_RUN_REPO_ID: deploymentId, WORKFLOW_RUN_REF: "refs/heads/main", - [STEP_INFERENCE_SOURCES_ENV_KEY]: JSON.stringify(projection.sources), }; - - // Materialize the workflow asset on the sidecar's local substrate - // so the workflow-process child's `loadWorkflowDefinition` can read - // `workflow.json` out of the workflow-asset repo's working tree - // (`packages/workflow-host/src/child/run-child.ts`). The hub's - // orchestrator writes the asset to the hub's repo store; the - // sidecar's substrate is a separate data dir on disk, and nothing - // replicates between the two today. - // - // The frame inlines the validated `WorkflowDefinition` already - // (see `AgentDeployFrame.workflow.definition`), so the router has - // the bytes in scope. The destination path mirrors the bare - // RepoStore's `getRepoDir` for `{ kind: "workflow", id }`: - // `${SIDECAR_DATA_DIR}/assets/workflow//workflow.json`. The - // workflow-asset repo kind handler's `directoryPrefix` is - // `assets/workflow` (see `packages/hub-sessions/src/workflow-kind.ts`). - // - // The child reads via `fs.readFile`, not via a git ref resolution, - // so writing the bytes outside any git operation is sufficient. - const sidecarDataDir = substrateEnv.SIDECAR_DATA_DIR; - if (typeof sidecarDataDir !== "string" || sidecarDataDir.length === 0) { - throw new Error( - "sidecar deploy router: SIDECAR_DATA_DIR must be present in the multi-step substrate env for the multi-step branch; the workflow-process child resolves the workflow-asset repo dir against this data dir", - ); - } - const workflowAssetPath = pathJoin( - sidecarDataDir, - "assets", - "workflow", - projection.definition.id, - "workflow.json", - ); - const workflowAssetBytes = JSON.stringify(projection.definition, null, 2); - try { - await mkdir(dirname(workflowAssetPath), { recursive: true }); - // Idempotent: only rewrite when the on-disk content differs from - // the projection. Treats a missing file as different. - let existing: string | null = null; - try { - existing = await readFile(workflowAssetPath, "utf8"); - } catch (cause) { - if ( - !( - cause instanceof Error && - "code" in cause && - (cause as { code: unknown }).code === "ENOENT" - ) - ) { - throw cause; - } - } - if (existing !== workflowAssetBytes) { - await writeFile(workflowAssetPath, workflowAssetBytes, "utf8"); - } - } catch (cause) { - const reason = cause instanceof Error ? cause.message : String(cause); - throw new Error( - `sidecar deploy router: failed to materialize workflow.json at ${workflowAssetPath}: ${reason}`, - { cause }, - ); - } + // Live-rotatable per-step inference sources. Seeded from the deploy + // spec, then revised in place by the single-step sources-rotation + // handler below. `STEP_INFERENCE_SOURCES` is NOT in the frozen + // `substrateEnv`: it is recomputed on every spawn and recycle respawn + // via `dynamicSpawnEnv`, so a rotation survives a recycle instead of + // reverting to the deploy-time list. + let currentSources = spec.sources; const wired = createSidecarWorkflowSupervisor({ - // Hold the constructed supervisor in the outer scope so the - // failure-path finally can `shutdown()` it; assignment - // happens immediately so even a synchronous throw inside - // the closure's call sites below is covered. - // (See the wiredForUnwind declaration above the try.) transport: deps.transport, repoStore: deps.repoStore, signingKeySeed: deps.signingKeySeed, @@ -1059,23 +1194,22 @@ export function createSidecarDeployRouter(deps: { }, workflowRunRef: "refs/heads/main", deploymentId, - deploymentMailAddress: frame.agentAddress, + stepCount: spec.definition.stepOrder.length, + deploymentMailAddress: spec.agentAddress, deriveStepAddress: stepStrategy.deriveStepAddress, deriveStepRepoId: stepStrategy.deriveStepRepoId, substrateEnv, + // Recomputed on every spawn AND recycle respawn. The rotation + // handler below revises `currentSources` in place, so a respawn + // re-serializes the current (possibly rotated) list rather than the + // frozen deploy-time value. + dynamicSpawnEnv: () => ({ + [STEP_INFERENCE_SOURCES_ENV_KEY]: JSON.stringify(currentSources), + }), subprocessSpawner: multistepSpawner, ...(deps.multistepBinaryPath !== undefined ? { binaryPath: deps.multistepBinaryPath } : {}), - // The multi-step branch never invokes trivialLaunch, but the - // supervisor's constructor requires the binding. Wire a sentinel - // that throws so a stray invocation surfaces loudly rather than - // silently succeeding. - trivialLaunch: () => { - throw new Error( - "sidecar deploy router: trivialLaunch invoked on the multi-step branch; this is a programming bug", - ); - }, ...(deps.onDispatchTiming !== undefined ? { onDispatchTiming: deps.onDispatchTiming } : {}), @@ -1085,133 +1219,110 @@ export function createSidecarDeployRouter(deps: { ...(deps.consumedRetentionMs !== undefined ? { consumedRetentionMs: deps.consumedRetentionMs } : {}), + ...(deps.readyTimeoutMs !== undefined + ? { readyTimeoutMs: deps.readyTimeoutMs } + : {}), }); - // Grants bridge (the sharp edge of the always-spawn convergence). - // - // The legacy agent-deploy path shipped grants in-band on the - // deploy frame; the in-process runtime loaded them straight into - // its grant store and never wrote them to disk. The spawned child - // does not see the frame: it reads each step's grants out of - // `state/grants.json` in the step's agent-state repo while the - // supervisor assembles the credentialsSnapshot inside `spawn()`. - // - // Write every step's grants to the step's agent-state repo BEFORE - // `spawn()` so the supervisor's read sees them. The hub is the - // source of truth -- `frame.config.grants` is the operator-approved - // grant set the hub shipped -- and the same `deriveStepRepoId` the - // supervisor reads with keys the write so the read and the write - // address the same repo. A missing or empty file would make every - // authorize fail closed, so the write must surface its failure. - await writeStepGrants({ - repoStore: deps.repoStore, - deploymentId, - stepOrder: projection.definition.stepOrder, - deriveStepRepoId: stepStrategy.deriveStepRepoId, - grants: frame.config.grants, - }); + // OUTBOUND half of mailbox ownership (§3a): register a signing key for + // the deployment mail address on the host transport so the supervisor + // signs the deployment's outbound mail. Every step -- single- or + // multi-step -- signs its outbound sends as `spec.agentAddress` (the + // one deployment mail address; no per-step sender reaches the host + // transport), so the transport MUST hold a `CryptoProvider` for it or + // `getTransportFor(senderAddress).send` throws "not registered". + // Registration happens before `spawn()` so the address is live the + // instant the first reply routes outbound. + const { keyPair } = await deps.keyStore.loadOrGenerateKey( + spec.agentAddress, + ); + deps.transport.register( + spec.agentAddress, + deps.createAgentCrypto(keyPair), + ); + agentTransportRegistered = true; - // OUTBOUND half of mailbox ownership (§3a): register the spawned - // agent's signing key on the host transport so the supervisor's - // outbound mail path (`MailBusBindings.sendOutbound`) signs the - // agent's replies as the AGENT's identity, with parity to the - // in-process path's `transport.register(address, crypto)`. - // - // Gated on the single-step launched-agent deploy: there the - // deployment mail address IS the legacy agent identity whose - // keypair lives in the keyStore, so the supervisor can sign - // outbound mail as that address. A genuine multi-step deploy - // derives a distinct per-step address with no keypair on the host; - // per-step outbound signing for multi-step is out of 4.3 scope - // (the unified single-agent path 4.3 targets is the single-step - // case). The registration happens before `spawn()` so the agent's - // address is live the instant the first reply routes outbound. - if (projection.definition.stepOrder.length === 1) { - const { keyPair } = await deps.keyStore.loadOrGenerateKey( - frame.agentAddress, - ); - deps.transport.register( - frame.agentAddress, - deps.createAgentCrypto(keyPair), - ); - agentTransportRegistered = true; + // The public key the deploy ack surfaces to the hub is the deployment + // address's own Ed25519 key -- the one `loadOrGenerateKey` minted above, + // which `AgentKeyStore.signChallenge(spec.agentAddress)` also signs + // reconnect challenges with. EVERY deployment acks it, single- and + // multi-step alike, so the hub can verify the reconnect ownership + // challenge for both: a single-step head records it into + // `agent_instance.publicKey`; a workflow-derived deployment records it on + // its `workflow_deployment` row. A multi-step deployment previously acked + // the supervisor principal key -- which the hub discarded and which does + // NOT match what `signChallenge` signs with -- so its address could be + // re-claimed on reconnect without proof; carrying the deployment key + // closes that. + const deploymentPublicKey = hexEncode(keyPair.publicKey); + if (spec.definition.stepOrder.length === 1) { + // A single-step workflow stages its deploy tree at the head (the + // lone step IS the head). Initialize the head's on-disk deploy-tree + // repo (idempotent) so the hub's deploy-pack push has a repo to + // apply into. The narrow `initRepo` (not `provisionAgent`) is + // deliberate: the supervised child mints its own keypair and + // persists no hub-agent config. + await deps.sessions.initRepo(spec.agentAddress); + + // Record the hub's public key at the head so the deploy-pack apply + // (and any inbound hub-signed frame) verifies against it. The + // verifier resolves the key from the in-memory key store's + // `recordHubKey` map, so a single-step deployment cannot stand up + // without it. + if (spec.hubPublicKey === undefined) { + throw new Error( + "sidecar deploy router: a single-step workflow deployment requires a hubPublicKey to record at the head; none was supplied", + ); + } + deps.keyStore.recordHubKey(spec.agentAddress, spec.hubPublicKey); + hubKeyRecorded = true; } - const stepOrder = [...projection.definition.stepOrder]; - // Warm-keep is the single-step launched-agent deploy (design §3b): - // the sole step IS the long-lived agent, so the child warm-keeps it - // across messages. A genuine multi-step deploy keeps - // instantiate-send-teardown per step -- warm-keeping N steps would - // hold N agents and N LSP subprocesses for no benefit. The signal - // is carried explicitly from this projection-level recognition down - // through the spawn env to the child's run-loop, never re-derived - // heuristically there. - const warmKeep = projection.definition.stepOrder.length === 1; + const stepOrder = [...spec.definition.stepOrder]; + // Warm-keep is the single-step launched-agent deploy: the sole step + // IS the long-lived agent, so the child warm-keeps it across + // messages. A multi-step deploy keeps instantiate-send-teardown per + // step. The signal is carried explicitly down through the spawn env. + const warmKeep = spec.definition.stepOrder.length === 1; const spawnOpts: SpawnOpts = { stepOrder, definitionHash, warmKeep, onInferenceEvent: (event) => { - // The event arrives already HMAC-verified and validated as an - // `EventPayload` over the child's event channel. Re-narrow it - // to the manually-defined `InferenceEvent` union the hub's - // `agent.event` sink consumes (the arktype-inferred - // `EventPayload` widens a few discriminants the hand-written - // type narrows). A parse failure here means the channel - // delivered something the validator rejects, which would be a - // corruption upstream -- drop it loudly rather than forwarding - // an unvalidated payload onto the hub timeline. + // The event arrives HMAC-verified over the child's event channel. + // Re-narrow it to the hub's `InferenceEvent` union; a parse + // failure means upstream corruption, so drop it loudly rather + // than forwarding an unvalidated payload onto the hub timeline. const validated = parseInferenceEvent(event); if (validated instanceof type.errors) { - logger.warn`dropping workflow inference event for ${frame.agentAddress}: ${validated.summary}`; + logger.warn`dropping workflow inference event for ${spec.agentAddress}: ${validated.summary}`; return; } - publishInferenceEvent( - frame.agentAddress, - validated, - frame.config.sessionId, - ); + publishInferenceEvent(spec.agentAddress, validated, spec.sessionId); }, }; - // Surface spawn-time errors structurally: if the subprocess - // spawner crashes immediately (binary missing, env malformed, - // EXEC error) the supervisor's `wireChild` races the child's - // `exited` against `readyPromise` and the rejection propagates - // here. The router lets it surface; the link's deploy handler - // converts the rejection into a structured failure frame. The - // supervisor is registered against the deployment address only - // after spawn succeeds; a spawn-time rejection leaves the - // registry untouched so the undeploy hook does not chase a - // supervisor that never owned a child. + // Surface spawn-time errors structurally: a subprocess spawner that + // crashes immediately rejects here, and the caller converts the + // rejection into a structured failure frame. The supervisor is + // registered against the deployment address only after spawn succeeds, + // so a spawn-time rejection leaves the registry untouched. await wired.supervisor.spawn(spawnOpts); - // Child process is live after `spawn` resolves; the failure - // unwind needs the supervisor handle from here on. wiredForUnwind = wired; - activeSupervisors.set(frame.agentAddress, wired); + activeSupervisors.set(spec.agentAddress, wired); supervisorRegistered = true; // Bind the deployment's mail address to this supervisor's - // `routeInbound` so the sidecar's hub-link dispatches inbound - // mail for the deployment address into the supervisor's mail-bus - // subscription rather than the legacy session path. The legacy - // path is the wrong receiver for multi-step deployments: the - // deployment address is never registered on `transport` (no - // `startSession` runs against it) and there is no `sessions` - // entry to satisfy `commitInboundMail`. Registration happens - // after `spawn` succeeds so a spawn-time rejection leaves the - // registry untouched. - deps.multistepMailRouter?.register(frame.agentAddress, (message) => { + // `routeInbound` so the hub-link dispatches inbound mail into the + // supervisor's mail-bus subscription. Registration happens after + // `spawn` succeeds so a spawn-time rejection leaves the registry + // untouched. + deps.multistepMailRouter?.register(spec.agentAddress, (message) => { wired.routeInbound(message); }); - // Register the signal-delivery handler against the deployment - // address so a hub-side `signal.deliver` frame dispatches through - // the supervisor's `deliverSignal`, which forwards a control IPC - // `signal.deliver` to the workflow-process child. The child writes - // the resulting `SignalReceived` event through its own substrate; - // the workflow-run pack-push pipeline then propagates the commit - // to the hub with no concurrent writer at the workflow-run ref. - deps.multistepSignalRouter?.register(frame.agentAddress, async (args) => { + // Register the signal-delivery handler so a hub `signal.deliver` frame + // dispatches through the supervisor's `deliverSignal`. + deps.multistepSignalRouter?.register(spec.agentAddress, async (args) => { await wired.supervisor.deliverSignal({ runId: args.runId, signalName: args.signalName, @@ -1219,47 +1330,117 @@ export function createSidecarDeployRouter(deps: { payload: args.payload, }); }); - // Register the drain handler against the deployment address so a - // hub-side `drain.deliver` frame dispatches through the - // supervisor's `drain`, which forwards a `drain` control IPC frame - // to the workflow-process child and arms one `drainTimeout` - // accumulator per in-flight run. Cancel-mode in-flight steps abort - // on the child side; wait-mode steps continue. Each accumulator - // commits a signed `CancelRequested{origin: "supervisor-drain"}` - // against the workflow-run repo when the deadline expires. - deps.multistepDrainRouter?.register(frame.agentAddress, async (args) => { + // Register the drain handler so a hub `drain.deliver` frame dispatches + // through the supervisor's `drain`. + deps.multistepDrainRouter?.register(spec.agentAddress, async (args) => { await wired.supervisor.drain({ deadlineMs: args.deadlineMs }); }); + // Register the sources-rotation handler ONLY for a single-step warm + // deployment: it has one long-lived agent whose sources can be + // swapped in place. A multi-step deployment has no single warm agent, + // so it registers no handler and `tryRoute` reports its address as + // unrouted. + if (warmKeep) { + // A single-step deployment's source table has exactly one entry, + // keyed by the head step. Derive that key once here (the layer that + // owns the single-key invariant); `deliverSources` stays flat and + // stepId-agnostic. + const rotationStepId = spec.definition.stepOrder[0]; + if (rotationStepId === undefined) { + throw new Error( + "single-step deploy has no step id for sources rotation", + ); + } + deps.multistepSourcesRouter?.register( + spec.agentAddress, + async (args) => { + const rotated = { [rotationStepId]: args.sources }; + // Swap `currentSources` synchronously BEFORE the durable persist. + // `currentSources` is the process-local respawn hint the + // supervisor reads synchronously through `dynamicSpawnEnv`, so a + // recycle that interleaves the persist `await` must respawn the + // child on the SAME sources being persisted, not the stale prior + // table. The obvious inverse -- persist first, then swap -- is + // rejected: it leaves the child on the OLD sources during the + // persist window while the record has already moved to NEW, so a + // recycle there respawns stale and a restart would "correct" it, + // i.e. the running child contradicts durable intent. Swapping + // first makes the only residual disagreement child-ahead-of- + // durable on a failed persist, which the next recycle heals down + // to the rolled-back durable truth -- the benign direction. The + // wire boundary guarantees `args.sources[0]` is the default, + // which the recycle env form pins as the active source. + const prevSources = currentSources; + currentSources = rotated; + // The durable write still precedes the LIVE swap + // (`deliverSources`), preserving persist-before-externally-visible + // for state that outlives the process; only the process-local + // respawn hint moves ahead. On a failed persist, roll the hint + // back so `currentSources` and the record stay in agreement in the + // common (no interleaved recycle) failure case -- the invariant + // restart consistency depends on. Persistence lets the rotation + // survive a full sidecar restart, not just a recycle: the boot + // scan reseeds spec.sources from record.sources. Overwrites the + // deploy-time record in place. Skipped when no data dir was wired + // (a test router that never persists), matching the restore guard. + if (stepStateDataDir !== undefined) { + try { + await persistDeploymentRecord( + stepStateDataDir, + deploymentId, + buildDeploymentRecord(spec, rotated), + ); + } catch (cause) { + // Restoring unconditionally is safe because rotations for one + // deployment are serialized by the sidecar's per-connection + // inbound-frame queue: each hub frame, sources.update + // included, runs its handler to completion on that queue + // before the next frame's handler starts, so no second + // rotation is in flight whose committed table this rollback + // could clobber. This does NOT rely on the hub pacing its + // sends -- the hub dispatches sources.update fire-and-forget; + // the sidecar frame queue is the sole serializer. Parallelizing + // inbound-frame dispatch would break this rollback. + currentSources = prevSources; + throw cause; + } + } + await wired.supervisor.deliverSources({ + sources: args.sources, + defaultSource: args.defaultSource, + }); + }, + ); + } routersRegistered = true; // Register the deployment-address mapping last so a failure in any - // earlier step (asset materialization, supervisor.spawn) leaves the - // boot-edge `DeploymentAddressRegistry` untouched. The link's - // `handleAgentDeploy` catches a rejection here and surfaces - // `agent.error` without invoking the undeploy hook; a partial - // registration would persist a `(deploymentId -> agentAddress)` - // entry for a deployment that never finished standing up. + // earlier step leaves the boot-edge `DeploymentAddressRegistry` + // untouched. Nothing fallible runs after it, so the finally unwind + // has no registry entry to reverse. deps.registerDeployment({ deploymentId, - agentAddress: frame.agentAddress, + agentAddress: spec.agentAddress, }); - claimedSlugSucceeded = true; - return { - publicKey: await derivePrincipalPublicKeyHex(deps.signingKeySeed), - }; + succeeded = true; + return { publicKey: deploymentPublicKey }; } finally { - if (!claimedSlugSucceeded) { - // Unwind in reverse registration order so each step undoes - // state the success path has confirmed; ordering matches - // the `undeploy` hook for consistency. + if (!succeeded) { + // Unwind in reverse registration order so each step undoes state + // the success path confirmed; ordering matches the `undeploy` hook. if (routersRegistered) { - deps.multistepMailRouter?.unregister(frame.agentAddress); - deps.multistepSignalRouter?.unregister(frame.agentAddress); - deps.multistepDrainRouter?.unregister(frame.agentAddress); + deps.multistepMailRouter?.unregister(spec.agentAddress); + deps.multistepSignalRouter?.unregister(spec.agentAddress); + deps.multistepDrainRouter?.unregister(spec.agentAddress); + // Unregister unconditionally: the sources handler was registered + // only for a single-step deploy, but `unregister` is a no-op for + // an address that never registered one, so a multi-step unwind + // safely calls it too. + deps.multistepSourcesRouter?.unregister(spec.agentAddress); } if (supervisorRegistered) { - activeSupervisors.delete(frame.agentAddress); + activeSupervisors.delete(spec.agentAddress); } if (wiredForUnwind !== undefined) { await wiredForUnwind.supervisor.shutdown().catch((cause) => { @@ -1269,131 +1450,237 @@ export function createSidecarDeployRouter(deps: { }); } if (agentTransportRegistered) { - // Drop the agent's transport registration so a failed deploy - // does not leave the address live on the host transport with a - // dangling `CryptoProvider`. `unregister` is safe to call even - // if the address was never registered. - deps.transport.unregister(frame.agentAddress); + // Drop the agent's transport registration so a failed deploy does + // not leave the address live with a dangling `CryptoProvider`. + deps.transport.unregister(spec.agentAddress); + } + if (hubKeyRecorded) { + // Reverse the single-step head's `recordHubKey` so a failed deploy + // leaves no in-memory hub key behind. `forgetAgent` also drops the + // agent keypair cache `loadOrGenerateKey` populated, which is safe: + // the transport registration is already unwound above, nothing reads + // that cache after unwind, and a redeploy reloads the keypair from + // disk. The on-disk deploy-tree repo `initRepo` created is + // deliberately NOT reversed. It is idempotent and the hub re-pushes + // the deploy pack on every redeploy, so it is benign residue; and + // decisively, the durable Ed25519 identity keypair lives inside that + // same directory (`keys/` nests under the agent repo dir), so + // removing the repo would destroy an identity a rerouted head must + // keep across a failed redeploy. + deps.keyStore.forgetAgent(spec.agentAddress); } - releaseSlug(deploymentId, frame.agentAddress); } } } + /** + * Provision one step of a multi-step deploy WITHOUT spawning. The hub + * stages each step's deploy tree before firing the deployment-level + * workflow frame; a full-closure deploy pack still needs an initialized + * agent-state repo to apply into and the hub key recorded to verify the + * pack commit signature. This does exactly those two things -- the same + * harness-free `initRepo` + `recordHubKey` seam the single-step head uses + * -- and constructs no supervisor or child. The deployment-level workflow + * frame (fired once after every step is provisioned) spawns the child, + * which reads each step's staged deploy tree from disk. + * + * Returns the sidecar's principal public key so the link's + * `agent.deploy.ack` carries a key, matching the multi-step ack. A + * per-step address is workflow-derived and records no `agent_instance` + * key, so the hub discards this value. + */ + async function provisionStep( + frame: AgentDeployFrame, + ): Promise { + await deps.sessions.initRepo(frame.agentAddress); + deps.keyStore.recordHubKey(frame.agentAddress, frame.hubPublicKey); + return { + publicKey: await derivePrincipalPublicKeyHex(deps.signingKeySeed), + }; + } + + async function deployMultiStep( + frame: AgentDeployFrame, + projection: NonNullable, + ): Promise { + // Boundary validation: a malformed projection is rejected at the + // router edge before the supervisor is constructed so the link + // surfaces a structured failure rather than a hung `starting` + // supervisor. + validateWorkflowProjection(projection); + + // Source-admission gate: reject a deploy where any step pins an + // inference provider this sidecar cannot build, BEFORE any state is + // claimed or the child is spawned. The throw propagates back through + // the deploy frame so the hub's `deployWorkflow` rejects synchronously + // at deploy time, rather than the child failing the run when the + // step's inference first resolves. Covers single- and multi-step: the + // projection's `narrow` guarantees every stepOrder entry has a + // `sources` entry. Every source in a step's failover chain must be + // buildable -- a chain with an unbuildable tail would fail only after + // the reactor failed over onto it -- so this iterates the whole list. + for (const stepId of projection.definition.stepOrder) { + const chain = projection.sources[stepId]; + if (chain !== undefined) { + for (const source of chain) deps.assertSourceBuildable(source); + } + } + + // Reject a re-deploy of an address already live OR mid-deploy in this + // process BEFORE touching any durable state. The durable writes below (the + // restore record, workflow.json, step grants) are destructive overwrites of + // state owned by whatever deployment currently holds the address; + // overwriting is only legal when this deploy owns the address. + // `activeSupervisors` catches an address whose deploy has completed; + // `reservingDeployAddresses` catches one whose deploy is still in flight. + // The map is populated only after `spawn` succeeds, so the has-check alone + // leaves a window in which two frames both pass and the loser's catch below + // deletes the winner's live record; the reservation set closes it. A + // re-deploy after `undeploy` passes: `undeploy` drops the + // `activeSupervisors` entry, and a failed or completed deploy has already + // cleared its reservation. + if ( + activeSupervisors.has(frame.agentAddress) || + reservingDeployAddresses.has(frame.agentAddress) + ) { + throw new Error( + `sidecar deploy router: ${frame.agentAddress} is already deployed; undeploy it before redeploying`, + ); + } + + const deploymentId = deriveDeploymentId(frame.agentAddress); + + // Single-step launched-agent deploy vs. derived multi-step deploy. + // + // A one-step projection is the agent-launch identity path: the sole + // step keeps the deployment's own (legacy) mail address, and its + // grants live in the legacy agent-state repo keyed by the legacy + // instance id (`parseAgentId(frame.agentAddress)`). This preserves + // the identity the legacy agent-deploy path established -- the + // workflow-run repo stays keyed by `deriveWorkflowRunRepoId(legacy)` + // and `agent_instance.address` remains the `ins_` legacy shape. + // + // A multi-step projection derives `-` per step + // for both the mail address and the agent-state repo id, isolating + // each step's grants in its own repo. + const stepStrategy = createStepStrategy({ + legacyAddress: frame.agentAddress, + stepOrder: projection.definition.stepOrder, + multistepDeriveStepAddress, + }); + + // Claim the deployment slug BEFORE any durable write so a colliding + // deploymentId (two distinct addresses projecting to the same slug) is + // rejected before `workflow.json`, the step grants, or the supervisor + // touch disk -- the router's "no repo state touched before rejection" + // guarantee. The claim is released on any failure below; a successful + // deploy keeps it (the undeploy hook releases it at teardown). The + // spawn core owns unwinding the supervisor and registrations it stands + // up; the slug is the caller's. + // Resolve the sidecar data dir once: the deployment record, workflow.json, + // and the per-step scratch all root under it. Required for any deployment + // that spawns a child. + const dataDir = stepStateDataDir; + if (typeof dataDir !== "string" || dataDir.length === 0) { + throw new Error( + "sidecar deploy router: SIDECAR_DATA_DIR must be present in the multi-step substrate env; the deployment record and workflow-process child root under it", + ); + } + + // The spec the shared spawn core consumes, and the durable record that + // lets a boot-time restore rebuild the SAME spec (definition re-read from + // workflow.json by id, grants from the step repos, and the record's + // frame/in-memory-only inputs: sources, session id, single-step hub key). + const spec: WorkflowDeploySpec = { + agentAddress: frame.agentAddress, + definition: projection.definition, + sources: projection.sources, + sessionId: frame.config.sessionId, + hubPublicKey: + projection.definition.stepOrder.length === 1 + ? frame.hubPublicKey + : undefined, + }; + const record = buildDeploymentRecord(spec, spec.sources); + + claimSlug(deploymentId, frame.agentAddress); + // Hold the single-flight reservation across the async body below and clear + // it in the finally. Everything above is synchronous and throws before any + // durable write, so the reservation is only needed from the first await + // here onward; the top-of-method guard already consults this set for a + // concurrent frame, and claimSlug/deploymentId derivation above cannot + // yield control before this point. + reservingDeployAddresses.add(frame.agentAddress); + try { + // Persist the deployment record BEFORE the spawn so a crash mid-spawn + // leaves a record the boot scan re-drives (an idempotent re-spawn; the + // child's in-flight-run discovery resumes any run). A soft-failed deploy + // deletes it below, so only a crash-interrupted deploy leaves one. + await persistDeploymentRecord(dataDir, deploymentId, record); + + // Materialize the deploy-only durable state the spawned child and the + // supervisor read from disk: the workflow definition (`workflow.json`) + // and each step's grants. The restore path finds both already on disk + // and skips this; both land before the shared spawn core runs. + await materializeWorkflowJson(dataDir, projection.definition); + + // Grants bridge: the spawned child does not see the frame; it reads + // each step's grants out of `state/grants.json` in the step's + // agent-state repo while the supervisor assembles the + // credentialsSnapshot. Write the operator-approved + // `frame.config.grants` to the same repo the supervisor reads via + // `deriveStepRepoId`, before the spawn core, so the read sees them. + await writeStepGrants({ + repoStore: deps.repoStore, + deploymentId, + stepOrder: projection.definition.stepOrder, + deriveStepRepoId: stepStrategy.deriveStepRepoId, + grants: frame.config.grants, + }); + + // Hand off to the shared spawn core. + return await spawnWorkflowDeployment(spec); + } catch (cause) { + // Soft failure (this process survived, the deploy threw): drop the + // record and release the slug so the failed deploy is neither restored + // nor leaks its slug. The record delete must not mask the real deploy + // error or skip releasing the slug: a rejecting delete is logged (the + // orphaned record is a durable-state leak the next boot scan re-drives) + // but `cause` is still what propagates and the slug is still released. + try { + await deleteWorkflowDeploymentRecord(dataDir, deploymentId); + } catch (cleanupError) { + const message = + cleanupError instanceof Error + ? cleanupError.message + : String(cleanupError); + logger.error`deploy cleanup: deleteWorkflowDeploymentRecord failed for ${deploymentId}: ${message}`; + } + releaseSlug(deploymentId, frame.agentAddress); + throw cause; + } finally { + // Release the single-flight reservation whether the deploy succeeded or + // threw. On success the address is now in `activeSupervisors`, which the + // guard also consults, so a later re-deploy is still rejected. + reservingDeployAddresses.delete(frame.agentAddress); + } + } + return { async deploy(frame): Promise { + if (frame.provisionStep === true) { + return await provisionStep(frame); + } if (frame.workflow !== undefined) { return await deployMultiStep(frame, frame.workflow); } - let publicKey: string | undefined; - const deploymentId = deriveTrivialDeploymentId(frame.agentAddress); - claimSlug(deploymentId, frame.agentAddress); - // See deployMultiStep's `claimedSlugSucceeded` guard: without - // it, any failure between `claimSlug` and the - // `registerDeployment` call below leaves the slug claimed - // forever (the undeploy hook never fires for failed deploys). - let trivialClaimedSlugSucceeded = false; - try { - const wired = createSidecarWorkflowSupervisor({ - transport: deps.transport, - repoStore: deps.repoStore, - signingKeySeed: deps.signingKeySeed, - workflowRunRepoId: { - kind: "workflow-run", - id: deploymentId, - }, - workflowRunRef: "refs/heads/main", - deploymentId, - deploymentMailAddress: frame.agentAddress, - deriveStepAddress: ({ deploymentId: dep, stepId }) => - `${dep}-${stepId}`, - substrateEnv: {}, - trivialLaunch: async (bindings) => { - const result = await deps.sessions.provisionAgent( - // The trivialLaunch contract treats `config` as opaque - // bytes the host minted; for the sidecar wiring the - // bytes are a `HarnessConfig` the frame carried - // verbatim, and `SessionManager.provisionAgent` - // expects exactly that. - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- frame.config is the validated HarnessConfig the link surfaced - bindings.config as Parameters< - SessionManager["provisionAgent"] - >[0], - ); - deps.keyStore.recordHubKey( - bindings.agentAddress, - bindings.hubPublicKey, - ); - await deps.sessions.persistHubPublicKey( - bindings.agentAddress, - bindings.hubPublicKey, - ); - publicKey = result.publicKey; - // Subscribe to per-agent InferenceEvents and project the - // reactor's run-bracket vocabulary onto the workflow-run - // event chain. The seam is a no-op until the harness - // dispatches the first `message.run.started`; the - // disposer is not held here because the trivial branch - // shares the agent's lifetime with the deployment and - // SessionManager prunes the listener set on - // destroySession through the closure's natural unbind. - // The reactor brackets one workflow-run per inbound - // mail; each `message.run.started` mints a fresh runId - // and brackets the chain. - const cell: TrivialRunCell = { - runId: null, - stepStarted: false, - }; - deps.onAgentEvent(bindings.agentAddress, (event) => { - driveTrivialRunChain(event, bindings.recordRunEvent, cell).catch( - (err: unknown) => { - // Capture rejections inside the listener so a substrate - // failure (e.g. the hub rejecting the workflow-run pack - // push) does not surface as an unhandled rejection on - // the host process. The trivial branch's audit chain - // is best-effort against the deploy path; persistent - // substrate or transport misconfigurations log loudly - // here without killing the agent's reactor. - const msg = err instanceof Error ? err.message : String(err); - logger.warn`trivial run-event recording failed for ${bindings.agentAddress}: ${msg}`; - }, - ); - }); - }, - ...(deps.consumedRetentionMs !== undefined - ? { consumedRetentionMs: deps.consumedRetentionMs } - : {}), - }); - await wired.supervisor.deploy({ - agentAddress: frame.agentAddress, - agentId: frame.agentId, - config: frame.config, - hubPublicKey: frame.hubPublicKey, - }); - if (publicKey === undefined) { - throw new Error( - "sidecar deploy router: trivialLaunch did not surface a public key", - ); - } - // Register the deployment-address mapping last so a failure in - // `supervisor.deploy` (e.g. the host-supplied `trivialLaunch` - // callback's `provisionAgent` throwing) leaves the boot-edge - // `DeploymentAddressRegistry` untouched. The link's - // `handleAgentDeploy` catches a rejection here and surfaces - // `agent.error` without invoking the undeploy hook; a partial - // registration would persist a `(deploymentId -> agentAddress)` - // entry for a deployment that never finished standing up. - deps.registerDeployment({ - deploymentId, - agentAddress: frame.agentAddress, - }); - trivialClaimedSlugSucceeded = true; - return { publicKey }; - } finally { - if (!trivialClaimedSlugSucceeded) { - releaseSlug(deploymentId, frame.agentAddress); - } - } + // Every deploy stages through the workflow-run substrate: a + // provision-step frame primes the per-step repo, and a workflow + // frame spawns the supervised child. A frame carrying neither is + // an unsupported shape -- there is no in-process fall-through. + throw new Error( + `sidecar deploy router: unsupported deploy frame for ${frame.agentAddress}; a deploy must carry provisionStep or a workflow definition`, + ); }, async undeploy(frame): Promise { // Symmetric teardown for `deploy`: release the per-deployment @@ -1401,19 +1688,20 @@ export function createSidecarDeployRouter(deps: { // / `drain.deliver` / `mail.inbound` aimed at the dead deployment // address is rejected by the router rather than dispatched into // an orphan supervisor handler. The unregister calls are - // idempotent and safe to invoke for both branches even though - // the trivial branch never registers against the multi-step - // routers; those calls are no-ops when no handler is registered. + // idempotent -- they are no-ops when no handler is registered. // // Routers come down BEFORE the supervisor's `shutdown()` so any // hub-side frame racing the undeploy is dropped at the router // boundary rather than dispatched into a supervisor that is in // the middle of tearing its child down. The pattern is: drop // racing frames first, then unwind the underlying resource. - const deploymentId = deriveTrivialDeploymentId(frame.agentAddress); + const deploymentId = deriveDeploymentId(frame.agentAddress); deps.multistepMailRouter?.unregister(frame.agentAddress); deps.multistepSignalRouter?.unregister(frame.agentAddress); deps.multistepDrainRouter?.unregister(frame.agentAddress); + // Unregister unconditionally (a no-op for a multi-step address that + // registered no sources handler), matching the sibling routers. + deps.multistepSourcesRouter?.unregister(frame.agentAddress); // Shut the per-deployment supervisor down so the workflow-process // child, its IPC pipes, and its event-channel fd are released. // The supervisor's `shutdown()` is idempotent (returns early when @@ -1421,18 +1709,16 @@ export function createSidecarDeployRouter(deps: { // kill + `exited` await internally. The map entry is removed // before the await so a subsequent re-deploy on the same address // cannot observe a stale handle even if `shutdown()` rejects. - // Trivial deploys never enter the map, so this lookup is a no-op - // for the trivial branch. const wired = activeSupervisors.get(frame.agentAddress); if (wired !== undefined) { activeSupervisors.delete(frame.agentAddress); await wired.supervisor.shutdown(); - // Drop the agent's transport registration installed at spawn for - // the single-step launched-agent deploy (OUTBOUND half of - // mailbox ownership, §3a). `unregister` is a no-op when the - // address was never registered (a genuine multi-step deploy - // whose derived per-step addresses carry no host keypair), so it - // is safe to call unconditionally for any spawned deployment. + // Drop the deployment address's transport registration installed at + // spawn (OUTBOUND half of mailbox ownership, §3a). Both single- and + // multi-step register the deployment address for outbound signing, so + // this tears down a real registration for either; `unregister` is a + // no-op only if the spawn failed before registering, so it is safe to + // call unconditionally for any spawned deployment. deps.transport.unregister(frame.agentAddress); // Reclaim the deployment's per-step local-disk scratch now that // its supervisor + workflow-process child are torn down. The @@ -1452,132 +1738,139 @@ export function createSidecarDeployRouter(deps: { ); } } + // Drop the deployment record so a boot-time restore does not re-spawn a + // torn-down deployment. Runs on every undeploy -- not only when a + // supervisor was active -- so a record left behind by a + // crash-interrupted deploy is reclaimed too. + if (stepStateDataDir !== undefined) { + await deleteWorkflowDeploymentRecord(stepStateDataDir, deploymentId); + } releaseSlug(deploymentId, frame.agentAddress); deps.unregisterDeployment({ deploymentId, agentAddress: frame.agentAddress, }); }, - }; -} + async restoreWorkflowDeployments(): Promise { + const dataDir = stepStateDataDir; + if (dataDir === undefined) { + // No substrate config was wired (a test router that never spawns a + // child): nothing was ever persisted under this data dir, so there + // is nothing to restore. + return; + } -/** - * Per-deployment cell tracking the active workflow-run bracket. The - * trivial branch holds one cell per agent: each `message.run.started` - * mints a fresh `runId`, the first `inference.start` after that flips - * `stepStarted` true, and the matching `message.run.ended` reads both - * back out before clearing the cell. Two outstanding brackets at once - * are not possible for the trivial path because the reactor - * serializes inbound messages. - */ -export interface TrivialRunCell { - runId: string | null; - stepStarted: boolean; -} + const scanned = await scanWorkflowDeploymentRecords(dataDir); + // Restore serially, not in parallel: deterministic boot-log ordering, + // one isolable warning per failed record, and no concurrent + // child-spawn / transport-register storm. Restore runs before + // `hubLink.connect()`, so there are no concurrent deploys to contend + // with. Each record's failure is caught so one bad deployment cannot + // strand the rest. + for (const { deploymentId, record } of scanned) { + try { + // Integrity: the stored address must re-derive to its own directory + // name. A mismatch means a corrupt or misplaced record; skip it + // rather than restore a deployment under the wrong slug. + const derived = deriveDeploymentId(record.agentAddress); + if (derived !== deploymentId) { + logger.warn`skipping workflow deployment restore: ${record.agentAddress} derives slug ${derived}, not its directory ${deploymentId}`; + continue; + } -/** - * Placeholder definition hash baked into the trivial workflow's - * `RunStarted` envelopes. The trivial workflow's content-addressed - * definition lands with the trivial-deploy capability walk; until - * then the on-disk envelope carries a stable sentinel so audit-log - * consumers see a consistent value across deployments. - */ -const TRIVIAL_DEFINITION_HASH = "trivial:v1"; + // Re-read and RE-VALIDATE the definition off disk with the exact + // gates the deploy path applies: the wire arktype + // (`AgentDeployWorkflow`) to narrow the untrusted on-disk shape, + // then `validateWorkflowProjection` for the structural invariants + // the arktype does not cover (non-empty stepOrder, every stepOrder + // entry backed by a `steps` entry). The on-disk `workflow.json` is + // untrusted at restore, so it must clear the same bar a fresh + // deploy frame clears -- no weaker. + const definitionRaw = await readWorkflowJson( + dataDir, + record.definitionId, + ); + const projection = AgentDeployWorkflow({ + definition: definitionRaw, + sources: record.sources, + }); + if (projection instanceof type.errors) { + logger.warn`skipping workflow deployment restore for ${record.agentAddress}: workflow.json failed validation: ${projection.summary}`; + continue; + } + validateWorkflowProjection(projection); -/** - * Map one InferenceEvent into `recordRunEvent` calls when the - * reactor's run-bracket vocabulary lines up with a workflow-run - * lifecycle moment. The mapping is: - * - * - `message.run.started` -> RunStarted (mints a new workflow runId) - * - first `inference.start` after RunStarted -> StepStarted (attempt 1) - * - `message.run.ended` status=completed -> StepCompleted + RunCompleted - * - `message.run.ended` status=failed -> StepCompleted - * - * The runId is the `messageRunId` the reactor minted. `StepStarted` - * is suppressed if a second `inference.start` arrives within the - * same bracket (the trivial workflow's single step does not retry - * inside one run; the next attempt would be a separate workflow-run - * instance). - * - * The supervisor's `SupervisorRunEvent` union covers `RunCompleted` - * but not `RunFailed`; the trivial branch surfaces a failed run by - * recording `StepCompleted` against the step that failed (so the - * per-step audit trail closes) and letting the absence of a - * subsequent `RunCompleted` mark the run as unsuccessful for - * downstream consumers. Widening the supervisor's - * `SupervisorRunEvent` union to carry `RunFailed` lands when the - * substrate kind handler grows the matching signature-verification - * path. - */ -export async function driveTrivialRunChain( - event: InferenceEvent, - recordRunEvent: RecordRunEvent, - cell: TrivialRunCell, -): Promise { - if (event.type === "message.run.started") { - cell.runId = event.data.messageRunId; - cell.stepStarted = false; - const runStarted: SupervisorRunEvent = { - kind: "RunStarted", - runId: event.data.messageRunId, - at: new Date().toISOString(), - definitionHash: TRIVIAL_DEFINITION_HASH, - trigger: { - type: "mail", - payload: { messageId: event.data.messageId }, - }, - consumedMessageId: event.data.messageId, - }; - await recordRunEvent(runStarted); - return; - } - if (event.type === "inference.start") { - if (cell.runId === null) return; - if (cell.stepStarted) return; - cell.stepStarted = true; - await recordRunEvent({ - kind: "StepStarted", - runId: cell.runId, - at: new Date().toISOString(), - stepId: TRIVIAL_STEP_ID, - attempt: 1, - input: { ref: "refs/heads/main" }, - }); - return; - } - if (event.type === "message.run.ended") { - const runId = cell.runId; - if (runId === null) return; - cell.runId = null; - cell.stepStarted = false; - const at = new Date().toISOString(); - await recordRunEvent({ - kind: "StepCompleted", - runId, - at, - stepId: TRIVIAL_STEP_ID, - attempt: 1, - output: { ref: "refs/heads/main" }, - }); - if (event.data.status === "completed") { - await recordRunEvent({ - kind: "RunCompleted", - runId, - at, - }); - } - } + // Re-run the source-admission gate: refuse to restore a deployment + // whose pinned provider this sidecar can no longer build. Every + // source in a step's failover chain must be buildable, so this + // iterates the whole list. The record is KEPT (not deleted) so a + // later boot with the provider restored retries it. + for (const stepId of projection.definition.stepOrder) { + const chain = projection.sources[stepId]; + if (chain !== undefined) { + for (const source of chain) deps.assertSourceBuildable(source); + } + } + + const spec: WorkflowDeploySpec = { + agentAddress: record.agentAddress, + definition: projection.definition, + sources: projection.sources, + sessionId: record.sessionId, + hubPublicKey: record.hubPublicKey, + }; + + // The slug is the caller's, matching `deployMultiStep`: claim before + // the spawn, release on failure. Unlike deploy's soft-fail, restore + // does NOT delete the record and does NOT re-materialize + // `workflow.json` or the step grants -- all of that is already on + // disk from the original deploy. A failed restore just warns and + // leaves the record for the next boot; there is deliberately no GC + // of a permanently-unrestorable record here (an operator reclaims it + // by undeploying the address). + // + // Release only a slug THIS pass newly claimed: if the address is + // already live (its slug still held by the running deployment), the + // core's double-spawn guard throws, and freeing the slug then would + // strand a live deployment's collision guard. `claimSlug` is a + // no-op for an already-held (deploymentId, address) pair, so the + // pre-claim check distinguishes the two. + const slugNewlyClaimed = + slugClaims.get(deploymentId) !== record.agentAddress; + claimSlug(deploymentId, record.agentAddress); + try { + await spawnWorkflowDeployment(spec); + logger.info`Restored workflow deployment for ${record.agentAddress}`; + } catch (cause) { + if (slugNewlyClaimed) { + releaseSlug(deploymentId, record.agentAddress); + } + throw cause; + } + } catch (cause) { + const reason = cause instanceof Error ? cause.message : String(cause); + logger.warn`Failed to restore workflow deployment ${deploymentId}: ${reason}`; + } + } + }, + activeAddresses(): string[] { + // `activeSupervisors` is keyed by deployment agent address and holds + // exactly the deployments with a live supervisor (deploy and restore + // add; undeploy and spawn-unwind remove), so its keys are the addresses + // this sidecar can currently route mail to. + return [...activeSupervisors.keys()]; + }, + }; } /** * Logical mail-audit reference the supervisor stamps onto every * inbox/processing/consumed envelope for sidecar-hosted deployments. * The substrate does not dereference the value; it is a host-side - * pointer the audit consumer joins on. The trivial-branch single-agent - * mail audit is keyed by the deployment id plus the parsed messageId, - * which is unique per inbound message and stable across the FIFO - * pipeline's enqueue/dequeue/markConsumed transitions. + * pointer the audit consumer joins on. The mail audit is keyed by the + * deployment id plus the parsed messageId, which is unique per inbound + * message and stable across the FIFO pipeline's + * enqueue/dequeue/markConsumed transitions. */ export function deriveSidecarMailAuditRef(deploymentId: string): ( messageId: string, @@ -1594,10 +1887,9 @@ export function deriveSidecarMailAuditRef(deploymentId: string): ( /** * Construct a per-deployment supervisor with the sidecar's bindings - * pre-wired. The host calls this once per `agent.deploy` frame; the - * supervisor's trivial branch routes the deploy through the - * host-supplied `trivialLaunch` callback so the on-wire and - * on-disk surfaces stay bit-identical to the pre-supervisor path. + * pre-wired. The router calls this once per multi-step `agent.deploy` + * frame to stand up the workflow-process child that hosts the + * deployment. */ export function createSidecarWorkflowSupervisor( opts: CreateSidecarWorkflowSupervisorOpts, @@ -1619,16 +1911,17 @@ export function createSidecarWorkflowSupervisor( subprocessSpawner: opts.subprocessSpawner ?? defaultSubprocessSpawner, binaryPath: opts.binaryPath ?? SIDECAR_WORKFLOW_CHILD_BINARY, substrateEnv: opts.substrateEnv, + dynamicSpawnEnv: opts.dynamicSpawnEnv, workflowRunRepoId: opts.workflowRunRepoId, workflowRunRef: opts.workflowRunRef, deploymentId: opts.deploymentId, + stepCount: opts.stepCount, deploymentMailAddress: opts.deploymentMailAddress, readPrincipal: supervisorPrincipal, deriveStepAddress: opts.deriveStepAddress, ...(opts.deriveStepRepoId !== undefined ? { deriveStepRepoId: opts.deriveStepRepoId } : {}), - trivialLaunch: opts.trivialLaunch, deriveMailAuditRef: deriveSidecarMailAuditRef(opts.deploymentId), ...(opts.onDispatchTiming !== undefined ? { onDispatchTiming: opts.onDispatchTiming } @@ -1639,6 +1932,9 @@ export function createSidecarWorkflowSupervisor( ...(opts.consumedRetentionMs !== undefined ? { consumedRetentionMs: opts.consumedRetentionMs } : {}), + ...(opts.readyTimeoutMs !== undefined + ? { readyTimeoutMs: opts.readyTimeoutMs } + : {}), }); return { supervisor, diff --git a/apps/sidecar/src/workflow-run-pack-client.test.ts b/apps/sidecar/src/workflow-run-pack-client.test.ts index eac38f09..1b328cbf 100644 --- a/apps/sidecar/src/workflow-run-pack-client.test.ts +++ b/apps/sidecar/src/workflow-run-pack-client.test.ts @@ -1,5 +1,6 @@ import { describe, test, expect } from "bun:test"; +import type { InferenceSource } from "@intx/types/runtime"; import type { RepoId, RepoStore } from "@intx/hub-sessions"; import { @@ -7,6 +8,7 @@ import { createMultistepDrainRouter, createMultistepMailRouter, createMultistepSignalRouter, + createMultistepSourcesRouter, createWorkflowRunPackClient, createWorkflowRunPackPushingRepoStore, } from "./workflow-run-pack-client"; @@ -449,6 +451,102 @@ describe("createMultistepMailRouter", () => { }); }); +describe("createMultistepSourcesRouter", () => { + const source: InferenceSource = { + id: "primary", + provider: "anthropic", + baseURL: "https://api.anthropic.com", + apiKey: "sk-x", + model: "claude-test", + }; + const frame = { + type: "sources.update" as const, + agentAddress: "dep@integration.interchange", + sources: [source], + defaultSource: "primary", + }; + + test("tryRoute returns false when no handler is registered", async () => { + const router = createMultistepSourcesRouter(); + expect(await router.tryRoute(frame)).toBe(false); + }); + + test("a registered handler receives the rotation and tryRoute returns true", async () => { + const router = createMultistepSourcesRouter(); + const received: { sources: InferenceSource[]; defaultSource: string }[] = + []; + router.register("dep@integration.interchange", async (args) => { + received.push(args); + }); + expect(await router.tryRoute(frame)).toBe(true); + expect(received).toHaveLength(1); + expect(received[0]?.sources).toEqual([source]); + expect(received[0]?.defaultSource).toBe("primary"); + }); + + test("registration is per-address; an unrelated address falls through", async () => { + const router = createMultistepSourcesRouter(); + router.register("dep-a@integration.interchange", async () => undefined); + expect( + await router.tryRoute({ + ...frame, + agentAddress: "dep-b@integration.interchange", + }), + ).toBe(false); + }); + + test("unregister removes the handler", async () => { + const router = createMultistepSourcesRouter(); + router.register("dep@integration.interchange", async () => undefined); + router.unregister("dep@integration.interchange"); + expect(await router.tryRoute(frame)).toBe(false); + }); + + test("rejects a rotation with duplicate ids for a registered address without dispatching", async () => { + const router = createMultistepSourcesRouter(); + let called = false; + router.register("dep@integration.interchange", async () => { + called = true; + }); + // Duplicate ids would crash the child's control-channel receiver on + // its narrow, so the router rejects before dispatch. + await expect( + router.tryRoute({ ...frame, sources: [source, { ...source }] }), + ).rejects.toThrow(/unique ids/); + expect(called).toBe(false); + }); + + test("rejects a rotation whose default is not the head source", async () => { + const router = createMultistepSourcesRouter(); + let called = false; + router.register("dep@integration.interchange", async () => { + called = true; + }); + const second = { ...source, id: "secondary" }; + await expect( + router.tryRoute({ + ...frame, + sources: [source, second], + defaultSource: "secondary", + }), + ).rejects.toThrow(/first element is the default/); + expect(called).toBe(false); + }); + + test("an invalid rotation for an unregistered address is unrouted, not rejected", async () => { + const router = createMultistepSourcesRouter(); + // Registration is checked before validation: an unregistered address + // reports `false` and its (here invalid) payload is never inspected. + expect( + await router.tryRoute({ + ...frame, + agentAddress: "unregistered@integration.interchange", + sources: [source, { ...source }], + }), + ).toBe(false); + }); +}); + describe("createMultistepDrainRouter", () => { test("tryRoute resolves to false when no handler is registered", async () => { const router = createMultistepDrainRouter(); diff --git a/apps/sidecar/src/workflow-run-pack-client.ts b/apps/sidecar/src/workflow-run-pack-client.ts index 7417d498..8278fe77 100644 --- a/apps/sidecar/src/workflow-run-pack-client.ts +++ b/apps/sidecar/src/workflow-run-pack-client.ts @@ -13,7 +13,11 @@ // shape the supervisor uses for `writeTreePreservingPrefix`; the // transferId is minted inside `HubLink.pushWorkflowRunPack`. +import { type } from "arktype"; + import { getLogger } from "@intx/log"; +import { SourcesUpdatedData } from "@intx/workflow-host"; +import type { InferenceSource } from "@intx/types/runtime"; import type { RepoId, RepoStore, @@ -81,7 +85,7 @@ export function createWorkflowRunPackClient( /** * Mapping registry the boot-edge substrate facade consults to resolve - * `repoId.id` (the workflow-run deploymentId, which the trivial branch + * `repoId.id` (the workflow-run deploymentId, which the deploy router * derives by slugging the agent's mail address) back into the * agentAddress carried on every outbound pack frame. Populated by the * deploy router as each `agent.deploy` frame lands. @@ -118,11 +122,9 @@ export type MultistepMailHandler = (message: Uint8Array) => void; /** * Per-deployment-address mail handler registry the sidecar hub-link - * consults before falling back to `transport.deliver` / - * `sessions.commitInboundMail`. The trivial deploy path never registers - * a handler -- its mail flows through the legacy session path. The - * multi-step deploy router registers a handler against the deployment's - * mail address after `wired.supervisor.spawn` succeeds, so an inbound + * consults before falling back to `transport.deliver`. The multi-step + * deploy router registers a handler against the deployment's mail + * address after `wired.supervisor.spawn` succeeds, so an inbound * `mail.inbound` frame for that address dispatches into the * supervisor's mail-bus subscription rather than the * never-provisioned-for-this-address transport mailbox. @@ -177,11 +179,11 @@ export type MultistepSignalHandler = (args: { /** * Per-deployment-address signal handler registry the sidecar hub-link - * consults on every inbound `signal.deliver` frame. The trivial deploy - * path never registers a handler. The multi-step deploy router + * consults on every inbound `signal.deliver` frame. The deploy router * registers a handler against the deployment's mail address after - * `wired.supervisor.spawn` succeeds; the handler dispatches the signal - * into the supervisor's `deliverSignal`. + * `wired.supervisor.spawn` succeeds, for single-step and multi-step + * deployments alike; the handler dispatches the signal into the + * supervisor's `deliverSignal`. * * The registry lives at the sidecar's host layer (not inside the * workflow-host library) for the same boundary reason as @@ -242,11 +244,11 @@ export type MultistepDrainHandler = (args: { /** * Per-deployment-address drain handler registry the sidecar hub-link - * consults on every inbound `drain.deliver` frame. The trivial deploy - * path never registers a handler. The multi-step deploy router + * consults on every inbound `drain.deliver` frame. The deploy router * registers a handler against the deployment's mail address after - * `wired.supervisor.spawn` succeeds; the handler dispatches into the - * supervisor's `drain`. + * `wired.supervisor.spawn` succeeds, for single-step and multi-step + * deployments alike; the handler dispatches into the supervisor's + * `drain`. * * The registry lives at the sidecar's host layer (not inside the * workflow-host library) for the same boundary reason as @@ -283,6 +285,85 @@ export function createMultistepDrainRouter(): MultistepDrainRouter { }; } +/** + * Per-deployment sources-rotation handler the deploy router installs + * against the `MultistepSourcesRouter` after a supervisor's `spawn` + * succeeds -- but ONLY for a single-step (warm launched-agent) + * deployment. The handler hands the rotated list off to the supervisor's + * `deliverSources`, which sends a `sources-updated` control IPC frame to + * the workflow-process child, where the warm agent's live sources are + * swapped in place. A multi-step deployment has no single warm agent to + * rotate, so the router registers no handler for it and an inbound + * `sources.update` for a multi-step address is unrouted. + */ +export type MultistepSourcesHandler = (args: { + sources: InferenceSource[]; + defaultSource: string; +}) => Promise; + +/** + * Per-deployment-address sources-rotation handler registry. Only a + * single-step warm deployment registers a handler (after + * `wired.supervisor.spawn` succeeds); a multi-step deployment never + * does, so `tryRoute` resolves a rotation only for a registered + * single-step address and returns `false` for any other. + * + * The registry lives at the sidecar's host layer for the same boundary + * reason as the mail/signal/drain routers: the routing decision is a + * concrete sidecar host concern, and the workflow-host package stays + * agnostic to which transport surface its supervisor handle rides on. + */ +export type MultistepSourcesRouter = { + register(address: string, handler: MultistepSourcesHandler): void; + unregister(address: string): void; + tryRoute(frame: { + type: "sources.update"; + agentAddress: string; + sources: InferenceSource[]; + defaultSource: string; + }): Promise; +}; + +export function createMultistepSourcesRouter(): MultistepSourcesRouter { + const handlers = new Map(); + return { + register(address, handler) { + handlers.set(address, handler); + }, + unregister(address) { + handlers.delete(address); + }, + async tryRoute(frame) { + const handler = handlers.get(frame.agentAddress); + // Registration check first: an unregistered (multi-step or torn-down) + // address is unrouted -- reported as `false`, its payload never + // inspected, because it would not be acted on regardless. + if (handler === undefined) return false; + // Validate the rotation BEFORE dispatch. This is the only inbound + // router that validates its frame, and deliberately so: a bad list + // (duplicate ids, or a default that is not the head element) would + // reach the child's control-channel receiver and crash it on + // `SourcesUpdatedData`'s narrow -- the sources-updated frame is the + // only inbound frame carrying a crash-on-invalid narrow downstream, + // and the only one that is request/ack. Rejecting here throws, and + // the hub-link turns the throw into a truthful `session.error` + // instead of acking and detonating the child. + const validated = SourcesUpdatedData({ + sources: frame.sources, + defaultSource: frame.defaultSource, + }); + if (validated instanceof type.errors) { + throw new Error(validated.summary); + } + await handler({ + sources: frame.sources, + defaultSource: frame.defaultSource, + }); + return true; + }, + }; +} + /** * Boot-edge facade around the substrate-shaped `RepoStore`. Forwards * every method to the underlying store; intercepts the @@ -502,6 +583,31 @@ export function createWorkflowRunPackPushingRepoStore( schedulePush(agentAddress, repoId, ref); return result; }, + async writeTreeDelta(principal, repoId, ref, args) { + if (repoId.kind === "workflow-run") { + const latched = takeLatchedError(repoId, ref); + if (latched !== null) { + throw latched; + } + } + const result = await underlying.writeTreeDelta( + principal, + repoId, + ref, + args, + ); + if (repoId.kind !== "workflow-run") { + return result; + } + const agentAddress = registry.resolve(repoId.id); + if (agentAddress === null) { + throw new Error( + `workflow-run pack push: no agent address registered for deployment ${repoId.id}; the deploy router must record the mapping before the supervisor commits run events`, + ); + } + schedulePush(agentAddress, repoId, ref); + return result; + }, }; return wrapped; } diff --git a/apps/sidecar/src/workflow-substrate-factory-step-storage.test.ts b/apps/sidecar/src/workflow-substrate-factory-step-storage.test.ts index f74f669f..28d96e76 100644 --- a/apps/sidecar/src/workflow-substrate-factory-step-storage.test.ts +++ b/apps/sidecar/src/workflow-substrate-factory-step-storage.test.ts @@ -31,7 +31,11 @@ import type { } from "@intx/types/runtime"; import type { RepoId } from "@intx/hub-sessions"; import { createBuiltinRegistry } from "@intx/inference/providers"; -import type { ChildOutboundMailBridge, StepEnvBase } from "@intx/workflow-host"; +import type { + ChildOutboundMailBridge, + SourcesSnapshotRef, + StepEnvBase, +} from "@intx/workflow-host"; import type { StepInvokeRequest } from "@intx/workflow"; import { @@ -88,11 +92,11 @@ function buildDeps(opts: { durableConversation?: DurableConversationRegistry; }): SidecarStepBuildEnvDeps { return { - table: { [STEP_ID]: SOURCE }, dataDir: opts.dataDir, workflowRunRepoId: WORKFLOW_RUN_REPO_ID, signer: (payload: string) => Promise.resolve(`sig:${payload.length}`), mailboxAddress: "ins_deployment-keying@example.com", + stepCount: 1, outboundMailBridge: stubOutboundMailBridge(), cache: { cacheMaxBytes: 1_000_000, registryMaxTarballBytes: 1_000_000 }, adapters: createBuiltinRegistry(), @@ -102,6 +106,12 @@ function buildDeps(opts: { }; } +function sourcesRefFor( + chain: InferenceSource[] = [SOURCE], +): SourcesSnapshotRef { + return { current: { [STEP_ID]: chain } }; +} + function requestForRun(runId: string): StepInvokeRequest { return { // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- buildEnv reads only authzContext; the agent definition is never consulted here @@ -126,12 +136,19 @@ describe("createSidecarStepBuildEnv per-step scratch keying", () => { }), ); - const env1: StepEnvBase = await buildEnv(requestForRun("run-1")); + const sourcesRef = sourcesRefFor(); + const env1: StepEnvBase = await buildEnv( + requestForRun("run-1"), + sourcesRef, + ); // A file the agent would write during run-1's turn. const marker = path.join(env1.workdir, "turn-1.txt"); await fs.writeFile(marker, "carried"); - const env2: StepEnvBase = await buildEnv(requestForRun("run-2")); + const env2: StepEnvBase = await buildEnv( + requestForRun("run-2"), + sourcesRef, + ); // Stable keying: a different runId resolves to the SAME workdir, so // the warm case is bounded to one dir per agent (not one-per-message) @@ -152,9 +169,16 @@ describe("createSidecarStepBuildEnv per-step scratch keying", () => { test("cold path keys a distinct per-run workdir under that run's subtree", async () => { const dataDir = await makeTempDir(); const buildEnv = createSidecarStepBuildEnv(buildDeps({ dataDir })); + const sourcesRef = sourcesRefFor(); - const env1: StepEnvBase = await buildEnv(requestForRun("run-1")); - const env2: StepEnvBase = await buildEnv(requestForRun("run-2")); + const env1: StepEnvBase = await buildEnv( + requestForRun("run-1"), + sourcesRef, + ); + const env2: StepEnvBase = await buildEnv( + requestForRun("run-2"), + sourcesRef, + ); // Per-run keying: each run gets its own workdir, rooted under that // run's `runs//` subtree -- exactly what the run-completion @@ -172,4 +196,62 @@ describe("createSidecarStepBuildEnv per-step scratch keying", () => { // The cold path never parks scratch under the warm sub-root. expect(env1.workdir).not.toContain(path.join("warm", STEP_ID)); }); + + test("feeds the reactor the whole failover chain with the head pinned as default", async () => { + // The reactor resolves its initial source by `defaultSource` and fails + // over forward across `sources`; the child must hand it the full ordered + // chain, not just the active source, or cross-provider failover is lost. + const failoverSource: InferenceSource = { + id: "failover", + provider: "openai", + baseURL: "https://api.openai.com", + apiKey: "sk-failover", + model: "gpt-failover", + }; + const chain = [SOURCE, failoverSource]; + const dataDir = await makeTempDir(); + const buildEnv = createSidecarStepBuildEnv(buildDeps({ dataDir })); + + const env: StepEnvBase = await buildEnv( + requestForRun("run-1"), + sourcesRefFor(chain), + ); + + // The whole chain reaches the reactor, ordered, with element 0 as the + // initial source. + expect(env.sources).toEqual(chain); + expect(env.defaultSource).toBe(SOURCE.id); + }); + + test("resolves sources from the mutable ref at build time", async () => { + // The build reads the source table through the ref, so a rotation that + // writes the ref before a build is reflected in the agent that build + // constructs (the cold-window path; a warm already-built agent swaps + // sources through the warm cache, not here). + const dataDir = await makeTempDir(); + const buildEnv = createSidecarStepBuildEnv(buildDeps({ dataDir })); + const sourcesRef = sourcesRefFor(); + + const before: StepEnvBase = await buildEnv( + requestForRun("run-1"), + sourcesRef, + ); + expect(before.sources).toEqual([SOURCE]); + + const rotated: InferenceSource = { + id: "rotated", + provider: "openai", + baseURL: "https://api.openai.com", + apiKey: "sk-rotated", + model: "gpt-rotated", + }; + sourcesRef.current = { [STEP_ID]: [rotated] }; + + const after: StepEnvBase = await buildEnv( + requestForRun("run-2"), + sourcesRef, + ); + expect(after.sources).toEqual([rotated]); + expect(after.defaultSource).toBe(rotated.id); + }); }); diff --git a/apps/sidecar/src/workflow-substrate-factory.ts b/apps/sidecar/src/workflow-substrate-factory.ts index 2decc305..e6060427 100644 --- a/apps/sidecar/src/workflow-substrate-factory.ts +++ b/apps/sidecar/src/workflow-substrate-factory.ts @@ -66,6 +66,7 @@ import { type GrantEvaluator, type RunChildWorkflow, type RunWorkflowChildBindings, + type SourcesSnapshotRef, type StepEnvBase, type SubstrateFactory, type SubstrateFactoryEnv, @@ -145,9 +146,9 @@ const SubstrateConfig = type({ STEP_INFERENCE_SOURCES: "string > 0", // Per-step tool-loader caps. The supervisor threads the boot edge's // resolved `SIDECAR_CACHE_MAX_BYTES` / `SIDECAR_REGISTRY_MAX_TARBALL_BYTES` - // through `substrateEnv`; the child's per-step tool materialization - // uses the same caps the in-process harness builder does. Validated - // as positive-finite-number strings at this boundary. + // through `substrateEnv` so the child's per-step tool materialization is + // bounded by the sidecar's boot-edge-resolved caps. Validated as + // positive-finite-number strings at this boundary. SIDECAR_CACHE_MAX_BYTES: "string > 0", SIDECAR_REGISTRY_MAX_TARBALL_BYTES: "string > 0", // JSON-encoded custom inference adapter manifest. Required: the boot @@ -179,15 +180,18 @@ function parseByteCap(raw: string, name: string): number { } /** - * Per-step `InferenceSource` table parsed from the spawn-time + * Per-step inference-source table parsed from the spawn-time * `STEP_INFERENCE_SOURCES` env entry. The deploy router serializes - * `frame.workflow.sources` (a `Record`) as + * `frame.workflow.sources` (a `Record`) as * JSON and threads it through the supervisor's `substrateEnv`; the - * factory parses and validates the table once at construction time - * and pins it for `buildEnv` lookups. + * factory parses and validates the table once at construction time and + * seeds it into the run loop's mutable sources reference, which each + * `buildEnv` reads. Each value is the step's ordered + * failover chain -- element 0 is the active source, the tail are the + * reactor's forward-only failover targets -- so the list is non-empty. */ const StepInferenceSourceTable = type({ - "[string]": InferenceSource, + "[string]": InferenceSource.array().atLeastLength(1), }); type StepInferenceSourceTable = typeof StepInferenceSourceTable.infer; @@ -260,24 +264,26 @@ export function parseAdapterManifest(raw: string): AdapterManifest { } /** - * Resolve the per-step `InferenceSource` pinned at factory - * construction. The supervisor's multi-step branch only invokes a - * step whose `stepId` appears in `frame.workflow.sources`; a lookup + * Resolve the per-step inference-source failover chain from the table a + * build reads. The supervisor's multi-step branch only invokes + * a step whose `stepId` appears in `frame.workflow.sources`; a lookup * miss here is a programmer error in the supervisor, not a wire-side * failure, and the resolver surfaces it with the missing `stepId` - * named. + * named. The returned list is the step's ordered chain (element 0 the + * active source, the tail the reactor's failover targets); the table's + * arktype guarantees it is non-empty. */ function createStepInferenceSourceResolver( table: StepInferenceSourceTable, -): (stepId: string) => InferenceSource { - return (stepId: string): InferenceSource => { - const source = table[stepId]; - if (source === undefined) { +): (stepId: string) => InferenceSource[] { + return (stepId: string): InferenceSource[] => { + const sources = table[stepId]; + if (sources === undefined) { throw new Error( `sidecar workflow-child step invoker buildEnv: no InferenceSource pinned for stepId ${JSON.stringify(stepId)}; the supervisor must populate frame.workflow.sources for every stepOrder entry`, ); } - return source; + return sources; }; } @@ -427,7 +433,6 @@ function warmStepStorageRoot(args: { } export interface SidecarStepBuildEnvDeps { - table: StepInferenceSourceTable; dataDir: string; workflowRunRepoId: RepoId; signer: (payload: string) => Promise; @@ -442,6 +447,15 @@ export interface SidecarStepBuildEnvDeps { * the agent's `CryptoProvider` against. */ mailboxAddress: string; + /** + * Step count of the deployed `WorkflowDefinition` (`stepOrder.length`), + * threaded from the host through the spawn-time env. Selects the + * head/step collapse when locating a step's deploy tree + * (`stepDeployTreeDir` -> `resolveStepAddress`): a single-step + * deployment reads at the head, a multi-step deployment at the per-step + * address, matching the host's producer push. + */ + stepCount: number; /** * Child-side outbound-mail bridge over the upstream control channel * (OUTBOUND half of mailbox ownership, §3a). The per-step env builder @@ -485,9 +499,10 @@ export interface SidecarStepBuildEnvDeps { * the per-step env construction is observable without standing up the * full substrate. * - * The closure pins the parsed per-step source table, derives the - * `stepId` / `runId` / `attempt` from the runtime's `AuthorizeContext`, - * resolves the per-step `InferenceSource` from the table, and stands up + * The closure reads the per-step source table from the mutable + * reference passed per build, derives the `stepId` / `runId` / `attempt` + * from the runtime's `AuthorizeContext`, resolves the per-step + * `InferenceSource` from that table, and stands up * a per-step isogit `ContextStore` (also serving as the audit store) * plus a per-step workspace directory rooted under the run. A * construction failure (mkdir, isogit init) surfaces here rather than @@ -496,11 +511,22 @@ export interface SidecarStepBuildEnvDeps { */ export function createSidecarStepBuildEnv( deps: SidecarStepBuildEnvDeps, -): (req: StepInvokeRequest) => Promise { - const resolveStepInferenceSource = createStepInferenceSourceResolver( - deps.table, - ); - return async (req: StepInvokeRequest): Promise => { +): ( + req: StepInvokeRequest, + sourcesRef: SourcesSnapshotRef, +) => Promise { + return async ( + req: StepInvokeRequest, + sourcesRef: SourcesSnapshotRef, + ): Promise => { + // Resolve against the live table each build so a source rotation that + // wrote `sourcesRef.current` before this build is reflected in the + // agent this build constructs. A warm agent that is already built does + // not pass through here again, so a rotation does not reach it through + // this path -- this ref covers only a build that has not happened yet. + const resolveStepInferenceSource = createStepInferenceSourceResolver( + sourcesRef.current, + ); const { stepId, runId, attempt } = req.authzContext; if (stepId === undefined) { throw new Error( @@ -517,7 +543,16 @@ export function createSidecarStepBuildEnv( "sidecar workflow-child step invoker buildEnv: AuthorizeContext.attempt is required to root per-step storage per attempt; the workflow runtime must populate attempt on every step-originated invocation", ); } - const source = resolveStepInferenceSource(stepId); + const sources = resolveStepInferenceSource(stepId); + // The resolver's arktype guarantees a non-empty chain; assert it here so + // the reactor's initial-source pin (element 0) is a checked fact rather + // than an unchecked index. + const activeSource = sources[0]; + if (activeSource === undefined) { + throw new Error( + `sidecar workflow-child step invoker buildEnv: empty InferenceSource chain pinned for stepId ${JSON.stringify(stepId)}`, + ); + } // Root the per-step scratch (workspace + tool tarball-cache + // apply-state). The cold (multi-step) path keys it per @@ -573,6 +608,7 @@ export function createSidecarStepBuildEnv( dataDir: deps.dataDir, mailboxAddress: deps.mailboxAddress, stepId, + stepCount: deps.stepCount, storeDir, cache: deps.cache, }); @@ -602,8 +638,12 @@ export function createSidecarStepBuildEnv( // narrower type). const env: StepEnvBase & { transport: MessageTransport; address: string } = { - sources: [source], - defaultSource: source.id, + // Feed the reactor the step's full ordered failover chain and pin + // its initial source to element 0. The reactor resolves the initial + // source by id and fails over forward through `sources`, so this + // restores cross-source failover inside the workflow-child. + sources, + defaultSource: activeSource.id, storage, workdir, audit: storage, @@ -1011,11 +1051,11 @@ export function createSidecarSubstrateFactory( : undefined; const buildStepEnv = createSidecarStepBuildEnv({ - table: stepInferenceSources, dataDir: validated.SIDECAR_DATA_DIR, workflowRunRepoId, signer: conversationSigner, mailboxAddress: env.spawn.mailboxAddress, + stepCount: env.spawn.stepCount, outboundMailBridge: env.outboundMailBridge, cache: stepToolCache, adapters: childAdapterRegistry, @@ -1098,12 +1138,14 @@ export function createSidecarSubstrateFactory( onEvent, authorize, warmCache, + sourcesRef, ) => createWorkflowStepInvoker({ workflowAuthorize: authorize, - buildEnv: buildStepEnv, + buildEnv: (buildReq) => buildStepEnv(buildReq, sourcesRef), agentFactory: stepAgentFactory, onEvent, + sourcesRef, ...(warmCache !== undefined ? { warmCache } : {}), ...(onRunBoundary !== undefined ? { onRunBoundary } : {}), })(req); @@ -1182,6 +1224,7 @@ export function createSidecarSubstrateFactory( workflowDefinitionRepoId, workflowDefinitionRef: validated.WORKFLOW_DEFINITION_REF, invokeStep, + initialSources: stepInferenceSources, spawnChild, scheduler, evaluateGrants: evaluateGrantsAdapter, diff --git a/docs/API.md b/docs/API.md index cf44e133..d0b692e1 100644 --- a/docs/API.md +++ b/docs/API.md @@ -51,7 +51,6 @@ | GET | /api/tenants/:tenantId/agents/instances/:instanceId/health | Get instance health | | GET | /api/tenants/:tenantId/agents/instances/:instanceId/offerings | List instance offerings | | GET | /api/tenants/:tenantId/agents/instances/:instanceId/events | SSE event stream | -| POST | /api/tenants/:tenantId/agents/instances/:instanceId/abort | Abort current operation | | POST | /api/tenants/:tenantId/agents/instances/:instanceId/mail | Send mail to the agent | | GET | /api/tenants/:tenantId/agents/instances/:instanceId/mail | List mail for an instance | | GET | /api/tenants/:tenantId/agents/instances/:instanceId/turns | List inference turns for an instance | @@ -603,18 +602,6 @@ Server-Sent Events stream for agent events. Use POST .../messages for client-to- 404: ErrorResponse -- Instance not found 410: ErrorResponse -- Instance stopped -### POST /api/tenants/:tenantId/agents/instances/:instanceId/abort -Abort current operation - -Aborts the agent's current inference or tool execution. - -Body: unknown - -204: (no content) -- Abort signal sent -404: ErrorResponse -- Instance not found -409: ErrorResponse -- Instance not running -502: ErrorResponse -- Sidecar unavailable - ### POST /api/tenants/:tenantId/agents/instances/:instanceId/mail Send mail to the agent diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 78714412..515be791 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -93,7 +93,7 @@ The reactor director selects which model to use for each inference call (per-cal Model provider selection follows the priority order established at launch. The highest-priority source is active; on a source-specific failure — an authentication failure, a protocol mismatch, or a transient network or timeout error the inference harness has already retried — the reactor fails over to the next source in the list. A rate limit is the exception: it waits out a short same-source backoff first, since it clears with time, and only then fails over. The error surfaces only once the list is exhausted. Source-invariant failures (a context-window overflow, a fatal request error, or an abort) do not trigger failover, since no other source would serve the call differently. Each new inference cycle restarts at the most-preferred source. v1 uses strict priority; equal-priority load-balancing and configurable selection strategies (weighted round-robin, latency-based) are a planned extension. -The model provider list is not fixed for the agent's lifetime. The control plane can push model provider updates to running agents — adding newly available model providers, removing revoked ones, or adjusting priorities — following the same pattern as dynamic grant updates. The harness applies updates without interrupting in-flight inference calls; the updated list takes effect on the next selection. +The model provider list is not fixed for the agent's lifetime. The control plane can push model provider updates to a running deployment — adding newly available model providers, removing revoked ones, or adjusting priorities. The sidecar routes the update to the deployment's supervisor, which swaps the running agent's sources in place. The harness applies updates without interrupting in-flight inference calls; the updated list takes effect on the next selection. The agent is unaware of which model provider is serving a given inference call. @@ -195,7 +195,7 @@ This three-source model enables important patterns: **Inherited capabilities** apply when agents create other agents. The parent agent can grant its child a subset of its own creator-sourced capabilities, establishing a delegation chain. Children cannot exceed their parent's authority, but they can carry creator-sourced capabilities that future invokers of the child would not have on their own. -**Grant revocation** is policy-driven with a default of fail-secure. If the creator's grants are revoked after agents have been launched with creator-sourced grants, running agents lose the affected grants immediately — the control plane pushes a grants update to the harness. Tenants can configure grace periods or notification-only behavior for specific grant types. Invoker-granted capabilities expire when the agent stops unless explicitly persisted. +**Grant revocation** is policy-driven with a default of fail-secure. If the creator's grants are revoked after agents have been launched with creator-sourced grants, running agents must lose the affected grants. Propagating a revocation to an already-running deployment is not currently implemented — the earlier grants-update wire mechanism has been retired, and its supervised replacement is designed separately; a change takes effect when the deployment next loads its grants. Tenants can configure grace periods or notification-only behavior for specific grant types. Invoker-granted capabilities expire when the agent stops unless explicitly persisted. Authorization grants are part of the agent's auditable state. The harness logs what was granted, by whom (tenant, creator, or invoker), when, and tracks all usage of delegated credentials. diff --git a/docs/AUTH.md b/docs/AUTH.md index 26d981d1..22e40f3f 100644 --- a/docs/AUTH.md +++ b/docs/AUTH.md @@ -203,11 +203,11 @@ Conditions are evaluated at runtime by the authorization engine. A grant with un Grant revocation is policy-driven with a default of fail-secure. -**Creator grant revocation**: If the creator's authority is revoked after agents have been launched with creator-sourced grants, running agents lose the affected grants. The control plane pushes a `grants.update` frame to the harness, which replaces its live materialized grant set. The harness authorizes each tool call against this set before the call executes, so a revoked capability is blocked at the next authorization check; a tool call already in flight is not interrupted. Tenants can configure grace periods or notification-only behavior for specific grant types. +**Creator grant revocation**: If the creator's authority is revoked after agents have been launched with creator-sourced grants, running agents must lose the affected grants. The harness authorizes each tool call against its live materialized grant set before the call executes, so a revoked capability is blocked once that set no longer contains it; a tool call already in flight is not interrupted. Propagating a revocation to an already-running deployment is not currently implemented — the earlier `grants.update` wire mechanism has been retired, and its supervised replacement is designed separately. Until then, the change takes effect when the deployment next loads its grants (the deploy pack at spawn and the supervisor's IPC credentials snapshot at recycle). Tenants can configure grace periods or notification-only behavior for specific grant types. **Invoker grant revocation**: Invoker-granted capabilities expire when the agent stops unless explicitly persisted. They are session-scoped by default. -**Tenant policy changes**: When tenant policies change (role modifications, system role updates), the control plane re-evaluates affected agents and pushes grant updates to their harnesses. +**Tenant policy changes**: When tenant policies change (role modifications, system role updates), the control plane re-evaluates affected agents. Propagating the resulting grant changes to already-running deployments shares the retired-mechanism gap described above; a change takes effect when each deployment next loads its grants. This parallels the credential revocation model described in CREDENTIALS.md — both follow the same fail-secure default with configurable tenant policies. diff --git a/docs/HARNESS_DESIGN.md b/docs/HARNESS_DESIGN.md index 31d7c236..6be03ba1 100644 --- a/docs/HARNESS_DESIGN.md +++ b/docs/HARNESS_DESIGN.md @@ -79,11 +79,11 @@ All communication between hub and sidecar is over a single persistent WebSocket | `agent.deploy.ack` | `agentAddress`, `publicKey` | Agent deployed, here is its public key | | `agent.error` | `agentAddress`, `error` | Deployment failed | -When the hub sends `agent.deploy`, the sidecar generates a key pair (if new) or loads the existing one, persists the `HarnessConfig` to `agent.json`, initializes the harness, and responds with `agent.deploy.ack` including the agent's hex-encoded public key. The hub stores this public key for reconnect verification. +When the hub sends `agent.deploy`, the sidecar spawns a supervised **workflow-process child** to host the deployment and responds with `agent.deploy.ack`. For a single-step (launched-agent) deployment the sidecar generates the agent's key pair (if new) or loads the existing one, records the hub's public key for later deploy-pack verification, initializes the head's on-disk deploy-tree repository (the narrow `initRepo`, not the retired `provisionAgent`), and acks the agent's hex-encoded public key — the hub stores it in `agent_instance.publicKey` for reconnect verification. A multi-step deployment's address is workflow-derived and has no `agent_instance` row, so the ack carries the deployment address's own key — the same Ed25519 key the sidecar signs reconnect challenges with — and the hub stores it on the deployment's `workflow_deployment` row (keyed by address), mirroring the launched-agent path so both are reconnect-verifiable. Before the child is spawned, the inputs a restart cannot otherwise recover — the per-step inference sources, the session id, and (single-step only) the hub public key — are written to a per-deployment `deployment.json` record. -When the hub sends `agent.undeploy`, the sidecar stops the harness, unregisters the agent from the transport, and clears the persisted config. +When the hub sends `agent.undeploy`, the sidecar shuts the deployment's supervisor down (killing the workflow-process child and releasing its IPC pipes and event-channel handle), unregisters the deployment address from the transport and from the mail/signal/drain routers, reclaims the deployment's per-step scratch, and deletes the `deployment.json` record so a later boot does not re-spawn a torn-down deployment. The agent's key pair and its durable agent-state / conversation repositories are left in place so a redeploy on the same address resumes them. -Credentials travel in the `agent.deploy` frame's `config.providers` array. There is no separate credential push endpoint. +Credentials travel in the `agent.deploy` frame's inference **sources** — `config.sources`, and the per-step `workflow.sources` failover chains — where each `InferenceSource` carries its own API key. There is no separate credential push endpoint. ## Per-Agent Key Pairs @@ -95,21 +95,30 @@ Directory layout under `SIDECAR_DATA_DIR`: ``` SIDECAR_DATA_DIR/ - agent-name_at_tenant_interchange_network/ - .git/ # isogit repository (context, audit records) - agent.json # persisted HarnessConfig for session restore + / # per-agent key custody + head deploy-tree repo + .git/ # isogit repository (deploy tree, context, audit records) keys/ - id_ed25519 # agent private key (raw 32 bytes) - id_ed25519.pub # agent public key (raw 32 bytes) + id_ed25519 # agent private key (raw 32 bytes, mode 0600) + id_ed25519.pub # agent public key (raw 32 bytes) + workflow-runs/ + / # workflow-run substrate for one deployment + deployment.json # per-deployment restore record (mode 0600); see below + workflow-step-state/ + / # ephemeral per-step scratch, reclaimed on undeploy + agent-conversation-state/ # durable per-agent conversation, survives undeploy ``` -The `agent.json` file stores the full harness configuration (system prompt, model, providers, session ID) so that the sidecar can restore agent sessions on restart without re-receiving the config from the hub. +The per-agent key directory is keyed by the sanitized agent address; the workflow subtrees are keyed by the derived deployment id. + +The `deployment.json` record stores only what a restart cannot otherwise recover: the deployment's `agentAddress`, the `definitionId` naming its workflow definition on disk, each step's ordered inference-**sources** failover chain (`sources`), the optional inference `sessionId`, and — for a single-step deployment — the `hubPublicKey`. A `version` field guards the schema so a stale record can be rejected rather than parsed blindly. The record deliberately does **not** duplicate the workflow definition (kept on disk under its `definitionId` and re-read at restore) or the step grants (kept in each step's agent-state repo). Because each source embeds its API key, the record is written owner-only (mode 0600). + +A live source rotation for a single-step deployment overwrites this record's `sources` before it takes effect. Persistence is what makes a rotation durable: a rotation whose write fails is not durable, and the deployment falls back to the last durably-recorded source list on the next recycle or restart. The directory name is the agent address with `@` replaced by `_at_` and non-alphanumeric characters (except `-` and `_`) replaced by `_`. ## Agent Deployment vs User Sessions -The sidecar manages agents, not user sessions. When the hub deploys an agent to a sidecar, the sidecar starts a harness for that agent. The harness runs continuously, receiving messages from any source — other agents, users, system signals. User sessions are a hub-side concept: the hub tracks which users are interacting with which agents and routes user messages to the agent's address accordingly, but the sidecar does not know or care about individual user sessions. +The sidecar manages agents, not user sessions. When the hub deploys an agent to a sidecar, the sidecar spawns a supervised **workflow-process child** for that deployment. The child runs continuously, receiving messages from any source — other agents, users, system signals — and builds the agent harness inside its own process. User sessions are a hub-side concept: the hub tracks which users are interacting with which agents and routes user messages to the agent's address accordingly, but the sidecar does not know or care about individual user sessions. The hub maintains a sidecar-to-agent mapping in its database. This mapping determines where to route messages for a given agent address. When a sidecar disconnects, the hub knows which agents are affected and queues messages for them until the sidecar reconnects. @@ -156,7 +165,7 @@ On reconnection (agents successfully restored from `SIDECAR_DATA_DIR`), the side ### Self-Restoration -When the WebSocket connection to the hub opens, the sidecar scans `SIDECAR_DATA_DIR` for agent directories containing both a key pair and an `agent.json` config. For each valid directory, the sidecar restores the harness from the persisted config. The `register` or `reconnect` frame is held until restoration completes. This restoration happens entirely on the sidecar side — the hub is not involved in session restoration. +At boot, **before** opening the WebSocket connection to the hub, the sidecar scans `SIDECAR_DATA_DIR/workflow-runs/` for `deployment.json` records (`scanWorkflowDeploymentRecords`). Each record is validated at the trust boundary, its stored `agentAddress` is cross-checked against its directory name, its workflow definition is re-read and re-validated off disk with the same gates a fresh deploy applies, and its pinned inference sources are re-admitted against the providers this sidecar can build; a record that fails any check is logged and skipped so one bad deployment cannot strand the rest. Each surviving record is restored through the **same** spawn path a live deploy uses — a supervised workflow-process child, with the single-step head's key pair and recorded hub key re-established. Restoration completes before the `register` or `reconnect` frame is sent, and happens entirely on the sidecar side; the hub is not involved in restoration. ### Challenge/Response Verification @@ -166,12 +175,13 @@ After self-restoration, the sidecar connects to the hub and proves ownership of 2. Hub generates a 32-byte random nonce per address and sends a `challenge` frame 3. Sidecar signs `nonce || agent_address` (concatenated bytes) with each agent's private key and sends a `challenge.response` frame 4. Hub verifies each signature against the stored public key for that address -5. Verified addresses are provisionally added to the routing table (required so the `grants.update` request/ack round-trip can reach the sidecar) -6. Hub sends `grants.update` with current grants for each verified address -7. Hub compares each agent's advertised deploy ref against its own. For agents whose ref is stale or absent, the hub creates and sends a fresh deploy pack (fire-and-forget, does not block reconnect completion) -8. On grant refresh success, the address remains in the routing table and queued messages are flushed -9. On failure (grant refresh rejected or timed out), the address is rolled back from the routing table — its queue is preserved for the next reconnect attempt and the sidecar receives `challenge.failed` -10. For addresses that fail cryptographic verification, hub sends `challenge.failed` with the address and reason +5. Verified addresses are provisionally added to the routing table so the hub's `agent.reconnected` reaction can run for each +6. Hub compares each agent's advertised deploy ref against its own. For agents whose ref is stale or absent, the hub creates and sends a fresh deploy pack (fire-and-forget, does not block reconnect completion) +7. On a successful reaction, the address remains in the routing table and queued messages are flushed +8. On failure (the reconnect reaction rejected by governance), the address is rolled back from the routing table — its queue is preserved for the next reconnect attempt and the sidecar receives `challenge.failed` +9. For addresses that fail cryptographic verification, hub sends `challenge.failed` with the address and reason + +A supervised deployment carries its grants in the deploy pack and refreshes them over the supervisor's IPC credentials snapshot at spawn and recycle, so reconnect no longer performs a grant-refresh round-trip over the wire. ### Reconnection Frames @@ -209,7 +219,7 @@ The sidecar's isogit repository is the source of truth for agent inference conte ## Security Model -Credentials travel in the `agent.deploy` frame's `config.providers` array and are held in memory by the harness. They are never persisted to disk (the `agent.json` file contains the full config including provider entries with API keys — this is a known limitation of the prototype that should be addressed before production use). +Credentials travel in the `agent.deploy` frame's inference `sources` (each `InferenceSource` carries its own API key) and are held in memory by the running deployment. They are also persisted to disk in the deployment's `deployment.json` record — the `sources` field embeds those API keys — so the sidecar can restore a deployment on restart without re-receiving them from the hub. The record is written owner-only (mode 0600), but storing provider API keys on the sidecar's disk at all is a known limitation of the prototype that should be addressed before production use. ## Key Rotation diff --git a/docs/IMPLEMENTATION.md b/docs/IMPLEMENTATION.md index c4e16eb8..b24c5d2a 100644 --- a/docs/IMPLEMENTATION.md +++ b/docs/IMPLEMENTATION.md @@ -237,9 +237,10 @@ The IMAP inbox is the source of truth for conversation history. Session channels **Prototype (hub-mediated sidecar transport):** 1. Sidecar reconnects to the hub and proves ownership of agent addresses via signed challenge (see HARNESS_DESIGN.md) -2. Hub refreshes grants via `grants.update` frame — the sidecar must have current grants before processing any messages -3. Hub flushes queued undelivered messages as `message.send` frames for verified agents -4. Sidecar loads agent context from isogit and resumes operation +2. Hub flushes queued undelivered messages as `message.send` frames for verified agents +3. Sidecar loads agent context from isogit and resumes operation + +A supervised deployment carries its grants in the deploy pack and refreshes them over the supervisor's IPC credentials snapshot at spawn and recycle, so reconnect does not refresh grants over the wire. In the prototype, the hub's database serves as the delivery queue for messages sent while the sidecar is disconnected. The sidecar's isogit repository is the source of truth for agent inference context. The hub maintains a sidecar-to-agent mapping so it knows which sidecar to route messages to for a given agent address. See HARNESS_DESIGN.md for the reconnection wire protocol. @@ -249,19 +250,17 @@ The sidecar WebSocket protocol includes frames for agent deployment, reconnectio **Agent deployment:** -| Direction | Frame | Purpose | -| ------------- | -------------------- | ------------------------------------------------------ | -| Hub → Sidecar | `agent.deploy` | Provision an agent (keys, directory, ephemeral config) | -| Sidecar → Hub | `agent.deploy.ack` | Confirm provisioning, provide agent public key | -| Hub → Sidecar | `session.start` | Start the inference harness for a provisioned agent | -| Sidecar → Hub | `session.start.ack` | Confirm the harness is running | -| Hub → Sidecar | `agent.undeploy` | Remove an agent from the sidecar | -| Sidecar → Hub | `agent.undeploy.ack` | Confirm teardown (includes state push status) | -| Sidecar → Hub | `agent.error` | Report a failure at any stage | +| Direction | Frame | Purpose | +| ------------- | -------------------- | ------------------------------------------------- | +| Hub → Sidecar | `agent.deploy` | Stage a deploy through the workflow-run substrate | +| Sidecar → Hub | `agent.deploy.ack` | Confirm the deploy staged, provide the public key | +| Hub → Sidecar | `agent.undeploy` | Remove an agent from the sidecar | +| Sidecar → Hub | `agent.undeploy.ack` | Confirm teardown (includes state push status) | +| Sidecar → Hub | `agent.error` | Report a failure at any stage | -Agent deployment is a three-phase operation: provision, pack delivery, session start. The hub sends `agent.deploy` to provision the agent (generate keys, create directory, persist config). After receiving `agent.deploy.ack` with the agent's public key, the hub streams the deploy tree via pack frames. After `pack.ack`, the hub sends `session.start` to start the inference harness. This ordering ensures the deploy tree (prompt, skills) is on disk before the harness reads it. +Agent deployment stages through the workflow-run substrate rather than a separate provision-then-start handshake. The hub sends `agent.deploy`; the sidecar's deploy router primes the per-step repo (for a provision-step frame) or spawns the supervised workflow-process child (for a workflow frame), then acks with `agent.deploy.ack` carrying the public key. The deploy tree (prompt, skills) rides in on the follow-up deploy pack, which the child reads from the substrate. The workflow-process child starts inference itself once spawned -- there is no separate `session.start` step. -Undeploy is an acknowledged operation. The sidecar stops the harness, pushes state to the hub (best-effort), deletes the agent directory, and responds with `agent.undeploy.ack`. The `statePushed` field indicates whether the state push was attempted. The hub defers routing table cleanup until the ack arrives. +Undeploy is an acknowledged operation. The sidecar shuts the deployment's supervisor down, pushes state to the hub (best-effort), deletes the agent directory, and responds with `agent.undeploy.ack`. The `statePushed` field indicates whether the state push was attempted. The hub defers routing table cleanup until the ack arrives. **Reconnection:** @@ -274,11 +273,9 @@ Undeploy is an acknowledged operation. The sidecar stops the harness, pushes sta **Grant management:** -| Direction | Frame | Purpose | -| ------------- | --------------- | ---------------------------------------------------- | -| Hub → Sidecar | `grants.update` | Push updated grants to a running agent (request/ack) | +The live `grants.update` wire path has been retired. A supervised deployment receives its grants in the deploy pack and refreshes them over the supervisor's IPC credentials snapshot at spawn and recycle; there is no wire frame that pushes a grant change to an already-running deployment. Propagating a mid-run grant change (for example, a revocation) to a running deployment is not currently implemented. -On first connection (no existing agents), the sidecar sends a `register` frame. On reconnection (agents in data directory), it sends `reconnect`. After successful challenge verification, verified addresses are provisionally added to the routing table so the `grants.update` round-trip can reach the sidecar. If grant refresh fails, the address is rolled back from the routing table and its queued messages are preserved for the next reconnect attempt. +On first connection (no existing agents), the sidecar sends a `register` frame. On reconnection (agents in data directory), it sends `reconnect`. After successful challenge verification, verified addresses are provisionally added to the routing table so the hub's `agent.reconnected` reaction can run for each; an address whose reaction is rejected by governance is rolled back from the routing table and its queued messages are preserved for the next reconnect attempt. ### Debug and Telemetry Streams @@ -738,88 +735,59 @@ live `subscribeKind` tail, but a resume-rehydrated queued signal needs the runtime body's `RunState` reader plumbing, which lands when the per-run state-machine reader is wired into the child. -### Deploy Routing (Option Z) - -The supervisor is the single ingress for inbound `agent.deploy` -frames. The host-side handler calls `supervisor.deploy(frame)` and -the supervisor decides between the trivial (1-step) passthrough -and the multi-step IPC-backed spawn. Routing lives on the -supervisor side of the seam; the host does not re-decide. The -sidecar's hub-link surface lives at -`packages/hub-agent/src/ws/hub-link.ts`; it consumes a -`DeployRouter` binding whose production implementation is the -workflow-host supervisor. - -Four invariants on the routing model are locked: - -1. **Supervisor owns `agent.deploy` framing.** The hub-side handler - in `packages/hub-agent/src/ws/hub-link.ts` calls - `deployRouter.deploy(frame)`; the production router is the - workflow-host supervisor's `supervisor.deploy(frame)`. No routing - logic leaks back into `hub-agent` or `hub-sessions`. -2. **Trivial branch is a process-topology passthrough.** The - supervisor invokes the host-injected `trivialLaunch` callback - directly. No IPC channel opens. No workflow-process child is - spawned. No mail-bus subscription registers. The bytes flowing - through the deploy-flow gate path are bit-identical to the - pre-supervisor surface. -3. **Trivial deploys emit the canonical workflow-run event chain.** - The supervisor commits `RunStarted` / `StepStarted` / - `StepCompleted` / `RunCompleted` to the workflow-run repo from - the supervisor process itself, signed via the host's - `signAsPrincipal("supervisor", ...)` callback. The chain fires - per inbound mail trigger (one run per fire) and is driven by - the host calling `bindings.recordRunEvent(...)` from the - trivialLaunch callback's reactor / harness lifecycle moments - (`message.run.started` / `message.run.ended` brackets). The - on-disk envelope is byte-identical to the one a multi-step - deployment's workflow-process child produces. Trivial and - multi-step deployments differ in process topology, not in - observability. The supervisor's signing key never leaves the - supervisor's address space; `signAsPrincipal` returns the raw - signature bytes for the canonical payload only after the - supervisor has serialized them. -4. **`credentialsSnapshot` is multi-step-only.** The trivial branch - does not assemble a snapshot; `getCredentialsSnapshot()` - continues to return `null` after a trivial deploy. The - multi-step branch (`steps.length >= 2`) provisions per-step - `agent-state` repos, mints keys, spawns the workflow-process - child via `subprocessSpawner`, registers the deployment's mail - address, waits for `ready`, and assembles the - `credentialsSnapshot` -- this is the body of `spawn(opts)`, - which the multi-step branch is the worker for. The - `agent.deploy` wire frame today carries only a `HarnessConfig` - (no workflow definition); every frame is therefore trivial. The - seam exists now so the frame-format extension that carries a - `WorkflowDefinition` lands as a pure data-shape change. - -The supervisor's per-event commit primitive lives in -`packages/workflow-host/src/supervisor/run-event-signing.ts` -alongside the analogous `commitCancelRequested` path. The trivial -branch's `recordRunEvent` is a thin closure over that primitive that -the supervisor hands into the trivialLaunch bindings; the multi-step -branch composes the same primitive from inside its in-supervisor -event-channel receiver. +### Deploy Routing + +The sidecar's deploy router is the single ingress for inbound +`agent.deploy` frames. The hub-link surface at +`packages/hub-agent/src/ws/hub-link.ts` consumes a `DeployRouter` +binding whose production implementation is `createSidecarDeployRouter` +in `apps/sidecar/src/workflow-host-wiring.ts`. Routing lives on the +router side of the seam; the hub-link hands the frame off and folds the +returned public key into the outbound `agent.deploy.ack`. + +Every deploy stages through the workflow-run substrate. The router +branches on the frame shape: + +1. **Provision-step frame (`provisionStep: true`).** The router primes + the frame's per-step `agent-state` repo and records the hub key, + without constructing a supervisor or spawning a child. The follow-up + full-closure deploy pack then applies into the primed repo and + verifies against the recorded key. +2. **Workflow frame (carries a `WorkflowDefinition`).** The router + constructs a fresh per-deployment workflow-host supervisor and drives + its `spawn(opts)` lifecycle: per-step `agent-state` repo + provisioning, key minting, workflow-process child spawn via + `subprocessSpawner`, mail-bus registration, the IPC ready handshake, + and `credentialsSnapshot` assembly. + +A frame carrying neither shape is rejected -- there is no in-process +deploy path. A single agent deploys as a single-step workflow: the +instance-deploy path wraps its `HarnessConfig` with +`wrapHarnessAsSingleStepWorkflow` +(`packages/workflow-deploy/src/orchestrator.ts`) and deploys it at the +workflow's head, so single- and multi-step deployments run the same +supervised-child topology and differ only in step count. + +Run-lifecycle events (`RunStarted` / `StepStarted` / `StepCompleted` / +`RunCompleted`) are committed by the workflow-process child from its own +address space against the workflow-run repo; the supervisor does not +commit them in-process. The supervisor's surviving run-event work is +compaction: once a run terminates it folds that run's per-event blobs +into one combined log +(`packages/workflow-host/src/supervisor/run-event-compaction.ts`). The workflow-run repo's substrate `repoId.id` is constrained to `/^[a-zA-Z0-9_-]+$/` (`SAFE_REPO_ID` in `packages/hub-sessions/src/repo-store/types.ts`), which the -agent-address shape (`ins_@`) does not satisfy. The -sidecar wiring derives the trivial branch's deployment id by -substituting disallowed characters with `-`; see -`deriveTrivialDeploymentId` in -`apps/sidecar/src/workflow-host-wiring.ts`. The supervisor -principal's `deploymentId` and the workflow-run `repoId.id` are kept -equal so the workflow-run kind handler's principal-vs-repo authz -check holds for every supervisor-authored event commit. - -The sidecar production wiring lives in -`apps/sidecar/src/workflow-host-wiring.ts`: -`createSidecarDeployRouter` constructs a fresh per-deployment -supervisor on every inbound frame whose `trivialLaunch` closes -over `SessionManager.provisionAgent` plus the hub-pairing-key -recording the legacy handler performed inline. The `HubTransport` -mail-bus adapter the supervisor consumes lives in the +agent-address shape (`ins_@`) does not satisfy. The sidecar +wiring derives the deployment id by substituting disallowed characters +with `-`; see `deriveDeploymentId` in +`apps/sidecar/src/workflow-host-wiring.ts`. The supervisor principal's +`deploymentId` and the workflow-run `repoId.id` are kept equal so the +workflow-run kind handler's principal-vs-repo authz check holds for +every supervisor-authored event commit. + +The `HubTransport` mail-bus adapter the supervisor consumes lives in the `@intx/workflow-host` package proper (`packages/workflow-host/src/mail-bus/`) so an alternative-sidecar implementation can reuse it without forking the wiring. @@ -844,8 +812,8 @@ snapshot. The implementation lives in returns a `CredentialsSnapshot` with per-step `address`, `grants`, and `contentHash`. The hash is stable across processes so the child can detect a stale snapshot pushed after a fresher one. -- A missing grants file is treated as an empty grant array (so the - trivial path with no operator-supplied grants does not crash); a +- A missing grants file is treated as an empty grant array (so a + single-step deploy with no operator-supplied grants does not crash); a malformed file fails loudly at the boundary. ### Signed CancelRequested Authority @@ -1104,14 +1072,13 @@ Health can also be reported via periodic heartbeat messages to the control plane ### Deployment Procedure -1. **Provision** - Hub sends `agent.deploy` with ephemeral config; sidecar creates directory, keys, and returns the public key +1. **Stage** - Hub sends `agent.deploy`; the sidecar's deploy router stages the deploy through the workflow-run substrate (priming the per-step repo or spawning the supervised workflow-process child) and returns the public key 2. **Assembly** - Hub resolves skill dependencies and assembles the deploy tree -3. **Pack transfer** - Hub streams the packfile to the sidecar (see Agent Deployment) -4. **Session start** - Hub sends `session.start`; sidecar reads deploy tree and starts the harness -5. **Health gate** - New version must pass health checks before receiving traffic -6. **Traffic shift** - Registry updates discovery to point to new version -7. **Drain** - Old version stops accepting new work, completes in-flight operations -8. **Retirement** - Old version shuts down; deploy tag remains for rollback +3. **Pack transfer** - Hub streams the packfile to the sidecar; the workflow-process child reads the deploy tree from the substrate and begins inference (see Agent Deployment) +4. **Health gate** - New version must pass health checks before receiving traffic +5. **Traffic shift** - Registry updates discovery to point to new version +6. **Drain** - Old version stops accepting new work, completes in-flight operations +7. **Retirement** - Old version shuts down; deploy tag remains for rollback ### Rollback @@ -1205,7 +1172,7 @@ Pack transfers share the WebSocket with live session traffic. To prevent interfe - Pack chunks are interleaved with other frames at the WebSocket message level (messages are atomic; the receiver processes each independently) - The sender limits unacknowledged pack data to a configurable window before pausing -- Session-critical frames (`mail.inbound`, `message.send`, `session.abort`) are never delayed by pack transfers +- Session-critical frames (`mail.inbound`, `message.send`) are never delayed by pack transfers - The receiver buffers incoming chunks in memory (or a temp file for large transfers) and only unpacks atomically on `pack.done` ### Rejection Reasons @@ -1222,16 +1189,14 @@ Pack transfers share the WebSocket with live session traffic. To prevent interfe ### Deploy Flow -Deployment is a three-phase operation: provision, pack delivery, session start. - -**Phase 1: Provision** +Deployment is a two-phase operation: stage and pack delivery. There is no separate session-start phase -- the workflow-process child begins inference once it is spawned and its deploy tree has landed. -1. Hub sends `agent.deploy` with ephemeral config (credentials, materialized grants, providers, session ID) -2. Sidecar creates the agent directory, generates an Ed25519 key pair, and persists the config -3. Sidecar responds with `agent.deploy.ack` containing the agent's public key (hex-encoded) -4. Hub stores the public key for future challenge/response verification +**Phase 1: Stage** -The sidecar is now provisioned but not running. It can receive pack data. +1. Hub sends `agent.deploy` with the deploy config (credentials, materialized grants, providers, session ID) and, for a multi-step deploy, a workflow definition +2. The sidecar's deploy router stages the deploy through the workflow-run substrate: a provision-step frame primes the per-step `agent-state` repo and records the hub key; a workflow frame spawns the supervised workflow-process child +3. Sidecar responds with `agent.deploy.ack` containing the public key (hex-encoded) +4. Hub stores the public key for future deploy-commit verification **Phase 2: Pack delivery** @@ -1239,19 +1204,12 @@ The sidecar is now provisioned but not running. It can receive pack data. 6. Hub sends `pack.push` frames (chunked, interleaved with other traffic) 7. Hub sends `pack.done` with target ref (`refs/heads/deploy`) and commit SHA 8. Sidecar validates the packfile integrity -9. Sidecar verifies the deploy commit signature against the hub's public key +9. Sidecar verifies the deploy commit signature against the recorded hub public key 10. Sidecar unpacks objects into the git object store 11. Sidecar updates `refs/heads/deploy` to the new commit -12. Sidecar merges `deploy` into its agent branch (trivial merge) +12. Sidecar merges `deploy` into its agent branch (fast-forward) 13. Sidecar checks out only `deploy/` paths via partial checkout (`filepaths: ["deploy/"], noUpdateHead: true`) — `state/` working-tree files are untouched because `filepaths` restricts the scope, `noUpdateHead` prevents moving the branch pointer -14. Sidecar responds with `pack.ack` - -**Phase 3: Session start** - -15. Hub sends `session.start` -16. Sidecar reads the deploy tree from disk (prompt, skills/tools) -17. Sidecar creates the inference harness with the deploy tree and ephemeral config -18. Sidecar starts the harness and responds with `session.start.ack` +14. Sidecar responds with `pack.ack`; the workflow-process child reads the deploy tree from the substrate and runs The agent is now running and can receive messages. @@ -1260,13 +1218,13 @@ On first deploy a full packfile is sent. On subsequent deploys, if the sidecar a ### Undeploy Flow 1. Hub sends `agent.undeploy` with a reason string -2. Sidecar stops the inference harness (or removes the agent from the provisioned set if session has not started) +2. Sidecar shuts the deployment's supervisor down, releasing the workflow-process child and its per-deployment routing state (a no-op if no supervisor is live for the address) 3. Sidecar pushes state to the hub via `pack.push`/`pack.done` (best-effort — the `statePushed` field in the ack indicates whether this was attempted, not whether the hub confirmed receipt) 4. Sidecar deletes the agent directory 5. Sidecar responds with `agent.undeploy.ack` 6. Hub removes the agent from the routing table -If the sidecar disconnects before sending the ack, the hub removes the agent from the routing table on disconnect. If `startSession` fails, the agent remains provisioned and can be retried or undeployed. +If the sidecar disconnects before sending the ack, the hub removes the agent from the routing table on disconnect. ### State Push Flow @@ -1284,21 +1242,18 @@ If the WebSocket disconnects mid-transfer, no git state is corrupted: the sideca ### Reconnect Sequencing -Pack transfers are sequenced after identity verification: +On reconnect the sidecar restores its workflow deployments from local disk (see the deployment restore path) and re-announces them so the hub restores their routes. A workflow-deployment address is **not** keyless: it carries the deployment's own Ed25519 key (minted at deploy, acked to the hub), so it proves ownership through the same challenge/response a launched agent does. -1. Sidecar sends `reconnect` with `agentAddresses` and `deployRefs` -2. Hub sends `challenge` per address -3. Sidecar sends `challenge.response` per address -4. Hub verifies signatures — only verified agents proceed -5. Hub sends `grants.update` per verified agent (sidecar must have current grants before processing messages) -6. For agents whose deploy ref is behind: hub initiates pack transfer, waits for `pack.ack` -7. Hub sends `session.start` per agent -8. After `session.start.ack`: hub flushes queued messages +1. Sidecar sends an empty `register` frame to establish the connection, then a `reconnect` frame carrying its live workflow-deployment addresses (in `agentAddresses`) +2. For each address the hub resolves the stored deployment key (`workflow_deployment.publicKey`, gated on a live `deployed` status), issues a random nonce, and routes the address only after the sidecar returns a valid signature over `nonce ‖ address`. An address with no live key fails closed and stays unrouted — a token-holding sidecar cannot reclaim a deployment's route without the deployment's key +3. The hub does not run its deploy-ref freshness catch-up for a workflow-deployment address — that pack-transfer path is launched-agent only (`isWorkflowDerivedAddress` short-circuits it). A deployment's in-flight run state is reconstructed sidecar-locally at restore, not re-fetched from the hub -Pack content is never sent to a sidecar that has not proved ownership of the agent's key. +The sidecar verifies every inbound deploy pack's commit signature against the hub key it recorded at deploy time before applying it, so pack content is never applied on an unverified signature. On first deploy (no prior key exists), the sidecar is authenticated by its registration token but cannot prove agent key ownership (the key does not exist yet). The hub sends `agent.deploy` to provision the agent, and the sidecar generates the key and returns it in `agent.deploy.ack`. The registration token and the authenticated WebSocket channel bound the trust for first-deploy; challenge/response protects all subsequent interactions. +The hub enforces that boundary structurally in `handleRegister`: a `register` frame routes an address only when `lookupPublicKey` returns null for it — a genuine keyless first-deploy. An address that already has a stored key is refused and must re-enter routing through the challenged reconnect, so a token-holding sidecar cannot reclaim a keyed address's route (its own or a victim's) on token auth alone. The check runs before any routing mutation, so a refused address never even evicts its current owner. If the key lookup is not configured, register fails closed (routes nothing and logs an error) rather than routing unverified. + ### State Push Policy The sidecar pushes state to the hub based on configurable policy: diff --git a/docs/unified-execution-host-design.md b/docs/unified-execution-host-design.md index 3aa20783..db31e089 100644 --- a/docs/unified-execution-host-design.md +++ b/docs/unified-execution-host-design.md @@ -737,8 +737,8 @@ oci-container`. A per-node declaration equal to or weaker than the current rung - **Workflow-level defaulting.** Unchanged from Decision 2: `defineWorkflow` normalizes an absent workflow-level `isolation` to `{ granularity: "per-tenant" }`. The single-agent launch path - (`wrapHarnessAsSingleStepAgent`) sets the workflow-level field from the launch - request / agent policy. + (`wrapHarnessAsSingleStepWorkflow`) sets the workflow-level field from the + launch request / agent policy. - **Validation.** `defineWorkflow`'s `normalize` validates every `isolation` occurrence — workflow-level and per-node — against the allowed `granularity` / `sandbox` sets, throwing a structured error on an unknown @@ -1013,17 +1013,16 @@ Every surviving "trivial"-named symbol is renamed to reflect that the single-step identity path is the canonical single-agent deploy, not a "trivial" special case — or deleted where the in-process branch it served is gone: -- `wrapHarnessAsTrivialAgent` -> `wrapHarnessAsSingleStepAgent` - (`packages/workflow-deploy/src/orchestrator.ts`), still used to build the - one-step definition. +- `wrapHarnessAsSingleStepWorkflow` + (`packages/workflow-deploy/src/orchestrator.ts`) kept its name; still used to + build the one-step definition. - `buildTrivialApprovalSet` -> `buildSingleStepApprovalSet` (`packages/hub-sessions/src/session-service.ts`). - `isTrivialDeploy` / `trivialBindings` / the trivial orchestrator branch: **deleted** if single-step routes through the multi-step branch (likely); otherwise renamed `isSingleStepDeploy`. Confirm during build. -- `deriveTrivialDeploymentId` (`apps/sidecar/src/workflow-host-wiring.ts`): - deleted; call `deriveWorkflowRunRepoId` directly at the call sites (it is a - thin delegator). +- `deriveDeploymentId` (`apps/sidecar/src/workflow-host-wiring.ts`): kept; still + called at the workflow-host wiring sites. - `TrivialLaunch` / `trivialLaunch` / `trivialClaimedSlugSucceeded`: deleted with the in-process branch. @@ -1495,7 +1494,7 @@ are explicitly **not** a go-live gate for INTR-209. `runWorkflowChildFromProcessEnv`). - Deploy router / wiring: `apps/sidecar/src/workflow-host-wiring.ts` (`createSidecarDeployRouter`, `deployMultiStep`, `activeSupervisors`, - `deriveTrivialDeploymentId`, `driveTrivialRunChain`, `TrivialRunCell`, + `deriveDeploymentId`, `driveTrivialRunChain`, `TrivialRunCell`, `publishWorkflowInferenceEvent`). - Child-launch / sandbox-boundary seam (Decision 1): `apps/sidecar/src/workflow-host-wiring.ts` (`SubprocessSpawner`, @@ -1521,7 +1520,7 @@ are explicitly **not** a go-live gate for INTR-209. `WorkflowRunWorkflowProcessPrincipal`. - Identity: `packages/workflow-deploy/src/orchestrator.ts` (`deriveWorkflowRunRepoId`, `isWorkflowDerivedAddress`, - `wrapHarnessAsTrivialAgent`); `packages/types/src/agent-address.ts` + `wrapHarnessAsSingleStepWorkflow`); `packages/types/src/agent-address.ts` (`formatAgentAddress`, `parseAgentAddress`); `packages/hub-sessions/src/hub-session-orchestrator.ts` (deploy-ack listener); `packages/hub-sessions/src/hub-session-lookups.ts` (`findInstance`, diff --git a/packages/db/migrations/0035_violet_giant_man.sql b/packages/db/migrations/0035_violet_giant_man.sql new file mode 100644 index 00000000..4d615f9e --- /dev/null +++ b/packages/db/migrations/0035_violet_giant_man.sql @@ -0,0 +1,3 @@ +ALTER TABLE "workflow_deployment" ADD COLUMN "address" text NOT NULL;--> statement-breakpoint +ALTER TABLE "workflow_deployment" ADD COLUMN "public_key" text;--> statement-breakpoint +CREATE UNIQUE INDEX "workflow_deployment_address_idx" ON "workflow_deployment" USING btree ("address"); \ No newline at end of file diff --git a/packages/db/migrations/meta/0035_snapshot.json b/packages/db/migrations/meta/0035_snapshot.json new file mode 100644 index 00000000..21845ce4 --- /dev/null +++ b/packages/db/migrations/meta/0035_snapshot.json @@ -0,0 +1,3225 @@ +{ + "id": "ddb550f1-b5b9-4094-941b-6d0ad963d01a", + "prevId": "b997db7b-976b-4511-8780-9052911edf5c", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.account": { + "name": "account", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": ["token"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": ["email"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification": { + "name": "verification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.federation_trust": { + "name": "federation_trust", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_tenant_id": { + "name": "target_tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "direction": { + "name": "direction", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "federation_trust_tenant_id_tenant_id_fk": { + "name": "federation_trust_tenant_id_tenant_id_fk", + "tableFrom": "federation_trust", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "federation_trust_target_tenant_id_tenant_id_fk": { + "name": "federation_trust_target_tenant_id_tenant_id_fk", + "tableFrom": "federation_trust", + "tableTo": "tenant", + "columnsFrom": ["target_tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "federation_trust_tenant_id_target_tenant_id_unique": { + "name": "federation_trust_tenant_id_target_tenant_id_unique", + "nullsNotDistinct": false, + "columns": ["tenant_id", "target_tenant_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tenant": { + "name": "tenant", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "domain": { + "name": "domain", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_id": { + "name": "parent_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "tenant_parent_id_tenant_id_fk": { + "name": "tenant_parent_id_tenant_id_fk", + "tableFrom": "tenant", + "tableTo": "tenant", + "columnsFrom": ["parent_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "tenant_slug_unique": { + "name": "tenant_slug_unique", + "nullsNotDistinct": false, + "columns": ["slug"] + }, + "tenant_domain_unique": { + "name": "tenant_domain_unique", + "nullsNotDistinct": false, + "columns": ["domain"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.principal": { + "name": "principal", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ref_id": { + "name": "ref_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "principal_tenant_id_tenant_id_fk": { + "name": "principal_tenant_id_tenant_id_fk", + "tableFrom": "principal", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "principal_tenant_id_kind_ref_id_unique": { + "name": "principal_tenant_id_kind_ref_id_unique", + "nullsNotDistinct": false, + "columns": ["tenant_id", "kind", "ref_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agent_role": { + "name": "agent_role", + "schema": "", + "columns": { + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role_id": { + "name": "role_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "agent_role_agent_id_agent_id_fk": { + "name": "agent_role_agent_id_agent_id_fk", + "tableFrom": "agent_role", + "tableTo": "agent", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_role_role_id_role_id_fk": { + "name": "agent_role_role_id_role_id_fk", + "tableFrom": "agent_role", + "tableTo": "role", + "columnsFrom": ["role_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "agent_role_agent_id_role_id_pk": { + "name": "agent_role_agent_id_role_id_pk", + "columns": ["agent_id", "role_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.principal_role": { + "name": "principal_role", + "schema": "", + "columns": { + "principal_id": { + "name": "principal_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role_id": { + "name": "role_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "principal_role_principal_id_principal_id_fk": { + "name": "principal_role_principal_id_principal_id_fk", + "tableFrom": "principal_role", + "tableTo": "principal", + "columnsFrom": ["principal_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "principal_role_role_id_role_id_fk": { + "name": "principal_role_role_id_role_id_fk", + "tableFrom": "principal_role", + "tableTo": "role", + "columnsFrom": ["role_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "principal_role_principal_id_role_id_pk": { + "name": "principal_role_principal_id_role_id_pk", + "columns": ["principal_id", "role_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.role": { + "name": "role", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_system": { + "name": "is_system", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "role_tenant_id_tenant_id_fk": { + "name": "role_tenant_id_tenant_id_fk", + "tableFrom": "role", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.grant": { + "name": "grant", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role_id": { + "name": "role_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "principal_id": { + "name": "principal_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resource": { + "name": "resource", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "effect": { + "name": "effect", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "conditions": { + "name": "conditions", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "origin": { + "name": "origin", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "grant_tenant_id_tenant_id_fk": { + "name": "grant_tenant_id_tenant_id_fk", + "tableFrom": "grant", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "grant_role_id_role_id_fk": { + "name": "grant_role_id_role_id_fk", + "tableFrom": "grant", + "tableTo": "role", + "columnsFrom": ["role_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "grant_principal_id_principal_id_fk": { + "name": "grant_principal_id_principal_id_fk", + "tableFrom": "grant", + "tableTo": "principal", + "columnsFrom": ["principal_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agent": { + "name": "agent", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "creator_principal_id": { + "name": "creator_principal_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "system_prompt": { + "name": "system_prompt", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "context_config": { + "name": "context_config", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "initial_state": { + "name": "initial_state", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "model_config": { + "name": "model_config", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "capabilities": { + "name": "capabilities", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "credential_requirements": { + "name": "credential_requirements", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "model_requirements": { + "name": "model_requirements", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "tool_packages": { + "name": "tool_packages", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "grant_requirements": { + "name": "grant_requirements", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "current_version": { + "name": "current_version", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'1'" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'deployed'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "agent_tenant_id_tenant_id_fk": { + "name": "agent_tenant_id_tenant_id_fk", + "tableFrom": "agent", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_creator_principal_id_principal_id_fk": { + "name": "agent_creator_principal_id_principal_id_fk", + "tableFrom": "agent", + "tableTo": "principal", + "columnsFrom": ["creator_principal_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agent_version": { + "name": "agent_version", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "agent_version_agent_id_agent_id_fk": { + "name": "agent_version_agent_id_agent_id_fk", + "tableFrom": "agent_version", + "tableTo": "agent", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.provider": { + "name": "provider", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "plugin": { + "name": "plugin", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "authorization_url": { + "name": "authorization_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "token_url": { + "name": "token_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_info_url": { + "name": "user_info_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "provider_tenant_id_tenant_id_fk": { + "name": "provider_tenant_id_tenant_id_fk", + "tableFrom": "provider", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "provider_tenant_name": { + "name": "provider_tenant_name", + "nullsNotDistinct": false, + "columns": ["tenant_id", "name"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client": { + "name": "oauth_client", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_secret": { + "name": "client_secret", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "redirect_uris": { + "name": "redirect_uris", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "default_scopes": { + "name": "default_scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "oauth_client_tenant_id_tenant_id_fk": { + "name": "oauth_client_tenant_id_tenant_id_fk", + "tableFrom": "oauth_client", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_client_provider_id_provider_id_fk": { + "name": "oauth_client_provider_id_provider_id_fk", + "tableFrom": "oauth_client", + "tableTo": "provider", + "columnsFrom": ["provider_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_client_tenant_provider": { + "name": "oauth_client_tenant_provider", + "nullsNotDistinct": false, + "columns": ["tenant_id", "provider_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.credential": { + "name": "credential", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "principal_id": { + "name": "principal_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "oauth_client_id": { + "name": "oauth_client_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "secret": { + "name": "secret", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "refresh_secret": { + "name": "refresh_secret", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "credential_tenant_id_tenant_id_fk": { + "name": "credential_tenant_id_tenant_id_fk", + "tableFrom": "credential", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_principal_id_principal_id_fk": { + "name": "credential_principal_id_principal_id_fk", + "tableFrom": "credential", + "tableTo": "principal", + "columnsFrom": ["principal_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "credential_provider_id_provider_id_fk": { + "name": "credential_provider_id_provider_id_fk", + "tableFrom": "credential", + "tableTo": "provider", + "columnsFrom": ["provider_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_oauth_client_id_oauth_client_id_fk": { + "name": "credential_oauth_client_id_oauth_client_id_fk", + "tableFrom": "credential", + "tableTo": "oauth_client", + "columnsFrom": ["oauth_client_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "credential_tenant_name": { + "name": "credential_tenant_name", + "nullsNotDistinct": false, + "columns": ["tenant_id", "name"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.asset": { + "name": "asset", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "creator_principal_id": { + "name": "creator_principal_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "asset_tenant_id_tenant_id_fk": { + "name": "asset_tenant_id_tenant_id_fk", + "tableFrom": "asset", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "asset_creator_principal_id_principal_id_fk": { + "name": "asset_creator_principal_id_principal_id_fk", + "tableFrom": "asset", + "tableTo": "principal", + "columnsFrom": ["creator_principal_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "asset_tenant_kind_name": { + "name": "asset_tenant_kind_name", + "nullsNotDistinct": false, + "columns": ["tenant_id", "kind", "name"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agent_asset": { + "name": "agent_asset", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "asset_id": { + "name": "asset_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ref": { + "name": "ref", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_mode": { + "name": "access_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'read-only'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "agent_asset_agent_id_agent_id_fk": { + "name": "agent_asset_agent_id_agent_id_fk", + "tableFrom": "agent_asset", + "tableTo": "agent", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_asset_asset_id_asset_id_fk": { + "name": "agent_asset_asset_id_asset_id_fk", + "tableFrom": "agent_asset", + "tableTo": "asset", + "columnsFrom": ["asset_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "agent_asset_agent_asset": { + "name": "agent_asset_agent_asset", + "nullsNotDistinct": false, + "columns": ["agent_id", "asset_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.transaction": { + "name": "transaction", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "wallet_id": { + "name": "wallet_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "direction": { + "name": "direction", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "amount": { + "name": "amount", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "currency": { + "name": "currency", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_id": { + "name": "recipient_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sender_id": { + "name": "sender_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "request_id": { + "name": "request_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "transaction_wallet_id_wallet_id_fk": { + "name": "transaction_wallet_id_wallet_id_fk", + "tableFrom": "transaction", + "tableTo": "wallet", + "columnsFrom": ["wallet_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "transaction_agent_id_agent_id_fk": { + "name": "transaction_agent_id_agent_id_fk", + "tableFrom": "transaction", + "tableTo": "agent", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.wallet": { + "name": "wallet", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "backend_type": { + "name": "backend_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "currency": { + "name": "currency", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "balance": { + "name": "balance", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "wallet_tenant_id_tenant_id_fk": { + "name": "wallet_tenant_id_tenant_id_fk", + "tableFrom": "wallet", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.offering": { + "name": "offering", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "pricing": { + "name": "pricing", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "schema": { + "name": "schema", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "offering_agent_id_agent_id_fk": { + "name": "offering_agent_id_agent_id_fk", + "tableFrom": "offering", + "tableTo": "agent", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "offering_tenant_id_tenant_id_fk": { + "name": "offering_tenant_id_tenant_id_fk", + "tableFrom": "offering", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.model": { + "name": "model", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "canonical_name": { + "name": "canonical_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "disabled": { + "name": "disabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "model_tenant_id_tenant_id_fk": { + "name": "model_tenant_id_tenant_id_fk", + "tableFrom": "model", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "model_tenant_canonical_name": { + "name": "model_tenant_canonical_name", + "nullsNotDistinct": false, + "columns": ["tenant_id", "canonical_name"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.model_offering": { + "name": "model_offering", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "model_id": { + "name": "model_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "deployment_tags": { + "name": "deployment_tags", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "capabilities": { + "name": "capabilities", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "disabled": { + "name": "disabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "model_offering_tenant_id_tenant_id_fk": { + "name": "model_offering_tenant_id_tenant_id_fk", + "tableFrom": "model_offering", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "model_offering_model_id_model_id_fk": { + "name": "model_offering_model_id_model_id_fk", + "tableFrom": "model_offering", + "tableTo": "model", + "columnsFrom": ["model_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "model_offering_provider_id_model_provider_id_fk": { + "name": "model_offering_provider_id_model_provider_id_fk", + "tableFrom": "model_offering", + "tableTo": "model_provider", + "columnsFrom": ["provider_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "model_offering_tenant_model_provider": { + "name": "model_offering_tenant_model_provider", + "nullsNotDistinct": false, + "columns": ["tenant_id", "model_id", "provider_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.model_pricing": { + "name": "model_pricing", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "offering_id": { + "name": "offering_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "currency": { + "name": "currency", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "input_token_price": { + "name": "input_token_price", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "output_token_price": { + "name": "output_token_price", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cache_read_token_price": { + "name": "cache_read_token_price", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cache_write_token_price": { + "name": "cache_write_token_price", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "thinking_token_price": { + "name": "thinking_token_price", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "per_request_fee": { + "name": "per_request_fee", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "per_image_fee": { + "name": "per_image_fee", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "per_audio_fee": { + "name": "per_audio_fee", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "effective_from": { + "name": "effective_from", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "model_pricing_tenant_id_tenant_id_fk": { + "name": "model_pricing_tenant_id_tenant_id_fk", + "tableFrom": "model_pricing", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "model_pricing_offering_id_model_offering_id_fk": { + "name": "model_pricing_offering_id_model_offering_id_fk", + "tableFrom": "model_pricing", + "tableTo": "model_offering", + "columnsFrom": ["offering_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "model_pricing_offering_currency_effective_from": { + "name": "model_pricing_offering_currency_effective_from", + "nullsNotDistinct": false, + "columns": ["offering_id", "currency", "effective_from"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.model_provider": { + "name": "model_provider", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "plugin": { + "name": "plugin", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "base_url": { + "name": "base_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "credential_id": { + "name": "credential_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "wallet_id": { + "name": "wallet_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "disabled": { + "name": "disabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "model_provider_tenant_id_tenant_id_fk": { + "name": "model_provider_tenant_id_tenant_id_fk", + "tableFrom": "model_provider", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "model_provider_credential_id_credential_id_fk": { + "name": "model_provider_credential_id_credential_id_fk", + "tableFrom": "model_provider", + "tableTo": "credential", + "columnsFrom": ["credential_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + }, + "model_provider_wallet_id_wallet_id_fk": { + "name": "model_provider_wallet_id_wallet_id_fk", + "tableFrom": "model_provider", + "tableTo": "wallet", + "columnsFrom": ["wallet_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "model_provider_tenant_name": { + "name": "model_provider_tenant_name", + "nullsNotDistinct": false, + "columns": ["tenant_id", "name"] + } + }, + "policies": {}, + "checkConstraints": { + "model_provider_auth_xor": { + "name": "model_provider_auth_xor", + "value": "(\"model_provider\".\"credential_id\" is not null) <> (\"model_provider\".\"wallet_id\" is not null)" + } + }, + "isRLSEnabled": false + }, + "public.sidecar": { + "name": "sidecar", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'online'" + }, + "last_heartbeat": { + "name": "last_heartbeat", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agent_session": { + "name": "agent_session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "principal_id": { + "name": "principal_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "ended_at": { + "name": "ended_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "agent_session_tenant_id_tenant_id_fk": { + "name": "agent_session_tenant_id_tenant_id_fk", + "tableFrom": "agent_session", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_session_agent_id_agent_id_fk": { + "name": "agent_session_agent_id_agent_id_fk", + "tableFrom": "agent_session", + "tableTo": "agent", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_session_principal_id_principal_id_fk": { + "name": "agent_session_principal_id_principal_id_fk", + "tableFrom": "agent_session", + "tableTo": "principal", + "columnsFrom": ["principal_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agent_instance": { + "name": "agent_instance", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "principal_id": { + "name": "principal_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version_id": { + "name": "version_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'deployed'" + }, + "sidecar_id": { + "name": "sidecar_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "public_key": { + "name": "public_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "kernel_id": { + "name": "kernel_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "model_preferences": { + "name": "model_preferences", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "ended_at": { + "name": "ended_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "agent_instance_agent_id_agent_id_fk": { + "name": "agent_instance_agent_id_agent_id_fk", + "tableFrom": "agent_instance", + "tableTo": "agent", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_instance_tenant_id_tenant_id_fk": { + "name": "agent_instance_tenant_id_tenant_id_fk", + "tableFrom": "agent_instance", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_instance_principal_id_principal_id_fk": { + "name": "agent_instance_principal_id_principal_id_fk", + "tableFrom": "agent_instance", + "tableTo": "principal", + "columnsFrom": ["principal_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "agent_instance_version_id_agent_version_id_fk": { + "name": "agent_instance_version_id_agent_version_id_fk", + "tableFrom": "agent_instance", + "tableTo": "agent_version", + "columnsFrom": ["version_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "agent_instance_session_id_agent_session_id_fk": { + "name": "agent_instance_session_id_agent_session_id_fk", + "tableFrom": "agent_instance", + "tableTo": "agent_session", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "agent_instance_sidecar_id_sidecar_id_fk": { + "name": "agent_instance_sidecar_id_sidecar_id_fk", + "tableFrom": "agent_instance", + "tableTo": "sidecar", + "columnsFrom": ["sidecar_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "agent_instance_address_unique": { + "name": "agent_instance_address_unique", + "nullsNotDistinct": false, + "columns": ["address"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session_asset": { + "name": "session_asset", + "schema": "", + "columns": { + "instance_id": { + "name": "instance_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_asset_id": { + "name": "agent_asset_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "mount_path": { + "name": "mount_path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "asset_pack_sha": { + "name": "asset_pack_sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_commit_sha": { + "name": "source_commit_sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "materialized_at": { + "name": "materialized_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "session_asset_pack_sha_idx": { + "name": "session_asset_pack_sha_idx", + "columns": [ + { + "expression": "asset_pack_sha", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_asset_instance_id_agent_instance_id_fk": { + "name": "session_asset_instance_id_agent_instance_id_fk", + "tableFrom": "session_asset", + "tableTo": "agent_instance", + "columnsFrom": ["instance_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "session_asset_agent_asset_id_agent_asset_id_fk": { + "name": "session_asset_agent_asset_id_agent_asset_id_fk", + "tableFrom": "session_asset", + "tableTo": "agent_asset", + "columnsFrom": ["agent_asset_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "session_asset_instance_id_mount_path_pk": { + "name": "session_asset_instance_id_mount_path_pk", + "columns": ["instance_id", "mount_path"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.inference_turn": { + "name": "inference_turn", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "instance_id": { + "name": "instance_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'running'" + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ended_at": { + "name": "ended_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "inference_turn_instance_id_started_at_idx": { + "name": "inference_turn_instance_id_started_at_idx", + "columns": [ + { + "expression": "instance_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "inference_turn_session_id_agent_session_id_fk": { + "name": "inference_turn_session_id_agent_session_id_fk", + "tableFrom": "inference_turn", + "tableTo": "agent_session", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "inference_turn_instance_id_agent_instance_id_fk": { + "name": "inference_turn_instance_id_agent_instance_id_fk", + "tableFrom": "inference_turn", + "tableTo": "agent_instance", + "columnsFrom": ["instance_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "inference_turn_tenant_id_tenant_id_fk": { + "name": "inference_turn_tenant_id_tenant_id_fk", + "tableFrom": "inference_turn", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session_mail": { + "name": "session_mail", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "instance_id": { + "name": "instance_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "direction": { + "name": "direction", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "raw": { + "name": "raw", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "session_mail_instance_id_created_at_idx": { + "name": "session_mail_instance_id_created_at_idx", + "columns": [ + { + "expression": "instance_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_mail_session_id_agent_session_id_fk": { + "name": "session_mail_session_id_agent_session_id_fk", + "tableFrom": "session_mail", + "tableTo": "agent_session", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "session_mail_instance_id_agent_instance_id_fk": { + "name": "session_mail_instance_id_agent_instance_id_fk", + "tableFrom": "session_mail", + "tableTo": "agent_instance", + "columnsFrom": ["instance_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "session_mail_tenant_id_tenant_id_fk": { + "name": "session_mail_tenant_id_tenant_id_fk", + "tableFrom": "session_mail", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.turn_part": { + "name": "turn_part", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "turn_id": { + "name": "turn_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "ordinal": { + "name": "ordinal", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "turn_part_turn_id_inference_turn_id_fk": { + "name": "turn_part_turn_id_inference_turn_id_fk", + "tableFrom": "turn_part", + "tableTo": "inference_turn", + "columnsFrom": ["turn_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "turn_part_session_id_agent_session_id_fk": { + "name": "turn_part_session_id_agent_session_id_fk", + "tableFrom": "turn_part", + "tableTo": "agent_session", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.git_token": { + "name": "git_token", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "principal_id": { + "name": "principal_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_hash_sha256": { + "name": "token_hash_sha256", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "resource": { + "name": "resource", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ref_pattern": { + "name": "ref_pattern", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actions": { + "name": "actions", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "git_token_user_id_name_active_idx": { + "name": "git_token_user_id_name_active_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"git_token\".\"revoked_at\" is null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "git_token_tenant_id_tenant_id_fk": { + "name": "git_token_tenant_id_tenant_id_fk", + "tableFrom": "git_token", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "git_token_user_id_user_id_fk": { + "name": "git_token_user_id_user_id_fk", + "tableFrom": "git_token", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "git_token_principal_id_principal_id_fk": { + "name": "git_token_principal_id_principal_id_fk", + "tableFrom": "git_token", + "tableTo": "principal", + "columnsFrom": ["principal_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "git_token_token_hash_sha256_unique": { + "name": "git_token_token_hash_sha256_unique", + "nullsNotDistinct": false, + "columns": ["token_hash_sha256"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_deployment": { + "name": "workflow_deployment", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "definition_asset_id": { + "name": "definition_asset_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_key": { + "name": "public_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'deployed'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_deployment_tenant_idx": { + "name": "workflow_deployment_tenant_idx", + "columns": [ + { + "expression": "tenant_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_deployment_address_idx": { + "name": "workflow_deployment_address_idx", + "columns": [ + { + "expression": "address", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_deployment_tenant_id_tenant_id_fk": { + "name": "workflow_deployment_tenant_id_tenant_id_fk", + "tableFrom": "workflow_deployment", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_deployment_definition_asset_id_asset_id_fk": { + "name": "workflow_deployment_definition_asset_id_asset_id_fk", + "tableFrom": "workflow_deployment", + "tableTo": "asset", + "columnsFrom": ["definition_asset_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/packages/db/migrations/meta/_journal.json b/packages/db/migrations/meta/_journal.json index daaaf1f2..c7a4aca2 100644 --- a/packages/db/migrations/meta/_journal.json +++ b/packages/db/migrations/meta/_journal.json @@ -246,6 +246,13 @@ "when": 1782164862412, "tag": "0034_sleepy_songbird", "breakpoints": true + }, + { + "idx": 35, + "version": "7", + "when": 1783373028980, + "tag": "0035_violet_giant_man", + "breakpoints": true } ] } diff --git a/packages/db/src/schema/workflow-deployments.ts b/packages/db/src/schema/workflow-deployments.ts index 5f6b58e7..a8e5dbd8 100644 --- a/packages/db/src/schema/workflow-deployments.ts +++ b/packages/db/src/schema/workflow-deployments.ts @@ -1,4 +1,10 @@ -import { index, pgTable, text, timestamp } from "drizzle-orm/pg-core"; +import { + index, + pgTable, + text, + timestamp, + uniqueIndex, +} from "drizzle-orm/pg-core"; import { asset } from "./assets"; import { tenant } from "./tenants"; @@ -25,11 +31,31 @@ export const workflowDeployment = pgTable( definitionAssetId: text("definition_asset_id") .notNull() .references(() => asset.id, { onDelete: "cascade" }), + // The deployment's routable address (`ins_@`), stored rather + // than re-derived at read time so the reconnect ownership challenge can + // look up the deployment's public key by address, symmetrically with the + // `agent_instance` path. Unique so a double-insert fails loud. + // + // Added NOT NULL with no default and no backfill. Like the repo's other + // such adds (e.g. `credential.provider_id`), the migration relies on the + // table being empty when it runs: this table and the `address` column + // land one migration apart, both unreleased, so no populated row predates + // the column and none needs a backfilled address. + address: text("address").notNull(), + // The Ed25519 public key the sidecar minted for this deployment address, + // persisted at deploy-ack. Nullable by design: the row is written at + // deploy-start and the key arrives at ack, so a not-yet-acked deployment + // reads `null` and its reconnect challenge fails closed -- the address + // stays unrouted rather than routing without ownership proof. + publicKey: text("public_key"), status: text("status") .$type() .notNull() .default("deployed"), createdAt: timestamp("created_at").notNull().defaultNow(), }, - (t) => [index("workflow_deployment_tenant_idx").on(t.tenantId, t.createdAt)], + (t) => [ + index("workflow_deployment_tenant_idx").on(t.tenantId, t.createdAt), + uniqueIndex("workflow_deployment_address_idx").on(t.address), + ], ); diff --git a/packages/hub-agent/src/agent-key-store.test.ts b/packages/hub-agent/src/agent-key-store.test.ts index e114a1c7..6a7d76c8 100644 --- a/packages/hub-agent/src/agent-key-store.test.ts +++ b/packages/hub-agent/src/agent-key-store.test.ts @@ -108,36 +108,3 @@ describe("AgentKeyStore — load-or-generate", () => { ); }); }); - -describe("AgentKeyStore — scanKeys", () => { - test("returns key entries whose directory has both files plus agent.json", async () => { - const dataDir = await tempDir(); - const store = createAgentKeyStore({ - dataDir, - generateKeyPair: async () => makeKeyPair(3), - ...stubCrypto, - }); - await store.loadOrGenerateKey("agent@local"); - await fs.writeFile( - path.join(dataDir, "agent_at_local", "agent.json"), - JSON.stringify({ version: 1, address: "agent@local" }), - ); - - const entries = await store.scanKeys(); - expect(entries).toHaveLength(1); - expect(entries[0]?.address).toBe("agent@local"); - }); - - test("skips an agent directory with no agent.json", async () => { - const dataDir = await tempDir(); - const store = createAgentKeyStore({ - dataDir, - generateKeyPair: async () => makeKeyPair(5), - ...stubCrypto, - }); - await store.loadOrGenerateKey("agent@local"); - - const entries = await store.scanKeys(); - expect(entries).toEqual([]); - }); -}); diff --git a/packages/hub-agent/src/agent-key-store.ts b/packages/hub-agent/src/agent-key-store.ts index 35d91a7f..62523f36 100644 --- a/packages/hub-agent/src/agent-key-store.ts +++ b/packages/hub-agent/src/agent-key-store.ts @@ -13,27 +13,10 @@ // for incoming deploy packs. import fsp from "node:fs/promises"; -import path from "node:path"; -import { getLogger } from "@intx/log"; import { hasCode, hexDecode } from "@intx/types"; import type { KeyPair } from "@intx/types/runtime"; -import { - KEYS_DIR_NAME, - META_FILE, - PRIVATE_KEY_FILE, - PUBLIC_KEY_FILE, - keysDir, - privateKeyPath, - publicKeyPath, -} from "./agent-paths"; - -const logger = getLogger(["interchange", "hub-agent", "key-store"]); - -export type AgentKeyEntry = { - address: string; - keyPair: KeyPair; -}; +import { keysDir, privateKeyPath, publicKeyPath } from "./agent-paths"; export type AgentKeyStoreDeps = { dataDir: string; @@ -67,14 +50,6 @@ export type AgentKeyStore = { loadOrGenerateKey( address: string, ): Promise<{ keyPair: KeyPair; isNew: boolean }>; - /** - * Read every persisted keypair in the data directory and warm the - * in-memory cache with each one. Used by the sidecar's restore path - * to recover agents across restarts. Directories with a partial - * keypair (one of the two files present) are skipped with a warning; - * the operator can re-pair or remove them manually. - */ - scanKeys(): Promise; /** * Sign the challenge payload with the agent's cached private key. * Returns null when no key is cached for the address — the caller @@ -85,9 +60,9 @@ export type AgentKeyStore = { payload: Uint8Array, ): Promise; /** - * Record the hub public key the agent has been paired with. Cached - * in memory only; on-disk persistence of the pairing record lives in - * AgentRepoStore.persistPairing. + * Record the hub public key the agent has been paired with. Cached in + * memory only; the deploy path re-records it on every deploy, so it + * does not need to survive a restart. */ recordHubKey(address: string, hexHubPublicKey: string): void; /** @@ -154,62 +129,6 @@ export function createAgentKeyStore(deps: AgentKeyStoreDeps): AgentKeyStore { return { keyPair, isNew: true }; } - async function scanKeys(): Promise { - let entries; - try { - entries = await fsp.readdir(dataDir, { withFileTypes: true }); - } catch (err: unknown) { - if (hasCode(err) && err.code === "ENOENT") return []; - throw err; - } - - const results: AgentKeyEntry[] = []; - for (const entry of entries) { - if (!entry.isDirectory()) continue; - const dir = path.join(dataDir, entry.name); - const privPath = path.join(dir, KEYS_DIR_NAME, PRIVATE_KEY_FILE); - const pubPath = path.join(dir, KEYS_DIR_NAME, PUBLIC_KEY_FILE); - const [privOk, pubOk] = await Promise.all([ - fileExists(privPath), - fileExists(pubPath), - ]); - if (!privOk && !pubOk) continue; - if (!privOk || !pubOk) { - logger.warn`Skipping ${entry.name}: incomplete key pair (private=${String(privOk)}, public=${String(pubOk)})`; - continue; - } - const metaRaw = await readOptional(path.join(dir, META_FILE)); - if (metaRaw === null) { - // A key pair without a metadata file means the agent was - // half-provisioned and crashed. The restore composer logs - // and skips it; the operator can pair the key with a new - // config or remove the directory. - continue; - } - let parsed: unknown; - try { - parsed = JSON.parse(metaRaw); - } catch { - continue; - } - if (typeof parsed !== "object" || parsed === null) continue; - if (!("address" in parsed)) continue; - const address = parsed.address; - if (typeof address !== "string") continue; - const [privateKey, publicKey] = await Promise.all([ - fsp.readFile(privPath), - fsp.readFile(pubPath), - ]); - const keyPair: KeyPair = { - privateKey: new Uint8Array(privateKey), - publicKey: new Uint8Array(publicKey), - }; - agentKeys.set(address, keyPair); - results.push({ address, keyPair }); - } - return results; - } - async function signChallenge( address: string, payload: Uint8Array, @@ -244,7 +163,6 @@ export function createAgentKeyStore(deps: AgentKeyStoreDeps): AgentKeyStore { return { loadOrGenerateKey, - scanKeys, signChallenge, recordHubKey, verifyDeployCommit, @@ -264,12 +182,3 @@ async function fileExists(filePath: string): Promise { throw err; } } - -async function readOptional(filePath: string): Promise { - try { - return await fsp.readFile(filePath, "utf-8"); - } catch (err: unknown) { - if (hasCode(err) && err.code === "ENOENT") return null; - throw err; - } -} diff --git a/packages/hub-agent/src/agent-paths.ts b/packages/hub-agent/src/agent-paths.ts index 2a553300..5cf836ec 100644 --- a/packages/hub-agent/src/agent-paths.ts +++ b/packages/hub-agent/src/agent-paths.ts @@ -9,15 +9,10 @@ import path from "node:path"; -// The single source of truth for the per-agent layout filenames. The -// scan paths in agent-repo-store and agent-key-store cannot use the -// address-keyed helpers below because they walk the data directory -// before any agent.json has been parsed — those callers join these -// constants against the on-disk directory name directly. -export const KEYS_DIR_NAME = "keys"; -export const PRIVATE_KEY_FILE = "id_ed25519"; -export const PUBLIC_KEY_FILE = "id_ed25519.pub"; -export const META_FILE = "agent.json"; +// The per-agent key-file layout the address-keyed helpers below build on. +const KEYS_DIR_NAME = "keys"; +const PRIVATE_KEY_FILE = "id_ed25519"; +const PUBLIC_KEY_FILE = "id_ed25519.pub"; export function sanitizeAddress(address: string): string { return address.replace(/@/g, "_at_").replace(/[^a-zA-Z0-9_-]/g, "_"); @@ -38,7 +33,3 @@ export function privateKeyPath(dataDir: string, address: string): string { export function publicKeyPath(dataDir: string, address: string): string { return path.join(keysDir(dataDir, address), PUBLIC_KEY_FILE); } - -export function metaPath(dataDir: string, address: string): string { - return path.join(agentDir(dataDir, address), META_FILE); -} diff --git a/packages/hub-agent/src/agent-repo-store.test.ts b/packages/hub-agent/src/agent-repo-store.test.ts deleted file mode 100644 index ef596406..00000000 --- a/packages/hub-agent/src/agent-repo-store.test.ts +++ /dev/null @@ -1,124 +0,0 @@ -import { describe, test, expect, afterEach } from "bun:test"; -import fs from "node:fs/promises"; -import path from "node:path"; -import os from "node:os"; -import type { HarnessConfig } from "@intx/types/runtime"; - -import { createAgentRepoStore } from "./agent-repo-store"; - -const tempDirs: string[] = []; - -async function tempDir(): Promise { - const d = await fs.mkdtemp(path.join(os.tmpdir(), "agent-repo-store-test-")); - tempDirs.push(d); - return d; -} - -afterEach(async () => { - const dirs = tempDirs.splice(0); - await Promise.all( - dirs.map((d) => fs.rm(d, { recursive: true, force: true })), - ); -}); - -function makeConfig(address: string): HarnessConfig { - return { - agentId: "test-agent", - agentAddress: address, - sessionId: "sess-1", - principalId: "principal-1", - tenantId: "tenant-1", - systemPrompt: "test", - tools: [], - grants: [], - sources: [ - { - id: "test:test-model", - provider: "test", - apiKey: "key", - baseURL: "http://localhost", - model: "test-model", - }, - ], - defaultSource: "test:test-model", - }; -} - -// Provision the on-disk shape `scanConfigs` looks for: a per-agent -// directory containing the keypair files (presence is checked, contents -// are not used here — repo-store tests do not need a real keypair). -async function provisionAgentDir( - dataDir: string, - agentName: string, -): Promise { - const dir = path.join(dataDir, agentName); - await fs.mkdir(path.join(dir, "keys"), { recursive: true }); - await fs.writeFile(path.join(dir, "keys", "id_ed25519"), new Uint8Array()); - await fs.writeFile( - path.join(dir, "keys", "id_ed25519.pub"), - new Uint8Array(), - ); - return dir; -} - -describe("AgentRepoStore — config + pairing persistence", () => { - test("persistPairing then persistConfig preserves the pairing key", async () => { - const dataDir = await tempDir(); - const address = "agent@local"; - await provisionAgentDir(dataDir, "agent_at_local"); - - const repo = createAgentRepoStore({ dataDir }); - await repo.persistConfig(address, makeConfig(address)); - await repo.persistPairing(address, "aabbcc"); - - const entries = await repo.scanConfigs(); - expect(entries).toHaveLength(1); - const entry = entries[0]; - if (entry === undefined) throw new Error("unreachable"); - expect(entry.hubPublicKey).toBe("aabbcc"); - }); - - test("persistConfig after persistPairing keeps the pairing key intact", async () => { - const dataDir = await tempDir(); - const address = "agent@local"; - await provisionAgentDir(dataDir, "agent_at_local"); - - const repo = createAgentRepoStore({ dataDir }); - await repo.persistConfig(address, makeConfig(address)); - await repo.persistPairing(address, "aabbcc"); - - const updatedConfig = { ...makeConfig(address), systemPrompt: "updated" }; - await repo.persistConfig(address, updatedConfig); - - const entries = await repo.scanConfigs(); - expect(entries).toHaveLength(1); - const entry = entries[0]; - if (entry === undefined) throw new Error("unreachable"); - expect(entry.hubPublicKey).toBe("aabbcc"); - expect(entry.config.systemPrompt).toBe("updated"); - }); - - test("absent pairing key returns undefined from scanConfigs", async () => { - const dataDir = await tempDir(); - const address = "agent@local"; - await provisionAgentDir(dataDir, "agent_at_local"); - - const repo = createAgentRepoStore({ dataDir }); - await repo.persistConfig(address, makeConfig(address)); - - const entries = await repo.scanConfigs(); - expect(entries).toHaveLength(1); - const entry = entries[0]; - if (entry === undefined) throw new Error("unreachable"); - expect(entry.hubPublicKey).toBeUndefined(); - }); - - test("persistPairing on an unknown address throws", async () => { - const dataDir = await tempDir(); - const repo = createAgentRepoStore({ dataDir }); - - await expect(repo.persistPairing("nobody@local", "aabbcc")).rejects.toThrow( - "no existing agent.json", - ); - }); -}); diff --git a/packages/hub-agent/src/agent-repo-store.ts b/packages/hub-agent/src/agent-repo-store.ts index e0e410a9..3c0ccc09 100644 --- a/packages/hub-agent/src/agent-repo-store.ts +++ b/packages/hub-agent/src/agent-repo-store.ts @@ -1,18 +1,14 @@ // Per-agent on-disk repository layout. // -// Owns the isogit repo wrapper, the deploy-pack apply / state-pack -// produce flow, and the persistence of the per-agent `agent.json` -// metadata blob. Key custody lives in AgentKeyStore alongside this +// Owns the isogit repo wrapper and the deploy-pack apply / state-pack +// produce flow. Key custody lives in AgentKeyStore alongside this // store; both share the directory-layout helpers in agent-paths. import fs from "node:fs"; import fsp from "node:fs/promises"; -import path from "node:path"; import git from "isomorphic-git"; -import { type } from "arktype"; import { getLogger } from "@intx/log"; import { hasCode } from "@intx/types"; -import { HarnessConfig } from "@intx/types/runtime"; import { initAgentRepo, applyPack, @@ -21,30 +17,10 @@ import { type CommitVerifier, } from "@intx/storage-isogit"; -import { META_FILE, agentDir, metaPath } from "./agent-paths"; +import { agentDir } from "./agent-paths"; const logger = getLogger(["interchange", "hub-agent", "repo-store"]); -const AgentMeta = type({ - version: "1", - address: "string", - config: HarnessConfig, - "hubPublicKey?": "string", -}); -type AgentMeta = typeof AgentMeta.infer; - -/** - * A per-agent metadata record as it appears on disk. `hubPublicKey` - * — the hub identity this agent has been paired with — is set once - * by `persistPairing` and is otherwise preserved across `persistConfig` - * updates. - */ -export type AgentConfigEntry = { - address: string; - config: HarnessConfig; - hubPublicKey?: string; -}; - export type ApplyDeployPackArgs = { address: string; pack: Uint8Array; @@ -68,25 +44,6 @@ export type AgentRepoStore = { ): Promise<{ pack: Uint8Array; commitSha: string; ref: string }>; getDeployRef(address: string): Promise; remove(address: string): Promise; - /** - * Write the agent's HarnessConfig to disk. Preserves the existing - * `hubPublicKey` field if one was previously recorded by - * `persistPairing`. To update the pairing key, call `persistPairing` - * separately. - */ - persistConfig(address: string, config: HarnessConfig): Promise; - /** - * Record the hub public key this agent has been paired with. Set-once - * in normal operation; rewriting an existing value is permitted but - * indicates the agent has been re-paired with a different hub. - */ - persistPairing(address: string, hubPublicKey: string): Promise; - /** - * Scan the data directory for agent.json files and return the - * persisted config + pairing key for each. Files that fail to - * parse are skipped with a warning. - */ - scanConfigs(): Promise; }; export function createAgentRepoStore(config: { @@ -142,94 +99,6 @@ export function createAgentRepoStore(config: { logger.info`Deleted agent directory for ${address}`; } - async function readMeta(address: string): Promise { - try { - const raw = await fsp.readFile(metaPath(dataDir, address), "utf-8"); - const parsed: unknown = JSON.parse(raw); - const validated = AgentMeta(parsed); - if (validated instanceof type.errors) { - logger.warn`agent.json invalid for ${address}: ${validated.summary}`; - return null; - } - return validated; - } catch (err: unknown) { - if (hasCode(err) && err.code === "ENOENT") { - return null; - } - throw err; - } - } - - async function writeMeta(meta: AgentMeta): Promise { - await fsp.writeFile(metaPath(dataDir, meta.address), JSON.stringify(meta)); - } - - async function persistConfig( - address: string, - config: HarnessConfig, - ): Promise { - const existing = await readMeta(address); - const meta: AgentMeta = { version: 1, address, config }; - if (existing?.hubPublicKey !== undefined) { - meta.hubPublicKey = existing.hubPublicKey; - } - await writeMeta(meta); - } - - async function persistPairing( - address: string, - hubPublicKey: string, - ): Promise { - const existing = await readMeta(address); - if (existing === null) { - throw new Error( - `Cannot persist hub pairing for "${address}": no existing agent.json`, - ); - } - await writeMeta({ ...existing, hubPublicKey }); - } - - async function scanConfigs(): Promise { - let entries: fs.Dirent[]; - try { - entries = await fsp.readdir(dataDir, { withFileTypes: true }); - } catch (err: unknown) { - if (hasCode(err) && err.code === "ENOENT") return []; - throw err; - } - - const results: AgentConfigEntry[] = []; - for (const entry of entries) { - if (!entry.isDirectory()) continue; - // Read agent.json directly by directory name — we cannot reverse - // sanitizeAddress, so the address comes from the file's `address` - // field, not the directory name. - const metaFile = path.join(dataDir, entry.name, META_FILE); - let raw: string; - try { - raw = await fsp.readFile(metaFile, "utf-8"); - } catch (err: unknown) { - if (hasCode(err) && err.code === "ENOENT") continue; - throw err; - } - const parsed: unknown = JSON.parse(raw); - const validated = AgentMeta(parsed); - if (validated instanceof type.errors) { - logger.warn`Skipping ${entry.name}: invalid agent.json: ${validated.summary}`; - continue; - } - const result: AgentConfigEntry = { - address: validated.address, - config: validated.config, - }; - if (validated.hubPublicKey !== undefined) { - result.hubPublicKey = validated.hubPublicKey; - } - results.push(result); - } - return results; - } - return { getAgentDir, initRepo, @@ -237,8 +106,5 @@ export function createAgentRepoStore(config: { createStatePack, getDeployRef, remove, - persistConfig, - persistPairing, - scanConfigs, }; } diff --git a/packages/hub-agent/src/deploy-tree.test.ts b/packages/hub-agent/src/deploy-tree.test.ts index 838ce9fb..ce158f69 100644 --- a/packages/hub-agent/src/deploy-tree.test.ts +++ b/packages/hub-agent/src/deploy-tree.test.ts @@ -131,8 +131,8 @@ describe("readDeployTree", () => { test("returns corrupt-JSON bytes as-is for the caller to handle", async () => { // The reader does not parse; the caller's loader gate produces - // the manifest.invalid failure and routes it through the same - // deploy.apply.error frame channel as every other category. + // the manifest.invalid failure the same apply-rejection way as + // every other category. const dir = await tempDir(); await fs.promises.mkdir(path.join(dir, "deploy"), { recursive: true }); const garbage = "{this is not valid json"; diff --git a/packages/hub-agent/src/deploy-tree.ts b/packages/hub-agent/src/deploy-tree.ts index df27849a..7afd24c4 100644 --- a/packages/hub-agent/src/deploy-tree.ts +++ b/packages/hub-agent/src/deploy-tree.ts @@ -29,8 +29,8 @@ export type DeployTree = { * JSON parsing and arktype validation are the caller's * responsibility — the sidecar's harness builder does both inside * `materializeToolPackages` so a corrupt or schema-invalid manifest - * routes through the same `deploy.apply.error` frame channel as - * every other apply-time failure (category `manifest.invalid`). + * fails the apply loudly (category `manifest.invalid`) the same way + * as every other apply-time failure. * * Undefined means the manifest file is not present. */ diff --git a/packages/hub-agent/src/harness-builder.ts b/packages/hub-agent/src/harness-builder.ts index a9acafe5..b5e42001 100644 --- a/packages/hub-agent/src/harness-builder.ts +++ b/packages/hub-agent/src/harness-builder.ts @@ -1,92 +1,18 @@ // HarnessBuilder seam. // -// The package declares the shape of the per-agent harness construction -// step; the host (apps/sidecar today, any custom sidecar tomorrow) -// supplies the concrete implementation. This keeps @intx/hub-agent -// free of dependencies on the concrete tool, storage, authz, and -// inference packages the harness is wired up against. +// The package declares the shape of the host's source-admission check; +// the host (apps/sidecar today, any custom sidecar tomorrow) supplies +// the concrete implementation. This keeps @intx/hub-agent free of +// dependencies on the concrete inference packages the check consults. -import type { Harness } from "@intx/harness"; -import type { MailAuditStore } from "@intx/storage-isogit"; -import type { GrantRule } from "@intx/types/authz"; -import type { - ConnectorThreadState, - CryptoProvider, - HarnessConfig as AgentConfig, - InferenceEvent, - InferenceSource, - MessageTransport, -} from "@intx/types/runtime"; -import type { DeployApplyErrorFrame } from "@intx/types/sidecar"; - -/** - * Per-attempt context the builder uses to emit a deploy-apply error - * frame back to the hub. The host (typically the sidecar app's wiring - * to hub-link) translates this into the wire-level frame. - */ -export type DeployApplyErrorEmitter = ( - payload: Omit, -) => void; - -/** - * Inputs the builder receives for one agent's session-start. The package - * pre-resolves every value that depends on per-agent disk layout, transport - * registration, or crypto bootstrap so the builder itself only handles - * harness construction. - */ -export type BuildHarnessArgs = { - agentAddress: string; - agentConfig: AgentConfig; - /** Ordered inference sources; the head (the source whose id is - * `defaultSource`) starts active, the tail is the failover chain. */ - sources: InferenceSource[]; - defaultSource: string; - /** Absolute path to the per-agent directory, from AgentRepoStore. */ - storeDir: string; - /** Per-agent view of the host's message transport. */ - agentTransport: MessageTransport; - /** Per-agent crypto provider, from the host's createAgentCrypto. */ - crypto: CryptoProvider; - onEvent: (event: InferenceEvent) => void; - onConnectorStateChanged: (state: ConnectorThreadState | null) => void; - /** - * Emit a `deploy.apply.error` frame back to the hub when the tool- - * package loader rejects an apply. The builder calls this just - * before throwing to abort harness construction; the host wires it - * to the sidecar's hub-link. Hosts that do not yet support - * tool-package distribution can omit this — a builder that - * encounters a manifest while this is undefined throws with no - * dedicated frame, which lands as a generic `agent.error` via the - * existing session-start error handler. - */ - emitDeployApplyError?: DeployApplyErrorEmitter; -}; - -/** - * What the builder returns. The harness drives inference; the mailStore - * is owned by SessionManager for inbound/outbound audit; updateGrants - * mutates the live grant ref the harness's authz closure reads; - * disposers tear down builder-allocated resources on session end. - * - * The builder is responsible for invoking `dispose` on any resource it - * allocates that survives a build failure. The bundle's disposers are - * the contract for the *success* path; mid-construction failure must - * be cleaned up by the builder before the throw propagates. - */ -export type HarnessBundle = { - harness: Harness; - mailStore: MailAuditStore; - updateGrants(grants: GrantRule[]): void; - disposers: (() => Promise)[]; -}; +import type { InferenceSource } from "@intx/types/runtime"; export type HarnessBuilder = { /** - * Throws if the supplied source cannot be built by this host. Called - * before `build` at session-start, and standalone at update-source - * time so the operator sees rejection on the control plane rather - * than during the next inference call. + * Throws if the supplied source cannot be built by this host. The + * sidecar deploy router calls this to admit a step's pinned inference + * source before spawning, so the operator sees rejection on the + * control plane rather than during the next inference call. */ canBuildSource(source: InferenceSource): void; - build(args: BuildHarnessArgs): Promise; }; diff --git a/packages/hub-agent/src/index.ts b/packages/hub-agent/src/index.ts index fc424d77..000e182e 100644 --- a/packages/hub-agent/src/index.ts +++ b/packages/hub-agent/src/index.ts @@ -1,32 +1,18 @@ export { createAgentRepoStore, type AgentRepoStore, - type AgentConfigEntry, type ApplyDeployPackArgs, } from "./agent-repo-store"; export { createAgentKeyStore, type AgentKeyStore, type AgentKeyStoreDeps, - type AgentKeyEntry, } from "./agent-key-store"; -export type { - HarnessBuilder, - HarnessBundle, - BuildHarnessArgs, - DeployApplyErrorEmitter, -} from "./harness-builder"; +export type { HarnessBuilder } from "./harness-builder"; export { createSessionManager, type SessionManager, type SessionManagerConfig, - type SessionEventSink, - type ConnectorStateSink, - type AgentSession, - type AgentEventListener, - type ProvisionResult, - type RestoreResult, - type RestoredAgent, } from "./session-manager"; export { createHubLink, diff --git a/packages/hub-agent/src/paths.ts b/packages/hub-agent/src/paths.ts index d1ef1607..e95bbeaa 100644 --- a/packages/hub-agent/src/paths.ts +++ b/packages/hub-agent/src/paths.ts @@ -13,6 +13,3 @@ export { readDeployTree, type DeployTree } from "./deploy-tree"; export { sanitizeAddress } from "./agent-paths"; -// Type-only re-export: `export type` is erased under `verbatimModuleSyntax`, -// so it adds no runtime import edge to `harness-builder`. -export type { DeployApplyErrorEmitter } from "./harness-builder"; diff --git a/packages/hub-agent/src/session-manager.test.ts b/packages/hub-agent/src/session-manager.test.ts index 1c00b383..00a2a359 100644 --- a/packages/hub-agent/src/session-manager.test.ts +++ b/packages/hub-agent/src/session-manager.test.ts @@ -1,26 +1,12 @@ import { describe, test, expect, afterEach } from "bun:test"; +import fs from "node:fs"; import fsp from "node:fs/promises"; import path from "node:path"; import os from "node:os"; -import { createInMemoryTransport } from "@intx/mail-memory"; -import type { MailAuditStore } from "@intx/storage-isogit"; -import type { Harness } from "@intx/harness"; -import type { - CryptoProvider, - HarnessConfig, - InferenceEvent, - InferenceSource, - KeyPair, -} from "@intx/types/runtime"; +import git from "isomorphic-git"; -import { createAgentKeyStore } from "./agent-key-store"; -import { createAgentRepoStore, type AgentRepoStore } from "./agent-repo-store"; import { createSessionManager } from "./session-manager"; -import type { - BuildHarnessArgs, - HarnessBuilder, - HarnessBundle, -} from "./harness-builder"; +import type { AgentRepoStore } from "./agent-repo-store"; const tempDirs: string[] = []; @@ -37,649 +23,115 @@ afterEach(async () => { ); }); -function makeKeyPair(seed: number): KeyPair { - const privateKey = new Uint8Array(32); - const publicKey = new Uint8Array(32); - for (let i = 0; i < 32; i++) { - privateKey[i] = (seed + i) & 0xff; - publicKey[i] = (seed * 2 + i) & 0xff; - } - return { privateKey, publicKey }; -} - -function makeConfig(address: string): HarnessConfig { - return { - agentId: "test-agent", - agentAddress: address, - sessionId: "sess-1", - principalId: "principal-1", - tenantId: "tenant-1", - systemPrompt: "test", - tools: [], - grants: [], - sources: [ - { - id: "test:test-model", - provider: "test", - apiKey: "key", - baseURL: "http://localhost", - model: "test-model", - }, - ], - defaultSource: "test:test-model", +function makeStubRepoStore(opts: { + dataDir: string; + createStatePack: AgentRepoStore["createStatePack"]; + remove: AgentRepoStore["remove"]; +}): AgentRepoStore { + const unused = (name: string) => (): never => { + throw new Error(`${name} is not exercised by this test`); }; -} - -function makeCrypto(kp: KeyPair): CryptoProvider { return { - async sign() { - return new Uint8Array(64); - }, - async signSSH() { - return "unused"; - }, - async verify() { - return true; - }, - getPublicKey: () => kp.publicKey, + getAgentDir: (address) => path.join(opts.dataDir, address), + initRepo: unused("initRepo"), + applyDeployPack: unused("applyDeployPack"), + createStatePack: opts.createStatePack, + getDeployRef: unused("getDeployRef"), + remove: opts.remove, }; } -function makeMailStoreStub(): MailAuditStore & { commits: string[] } { - const commits: string[] = []; - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- MailAuditStore from storage-isogit is a structural type whose unrelated methods are unused in these tests - const store = { - commits, - async commitMail() { - commits.push("commit"); - return null; - }, - } as unknown as MailAuditStore & { commits: string[] }; - return store; -} - -function makeHarnessStub( - onSetSources?: (sources: InferenceSource[], defaultSource: string) => void, -): Harness { - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- Harness is a library type whose unused members are not structurally satisfied by the stub - return { - start: () => { - /* no-op: harness.start is invoked by SessionManager but the tests - do not observe inference activity */ - }, - stop: () => { - /* no-op */ - }, - deliver: () => { - /* no-op */ - }, - setSource: () => { - /* no-op */ - }, - setSources: (sources: InferenceSource[], defaultSource: string) => { - onSetSources?.(sources, defaultSource); - }, - get blobReader() { - throw new Error("blobReader unused in these tests"); - }, - } as unknown as Harness; +function makeManagerWithRepoStore( + repoStore: AgentRepoStore, +): ReturnType { + return createSessionManager({ repoStore }); } -type SetSourcesRecord = { - sources: InferenceSource[]; - defaultSource: string; -}; -type RecordingBuilder = HarnessBuilder & { - buildCalls: BuildHarnessArgs[]; - grants: GrantRecord[]; - setSourcesCalls: SetSourcesRecord[]; -}; -type GrantRecord = { address: string; count: number }; - -function makeRecordingBuilder(opts: { - rejectSource?: (s: { provider: string }) => boolean; - throwOnBuild?: boolean; -}): RecordingBuilder { - const buildCalls: BuildHarnessArgs[] = []; - const grants: GrantRecord[] = []; - const setSourcesCalls: SetSourcesRecord[] = []; +async function buildAssetPack( + files: Record, +): Promise<{ pack: Uint8Array; commitSha: string }> { + const sourceDir = await tempDir(); + await git.init({ fs, dir: sourceDir, defaultBranch: "main" }); + + for (const [rel, content] of Object.entries(files)) { + const abs = path.join(sourceDir, rel); + await fsp.mkdir(path.dirname(abs), { recursive: true }); + await fsp.writeFile(abs, content); + await git.add({ fs, dir: sourceDir, filepath: rel }); + } - return { - buildCalls, - grants, - setSourcesCalls, - canBuildSource(source) { - if (opts.rejectSource?.(source)) { - throw new Error(`source rejected: ${source.provider}`); - } - }, - async build(args) { - buildCalls.push(args); - if (opts.throwOnBuild) { - throw new Error("builder boom"); - } - const harness = makeHarnessStub((sources, defaultSource) => { - setSourcesCalls.push({ sources, defaultSource }); - }); - const mailStore = makeMailStoreStub(); - const bundle: HarnessBundle = { - harness, - mailStore, - updateGrants(g) { - grants.push({ address: args.agentAddress, count: g.length }); - }, - disposers: [], - }; - return bundle; - }, - }; -} + const commitSha = await git.commit({ + fs, + dir: sourceDir, + message: "asset", + author: { name: "test", email: "test@test.dev" }, + }); + + const oids = new Set([commitSha]); + const { commit } = await git.readCommit({ + fs, + dir: sourceDir, + oid: commitSha, + }); + oids.add(commit.tree); + async function walkTree(treeOid: string): Promise { + const { tree } = await git.readTree({ fs, dir: sourceDir, oid: treeOid }); + for (const entry of tree) { + oids.add(entry.oid); + if (entry.type === "tree") await walkTree(entry.oid); + } + } + await walkTree(commit.tree); -function makeManagerHarness( - dataDir: string, - opts: { - rejectSource?: (s: { provider: string }) => boolean; - throwOnBuild?: boolean; - } = {}, -): { - repoStore: ReturnType; - keyStore: ReturnType; - builder: RecordingBuilder; - transport: ReturnType; - manager: ReturnType; - events: { addr: string; sid: string }[]; -} { - const repoStore = createAgentRepoStore({ dataDir }); - const keyStore = createAgentKeyStore({ - dataDir, - generateKeyPair: async () => makeKeyPair(11), - signEd25519: async () => new Uint8Array(64), - verifySSHSig: async () => true, - }); - const builder = makeRecordingBuilder(opts); - const transport = createInMemoryTransport(); - const events: { addr: string; sid: string }[] = []; - const manager = createSessionManager({ - transport, - repoStore, - keyStore, - buildHarness: builder, - createAgentCrypto: (kp) => makeCrypto(kp), - onEvent: (addr, sid) => events.push({ addr, sid }), - onConnectorStateChanged: () => { - /* no-op: tests do not assert on connector state */ - }, + const result = await git.packObjects({ + fs, + dir: sourceDir, + oids: [...oids], + write: false, }); - return { repoStore, keyStore, builder, transport, manager, events }; + if (result.packfile === undefined) { + throw new Error("packObjects produced no packfile"); + } + return { pack: result.packfile, commitSha }; } -describe("SessionManager.provisionAgent + startSession happy path", () => { - test("invokes the builder with the resolved per-agent context", async () => { - const dataDir = await tempDir(); - const { manager, builder } = makeManagerHarness(dataDir); - const cfg = makeConfig("agent@local"); - - await manager.provisionAgent(cfg); - await manager.startSession("agent@local"); - - expect(manager.hasSession("agent@local")).toBe(true); - expect(builder.buildCalls).toHaveLength(1); - const call = builder.buildCalls[0]; - if (call === undefined) throw new Error("unreachable"); - expect(call.agentAddress).toBe("agent@local"); - expect(call.agentConfig.agentId).toBe("test-agent"); - expect(call.defaultSource).toBe("test:test-model"); - expect(call.sources.map((s) => s.id)).toContain("test:test-model"); - expect(call.storeDir).toContain("agent_at_local"); - }); -}); - -describe("SessionManager.updateSources", () => { - const NEW_SOURCES: InferenceSource[] = [ - { - id: "test:test-model", - provider: "test", - apiKey: "rotated-key", - baseURL: "http://localhost", - model: "test-model", - }, - { - id: "test:fallback", - provider: "test", - apiKey: "fallback-key", - baseURL: "http://localhost", - model: "fallback-model", - }, - ]; - - test("installs the full ordered list on the running harness", async () => { +describe("SessionManager.applyAssetPack", () => { + test("materializes the pack under /workspace//", async () => { const dataDir = await tempDir(); - const { manager, builder } = makeManagerHarness(dataDir); - await manager.provisionAgent(makeConfig("agent@local")); - await manager.startSession("agent@local"); - - await manager.updateSources("agent@local", NEW_SOURCES, "test:test-model"); - - expect(builder.setSourcesCalls).toHaveLength(1); - const recorded = builder.setSourcesCalls[0]; - if (recorded === undefined) throw new Error("unreachable"); - expect(recorded.defaultSource).toBe("test:test-model"); - expect(recorded.sources.map((s) => s.id)).toEqual([ - "test:test-model", - "test:fallback", - ]); - }); - - test("rejects a default that names no source in the new list", async () => { - const dataDir = await tempDir(); - const { manager } = makeManagerHarness(dataDir); - await manager.provisionAgent(makeConfig("agent@local")); - await manager.startSession("agent@local"); - - await expect( - manager.updateSources("agent@local", NEW_SOURCES, "test:missing"), - ).rejects.toThrow(); - }); -}); - -describe("SessionManager.startSession rollback when builder throws", () => { - test("restores provisioned state and unregisters the transport", async () => { - const dataDir = await tempDir(); - const { manager, transport } = makeManagerHarness(dataDir, { - throwOnBuild: true, + const { pack, commitSha } = await buildAssetPack({ + "greet/SKILL.md": "---\nname: greet\n---\nbody\n", }); - const cfg = makeConfig("agent@local"); - - await manager.provisionAgent(cfg); - await expect(manager.startSession("agent@local")).rejects.toThrow( - "builder boom", - ); - - expect(manager.hasSession("agent@local")).toBe(false); - expect(manager.isProvisioned("agent@local")).toBe(true); - // Transport's per-agent view should be gone; getTransportFor throws - // for unregistered addresses. - expect(() => transport.getTransportFor("agent@local")).toThrow(); - }); -}); - -describe("SessionManager.restoreSessions mismatch handling", () => { - test("config without a key pair is reported as failed", async () => { - const dataDir = await tempDir(); - const { manager, repoStore } = makeManagerHarness(dataDir); - - // Agent with a config but no on-disk keypair → goes to `failed`. - await fsp.mkdir(path.join(dataDir, "ghost_at_local"), { recursive: true }); - await repoStore.persistConfig("ghost@local", makeConfig("ghost@local")); - - const result = await manager.restoreSessions(); - - expect(result.failed).toContain("ghost@local"); - expect(result.restored.map((r) => r.address)).not.toContain("ghost@local"); - }); -}); - -describe("SessionManager.applyAssetPack materializes under the workspace dir", () => { - test("writes pack contents under /workspace//", async () => { - const dataDir = await tempDir(); - const { manager, repoStore } = makeManagerHarness(dataDir); - const address = "agent@local"; - await manager.provisionAgent(makeConfig(address)); - - // Build a real pack from a tiny source repo so applyAssetPack - // can index it. Reuses isomorphic-git directly to avoid pulling - // in the asset-service surface for this test. - const sourceDir = path.join(dataDir, "asset-source"); - await fsp.mkdir(sourceDir, { recursive: true }); - const git = await import("isomorphic-git"); - const fs = await import("node:fs"); - await git.default.init({ fs, dir: sourceDir, defaultBranch: "main" }); - await fsp.writeFile(path.join(sourceDir, "hello.txt"), "hello from asset"); - await git.default.add({ fs, dir: sourceDir, filepath: "hello.txt" }); - const commitSha = await git.default.commit({ - fs, - dir: sourceDir, - message: "asset", - author: { name: "t", email: "t@t" }, - }); - const oids = new Set([commitSha]); - const { commit } = await git.default.readCommit({ - fs, - dir: sourceDir, - oid: commitSha, - }); - oids.add(commit.tree); - const { tree } = await git.default.readTree({ - fs, - dir: sourceDir, - oid: commit.tree, - }); - for (const entry of tree) oids.add(entry.oid); - const packResult = await git.default.packObjects({ - fs, - dir: sourceDir, - oids: [...oids], - write: false, + // The wrapper's only logic over `applyAssetPackFn` is the workspace-root + // composition: `/workspace`. Prove the pack lands there. + const repoStore = makeStubRepoStore({ + dataDir, + createStatePack: () => + Promise.reject(new Error("createStatePack not exercised by this test")), + remove: () => + Promise.reject(new Error("remove not exercised by this test")), }); - if (packResult.packfile === undefined) { - throw new Error("packObjects produced no pack"); - } + const manager = makeManagerWithRepoStore(repoStore); await manager.applyAssetPack( - address, - "skills/greet/", - packResult.packfile, + "agent@local", + "skills/example/", + pack, "refs/heads/main", commitSha, ); - const expected = path.join( - repoStore.getAgentDir(address), - "workspace", - "skills", - "greet", - "hello.txt", - ); - const contents = await fsp.readFile(expected, "utf-8"); - expect(contents).toBe("hello from asset"); - }); -}); - -describe("SessionManager.updateGrants routes through the bundle", () => { - test("calls bundle.updateGrants with the new grants and persists config", async () => { - const dataDir = await tempDir(); - const { manager, builder, repoStore } = makeManagerHarness(dataDir); - const cfg = makeConfig("agent@local"); - - await manager.provisionAgent(cfg); - await manager.startSession("agent@local"); - - await manager.updateGrants("agent@local", [ - { - id: "g-1", - resource: "*", - action: "*", - effect: "allow", - origin: "system", - conditions: null, - expiresAt: null, - roleId: null, - principalId: null, - }, - ]); - - expect(builder.grants).toEqual([{ address: "agent@local", count: 1 }]); - - // Persisted config should now carry the new grants array length. - const configs = await repoStore.scanConfigs(); - const entry = configs.find((c) => c.address === "agent@local"); - expect(entry?.config.grants).toHaveLength(1); - }); -}); - -describe("SessionManager.onAgentEvent per-agent fan-out", () => { - test("delivers events only to the subscribed agent's listeners and respects the disposer", async () => { - const dataDir = await tempDir(); - const { manager, builder } = makeManagerHarness(dataDir); - - await manager.provisionAgent(makeConfig("alice@local")); - await manager.startSession("alice@local"); - await manager.provisionAgent(makeConfig("bob@local")); - await manager.startSession("bob@local"); - - const aliceBuild = builder.buildCalls.find( - (c) => c.agentAddress === "alice@local", - ); - const bobBuild = builder.buildCalls.find( - (c) => c.agentAddress === "bob@local", - ); - if (aliceBuild === undefined || bobBuild === undefined) { - throw new Error("unreachable: builder did not capture both addresses"); - } - - const aliceEvents: InferenceEvent[] = []; - const bobEvents: InferenceEvent[] = []; - const dispose = manager.onAgentEvent("alice@local", (e) => - aliceEvents.push(e), - ); - manager.onAgentEvent("bob@local", (e) => bobEvents.push(e)); - - const aliceTick: InferenceEvent = { - type: "inference.start", - seq: 0, - data: { model: "alice-m" }, - }; - const bobTick: InferenceEvent = { - type: "inference.start", - seq: 0, - data: { model: "bob-m" }, - }; - aliceBuild.onEvent(aliceTick); - bobBuild.onEvent(bobTick); - - expect(aliceEvents).toEqual([aliceTick]); - expect(bobEvents).toEqual([bobTick]); - - dispose(); - aliceBuild.onEvent(aliceTick); - expect(aliceEvents).toEqual([aliceTick]); - // bob continues to receive after alice's disposer runs. - bobBuild.onEvent(bobTick); - expect(bobEvents).toEqual([bobTick, bobTick]); - }); -}); - -async function seedAgentsOnDisk( - dataDir: string, - addresses: string[], -): Promise { - const seed = makeManagerHarness(dataDir); - for (const address of addresses) { - await seed.manager.provisionAgent(makeConfig(address)); - await seed.manager.startSession(address); - } -} - -function makeRestoredBundle(): HarnessBundle { - return { - harness: makeHarnessStub(), - mailStore: makeMailStoreStub(), - updateGrants() { - /* no-op: restore tests do not assert on grants */ - }, - disposers: [], - }; -} - -function makeRestoreStores(dataDir: string): { - repoStore: ReturnType; - keyStore: ReturnType; -} { - return { - repoStore: createAgentRepoStore({ dataDir }), - keyStore: createAgentKeyStore({ + const materialized = path.join( dataDir, - generateKeyPair: async () => makeKeyPair(11), - signEd25519: async () => new Uint8Array(64), - verifySSHSig: async () => true, - }), - }; -} - -describe("SessionManager.restoreSessions bounded parallelism", () => { - test("rejects a non-positive restoreConcurrency", async () => { - const dataDir = await tempDir(); - const { repoStore, keyStore } = makeRestoreStores(dataDir); - expect(() => - createSessionManager({ - transport: createInMemoryTransport(), - repoStore, - keyStore, - buildHarness: makeRecordingBuilder({}), - createAgentCrypto: (kp) => makeCrypto(kp), - onEvent: () => { - /* no-op */ - }, - onConnectorStateChanged: () => { - /* no-op */ - }, - restoreConcurrency: 0, - }), - ).toThrow(/restoreConcurrency must be a positive integer/); - }); - - test("restores up to restoreConcurrency agents at once, no more", async () => { - const dataDir = await tempDir(); - const addresses = ["a@local", "b@local", "c@local"]; - await seedAgentsOnDisk(dataDir, addresses); - - // Fresh stores + transport stand in for a restarted process that - // restores the seeded agents from disk. - const { repoStore, keyStore } = makeRestoreStores(dataDir); - - const buildStarts: string[] = []; - let startedCount = 0; - let signalBoundReached!: () => void; - const boundReached = new Promise((resolve) => { - signalBoundReached = resolve; - }); - let releaseBuilds!: () => void; - const gate = new Promise((resolve) => { - releaseBuilds = resolve; - }); - const builder: HarnessBuilder = { - canBuildSource() { - /* accept every source */ - }, - async build(args) { - buildStarts.push(args.agentAddress); - startedCount += 1; - if (startedCount === 2) signalBoundReached(); - await gate; - return makeRestoredBundle(); - }, - }; - - const manager = createSessionManager({ - transport: createInMemoryTransport(), - repoStore, - keyStore, - buildHarness: builder, - createAgentCrypto: (kp) => makeCrypto(kp), - onEvent: () => { - /* no-op */ - }, - onConnectorStateChanged: () => { - /* no-op */ - }, - restoreConcurrency: 2, - }); - - const restorePromise = manager.restoreSessions(); - // Two builds run at once; the third cannot start until a worker - // slot frees, which needs the gate to release. A serial restore - // would hold at one started build and this await would never settle. - await boundReached; - expect(buildStarts.length).toBe(2); - - releaseBuilds(); - const result = await restorePromise; - // All three eventually build once slots free up. - expect(buildStarts.length).toBe(3); - expect(result.failed).toEqual([]); - expect(result.restored.map((r) => r.address).sort()).toEqual( - [...addresses].sort(), + "agent@local", + "workspace", + "skills/example", + "greet/SKILL.md", ); - }); - - test("reports a failing agent as failed without blocking the rest", async () => { - const dataDir = await tempDir(); - const addresses = ["healthy1@local", "wedged@local", "healthy2@local"]; - await seedAgentsOnDisk(dataDir, addresses); - - const { repoStore, keyStore } = makeRestoreStores(dataDir); - - const builder: HarnessBuilder = { - canBuildSource() { - /* accept every source */ - }, - async build(args) { - if (args.agentAddress === "wedged@local") { - throw new Error("registry fetch exceeded the timeout"); - } - return makeRestoredBundle(); - }, - }; - - const manager = createSessionManager({ - transport: createInMemoryTransport(), - repoStore, - keyStore, - buildHarness: builder, - createAgentCrypto: (kp) => makeCrypto(kp), - onEvent: () => { - /* no-op */ - }, - onConnectorStateChanged: () => { - /* no-op */ - }, - }); - - const result = await manager.restoreSessions(); - expect(result.failed).toEqual(["wedged@local"]); - expect(result.restored.map((r) => r.address).sort()).toEqual([ - "healthy1@local", - "healthy2@local", - ]); + expect(fs.existsSync(materialized)).toBe(true); }); }); -function makeStubRepoStore(opts: { - dataDir: string; - createStatePack: AgentRepoStore["createStatePack"]; - remove: AgentRepoStore["remove"]; -}): AgentRepoStore { - const unused = (name: string) => (): never => { - throw new Error(`${name} is not exercised by this test`); - }; - return { - getAgentDir: (address) => path.join(opts.dataDir, address), - initRepo: unused("initRepo"), - applyDeployPack: unused("applyDeployPack"), - createStatePack: opts.createStatePack, - getDeployRef: unused("getDeployRef"), - remove: opts.remove, - persistConfig: unused("persistConfig"), - persistPairing: unused("persistPairing"), - scanConfigs: unused("scanConfigs"), - }; -} - -function makeManagerWithRepoStore( - dataDir: string, - repoStore: AgentRepoStore, -): ReturnType { - return createSessionManager({ - transport: createInMemoryTransport(), - repoStore, - keyStore: createAgentKeyStore({ - dataDir, - generateKeyPair: async () => makeKeyPair(11), - signEd25519: async () => new Uint8Array(64), - verifySSHSig: async () => true, - }), - buildHarness: makeRecordingBuilder({}), - createAgentCrypto: (kp) => makeCrypto(kp), - onEvent: () => { - /* no-op */ - }, - onConnectorStateChanged: () => { - /* no-op */ - }, - }); -} - describe("SessionManager repo-operation serialization", () => { test("deleteAgentDir removes the directory only after an in-flight state-pack read completes", async () => { const dataDir = await tempDir(); @@ -707,7 +159,7 @@ describe("SessionManager repo-operation serialization", () => { }, }); - const manager = makeManagerWithRepoStore(dataDir, repoStore); + const manager = makeManagerWithRepoStore(repoStore); const rejections: unknown[] = []; const onRejection = (reason: unknown) => rejections.push(reason); @@ -760,7 +212,7 @@ describe("SessionManager repo-operation serialization", () => { }, }); - const manager = makeManagerWithRepoStore(dataDir, repoStore); + const manager = makeManagerWithRepoStore(repoStore); const rejections: unknown[] = []; const onRejection = (reason: unknown) => rejections.push(reason); diff --git a/packages/hub-agent/src/session-manager.ts b/packages/hub-agent/src/session-manager.ts index db0a14f1..6b42aad2 100644 --- a/packages/hub-agent/src/session-manager.ts +++ b/packages/hub-agent/src/session-manager.ts @@ -1,159 +1,30 @@ -// Per-agent harness lifecycle. +// Per-agent on-disk repo operations for supervised deployments. // -// SessionManager owns the cross-agent state (provisioned/sessions maps, -// the per-agent mail-commit queue, the last-checkpoint-hash buffer used -// to thread connector reply hashes into outbound mail commits) and the -// global transport handlers (addMessageSentHandler). Per-agent harness -// construction lives behind the HarnessBuilder seam, supplied by the -// host. The package itself depends only on the lifecycle infrastructure -// (transport interface, mail-audit store type, crypto provider type) -// and the stores from this same package — it does not pin the concrete -// tool, storage, authz, or inference packages the harness is wired up -// against. The host owns those. -// -// Construction split between this module and the builder: -// - SessionManager handles: provisioned/sessions bookkeeping, -// transport.register/unregister/getTransportFor, AgentCrypto -// instantiation, the mail-commit queue, the addMessageSentHandler -// subscription, the rollback path on builder failure. -// - HarnessBuilder handles: storage + mailStore construction (from -// the per-agent signer), authz wiring (grantsRef + authorize -// closure), tool composition, harness construction. -// The boundary keeps the cross-agent state owned by SessionManager and -// the per-agent construction owned by the host. Pushing transport -// registration into the builder would break the invariant; pushing -// authz out of the builder would re-couple the package to authz. +// The in-process session runtime -- harness construction, agent +// provisioning, disk restore, and per-agent mail audit -- has been +// retired: every agent now runs as a supervised workflow-process child +// on the workflow-run substrate. What remains here is the thin +// serialization layer over the agent repo store that the deploy path +// and the hub-link still call: deploy/asset-pack applies, state-pack +// reads, deploy-ref reads, and directory teardown, each run +// one-at-a-time per agent so a teardown never races an in-flight git op. import path from "node:path"; -import { getLogger } from "@intx/log"; -import { hexEncode } from "@intx/types"; -import type { HubTransport } from "@intx/mail-memory"; -import type { GrantRule } from "@intx/types/authz"; -import type { DeployApplyErrorFrame } from "@intx/types/sidecar"; -import type { - ConnectorThreadState, - CryptoProvider, - HarnessConfig as AgentConfig, - InboundMessage, - InferenceEvent, - InferenceSource, - KeyPair, -} from "@intx/types/runtime"; -import type { Harness } from "@intx/harness"; - -import type { AgentKeyEntry, AgentKeyStore } from "./agent-key-store"; import type { AgentRepoStore } from "./agent-repo-store"; -import type { HarnessBuilder, HarnessBundle } from "./harness-builder"; import { applyAssetPack as applyAssetPackFn } from "./apply-asset-pack"; -const logger = getLogger(["interchange", "hub-agent", "session"]); - -/** - * Public session record. The grants ref and disposers live inside the - * HarnessBundle the builder produced — they are not part of this type. - */ -export type AgentSession = { - agentAddress: string; - agentId: string; - config: AgentConfig; -}; - -export type SessionEventSink = ( - agentAddress: string, - sessionId: string, - event: InferenceEvent, -) => void; - -export type ConnectorStateSink = ( - agentAddress: string, - state: ConnectorThreadState | null, -) => void; - -export type DeployApplyErrorSink = ( - agentAddress: string, - payload: Omit, -) => void; - export type SessionManagerConfig = { - transport: HubTransport; repoStore: AgentRepoStore; - keyStore: AgentKeyStore; - buildHarness: HarnessBuilder; - /** - * Per-agent crypto factory. Receives the agent's raw key pair and - * returns a CryptoProvider bound to it. Keeps the package free of - * `@intx/crypto`. - */ - createAgentCrypto: (keyPair: KeyPair) => CryptoProvider; - onEvent: SessionEventSink; - onConnectorStateChanged: ConnectorStateSink; - /** - * Optional: emit a deploy-apply error frame to the hub when the - * harness builder rejects an apply attempt. Wired by the sidecar - * app to hub-link's frame channel. Hosts without tool-package - * distribution can omit this. - */ - onDeployApplyError?: DeployApplyErrorSink; - /** - * Maximum number of agents whose sessions `restoreSessions` restores - * concurrently. Restore work is per-agent isolated (per-agent git dir, - * per-address repo lock), so agents restore in parallel up to this - * bound and a single slow or wedged agent no longer gates the rest. - * Defaults to `DEFAULT_RESTORE_CONCURRENCY`. - */ - restoreConcurrency?: number; -}; - -/** - * Default `restoreConcurrency`: how many agents `restoreSessions` - * restores in parallel when the caller does not override it. - */ -const DEFAULT_RESTORE_CONCURRENCY = 4; - -export type ProvisionResult = { - publicKey: string; - keyPair: KeyPair; -}; - -export type RestoredAgent = { - address: string; - keyPair: KeyPair; - hubPublicKey?: string; }; -export type RestoreResult = { - restored: RestoredAgent[]; - failed: string[]; -}; - -export type AgentEventListener = (event: InferenceEvent) => void; - export type SessionManager = { - provisionAgent(config: AgentConfig): Promise; - startSession(agentAddress: string): Promise; - destroySession(agentAddress: string): Promise; - abortSession(agentAddress: string, reason: string): Promise; - deliverMessage(agentAddress: string, message: InboundMessage): void; - updateGrants(agentAddress: string, grants: GrantRule[]): Promise; /** - * Subscribe to InferenceEvents scoped to a specific agent address. - * Returns a disposer that removes the listener. Listeners fire - * synchronously before the global `onEvent` sink during dispatch. - * Production wires this against the workflow-host supervisor's - * trivial-launch path so the per-message reactor brackets the - * trivial workflow's run-event chain through `recordRunEvent`. + * Initialize the on-disk deploy-tree repo for an address. A single-step + * workflow deploy uses this at the head so the follow-up deploy-pack + * apply has a repo to apply into. */ - onAgentEvent(agentAddress: string, listener: AgentEventListener): () => void; - updateSources( - agentAddress: string, - sources: InferenceSource[], - defaultSource: string, - ): Promise; - hasSession(agentAddress: string): boolean; - isProvisioned(agentAddress: string): boolean; - getAddresses(): string[]; - restoreSessions(): Promise; + initRepo(address: string): Promise; /** * Apply a deploy pack to the agent's repo. Thin wrapper around * AgentRepoStore for callers that already have a SessionManager handle. @@ -169,7 +40,7 @@ export type SessionManager = { /** * Materialize an asset pack at `//` for the * agent. The workspace root is per-agent; this is distinct from the - * agent's deploy git tree. Asset packs are unsigned in v1 — no + * agent's deploy git tree. Asset packs are unsigned in v1 -- no * `verifyCommit` parameter. */ applyAssetPack( @@ -184,97 +55,32 @@ export type SessionManager = { ): Promise<{ pack: Uint8Array; commitSha: string; ref: string }>; deleteAgentDir(agentAddress: string): Promise; getDeployRef(agentAddress: string): Promise; - persistHubPublicKey( - agentAddress: string, - hubPublicKey: string, - ): Promise; - commitInboundMail( - agentAddress: string, - rawMessage: Uint8Array, - ): Promise; + /** + * Session addresses this manager hosts. The in-process session runtime + * is retired, so this is always empty; the hub-link ships it in the + * register frame alongside the sidecar's workflow-deployment addresses. + */ + getAddresses(): string[]; + /** + * Session id for an address' outbound mail forwarding. Always undefined + * now that no in-process sessions exist; the hub-link tolerates a + * missing id and forwards the mail without one. + */ getSessionId(agentAddress: string): string | undefined; }; -type ProvisionedAgent = { - config: AgentConfig; - keyPair: KeyPair; -}; - -type LiveSession = AgentSession & { - harness: Harness; - bundle: HarnessBundle; -}; - export function createSessionManager( config: SessionManagerConfig, ): SessionManager { - const { - transport, - repoStore, - keyStore, - buildHarness, - createAgentCrypto, - onEvent, - onConnectorStateChanged, - onDeployApplyError, - restoreConcurrency = DEFAULT_RESTORE_CONCURRENCY, - } = config; - if (!Number.isInteger(restoreConcurrency) || restoreConcurrency < 1) { - throw new Error( - `createSessionManager: restoreConcurrency must be a positive integer; got ${String(restoreConcurrency)}`, - ); - } - - const sessions = new Map(); - const provisioned = new Map(); - const pending = new Set(); - - // Per-agent InferenceEvent fan-out. Subscribers register against a - // specific agentAddress; the dispatch site looks the listener set up - // by the event's owning address (the closure captured in startSession) - // and fires every listener synchronously before the global `onEvent` - // sink runs. Disposers remove a single listener and prune the set on - // emptiness so addresses without subscribers cost a single map miss. - const agentEventListeners = new Map>(); - - function onAgentEvent( - agentAddress: string, - listener: AgentEventListener, - ): () => void { - let set = agentEventListeners.get(agentAddress); - if (set === undefined) { - set = new Set(); - agentEventListeners.set(agentAddress, set); - } - set.add(listener); - return () => { - const current = agentEventListeners.get(agentAddress); - if (current === undefined) return; - current.delete(listener); - if (current.size === 0) agentEventListeners.delete(agentAddress); - }; - } - - // Checkpoint hash captured from the most recent connector.reply event for - // each agent. Consumed (deleted) by the MessageSentHandler so that only - // the connector reply mail commit receives the linkage — subsequent - // tool-initiated sends do not carry a stale hash. - // - // Ordering guarantee: the harness calls onEvent() synchronously inside - // handleEvent(), which fires before the detached transport.send() resolves. - // Because executeSend() always awaits at least once (signature generation) - // before calling MessageSentHandler, the map write in onEvent completes - // before the handler reads it. - const lastCheckpointHashes = new Map(); + const { repoStore } = config; // Per-agent promise chain that serializes the operations against an agent's - // on-disk directory that can be in flight during a live session -- mail-audit - // commits, state-pack and deploy-ref reads, and deploy/asset-pack applies all - // run one-at-a-time per agent. The chain exists for teardown: drainRepoOps - // awaits it before deleting the directory, so an operation that was valid - // when it started never runs against a path that has since vanished - // underneath it. Serializing additionally avoids corruption for the members - // that share the agent's `.git/` object store (mail commits, state-pack and + // on-disk directory -- state-pack and deploy-ref reads and deploy/asset-pack + // applies all run one-at-a-time per agent. The chain exists for teardown: + // drainRepoOps awaits it before deleting the directory, so an operation that + // was valid when it started never runs against a path that has since + // vanished underneath it. Serializing additionally avoids corruption for the + // members that share the agent's `.git/` object store (state-pack and // deploy-ref reads, deploy-pack applies), which isogit, lacking a // cross-process lock, would otherwise let interleave. Asset-pack applies are // on the chain only for the teardown reason -- they materialize into a @@ -305,16 +111,6 @@ export function createSessionManager( return result; } - function enqueueMailCommit( - agentAddress: string, - fn: () => Promise, - ): void { - void runRepoOp(agentAddress, fn).catch((err: unknown) => { - const msg = err instanceof Error ? err.message : String(err); - logger.error`Mail audit commit failed for ${agentAddress}: ${msg}`; - }); - } - // Await the agent's current operation chain so teardown removes the // directory only after in-flight git work finishes. Capturing the tail and // clearing the entry means an op enqueued AFTER this point starts a fresh @@ -330,393 +126,6 @@ export function createSessionManager( if (inflight !== undefined) await inflight; } - transport.addMessageSentHandler( - async ({ senderAddress, rawMessage, messageId }) => { - const session = sessions.get(senderAddress); - if (session === undefined) { - // Outbound mail for an address with no active session is a - // protocol violation — the transport should not have accepted - // the send. Surfacing this loudly catches the contract break - // before it propagates as silently-dropped audit records. - throw new Error( - `No active session for sender "${senderAddress}" — cannot audit outbound mail ${messageId}`, - ); - } - const mailStore = session.bundle.mailStore; - const checkpointHash = lastCheckpointHashes.get(senderAddress); - lastCheckpointHashes.delete(senderAddress); - enqueueMailCommit(senderAddress, async () => { - const result = await mailStore.commitMail(rawMessage, "out", { - ignoreDuplicate: true, - ...(checkpointHash !== undefined ? { checkpointHash } : {}), - }); - if (result !== null) { - logger.info`Committed outbound mail ${messageId} for ${senderAddress}`; - } - }); - }, - ); - - async function provisionAgent( - agentConfig: AgentConfig, - ): Promise { - const { agentAddress } = agentConfig; - - if ( - sessions.has(agentAddress) || - provisioned.has(agentAddress) || - pending.has(agentAddress) - ) { - throw new Error(`Agent already exists for address "${agentAddress}"`); - } - - pending.add(agentAddress); - - try { - const { keyPair, isNew } = await keyStore.loadOrGenerateKey(agentAddress); - - if (isNew) { - logger.info`Generated new key pair for ${agentAddress}`; - } - - await repoStore.initRepo(agentAddress); - await repoStore.persistConfig(agentAddress, agentConfig); - - provisioned.set(agentAddress, { config: agentConfig, keyPair }); - - const publicKey = hexEncode(keyPair.publicKey); - logger.info`Provisioned agent ${agentAddress}`; - return { publicKey, keyPair }; - } finally { - pending.delete(agentAddress); - } - } - - async function startSession(agentAddress: string): Promise { - const entry = provisioned.get(agentAddress); - if (entry === undefined) { - throw new Error(`No provisioned agent for address "${agentAddress}"`); - } - if (sessions.has(agentAddress)) { - throw new Error(`Session already running for agent "${agentAddress}"`); - } - - const { config: agentConfig, keyPair } = entry; - - const source = agentConfig.sources.find( - (s) => s.id === agentConfig.defaultSource, - ); - if (source === undefined) { - throw new Error( - `No source matches defaultSource "${agentConfig.defaultSource}" for agent "${agentAddress}"`, - ); - } - buildHarness.canBuildSource(source); - - provisioned.delete(agentAddress); - - try { - const crypto = createAgentCrypto(keyPair); - transport.register(agentAddress, crypto); - const agentTransport = transport.getTransportFor(agentAddress); - - const sessionId = agentConfig.sessionId; - const storeDir = repoStore.getAgentDir(agentAddress); - - const bundle = await buildHarness.build({ - agentAddress, - agentConfig, - sources: agentConfig.sources, - defaultSource: agentConfig.defaultSource, - storeDir, - agentTransport, - crypto, - onEvent(event: InferenceEvent) { - if ( - event.type === "connector.reply" && - event.data.checkpointHash !== undefined - ) { - lastCheckpointHashes.set(agentAddress, event.data.checkpointHash); - } - // Per-agent listeners fire before the global sink so an - // in-process consumer (the workflow-host trivial-launch - // subscriber) sees events at the same instant the hub - // forwarder does. Exceptions from a listener must not - // suppress the global forwarder; collect and rethrow only - // after `onEvent` has run. - let firstError: unknown; - const set = agentEventListeners.get(agentAddress); - if (set !== undefined) { - for (const listener of set) { - try { - listener(event); - } catch (err: unknown) { - if (firstError === undefined) firstError = err; - else - logger.error`agent-event listener for ${agentAddress} threw: ${String(err)}`; - } - } - } - onEvent(agentAddress, sessionId, event); - if (firstError !== undefined) throw firstError; - }, - onConnectorStateChanged(state) { - onConnectorStateChanged(agentAddress, state); - }, - ...(onDeployApplyError !== undefined - ? { - emitDeployApplyError: (payload) => { - onDeployApplyError(agentAddress, payload); - }, - } - : {}), - }); - - sessions.set(agentAddress, { - agentAddress, - agentId: agentConfig.agentId, - config: agentConfig, - harness: bundle.harness, - bundle, - }); - - // The composition-layer harness is started by `createHarness` -- - // by the time the builder returns the bundle, the agent's - // reactor is already running. No separate start() step. - logger.info`Started session for ${agentAddress} (session ${sessionId})`; - } catch (err) { - sessions.delete(agentAddress); - repoOpQueues.delete(agentAddress); - try { - transport.unregister(agentAddress); - } catch (cleanupErr) { - // Best-effort cleanup; don't mask the original error. - logger.error`Failed to unregister transport for ${agentAddress}: ${String(cleanupErr)}`; - } - provisioned.set(agentAddress, entry); - throw err; - } - } - - async function runDisposers( - session: LiveSession, - agentAddress: string, - ): Promise { - // Run every disposer even if some fail; an exception from one must - // not leak the others. Errors are collected and returned so the - // caller (destroy / abort) can decide whether to surface a - // partial-teardown condition; today the caller logs the summary - // and continues, since session teardown is also invoked from - // recovery paths where a thrown aggregate would obscure the - // original failure. - const errors: Error[] = []; - for (const disposer of session.bundle.disposers) { - try { - await disposer(); - } catch (err: unknown) { - const e = err instanceof Error ? err : new Error(String(err)); - logger.error`Disposer failed for ${agentAddress}: ${e.message}`; - errors.push(e); - } - } - return errors; - } - - async function destroySession(agentAddress: string): Promise { - if (provisioned.has(agentAddress)) { - provisioned.delete(agentAddress); - logger.info`Removed provisioned agent ${agentAddress}`; - return; - } - const session = sessions.get(agentAddress); - if (session === undefined) { - throw new Error(`No session exists for agent "${agentAddress}"`); - } - await session.harness.close(); - const disposerErrors = await runDisposers(session, agentAddress); - await drainRepoOps(agentAddress); - sessions.delete(agentAddress); - transport.unregister(agentAddress); - if (disposerErrors.length > 0) { - logger.warn`Stopped session for ${agentAddress} with ${String(disposerErrors.length)} disposer failure(s)`; - } else { - logger.info`Stopped session for ${agentAddress}`; - } - } - - async function abortSession( - agentAddress: string, - reason: string, - ): Promise { - if (provisioned.has(agentAddress)) { - provisioned.delete(agentAddress); - logger.info`Aborted provisioned agent ${agentAddress}: ${reason}`; - return; - } - const session = sessions.get(agentAddress); - if (session === undefined) { - throw new Error(`No session exists for agent "${agentAddress}"`); - } - await session.harness.close(); - const disposerErrors = await runDisposers(session, agentAddress); - await drainRepoOps(agentAddress); - sessions.delete(agentAddress); - transport.unregister(agentAddress); - if (disposerErrors.length > 0) { - logger.warn`Aborted agent ${agentAddress} (${reason}) with ${String(disposerErrors.length)} disposer failure(s)`; - } else { - logger.info`Aborted agent ${agentAddress}: ${reason}`; - } - } - - function deliverMessage(agentAddress: string, message: InboundMessage): void { - const session = sessions.get(agentAddress); - if (session === undefined) { - throw new Error(`No session exists for agent "${agentAddress}"`); - } - session.harness.deliver(message); - } - - function hasSession(agentAddress: string): boolean { - return sessions.has(agentAddress); - } - - function isProvisioned(agentAddress: string): boolean { - return provisioned.has(agentAddress); - } - - function getAddresses(): string[] { - return [...sessions.keys()]; - } - - async function restoreSessions(): Promise { - // Restore policy: an agent is restorable only when the on-disk - // state from both stores is consistent. Scan keys and configs - // independently, then join on address. A config without a key is - // surfaced as a hard failure — the agent's identity cannot be - // recovered without the key pair on disk. - // - // Key-without-config does not appear here because AgentKeyStore's - // scanKeys already requires a parseable agent.json before - // surfacing the directory, so the orphan-key case is filtered at - // the store boundary and never reaches this join. - const [keysByAddress, configEntries] = await Promise.all([ - keyStore.scanKeys().then((k) => new Map(k.map((e) => [e.address, e]))), - repoStore.scanConfigs(), - ]); - - // Restore agents with bounded parallelism. Each agent's provision + - // startSession is per-agent isolated (per-agent git dir, per-address - // repo lock), so running several at once cannot corrupt shared - // state, and one slow or wedged agent no longer gates the rest. - // Outcomes are written into per-index slots so the returned order is - // independent of completion order. - type Outcome = - | { kind: "restored"; agent: RestoredAgent } - | { kind: "failed"; address: string }; - const outcomes = new Array(configEntries.length); - - async function restoreOne(index: number): Promise { - const entry = configEntries[index]; - if (entry === undefined) return; - const keyEntry: AgentKeyEntry | undefined = keysByAddress.get( - entry.address, - ); - if (keyEntry === undefined) { - logger.error`Cannot restore "${entry.address}": agent.json exists but key pair is missing`; - outcomes[index] = { kind: "failed", address: entry.address }; - return; - } - - const agent: RestoredAgent = { - address: entry.address, - keyPair: keyEntry.keyPair, - }; - if (entry.hubPublicKey !== undefined) { - agent.hubPublicKey = entry.hubPublicKey; - } - - if (sessions.has(entry.address)) { - outcomes[index] = { kind: "restored", agent }; - return; - } - try { - await provisionAgent(entry.config); - if (entry.hubPublicKey !== undefined) { - await repoStore.persistPairing(entry.address, entry.hubPublicKey); - } - await startSession(entry.address); - outcomes[index] = { kind: "restored", agent }; - logger.info`Restored session for ${entry.address}`; - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - outcomes[index] = { kind: "failed", address: entry.address }; - logger.error`Failed to restore session for ${entry.address}: ${msg}`; - } - } - - // Worker pool: each worker pulls the next index until the list - // drains. Reading and advancing `cursor` is synchronous (no await - // between), so each index is handed to exactly one worker. - let cursor = 0; - async function worker(): Promise { - for (;;) { - const index = cursor; - cursor += 1; - if (index >= configEntries.length) return; - await restoreOne(index); - } - } - const workerCount = Math.min(restoreConcurrency, configEntries.length); - await Promise.all(Array.from({ length: workerCount }, () => worker())); - - const restored: RestoredAgent[] = []; - const failed: string[] = []; - for (const outcome of outcomes) { - if (outcome === undefined) continue; - if (outcome.kind === "restored") restored.push(outcome.agent); - else failed.push(outcome.address); - } - - return { restored, failed }; - } - - async function updateGrants( - agentAddress: string, - grants: GrantRule[], - ): Promise { - const session = sessions.get(agentAddress); - if (session === undefined) { - throw new Error(`No session exists for agent "${agentAddress}"`); - } - session.bundle.updateGrants(grants); - session.config = { ...session.config, grants }; - await repoStore.persistConfig(agentAddress, session.config); - logger.info`Updated grants for ${agentAddress} (${String(grants.length)} rules)`; - } - - async function updateSources( - agentAddress: string, - sources: InferenceSource[], - defaultSource: string, - ): Promise { - const session = sessions.get(agentAddress); - if (session === undefined) { - throw new Error(`No session exists for agent "${agentAddress}"`); - } - const source = sources.find((s) => s.id === defaultSource); - if (source === undefined) { - throw new Error( - `No source matches defaultSource "${defaultSource}" in update for agent "${agentAddress}"`, - ); - } - buildHarness.canBuildSource(source); - session.harness.setSources(sources, defaultSource); - session.config = { ...session.config, sources, defaultSource }; - await repoStore.persistConfig(agentAddress, session.config); - logger.info`Updated sources for ${agentAddress}`; - } - async function applyDeployPack( agentAddress: string, pack: Uint8Array, @@ -774,62 +183,18 @@ export function createSessionManager( await repoStore.remove(agentAddress); } - async function persistHubPublicKey( - agentAddress: string, - hubPublicKey: string, - ): Promise { - await repoStore.persistPairing(agentAddress, hubPublicKey); - } - - async function commitInboundMail( - agentAddress: string, - rawMessage: Uint8Array, - ): Promise { - const session = sessions.get(agentAddress); - if (session === undefined) { - throw new Error( - `No active session for "${agentAddress}" — cannot audit inbound mail`, - ); - } - const mailStore = session.bundle.mailStore; - enqueueMailCommit(agentAddress, async () => { - const result = await mailStore.commitMail(rawMessage, "in", { - ignoreDuplicate: true, - }); - if (result !== null) { - logger.info`Committed inbound mail ${result.messageId} for ${agentAddress}`; - } - }); - } - - function getSessionId(agentAddress: string): string | undefined { - return sessions.get(agentAddress)?.config.sessionId; - } - async function getDeployRef(agentAddress: string): Promise { return runRepoOp(agentAddress, () => repoStore.getDeployRef(agentAddress)); } return { - provisionAgent, - startSession, - destroySession, - abortSession, - deliverMessage, - updateGrants, - onAgentEvent, - updateSources, - hasSession, - isProvisioned, - getAddresses, - restoreSessions, + initRepo: (address: string) => repoStore.initRepo(address), applyDeployPack, applyAssetPack, createStatePack, deleteAgentDir, getDeployRef, - persistHubPublicKey, - commitInboundMail, - getSessionId, + getAddresses: () => [], + getSessionId: () => undefined, }; } diff --git a/packages/hub-agent/src/sidecar-orchestrator.ts b/packages/hub-agent/src/sidecar-orchestrator.ts index 435a342a..33d5b554 100644 --- a/packages/hub-agent/src/sidecar-orchestrator.ts +++ b/packages/hub-agent/src/sidecar-orchestrator.ts @@ -2,31 +2,22 @@ // of the sidecar runtime — stores, SessionManager, HubLink — and // returns a single start/close handle the host driver uses. // -// The host supplies policy (the data directory, the harness builder -// implementation, the per-agent crypto factory, the low-level crypto -// primitives, the hub credentials); the orchestrator does the -// composition. SessionManager and HubLink reference each other through -// SessionEventSink / ConnectorStateSink callbacks, but SessionManager -// is constructed first so the sinks initially point at no-op closures -// the orchestrator owns. After HubLink is constructed those closures -// are rewired to hubLink.sendEvent and hubLink.sendConnectorState, so -// the cross-reference is contained inside this module rather than +// The host supplies policy (the data directory, the low-level crypto +// primitives, the hub credentials, the deploy-router factory); the +// orchestrator does the composition. The multi-step deploy path +// forwards a spawned child's verified InferenceEvents to the hub +// through a sink the orchestrator owns: it points at a no-op closure +// until HubLink is constructed, then is rewired to hubLink.sendEvent, +// so the cross-reference is contained inside this module rather than // leaking up to the host entry point. import { getLogger } from "@intx/log"; import type { HubTransport } from "@intx/mail-memory"; -import type { DeployApplyErrorFrame } from "@intx/types/sidecar"; -import type { - ConnectorThreadState, - CryptoProvider, - InferenceEvent, - KeyPair, -} from "@intx/types/runtime"; +import type { InferenceEvent, KeyPair } from "@intx/types/runtime"; import { createAgentKeyStore, type AgentKeyStore } from "./agent-key-store"; import { createAgentRepoStore, type AgentRepoStore } from "./agent-repo-store"; import { createSessionManager, type SessionManager } from "./session-manager"; -import type { HarnessBuilder } from "./harness-builder"; import { createHubLink, type DeployRouter, @@ -34,6 +25,7 @@ import { type MailInboundRouter, type SignalInboundRouter, type DrainInboundRouter, + type SourcesInboundRouter, type ReconnectScheduler, } from "./ws/hub-link"; @@ -53,22 +45,13 @@ export type SidecarCryptoOps = { * Factory the orchestrator invokes once `sessions` and `keyStore` * are constructed. The host returns the `DeployRouter` the link * routes every inbound `agent.deploy` through; production wires - * this against a workflow-host supervisor whose `trivialLaunch` - * closes over `sessions.provisionAgent`. The host is responsible - * for closing over any other state the router needs (transport, - * substrate handle, signing keys) at the call site. - * - * `onAgentEvent` is the per-agent InferenceEvent subscription seam - * the trivial-launch closure uses to bracket the workflow-run event - * chain against the existing reactor moments - * (`message.run.started` / `inference.start` / `message.run.ended`). - * The orchestrator hands it through unchanged so the supervisor's - * `recordRunEvent` callback fires for the right address. + * this against the sidecar's workflow-run deploy router. The host is + * responsible for closing over any other state the router needs + * (transport, substrate handle, signing keys) at the call site. */ export type CreateDeployRouter = (deps: { sessions: SessionManager; keyStore: AgentKeyStore; - onAgentEvent: SessionManager["onAgentEvent"]; /** * Per-event sink the multi-step branch routes a spawned child's * verified `InferenceEvent`s through, keyed by the deployment's agent @@ -92,8 +75,6 @@ export type SidecarOrchestratorConfig = { token: string; dataDir: string; transport: HubTransport; - buildHarness: HarnessBuilder; - createAgentCrypto: (keyPair: KeyPair) => CryptoProvider; cryptoOps: SidecarCryptoOps; /** * Host-injected `DeployRouter` factory. The orchestrator calls it @@ -127,6 +108,23 @@ export type SidecarOrchestratorConfig = { * orchestrator forwards the binding unchanged to `createHubLink`. */ drainInboundRouter?: DrainInboundRouter; + /** + * Optional inbound sources-rotation dispatcher the link consults on + * every inbound `sources.update` frame. Production wires this against + * the sidecar's single-step deployment sources handler registry so a + * deployment-address rotation flows into the supervisor's + * `deliverSources`. The orchestrator forwards the binding unchanged to + * `createHubLink`. + */ + sourcesInboundRouter?: SourcesInboundRouter; + /** + * Returns the workflow-substrate deployment addresses this sidecar + * currently hosts. Forwarded to the hub link, which announces them on + * every (re)connect so the hub re-registers them for routing without a + * challenge. Production wires this to the deploy router's + * `activeAddresses`; omitted, the link announces none. + */ + getWorkflowAddresses?: () => string[]; pingIntervalMs?: number; reconnectDelayMs?: number; scheduleReconnect?: ReconnectScheduler; @@ -153,13 +151,13 @@ export function createSidecarOrchestrator( token, dataDir, transport, - buildHarness, - createAgentCrypto, cryptoOps, createDeployRouter, mailInboundRouter, signalInboundRouter, drainInboundRouter, + sourcesInboundRouter, + getWorkflowAddresses, pingIntervalMs, reconnectDelayMs, scheduleReconnect, @@ -173,10 +171,10 @@ export function createSidecarOrchestrator( verifySSHSig: cryptoOps.verifySSHSig, }); - // Pre-declare the sinks. SessionManager dispatches events into them - // synchronously; the closures point at no-ops until HubLink is - // constructed below, at which point they are swapped to the link's - // sendEvent / sendConnectorState methods. + // Sink the multi-step deploy path routes a spawned child's verified + // InferenceEvents through. It points at a no-op until HubLink is + // constructed below, at which point it is swapped to the link's + // sendEvent method. let dispatchEvent: ( agentAddress: string, sessionId: string, @@ -184,40 +182,12 @@ export function createSidecarOrchestrator( ) => void = () => { /* replaced after HubLink construction */ }; - let dispatchConnectorState: ( - agentAddress: string, - state: ConnectorThreadState | null, - ) => void = () => { - /* replaced after HubLink construction */ - }; - let dispatchDeployApplyError: ( - agentAddress: string, - payload: Omit, - ) => void = () => { - /* replaced after HubLink construction */ - }; - const sessions = createSessionManager({ - transport, - repoStore, - keyStore, - buildHarness, - createAgentCrypto, - onEvent(agentAddress, sessionId, event) { - dispatchEvent(agentAddress, sessionId, event); - }, - onConnectorStateChanged(agentAddress, state) { - dispatchConnectorState(agentAddress, state); - }, - onDeployApplyError(agentAddress, payload) { - dispatchDeployApplyError(agentAddress, payload); - }, - }); + const sessions = createSessionManager({ repoStore }); const deployRouter = createDeployRouter({ sessions, keyStore, - onAgentEvent: sessions.onAgentEvent, // Route a spawned child's verified InferenceEvents up the same // hub-link `agent.event` sink the in-process path uses, so step // agent events reach the hub timeline keyed to the deploy's @@ -249,14 +219,14 @@ export function createSidecarOrchestrator( ...(mailInboundRouter !== undefined ? { mailInboundRouter } : {}), ...(signalInboundRouter !== undefined ? { signalInboundRouter } : {}), ...(drainInboundRouter !== undefined ? { drainInboundRouter } : {}), + ...(sourcesInboundRouter !== undefined ? { sourcesInboundRouter } : {}), + ...(getWorkflowAddresses !== undefined ? { getWorkflowAddresses } : {}), ...(pingIntervalMs !== undefined ? { pingIntervalMs } : {}), ...(reconnectDelayMs !== undefined ? { reconnectDelayMs } : {}), ...(scheduleReconnect !== undefined ? { scheduleReconnect } : {}), }); dispatchEvent = hubLink.sendEvent; - dispatchConnectorState = hubLink.sendConnectorState; - dispatchDeployApplyError = hubLink.sendDeployApplyError; function start(): void { hubLink.connect(); diff --git a/packages/hub-agent/src/ws/hub-link-bootstrap-prune.test.ts b/packages/hub-agent/src/ws/hub-link-bootstrap-prune.test.ts index 4f6da257..6b6ec28c 100644 --- a/packages/hub-agent/src/ws/hub-link-bootstrap-prune.test.ts +++ b/packages/hub-agent/src/ws/hub-link-bootstrap-prune.test.ts @@ -31,15 +31,13 @@ import { signEd25519, verifySSHSignature } from "@intx/crypto"; import type { HarnessConfig, InboundMessage, - InferenceSource, KeyPair, } from "@intx/types/runtime"; -import type { GrantRule } from "@intx/types/authz"; import { hexDecode } from "@intx/types"; import { createHubLink, type DeployRouter } from "./hub-link"; import type { AgentKeyStore } from "../agent-key-store"; -import type { AgentEventListener, SessionManager } from "../session-manager"; +import type { SessionManager } from "../session-manager"; function createTestKeyStore(): AgentKeyStore & { registerKey(address: string, kp: KeyPair): void; @@ -55,12 +53,6 @@ function createTestKeyStore(): AgentKeyStore & { if (existing !== undefined) return { keyPair: existing, isNew: false }; throw new Error(`No key registered for ${address} in test store`); }, - async scanKeys() { - return [...agentKeys.entries()].map(([address, keyPair]) => ({ - address, - keyPair, - })); - }, async signChallenge(address, payload) { const kp = agentKeys.get(address); if (kp === undefined) return null; @@ -85,19 +77,11 @@ function createTestKeyStore(): AgentKeyStore & { }; } -function createTestDeployRouter( - sessions: SessionManager, - keyStore: AgentKeyStore, -): DeployRouter { +function createTestDeployRouter(keyStore: AgentKeyStore): DeployRouter { return { async deploy(frame) { - const result = await sessions.provisionAgent(frame.config); keyStore.recordHubKey(frame.agentAddress, frame.hubPublicKey); - await sessions.persistHubPublicKey( - frame.agentAddress, - frame.hubPublicKey, - ); - return { publicKey: result.publicKey }; + return { publicKey: "aa".repeat(32) }; }, }; } @@ -112,54 +96,10 @@ function createMockSessionManager(): SessionManager & { provisioned: [] as HarnessConfig[], addresses: [] as string[], - async provisionAgent(config: HarnessConfig) { - mock.provisioned.push(config); - mock.addresses.push(config.agentAddress); - return { - publicKey: "deadbeef", - keyPair: { - publicKey: new Uint8Array(32), - privateKey: new Uint8Array(32), - }, - }; - }, - async startSession(_agentAddress: string): Promise { - /* unused */ - }, - async destroySession(agentAddress: string): Promise { - mock.addresses = mock.addresses.filter((a) => a !== agentAddress); - }, - async abortSession(_agentAddress: string, _reason: string): Promise { - /* unused */ - }, - deliverMessage(_agentAddress: string, _message: InboundMessage): void { - /* unused */ - }, - async updateGrants( - _agentAddress: string, - _grants: GrantRule[], - ): Promise { - /* unused */ - }, - async updateSources( - _agentAddress: string, - _sources: InferenceSource[], - _defaultSource: string, - ): Promise { - /* unused */ - }, - hasSession(agentAddress: string): boolean { - return mock.addresses.includes(agentAddress); - }, - isProvisioned(agentAddress: string): boolean { - return mock.addresses.includes(agentAddress); - }, + initRepo: (_address: string) => Promise.resolve(), getAddresses(): string[] { return [...mock.addresses]; }, - async restoreSessions() { - return { restored: [], failed: [] }; - }, applyDeployPack: () => Promise.resolve(), applyAssetPack: () => Promise.resolve(), createStatePack: () => @@ -170,15 +110,7 @@ function createMockSessionManager(): SessionManager & { }), deleteAgentDir: () => Promise.resolve(), getDeployRef: (_agentAddress: string) => Promise.resolve(null), - persistHubPublicKey: (_agentAddress: string, _hubPublicKey: string) => - Promise.resolve(), - commitInboundMail: (_agentAddress: string, _rawMessage: Uint8Array) => - Promise.resolve(), getSessionId: (_agentAddress: string) => undefined, - onAgentEvent: - (_agentAddress: string, _listener: AgentEventListener) => () => { - /* unused */ - }, } satisfies SessionManager & { provisioned: HarnessConfig[]; addresses: string[]; @@ -312,7 +244,7 @@ describe("hub-link workflow-run pack bootstrap prune", () => { transport, sessions, keyStore, - deployRouter: createTestDeployRouter(sessions, keyStore), + deployRouter: createTestDeployRouter(keyStore), }); client.connect(); diff --git a/packages/hub-agent/src/ws/hub-link-mail-router.test.ts b/packages/hub-agent/src/ws/hub-link-mail-router.test.ts index de24b281..31d62346 100644 --- a/packages/hub-agent/src/ws/hub-link-mail-router.test.ts +++ b/packages/hub-agent/src/ws/hub-link-mail-router.test.ts @@ -21,20 +21,18 @@ import { type WsHandle, } from "@intx/hub-sessions"; import { createInMemoryTransport } from "@intx/mail-memory"; -import { base64Encode } from "@intx/types"; +import { base64Encode, hexEncode } from "@intx/types"; import type { HarnessConfig, InboundMessage, - InferenceSource, KeyPair, } from "@intx/types/runtime"; -import type { GrantRule } from "@intx/types/authz"; -import { signEd25519, verifySSHSignature } from "@intx/crypto"; +import { generateKeyPair, signEd25519, verifySSHSignature } from "@intx/crypto"; import { hexDecode } from "@intx/types"; import { createHubLink, type DeployRouter } from "./hub-link"; import type { AgentKeyStore } from "../agent-key-store"; -import type { AgentEventListener, SessionManager } from "../session-manager"; +import type { SessionManager } from "../session-manager"; function createTestKeyStore(): AgentKeyStore & { registerKey(address: string, kp: KeyPair): void; @@ -50,12 +48,6 @@ function createTestKeyStore(): AgentKeyStore & { if (existing !== undefined) return { keyPair: existing, isNew: false }; throw new Error(`No key registered for ${address} in test store`); }, - async scanKeys() { - return [...agentKeys.entries()].map(([address, keyPair]) => ({ - address, - keyPair, - })); - }, async signChallenge(address, payload) { const kp = agentKeys.get(address); if (kp === undefined) return null; @@ -80,31 +72,23 @@ function createTestKeyStore(): AgentKeyStore & { }; } -function createTestDeployRouter( - sessions: SessionManager, - keyStore: AgentKeyStore, -): DeployRouter { +function createTestDeployRouter(keyStore: AgentKeyStore): DeployRouter { return { async deploy(frame) { - const result = await sessions.provisionAgent(frame.config); keyStore.recordHubKey(frame.agentAddress, frame.hubPublicKey); - await sessions.persistHubPublicKey( - frame.agentAddress, - frame.hubPublicKey, - ); - return { publicKey: result.publicKey }; + return { publicKey: "aa".repeat(32) }; }, }; } -function withTestDeployBindings(sessions: SessionManager): { +function withTestDeployBindings(): { keyStore: AgentKeyStore & { registerKey(address: string, kp: KeyPair): void }; deployRouter: DeployRouter; } { const keyStore = createTestKeyStore(); return { keyStore, - deployRouter: createTestDeployRouter(sessions, keyStore), + deployRouter: createTestDeployRouter(keyStore), }; } @@ -143,65 +127,10 @@ function createMockSessionManager(): SessionManager & { provisionedAddresses: [] as string[], shouldThrow: null as string | null, - async provisionAgent(config: HarnessConfig) { - if (mock.shouldThrow !== null) throw new Error(mock.shouldThrow); - mock.provisioned.push(config); - mock.provisionedAddresses.push(config.agentAddress); - return { - publicKey: "deadbeef", - keyPair: { - publicKey: new Uint8Array(32), - privateKey: new Uint8Array(32), - }, - }; - }, - async startSession(agentAddress: string): Promise { - if (mock.shouldThrow !== null) throw new Error(mock.shouldThrow); - mock.started.push(agentAddress); - mock.provisionedAddresses = mock.provisionedAddresses.filter( - (a) => a !== agentAddress, - ); - mock.addresses.push(agentAddress); - }, - async destroySession(agentAddress: string): Promise { - if (mock.shouldThrow !== null) throw new Error(mock.shouldThrow); - mock.destroyed.push(agentAddress); - mock.addresses = mock.addresses.filter((a) => a !== agentAddress); - }, - async abortSession(agentAddress: string, reason: string): Promise { - if (mock.shouldThrow !== null) throw new Error(mock.shouldThrow); - mock.aborted.push({ address: agentAddress, reason }); - mock.addresses = mock.addresses.filter((a) => a !== agentAddress); - }, - deliverMessage(agentAddress: string, message: InboundMessage): void { - if (mock.shouldThrow !== null) throw new Error(mock.shouldThrow); - mock.delivered.push({ agentAddress, message }); - }, - async updateGrants( - _agentAddress: string, - _grants: GrantRule[], - ): Promise { - if (mock.shouldThrow !== null) throw new Error(mock.shouldThrow); - }, - async updateSources( - _agentAddress: string, - _sources: InferenceSource[], - _defaultSource: string, - ): Promise { - if (mock.shouldThrow !== null) throw new Error(mock.shouldThrow); - }, - hasSession(agentAddress: string): boolean { - return mock.addresses.includes(agentAddress); - }, - isProvisioned(agentAddress: string): boolean { - return mock.provisionedAddresses.includes(agentAddress); - }, + initRepo: (_address: string) => Promise.resolve(), getAddresses(): string[] { return [...mock.addresses]; }, - async restoreSessions() { - return { restored: [], failed: [] }; - }, applyDeployPack: () => Promise.resolve(), applyAssetPack: () => Promise.resolve(), createStatePack: () => @@ -212,15 +141,7 @@ function createMockSessionManager(): SessionManager & { }), deleteAgentDir: () => Promise.resolve(), getDeployRef: (_agentAddress: string) => Promise.resolve(null), - persistHubPublicKey: (_agentAddress: string, _hubPublicKey: string) => - Promise.resolve(), - commitInboundMail: (_agentAddress: string, _rawMessage: Uint8Array) => - Promise.resolve(), getSessionId: (_agentAddress: string) => undefined, - onAgentEvent: - (_agentAddress: string, _listener: AgentEventListener) => () => { - /* no-op disposer: this test does not exercise per-agent events */ - }, }; return mock; } @@ -241,12 +162,20 @@ const VALID_MESSAGE = new TextEncoder().encode( type TestEnv = { server: ReturnType; router: SidecarRouter; + // address -> hex-encoded Ed25519 public key, backing the hub's fail-closed + // `lookupPublicKey`. A workflow deployment routes only after signing the + // hub's reconnect nonce with the key registered here. + deploymentKeys: Map; }; function startTestServer(): TestEnv { + const deploymentKeys = new Map(); const router = createSidecarRouter({ requestTimeoutMs: 5000, hubPublicKey: "a".repeat(64), + lookups: { + lookupPublicKey: async (address) => deploymentKeys.get(address) ?? null, + }, }); const app = new Hono(); @@ -284,7 +213,7 @@ function startTestServer(): TestEnv { port: 0, }); - return { server, router }; + return { server, router, deploymentKeys }; } const env = startTestServer(); @@ -293,11 +222,27 @@ afterAll(async () => { await env.server.stop(true); }); +/** + * Wire a workflow deployment for the challenged reconnect path: mint a + * keypair, register it in the sidecar keyStore (so `signChallenge` answers the + * hub nonce) and in the hub's `deploymentKeys` lookup (so the hub challenges + * and verifies). The deployment address then routes once the reconnect + * challenge round-trips. + */ +async function provisionDeploymentKey( + keyStore: ReturnType, + address: string, +): Promise { + const kp = await generateKeyPair(); + keyStore.registerKey(address, kp); + env.deploymentKeys.set(address, hexEncode(kp.publicKey)); +} + describe("hub-link mail.inbound throwing router", () => { test("a throwing mailInboundRouter does not wedge subsequent frames", async () => { const transport = createInMemoryTransport(); const sessions = createMockSessionManager(); - const deploymentAddress = "dep_wedge-1@integration.interchange"; + const deploymentAddress = "ins_dep_wedge1@integration.interchange"; sessions.addresses.push(deploymentAddress); let calls = 0; @@ -313,14 +258,17 @@ describe("hub-link mail.inbound throwing router", () => { }, }; + const bindings = withTestDeployBindings(); + await provisionDeploymentKey(bindings.keyStore, deploymentAddress); const client = createHubLink({ hubURL: `ws://localhost:${env.server.port}/ws`, sidecarId: "sc-mail-wedge", token: "test-token", transport, sessions, - ...withTestDeployBindings(sessions), + ...bindings, mailInboundRouter, + getWorkflowAddresses: () => [deploymentAddress], }); client.connect(); diff --git a/packages/hub-agent/src/ws/hub-link.test.ts b/packages/hub-agent/src/ws/hub-link.test.ts index b1a2ca86..d9ab76a4 100644 --- a/packages/hub-agent/src/ws/hub-link.test.ts +++ b/packages/hub-agent/src/ws/hub-link.test.ts @@ -8,44 +8,36 @@ import { type WsHandle, } from "@intx/hub-sessions"; import { createInMemoryTransport } from "@intx/mail-memory"; -import { signEd25519, verifySSHSignature } from "@intx/crypto"; +import { generateKeyPair, signEd25519, verifySSHSignature } from "@intx/crypto"; import { base64Encode, hexEncode } from "@intx/types"; -import type { - HarnessConfig, - InboundMessage, - InferenceSource, -} from "@intx/types/runtime"; -import type { GrantRule } from "@intx/types/authz"; +import type { HarnessConfig } from "@intx/types/runtime"; import { + answerMalformedRequestFrame, createHubLink, type DeployRouter, type ReconnectScheduler, } from "./hub-link"; +import type { + AgentErrorFrame, + PackRejectFrame, + SessionErrorFrame, +} from "@intx/types/sidecar"; import type { AgentKeyStore } from "../agent-key-store"; -import type { AgentEventListener, SessionManager } from "../session-manager"; +import type { SessionManager } from "../session-manager"; /** - * Test-only deploy router that mirrors the pre-supervisor inline - * flow: provisionAgent + recordHubKey + persistHubPublicKey. The - * production wiring lifts this same closure into a workflow-host - * supervisor's `trivialLaunch`; the tests here exercise the link's - * surface against the closure directly so the SessionManager mock - * assertions are unchanged. + * Test-only deploy router modelling the current deploy path: record the + * hub pairing key so `verifyDeployCommit` can accept the deployment's + * packs, and surface a public key on the ack. Production stages the + * deploy through the workflow-run substrate; the tests here exercise the + * link's surface against the router directly. */ -function createTestDeployRouter( - sessions: SessionManager, - keyStore: AgentKeyStore, -): DeployRouter { +function createTestDeployRouter(keyStore: AgentKeyStore): DeployRouter { return { async deploy(frame) { - const result = await sessions.provisionAgent(frame.config); keyStore.recordHubKey(frame.agentAddress, frame.hubPublicKey); - await sessions.persistHubPublicKey( - frame.agentAddress, - frame.hubPublicKey, - ); - return { publicKey: result.publicKey }; + return { publicKey: "aa".repeat(32) }; }, }; } @@ -57,14 +49,14 @@ function createTestDeployRouter( * call site does not have to name a temporary binding for the * keyStore-router pairing. */ -function withTestDeployBindings(sessions: SessionManager): { +function withTestDeployBindings(): { keyStore: AgentKeyStore & { registerKey(address: string, kp: KeyPair): void }; deployRouter: DeployRouter; } { const keyStore = createTestKeyStore(); return { keyStore, - deployRouter: createTestDeployRouter(sessions, keyStore), + deployRouter: createTestDeployRouter(keyStore), }; } import type { KeyPair } from "@intx/types/runtime"; @@ -89,12 +81,6 @@ function createTestKeyStore(): AgentKeyStore & { if (existing !== undefined) return { keyPair: existing, isNew: false }; throw new Error(`No key registered for ${address} in test store`); }, - async scanKeys() { - return [...agentKeys.entries()].map(([address, keyPair]) => ({ - address, - keyPair, - })); - }, async signChallenge(address, payload) { const kp = agentKeys.get(address); if (kp === undefined) return null; @@ -136,87 +122,9 @@ async function waitFor( } } -type DeliveredMessage = { agentAddress: string; message: InboundMessage }; - -function createMockSessionManager(): SessionManager & { - provisioned: HarnessConfig[]; - started: string[]; - destroyed: string[]; - aborted: { address: string; reason: string }[]; - delivered: DeliveredMessage[]; - addresses: string[]; - provisionedAddresses: string[]; - shouldThrow: string | null; -} { - const mock = { - provisioned: [] as HarnessConfig[], - started: [] as string[], - destroyed: [] as string[], - aborted: [] as { address: string; reason: string }[], - delivered: [] as DeliveredMessage[], - addresses: [] as string[], - provisionedAddresses: [] as string[], - shouldThrow: null as string | null, - - async provisionAgent(config: HarnessConfig) { - if (mock.shouldThrow !== null) throw new Error(mock.shouldThrow); - mock.provisioned.push(config); - mock.provisionedAddresses.push(config.agentAddress); - return { - publicKey: "deadbeef", - keyPair: { - publicKey: new Uint8Array(32), - privateKey: new Uint8Array(32), - }, - }; - }, - async startSession(agentAddress: string): Promise { - if (mock.shouldThrow !== null) throw new Error(mock.shouldThrow); - mock.started.push(agentAddress); - mock.provisionedAddresses = mock.provisionedAddresses.filter( - (a) => a !== agentAddress, - ); - mock.addresses.push(agentAddress); - }, - async destroySession(agentAddress: string): Promise { - if (mock.shouldThrow !== null) throw new Error(mock.shouldThrow); - mock.destroyed.push(agentAddress); - mock.addresses = mock.addresses.filter((a) => a !== agentAddress); - }, - async abortSession(agentAddress: string, reason: string): Promise { - if (mock.shouldThrow !== null) throw new Error(mock.shouldThrow); - mock.aborted.push({ address: agentAddress, reason }); - mock.addresses = mock.addresses.filter((a) => a !== agentAddress); - }, - deliverMessage(agentAddress: string, message: InboundMessage): void { - if (mock.shouldThrow !== null) throw new Error(mock.shouldThrow); - mock.delivered.push({ agentAddress, message }); - }, - async updateGrants( - _agentAddress: string, - _grants: GrantRule[], - ): Promise { - if (mock.shouldThrow !== null) throw new Error(mock.shouldThrow); - }, - async updateSources( - _agentAddress: string, - _sources: InferenceSource[], - _defaultSource: string, - ): Promise { - if (mock.shouldThrow !== null) throw new Error(mock.shouldThrow); - }, - hasSession(agentAddress: string): boolean { - return mock.addresses.includes(agentAddress); - }, - isProvisioned(agentAddress: string): boolean { - return mock.provisionedAddresses.includes(agentAddress); - }, - getAddresses(): string[] { - return [...mock.addresses]; - }, - async restoreSessions() { - return { restored: [], failed: [] }; - }, +function createMockSessionManager(): SessionManager { + return { + initRepo: (_address: string) => Promise.resolve(), applyDeployPack: () => Promise.resolve(), applyAssetPack: () => Promise.resolve(), createStatePack: () => @@ -227,17 +135,9 @@ function createMockSessionManager(): SessionManager & { }), deleteAgentDir: () => Promise.resolve(), getDeployRef: (_agentAddress: string) => Promise.resolve(null), - persistHubPublicKey: (_agentAddress: string, _hubPublicKey: string) => - Promise.resolve(), - commitInboundMail: (_agentAddress: string, _rawMessage: Uint8Array) => - Promise.resolve(), + getAddresses: () => [], getSessionId: (_agentAddress: string) => undefined, - onAgentEvent: - (_agentAddress: string, _listener: AgentEventListener) => () => { - /* no-op disposer: the link tests do not exercise per-agent events */ - }, }; - return mock; } const TEST_CONFIG: HarnessConfig = { @@ -283,15 +183,25 @@ type TestEnv = { router: SidecarRouter; agentEvents: { addr: string; sid: string; event: unknown }[]; outboundMail: { rawMessage: string; recipients: string[] }[]; + // address -> hex-encoded Ed25519 public key, backing the hub's + // `lookupPublicKey`. A workflow deployment announced through the + // challenged reconnect frame routes only after signing the hub's nonce + // with the key registered here; tests populate it via + // `provisionDeploymentKey`. + deploymentKeys: Map; }; function startTestServer(): TestEnv { const agentEvents: TestEnv["agentEvents"] = []; const outboundMail: TestEnv["outboundMail"] = []; + const deploymentKeys = new Map(); const router = createSidecarRouter({ requestTimeoutMs: 5000, hubPublicKey: "a".repeat(64), + lookups: { + lookupPublicKey: async (address) => deploymentKeys.get(address) ?? null, + }, }); router.events.on("agent.event", ({ agentAddress, sessionId, event }) => { agentEvents.push({ addr: agentAddress, sid: sessionId, event }); @@ -338,7 +248,7 @@ function startTestServer(): TestEnv { port: 0, }); - return { server, router, agentEvents, outboundMail }; + return { server, router, agentEvents, outboundMail, deploymentKeys }; } // --------------------------------------------------------------------------- @@ -351,6 +261,23 @@ afterAll(async () => { await env.server.stop(true); }); +/** + * Wire a workflow deployment for the challenged reconnect path: mint an + * Ed25519 keypair, register it in the sidecar's keyStore (so `signChallenge` + * can answer the hub's nonce) and in the hub's `deploymentKeys` lookup (so the + * hub issues a challenge and verifies the signature). After this, the + * deployment address named in `getWorkflowAddresses` routes once the + * reconnect challenge round-trips -- the same proof a launched agent makes. + */ +async function provisionDeploymentKey( + keyStore: ReturnType, + address: string, +): Promise { + const kp = await generateKeyPair(); + keyStore.registerKey(address, kp); + env.deploymentKeys.set(address, hexEncode(kp.publicKey)); +} + describe("sidecar↔hub integration", () => { test("sidecar registers with hub on connect", async () => { const transport = createInMemoryTransport(); @@ -362,7 +289,7 @@ describe("sidecar↔hub integration", () => { transport, sessions, - ...withTestDeployBindings(sessions), + ...withTestDeployBindings(), }); client.connect(); @@ -379,7 +306,7 @@ describe("sidecar↔hub integration", () => { } }); - test("hub sends session.create and sidecar acks", async () => { + test("hub sends a deploy and the sidecar acks", async () => { const transport = createInMemoryTransport(); const sessions = createMockSessionManager(); const client = createHubLink({ @@ -389,7 +316,7 @@ describe("sidecar↔hub integration", () => { transport, sessions, - ...withTestDeployBindings(sessions), + ...withTestDeployBindings(), }); client.connect(); @@ -400,10 +327,9 @@ describe("sidecar↔hub integration", () => { await env.router.sendAgentDeploy("agent-1@test.interchange", TEST_CONFIG); - expect(sessions.provisioned).toHaveLength(1); - expect(sessions.provisioned[0]?.agentAddress).toBe( - "agent-1@test.interchange", - ); + // The deploy resolves against the ack; the sidecar stays connected + // after handling it. + expect(env.router.getConnectedSidecars()).toContain("sc-create"); } finally { client.close(); await waitFor( @@ -412,114 +338,6 @@ describe("sidecar↔hub integration", () => { } }); - test("hub receives session.error when create fails", async () => { - const transport = createInMemoryTransport(); - const sessions = createMockSessionManager(); - sessions.shouldThrow = "provider not configured"; - const client = createHubLink({ - hubURL: `ws://localhost:${env.server.port}/ws`, - sidecarId: "sc-fail", - token: "test-token", - - transport, - sessions, - ...withTestDeployBindings(sessions), - }); - - client.connect(); - try { - await waitFor(() => - env.router.getConnectedSidecars().includes("sc-fail"), - ); - - await expect( - env.router.sendAgentDeploy("fail-agent@test", TEST_CONFIG), - ).rejects.toThrow("provider not configured"); - } finally { - client.close(); - await waitFor( - () => !env.router.getConnectedSidecars().includes("sc-fail"), - ); - } - }); - - test("session.start starts the harness", async () => { - const transport = createInMemoryTransport(); - const sessions = createMockSessionManager(); - const client = createHubLink({ - hubURL: `ws://localhost:${env.server.port}/ws`, - sidecarId: "sc-start", - token: "test-token", - transport, - sessions, - ...withTestDeployBindings(sessions), - }); - - client.connect(); - try { - await waitFor(() => - env.router.getConnectedSidecars().includes("sc-start"), - ); - - await env.router.sendAgentDeploy( - "start-agent@test.interchange", - TEST_CONFIG, - ); - expect(sessions.started).toHaveLength(0); - - await env.router.sendSessionStart("start-agent@test.interchange"); - expect(sessions.started).toHaveLength(1); - expect(sessions.started[0]).toBe("start-agent@test.interchange"); - } finally { - client.close(); - await waitFor( - () => !env.router.getConnectedSidecars().includes("sc-start"), - ); - } - }); - - test("session.start failure removes agent from routing table", async () => { - const transport = createInMemoryTransport(); - const sessions = createMockSessionManager(); - const client = createHubLink({ - hubURL: `ws://localhost:${env.server.port}/ws`, - sidecarId: "sc-start-fail", - token: "test-token", - transport, - sessions, - ...withTestDeployBindings(sessions), - }); - - client.connect(); - try { - await waitFor(() => - env.router.getConnectedSidecars().includes("sc-start-fail"), - ); - - await env.router.sendAgentDeploy( - "start-fail@test.interchange", - TEST_CONFIG, - ); - - // Make startSession throw on the next call. - sessions.shouldThrow = "deploy tree missing"; - - await expect( - env.router.sendSessionStart("start-fail@test.interchange"), - ).rejects.toThrow("deploy tree missing"); - - // Failed session start should remove the agent from routing. - expect(env.router.getRoutableAddresses()).not.toContain( - "start-fail@test.interchange", - ); - } finally { - client.close(); - await waitFor( - () => !env.router.getConnectedSidecars().includes("sc-start-fail"), - ); - } - }); - test("sidecar forwards agent events to hub", async () => { const transport = createInMemoryTransport(); const sessions = createMockSessionManager(); @@ -531,7 +349,7 @@ describe("sidecar↔hub integration", () => { transport, sessions, - ...withTestDeployBindings(sessions), + ...withTestDeployBindings(), }); client.connect(); @@ -563,59 +381,6 @@ describe("sidecar↔hub integration", () => { } }); - test("hub routes mail inbound to sidecar", async () => { - const transport = createInMemoryTransport(); - const sessions = createMockSessionManager(); - // The agent must be in the session manager's address list so the - // register frame includes it in the routing table. - sessions.addresses.push("agent-1@test.interchange"); - const { generateKeyPair, createEd25519Crypto } = await import( - "@intx/crypto" - ); - const kp = await generateKeyPair(); - transport.register("agent-1@test.interchange", createEd25519Crypto(kp)); - - const client = createHubLink({ - hubURL: `ws://localhost:${env.server.port}/ws`, - sidecarId: "sc-mail-in", - token: "test-token", - - transport, - sessions, - ...withTestDeployBindings(sessions), - }); - - client.connect(); - try { - await waitFor(() => - env.router.getRoutableAddresses().includes("agent-1@test.interchange"), - ); - - const encoded = base64Encode(VALID_MESSAGE); - const routed = env.router.routeMail("agent-1@test.interchange", encoded); - expect(routed).toBe(true); - - // Wait for the message to be delivered to the agent's INBOX. - const agentTransport = transport.getTransportFor( - "agent-1@test.interchange", - ); - await waitFor(async () => { - const refs = await agentTransport.search("INBOX", {}); - return refs.length > 0; - }); - - const refs = await agentTransport.search("INBOX", {}); - expect(refs).toHaveLength(1); - const headers = await agentTransport.fetchHeaders(refs[0]!); - expect(headers.from).toBe("external@remote.interchange"); - } finally { - client.close(); - await waitFor( - () => !env.router.getConnectedSidecars().includes("sc-mail-in"), - ); - } - }); - test("sidecar forwards outbound mail to hub", async () => { const transport = createInMemoryTransport(); const sessions = createMockSessionManager(); @@ -633,7 +398,7 @@ describe("sidecar↔hub integration", () => { transport, sessions, - ...withTestDeployBindings(sessions), + ...withTestDeployBindings(), }); client.connect(); @@ -662,115 +427,97 @@ describe("sidecar↔hub integration", () => { } }); - test("mail routes between two sidecars via hub", async () => { - const { generateKeyPair, createEd25519Crypto } = await import( - "@intx/crypto" - ); - - // Sidecar A - const transportA = createInMemoryTransport(); - const sessionsA = createMockSessionManager(); - sessionsA.addresses.push("alice@test.interchange"); - const kpA = await generateKeyPair(); - transportA.register("alice@test.interchange", createEd25519Crypto(kpA)); - - // Sidecar B - const transportB = createInMemoryTransport(); - const sessionsB = createMockSessionManager(); - sessionsB.addresses.push("bob@test.interchange"); - const kpB = await generateKeyPair(); - transportB.register("bob@test.interchange", createEd25519Crypto(kpB)); - - const clientA = createHubLink({ - hubURL: `ws://localhost:${env.server.port}/ws`, - sidecarId: "sc-alice", - token: "test-token", - transport: transportA, - sessions: sessionsA, - ...withTestDeployBindings(sessionsA), - }); - const clientB = createHubLink({ - hubURL: `ws://localhost:${env.server.port}/ws`, - sidecarId: "sc-bob", - token: "test-token", - transport: transportB, - sessions: sessionsB, - ...withTestDeployBindings(sessionsB), - }); - - clientA.connect(); - clientB.connect(); - try { - await waitFor(() => - env.router.getRoutableAddresses().includes("alice@test.interchange"), - ); - await waitFor(() => - env.router.getRoutableAddresses().includes("bob@test.interchange"), - ); - - // Alice sends a message to Bob. - const aliceTransport = transportA.getTransportFor( - "alice@test.interchange", - ); - await aliceTransport.send({ - to: "bob@test.interchange", - type: "conversation.message", - content: "Hello Bob", - }); - - // Bob should receive it in his INBOX. - const bobTransport = transportB.getTransportFor("bob@test.interchange"); - await waitFor(async () => { - const refs = await bobTransport.search("INBOX", {}); - return refs.length > 0; - }); - - const refs = await bobTransport.search("INBOX", {}); - expect(refs).toHaveLength(1); - const headers = await bobTransport.fetchHeaders(refs[0]!); - expect(headers.from).toBe("alice@test.interchange"); - } finally { - clientA.close(); - clientB.close(); - await waitFor( - () => !env.router.getConnectedSidecars().includes("sc-alice"), - ); - await waitFor( - () => !env.router.getConnectedSidecars().includes("sc-bob"), - ); - } - }); - test("disconnect cleans up routing table", async () => { const transport = createInMemoryTransport(); const sessions = createMockSessionManager(); - sessions.addresses.push("tracked@test.interchange"); + const deploymentAddress = "ins_dep_disc1@integration.interchange"; + const bindings = withTestDeployBindings(); + await provisionDeploymentKey(bindings.keyStore, deploymentAddress); const client = createHubLink({ hubURL: `ws://localhost:${env.server.port}/ws`, sidecarId: "sc-disconnect", token: "test-token", - transport, sessions, - ...withTestDeployBindings(sessions), + ...bindings, + getWorkflowAddresses: () => [deploymentAddress], }); client.connect(); + // Routability lags the connection: it lands only after the reconnect + // challenge round-trips, so wait on the routable address directly. await waitFor(() => - env.router.getConnectedSidecars().includes("sc-disconnect"), - ); - expect(env.router.getRoutableAddresses()).toContain( - "tracked@test.interchange", + env.router.getRoutableAddresses().includes(deploymentAddress), ); client.close(); await waitFor( () => !env.router.getConnectedSidecars().includes("sc-disconnect"), ); - expect(env.router.getRoutableAddresses()).not.toContain( - "tracked@test.interchange", - ); + expect(env.router.getRoutableAddresses()).not.toContain(deploymentAddress); + }); + + test("a failing deploy-ref read closes the socket instead of silently dropping the reconnect", async () => { + const failEnv = startTestServer(); + + // Capture the reconnect callback so no real timer fires during the + // test. Its presence is also the proof we assert on: the re-announce + // catch closes the socket, and the close handler schedules a reconnect + // through this scheduler. Without the fix the deploy-ref rejection is + // an unhandled promise, no close occurs, and this stays null. + let pendingReconnect: (() => void) | null = null; + const fakeScheduleReconnect: ReconnectScheduler = (cb) => { + pendingReconnect = cb; + return () => { + pendingReconnect = null; + }; + }; + + const transport = createInMemoryTransport(); + const deploymentAddress = "ins_dep_reffail1@integration.interchange"; + // The deploy-ref read rejects the way a corrupt or unreadable ref + // would. The re-announce IIFE must catch it and close the socket + // rather than let the reconnect vanish with nothing logged. + const sessions: SessionManager = { + ...createMockSessionManager(), + getDeployRef: () => + Promise.reject(new Error("simulated corrupt deploy ref")), + }; + + const client = createHubLink({ + hubURL: `ws://localhost:${failEnv.server.port}/ws`, + sidecarId: "sc-ref-fail", + token: "test-token", + transport, + sessions, + ...withTestDeployBindings(), + getWorkflowAddresses: () => [deploymentAddress], + scheduleReconnect: fakeScheduleReconnect, + }); + + try { + client.connect(); + // On open the register frame is sent, then the re-announce IIFE + // reads the deploy ref and the read rejects. The catch closes the + // socket, and the close handler schedules a reconnect through the + // fake scheduler. Observing that pending callback is the proof the + // socket was closed rather than the failure swallowed. Without the + // fix the rejection is an unhandled promise, no close occurs, and + // this stays null until the wait times out. (The connect→reject→ + // close cycle is faster than a poll interval, so the transient + // connected state is deliberately not asserted -- the scheduled + // reconnect is the durable signal.) + await waitFor(() => pendingReconnect !== null); + + // No reconnect frame was ever sent, so the address never routed. + expect(failEnv.router.getRoutableAddresses()).not.toContain( + deploymentAddress, + ); + } finally { + client.close(); + await failEnv.server.stop(true); + } }); test("repo.pack.reject sent when applyDeployPack throws signature_invalid", async () => { @@ -782,7 +529,7 @@ describe("sidecar↔hub integration", () => { token: "test-token", transport, sessions, - ...withTestDeployBindings(sessions), + ...withTestDeployBindings(), }); client.connect(); @@ -825,7 +572,7 @@ describe("sidecar↔hub integration", () => { token: "test-token", transport, sessions, - ...withTestDeployBindings(sessions), + ...withTestDeployBindings(), }); client.connect(); @@ -910,7 +657,7 @@ describe("sidecar↔hub integration", () => { token: "test-token", transport, sessions, - ...withTestDeployBindings(sessions), + ...withTestDeployBindings(), }); client.connect(); @@ -929,34 +676,25 @@ describe("sidecar↔hub integration", () => { } }); - test("reconnect restores hubPublicKey into hubKeys map", async () => { + test("a deploy records the hub key so verifyCommit is bound to it", async () => { const { generateKeyPair, createSSHSignature } = await import( "@intx/crypto" ); - // Agent keypair — used for challenge/response signing. - const agentKp = await generateKeyPair(); // Hub keypair — the key whose public half the sidecar stores to verify - // deploy commit signatures. Distinct from the agent keypair. + // deploy-commit signatures. const hubKp = await generateKeyPair(); - const fakeAddress = "restored@test.interchange"; - - const agentPublicKeyHex = hexEncode(agentKp.publicKey); + const deployedAddress = "deployed@test.interchange"; const hubPublicKeyHex = hexEncode(hubKp.publicKey); - // Stand up a hub that supports reconnect (lookupPublicKey configured). - const reconnectRouter = createSidecarRouter({ + const deployHubRouter = createSidecarRouter({ requestTimeoutMs: 5000, challengeTimeoutMs: 5000, hubPublicKey: hubPublicKeyHex, - lookups: { - lookupPublicKey: async (addr) => - addr === fakeAddress ? agentPublicKeyHex : null, - }, }); - const reconnectApp = new Hono(); - reconnectApp.get( + const deployApp = new Hono(); + deployApp.get( "/ws", upgradeWebSocket((_c) => { let handle: WsHandle; @@ -970,69 +708,54 @@ describe("sidecar↔hub integration", () => { ws.close(); }, }; - reconnectRouter.handleOpen(handle); + deployHubRouter.handleOpen(handle); }, onMessage(evt, _ws) { if (typeof evt.data === "string") { - reconnectRouter.handleMessage(handle, evt.data); + deployHubRouter.handleMessage(handle, evt.data); } }, onClose(_evt, _ws) { - reconnectRouter.handleClose(handle); + deployHubRouter.handleClose(handle); }, }; }), ); - const reconnectServer = Bun.serve({ - fetch: reconnectApp.fetch, + const deployServer = Bun.serve({ + fetch: deployApp.fetch, websocket, port: 0, }); const transport = createInMemoryTransport(); const sessions = createMockSessionManager(); - - sessions.addresses.push(fakeAddress); - sessions.restoreSessions = async () => ({ - restored: [ - { - address: fakeAddress, - keyPair: agentKp, - config: { ...TEST_CONFIG, agentAddress: fakeAddress }, - hubPublicKey: hubPublicKeyHex, - }, - ], - failed: [], - }); - sessions.getDeployRef = async () => "a".repeat(40); - - // In production, SessionManager.restoreSessions populates - // AgentKeyStore's in-memory cache via scanKeys before HubLink - // iterates the restored agents. The mock SessionManager here does - // not exercise that path, so we seed the keyStore directly. const keyStore = createTestKeyStore(); - keyStore.registerKey(fakeAddress, agentKp); const client = createHubLink({ - hubURL: `ws://localhost:${reconnectServer.port}/ws`, - sidecarId: "sc-restore-key", + hubURL: `ws://localhost:${deployServer.port}/ws`, + sidecarId: "sc-deploy-key", token: "test-token", transport, sessions, keyStore, - deployRouter: createTestDeployRouter(sessions, keyStore), + deployRouter: createTestDeployRouter(keyStore), }); client.connect(); try { - await waitFor( - () => reconnectRouter.getRoutableAddresses().includes(fakeAddress), - 5000, + await waitFor(() => + deployHubRouter.getConnectedSidecars().includes("sc-deploy-key"), ); - // Capture the verifyCommit callback to prove the correct hub key - // was restored — not just any key. + // The hub's deploy frame carries hubPublicKeyHex; the deploy router + // records it via keyStore.recordHubKey, so a later pack's verifyCommit + // is bound to the hub key. + await deployHubRouter.sendAgentDeploy(deployedAddress, { + ...TEST_CONFIG, + agentAddress: deployedAddress, + }); + let capturedVerifyCommit: | ((p: string, s: string) => Promise) | undefined; @@ -1047,8 +770,8 @@ describe("sidecar↔hub integration", () => { capturedVerifyCommit = verifyCommit; }; - await reconnectRouter.sendPack( - fakeAddress, + await deployHubRouter.sendPack( + deployedAddress, new Uint8Array([1, 2, 3]), "refs/heads/deploy", "b".repeat(40), @@ -1056,8 +779,8 @@ describe("sidecar↔hub integration", () => { expect(capturedVerifyCommit).toBeFunction(); - // Create a real signature with the hub's private key and verify - // it round-trips through the restored verifyCommit callback. + // A signature from the hub's key round-trips through the recorded + // verifyCommit callback. const payload = "tree abc\nauthor t 0 +0000\n\ntest\n"; const sig = await createSSHSignature( payload, @@ -1066,8 +789,8 @@ describe("sidecar↔hub integration", () => { ); expect(await capturedVerifyCommit!(payload, sig)).toBe(true); - // A signature from a different key must fail, proving the callback - // is bound to the specific hub key that was restored. + // A signature from a different key fails, proving the callback is bound + // to the specific hub key the deploy recorded. const wrongKp = await generateKeyPair(); const wrongSig = await createSSHSignature( payload, @@ -1077,7 +800,7 @@ describe("sidecar↔hub integration", () => { expect(await capturedVerifyCommit!(payload, wrongSig)).toBe(false); } finally { client.close(); - await reconnectServer.stop(true); + await deployServer.stop(true); } }); @@ -1094,7 +817,7 @@ describe("sidecar↔hub integration", () => { transport, sessions, - ...withTestDeployBindings(sessions), + ...withTestDeployBindings(), pingIntervalMs: 100, }); @@ -1135,7 +858,7 @@ describe("sidecar↔hub integration", () => { token: "test-token", transport, sessions, - ...withTestDeployBindings(sessions), + ...withTestDeployBindings(), }); client.connect(); @@ -1195,7 +918,7 @@ describe("sidecar↔hub integration", () => { token: "test-token", transport, sessions, - ...withTestDeployBindings(sessions), + ...withTestDeployBindings(), }); client.connect(); @@ -1249,7 +972,7 @@ describe("sidecar↔hub integration", () => { transport, sessions, - ...withTestDeployBindings(sessions), + ...withTestDeployBindings(), scheduleReconnect: fakeScheduleReconnect, }); @@ -1287,8 +1010,7 @@ describe("sidecar↔hub integration", () => { // mail.inbound for it goes through the link's switch case, which // must consult mailInboundRouter first and -- on a `true` return // -- skip transport.deliver and sessions.commitInboundMail. - const deploymentAddress = "dep_multistep-1@integration.interchange"; - sessions.addresses.push(deploymentAddress); + const deploymentAddress = "ins_dep_mail1@integration.interchange"; const routed: { address: string; bytes: Uint8Array }[] = []; const mailInboundRouter = { @@ -1298,14 +1020,17 @@ describe("sidecar↔hub integration", () => { }, }; + const bindings = withTestDeployBindings(); + await provisionDeploymentKey(bindings.keyStore, deploymentAddress); const client = createHubLink({ hubURL: `ws://localhost:${env.server.port}/ws`, sidecarId: "sc-multistep-mail", token: "test-token", transport, sessions, - ...withTestDeployBindings(sessions), + ...bindings, mailInboundRouter, + getWorkflowAddresses: () => [deploymentAddress], }); client.connect(); @@ -1323,21 +1048,6 @@ describe("sidecar↔hub integration", () => { expect(routed).toHaveLength(1); expect(routed[0]?.address).toBe(deploymentAddress); expect(routed[0]?.bytes).toEqual(VALID_MESSAGE); - - // Legacy fallback paths must not have been reached. The mock - // sessions tracks delivered messages and `commitInboundMail` - // returns a resolved promise without recording, so the sentinel - // we check is the transport-side delivery. The transport's - // `deliver` would throw because no address is registered, and - // the link's `deliverLocalMail` catches and logs that throw. - // The cleaner check: sessions.delivered stays empty. (Even if - // the mock's commitInboundMail did nothing, `deliverLocalMail` - // would attempt transport.deliver -- but the test transport has - // no registered mailbox for `deploymentAddress`, so that would - // surface as a logged warn rather than a delivery. Asserting - // sessions.delivered.length === 0 makes the no-fallback - // intent explicit.) - expect(sessions.delivered).toHaveLength(0); } finally { client.close(); await waitFor( @@ -1346,85 +1056,94 @@ describe("sidecar↔hub integration", () => { } }); - test("mailInboundRouter that returns false falls through to the legacy path", async () => { + test("drainInboundRouter dispatches an inbound drain.deliver frame", async () => { const transport = createInMemoryTransport(); const sessions = createMockSessionManager(); - const address = "agent-fallback@test.interchange"; - sessions.addresses.push(address); - const { generateKeyPair, createEd25519Crypto } = await import( - "@intx/crypto" - ); - const kp = await generateKeyPair(); - transport.register(address, createEd25519Crypto(kp)); + const deploymentAddress = "ins_dep_drain1@integration.interchange"; - const consulted: string[] = []; - const mailInboundRouter = { - tryRoute(addr: string, _message: Uint8Array): boolean { - consulted.push(addr); - return false; + const routed: { agentAddress: string; deadlineMs: number }[] = []; + const drainInboundRouter = { + async tryRoute(frame: { + agentAddress: string; + deadlineMs: number; + }): Promise { + routed.push({ + agentAddress: frame.agentAddress, + deadlineMs: frame.deadlineMs, + }); + return true; }, }; + const bindings = withTestDeployBindings(); + await provisionDeploymentKey(bindings.keyStore, deploymentAddress); const client = createHubLink({ hubURL: `ws://localhost:${env.server.port}/ws`, - sidecarId: "sc-fallback-mail", + sidecarId: "sc-drain-router", token: "test-token", transport, sessions, - ...withTestDeployBindings(sessions), - mailInboundRouter, + ...bindings, + drainInboundRouter, + getWorkflowAddresses: () => [deploymentAddress], }); client.connect(); try { - await waitFor(() => env.router.getRoutableAddresses().includes(address)); - - const encoded = base64Encode(VALID_MESSAGE); - expect(env.router.routeMail(address, encoded)).toBe(true); + await waitFor(() => + env.router.getRoutableAddresses().includes(deploymentAddress), + ); - const agentTransport = transport.getTransportFor(address); - await waitFor(async () => { - const refs = await agentTransport.search("INBOX", {}); - return refs.length > 0; + env.router.sendDrain({ + agentAddress: deploymentAddress, + deadlineMs: 4_321, }); - expect(consulted).toContain(address); + await waitFor(() => routed.length > 0); + + expect(routed).toHaveLength(1); + expect(routed[0]?.agentAddress).toBe(deploymentAddress); + expect(routed[0]?.deadlineMs).toBe(4_321); } finally { client.close(); await waitFor( - () => !env.router.getConnectedSidecars().includes("sc-fallback-mail"), + () => !env.router.getConnectedSidecars().includes("sc-drain-router"), ); } }); - test("drainInboundRouter dispatches an inbound drain.deliver frame", async () => { + const ROTATION_SOURCE = { + id: "primary", + provider: "anthropic", + baseURL: "https://api.anthropic.com", + apiKey: "sk-rotation", + model: "claude-rotation", + }; + + test("sourcesInboundRouter acks an inbound sources.update round-trip", async () => { const transport = createInMemoryTransport(); const sessions = createMockSessionManager(); - const deploymentAddress = "dep_drain-1@integration.interchange"; - sessions.addresses.push(deploymentAddress); + const deploymentAddress = "ins_dep_srcack@integration.interchange"; - const routed: { agentAddress: string; deadlineMs: number }[] = []; - const drainInboundRouter = { - async tryRoute(frame: { - agentAddress: string; - deadlineMs: number; - }): Promise { - routed.push({ - agentAddress: frame.agentAddress, - deadlineMs: frame.deadlineMs, - }); + const routed: { agentAddress: string }[] = []; + const sourcesInboundRouter = { + async tryRoute(frame: { agentAddress: string }): Promise { + routed.push({ agentAddress: frame.agentAddress }); return true; }, }; + const bindings = withTestDeployBindings(); + await provisionDeploymentKey(bindings.keyStore, deploymentAddress); const client = createHubLink({ hubURL: `ws://localhost:${env.server.port}/ws`, - sidecarId: "sc-drain-router", + sidecarId: "sc-sources-ack", token: "test-token", transport, sessions, - ...withTestDeployBindings(sessions), - drainInboundRouter, + ...bindings, + sourcesInboundRouter, + getWorkflowAddresses: () => [deploymentAddress], }); client.connect(); @@ -1432,21 +1151,149 @@ describe("sidecar↔hub integration", () => { await waitFor(() => env.router.getRoutableAddresses().includes(deploymentAddress), ); + // Resolves on the sidecar's session.ack. Without a reply this would + // await the full request timeout, so a prompt resolution is the proof + // the round-trip no longer hangs. + await env.router.sendSourcesUpdate( + deploymentAddress, + [ROTATION_SOURCE], + "primary", + ); + expect(routed).toHaveLength(1); + expect(routed[0]?.agentAddress).toBe(deploymentAddress); + } finally { + client.close(); + await waitFor( + () => !env.router.getConnectedSidecars().includes("sc-sources-ack"), + ); + } + }); - env.router.sendDrain({ - agentAddress: deploymentAddress, - deadlineMs: 4_321, - }); + test("an unrouted sources.update is answered with session.error", async () => { + const transport = createInMemoryTransport(); + const sessions = createMockSessionManager(); + const deploymentAddress = "ins_dep_srcunrouted@integration.interchange"; - await waitFor(() => routed.length > 0); + const sourcesInboundRouter = { + async tryRoute(): Promise { + return false; + }, + }; - expect(routed).toHaveLength(1); - expect(routed[0]?.agentAddress).toBe(deploymentAddress); - expect(routed[0]?.deadlineMs).toBe(4_321); + const bindings = withTestDeployBindings(); + await provisionDeploymentKey(bindings.keyStore, deploymentAddress); + const client = createHubLink({ + hubURL: `ws://localhost:${env.server.port}/ws`, + sidecarId: "sc-sources-unrouted", + token: "test-token", + transport, + sessions, + ...bindings, + sourcesInboundRouter, + getWorkflowAddresses: () => [deploymentAddress], + }); + + client.connect(); + try { + await waitFor(() => + env.router.getRoutableAddresses().includes(deploymentAddress), + ); + await expect( + env.router.sendSourcesUpdate( + deploymentAddress, + [ROTATION_SOURCE], + "primary", + ), + ).rejects.toThrow(/no deployment registered/); } finally { client.close(); await waitFor( - () => !env.router.getConnectedSidecars().includes("sc-drain-router"), + () => + !env.router.getConnectedSidecars().includes("sc-sources-unrouted"), + ); + } + }); + + test("a rejected sources.update surfaces the reason as session.error", async () => { + const transport = createInMemoryTransport(); + const sessions = createMockSessionManager(); + const deploymentAddress = "ins_dep_srcreject@integration.interchange"; + + const sourcesInboundRouter = { + async tryRoute(): Promise { + throw new Error("supervisor is recycling"); + }, + }; + + const bindings = withTestDeployBindings(); + await provisionDeploymentKey(bindings.keyStore, deploymentAddress); + const client = createHubLink({ + hubURL: `ws://localhost:${env.server.port}/ws`, + sidecarId: "sc-sources-reject", + token: "test-token", + transport, + sessions, + ...bindings, + sourcesInboundRouter, + getWorkflowAddresses: () => [deploymentAddress], + }); + + client.connect(); + try { + await waitFor(() => + env.router.getRoutableAddresses().includes(deploymentAddress), + ); + await expect( + env.router.sendSourcesUpdate( + deploymentAddress, + [ROTATION_SOURCE], + "primary", + ), + ).rejects.toThrow(/supervisor is recycling/); + } finally { + client.close(); + await waitFor( + () => !env.router.getConnectedSidecars().includes("sc-sources-reject"), + ); + } + }); + + test("a sources.update with no router wired is answered with session.error", async () => { + const transport = createInMemoryTransport(); + const sessions = createMockSessionManager(); + const deploymentAddress = "ins_dep_srcnorouter@integration.interchange"; + + const bindings = withTestDeployBindings(); + await provisionDeploymentKey(bindings.keyStore, deploymentAddress); + const client = createHubLink({ + hubURL: `ws://localhost:${env.server.port}/ws`, + sidecarId: "sc-sources-norouter", + token: "test-token", + transport, + sessions, + ...bindings, + // No sourcesInboundRouter: a request/ack frame must still be answered + // or the hub hangs on its request timeout. + getWorkflowAddresses: () => [deploymentAddress], + }); + + client.connect(); + try { + await waitFor(() => + env.router.getRoutableAddresses().includes(deploymentAddress), + ); + await expect( + env.router.sendSourcesUpdate( + deploymentAddress, + [ROTATION_SOURCE], + "primary", + ), + ).rejects.toThrow(/no sourcesInboundRouter/); + } finally { + client.close(); + await waitFor( + () => + !env.router.getConnectedSidecars().includes("sc-sources-norouter"), ); } }); @@ -1552,7 +1399,7 @@ describe("sidecar↔hub integration", () => { token: "test-token", transport, sessions, - ...withTestDeployBindings(sessions), + ...withTestDeployBindings(), }); client.connect(); @@ -1624,8 +1471,8 @@ describe("sidecar↔hub integration", () => { }); }); -describe("routability decoupled from restore", () => { - test("empty register ships before restore resolves; reconnect-with-addresses only after", async () => { +describe("register + reconnect frames on connect", () => { + test("ships an empty register then a challenged reconnect carrying the workflow addresses", async () => { const frames: string[] = []; const app = new Hono(); app.get( @@ -1642,79 +1489,236 @@ describe("routability decoupled from restore", () => { const transport = createInMemoryTransport(); const sessions = createMockSessionManager(); - - type RestoreResult = Awaited>; - let releaseRestore!: (result: RestoreResult) => void; - const pendingRestore = new Promise((resolve) => { - releaseRestore = resolve; - }); - sessions.restoreSessions = () => pendingRestore; - sessions.getDeployRef = async () => "a".repeat(40); + const workflowAddresses = ["ins_dep_reg1@integration.interchange"]; const client = createHubLink({ hubURL: `ws://localhost:${server.port}/ws`, - sidecarId: "sc-early-register", + sidecarId: "sc-register", token: "test-token", transport, sessions, - ...withTestDeployBindings(sessions), + ...withTestDeployBindings(), + getWorkflowAddresses: () => workflowAddresses, }); client.connect(); try { - // The empty-address register reaches the hub while restoreSessions is - // still pending: routability no longer waits on restore. - await waitFor(() => - frames - .map((s) => JSON.parse(s)) - .some((f: { type: string }) => f.type === "register"), - ); - const beforeResolve = frames.map((s) => JSON.parse(s)); - const registerFrames = beforeResolve.filter( - (f: { type: string }) => f.type === "register", - ); - expect(registerFrames).toHaveLength(1); - expect(registerFrames[0].agentAddresses).toEqual([]); - // No reconnect yet: addresses enter addressIndex only after restore. - expect( - beforeResolve.some((f: { type: string }) => f.type === "reconnect"), - ).toBe(false); - - releaseRestore({ - restored: [ - { - address: "agent-1@test.interchange", - keyPair: { - publicKey: new Uint8Array(32), - privateKey: new Uint8Array(32), - }, - }, - ], - failed: [], + await waitFor(() => { + const types = frames.map((s) => JSON.parse(s).type); + return types.includes("register") && types.includes("reconnect"); }); - - // The challenged reconnect frame, carrying the restored address, ships - // only after restore resolves. - await waitFor(() => - frames - .map((s) => JSON.parse(s)) - .some((f: { type: string }) => f.type === "reconnect"), - ); const parsed = frames.map((s) => JSON.parse(s)); - const reconnectFrame = parsed.find( - (f: { type: string }) => f.type === "reconnect", + const registerFrames = parsed.filter( + (f: { type: string }) => f.type === "register", ); - expect(reconnectFrame.agentAddresses).toEqual([ - "agent-1@test.interchange", - ]); - expect( - parsed.findIndex((f: { type: string }) => f.type === "register"), - ).toBeLessThan( - parsed.findIndex((f: { type: string }) => f.type === "reconnect"), + const reconnectFrames = parsed.filter( + (f: { type: string }) => f.type === "reconnect", ); + // First-connect register carries no addresses and no workflow field: + // a workflow deployment is announced only through the challenged + // reconnect, never as a keyless register route. + expect(registerFrames).toHaveLength(1); + expect(registerFrames[0].agentAddresses).toEqual([]); + expect(registerFrames[0].workflowAddresses).toBeUndefined(); + // The reconnect frame announces the workflow addresses in + // agentAddresses, so each proves ownership via the Ed25519 challenge. + expect(reconnectFrames).toHaveLength(1); + expect(reconnectFrames[0].agentAddresses).toEqual(workflowAddresses); } finally { client.close(); await server.stop(true); } }); }); + +describe("answerMalformedRequestFrame", () => { + test("answers a malformed sources.update with session.error carrying the requestId", () => { + const sent: (SessionErrorFrame | AgentErrorFrame | PackRejectFrame)[] = []; + // A structurally-invalid sources list (empty source object) that failed + // the top-level parse but kept its type + requestId. + const answered = answerMalformedRequestFrame( + { + type: "sources.update", + requestId: "req-1", + agentAddress: "ins_x@example.com", + sources: [{}], + defaultSource: "x", + }, + "sources[0].id must be a string", + (frame) => sent.push(frame), + ); + expect(answered).toBe(true); + expect(sent).toHaveLength(1); + expect(sent[0]).toMatchObject({ + type: "session.error", + requestId: "req-1", + error: expect.stringMatching(/malformed sources.update frame/), + }); + }); + + test("answers a malformed agent.deploy with agent.error carrying the agentAddress", () => { + const sent: (SessionErrorFrame | AgentErrorFrame | PackRejectFrame)[] = []; + const answered = answerMalformedRequestFrame( + { + type: "agent.deploy", + agentAddress: "ins_deploy@example.com", + agentId: "x", + }, + "config must be an object", + (frame) => sent.push(frame), + ); + expect(answered).toBe(true); + expect(sent).toHaveLength(1); + expect(sent[0]).toMatchObject({ + type: "agent.error", + agentAddress: "ins_deploy@example.com", + error: expect.stringMatching(/malformed agent.deploy frame/), + }); + }); + + test("drops an unhandled request-shaped frame instead of answering it", () => { + // A request-shaped frame whose type is in none of the answerable sets + // (SESSION_ERROR / AGENT_ERROR / PACK_REJECT) has no requester to + // answer, so a malformed one is dropped rather than answered. + for (const frameType of ["some.unhandled.request", "another.unknown"]) { + const sent: (SessionErrorFrame | AgentErrorFrame | PackRejectFrame)[] = + []; + const answered = answerMalformedRequestFrame( + { + type: frameType, + requestId: "req-2", + agentAddress: "ins_x@example.com", + }, + "payload must be valid", + (frame) => sent.push(frame), + ); + expect(answered).toBe(false); + expect(sent).toHaveLength(0); + } + }); + + test("answers a malformed agent.undeploy with agent.error", () => { + const sent: (SessionErrorFrame | AgentErrorFrame | PackRejectFrame)[] = []; + const answered = answerMalformedRequestFrame( + { type: "agent.undeploy", agentAddress: "ins_undeploy@example.com" }, + "reason must be a string", + (frame) => sent.push(frame), + ); + expect(answered).toBe(true); + expect(sent[0]).toMatchObject({ + type: "agent.error", + agentAddress: "ins_undeploy@example.com", + error: expect.stringMatching(/malformed agent.undeploy frame/), + }); + }); + + test("answers a malformed repo.pack frame with repo.pack.reject on its transferId", () => { + for (const frameType of ["repo.pack.push", "repo.pack.done"]) { + const sent: (SessionErrorFrame | AgentErrorFrame | PackRejectFrame)[] = + []; + const answered = answerMalformedRequestFrame( + { + type: frameType, + agentAddress: "ins_pack@example.com", + repoId: { kind: "workflow-run", id: "dep-1" }, + transferId: "xfer-1", + seq: "not-a-number", + }, + "a nested field is malformed", + (frame) => sent.push(frame), + ); + expect(answered).toBe(true); + expect(sent[0]).toMatchObject({ + type: "repo.pack.reject", + transferId: "xfer-1", + reason: "corrupt", + }); + } + }); + + test("does not answer a repo.pack frame with no recoverable transferId", () => { + const sent: (SessionErrorFrame | AgentErrorFrame | PackRejectFrame)[] = []; + const answered = answerMalformedRequestFrame( + { + type: "repo.pack.push", + agentAddress: "ins_pack@example.com", + repoId: { kind: "workflow-run", id: "dep-1" }, + }, + "bad", + (frame) => sent.push(frame), + ); + expect(answered).toBe(false); + expect(sent).toHaveLength(0); + }); + + test("does not answer a repo.pack frame whose repoId is itself malformed", () => { + const sent: (SessionErrorFrame | AgentErrorFrame | PackRejectFrame)[] = []; + const answered = answerMalformedRequestFrame( + { + type: "repo.pack.push", + agentAddress: "ins_pack@example.com", + transferId: "xfer-3", + repoId: { kind: "not-a-real-kind" }, + }, + "bad", + (frame) => sent.push(frame), + ); + expect(answered).toBe(false); + expect(sent).toHaveLength(0); + }); + + test("recovers a non-pack frame that carries a malformed repoId-shaped field", () => { + // repoId is validated only inside the pack branch, so a requestId- or + // agentAddress-correlated frame that happens to carry a garbage + // repoId still recovers through its own correlation key. + const sent: (SessionErrorFrame | AgentErrorFrame | PackRejectFrame)[] = []; + const answered = answerMalformedRequestFrame( + { + type: "sources.update", + requestId: "req-9", + repoId: { kind: "not-a-real-kind" }, + sources: [{}], + }, + "sources invalid", + (frame) => sent.push(frame), + ); + expect(answered).toBe(true); + expect(sent[0]).toMatchObject({ + type: "session.error", + requestId: "req-9", + }); + }); + + test("does not answer a sources.update with no recoverable requestId", () => { + const sent: (SessionErrorFrame | AgentErrorFrame | PackRejectFrame)[] = []; + const answered = answerMalformedRequestFrame( + { type: "sources.update", sources: [{}] }, + "bad", + (frame) => sent.push(frame), + ); + expect(answered).toBe(false); + expect(sent).toHaveLength(0); + }); + + test("does not answer a fire-and-forget frame even with a requestId present", () => { + const sent: (SessionErrorFrame | AgentErrorFrame | PackRejectFrame)[] = []; + const answered = answerMalformedRequestFrame( + { type: "signal.deliver", agentAddress: "x", requestId: "r" }, + "bad", + (frame) => sent.push(frame), + ); + expect(answered).toBe(false); + expect(sent).toHaveLength(0); + }); + + test("does not answer a frame with no recognizable type", () => { + const sent: (SessionErrorFrame | AgentErrorFrame | PackRejectFrame)[] = []; + const answered = answerMalformedRequestFrame( + { garbage: true }, + "bad", + (frame) => sent.push(frame), + ); + expect(answered).toBe(false); + expect(sent).toHaveLength(0); + }); +}); diff --git a/packages/hub-agent/src/ws/hub-link.ts b/packages/hub-agent/src/ws/hub-link.ts index 5eb2544f..b5114aa1 100644 --- a/packages/hub-agent/src/ws/hub-link.ts +++ b/packages/hub-agent/src/ws/hub-link.ts @@ -1,8 +1,8 @@ // HubLink: the sidecar-side WebSocket protocol. // -// Connects to the hub, sends the register or reconnect frame, forwards -// outbound mail and inference events, and handles inbound agent -// lifecycle commands. Per-agent key material lives on AgentKeyStore; +// Connects to the hub, sends the register frame, forwards outbound +// mail and inference events, and handles inbound agent lifecycle +// commands. Per-agent key material lives on AgentKeyStore; // the link calls into the store for challenge signing, deploy-commit // verification, and hub-key bookkeeping. The wire layer itself never // touches raw key bytes. @@ -14,35 +14,176 @@ import { HubFrame, type SidecarFrame, type AgentDeployFrame, + type AgentErrorFrame, + type SessionErrorFrame, type AgentUndeployFrame, type ChallengeFrame, type ChallengeFailedFrame, - type SessionAbortFrame, - type SessionStartFrame, - type GrantsUpdateFrame, - type SourcesUpdateFrame, type PackPushFrame, type PackDoneFrame, type PackAckFrame, type PackRejectFrame, - type RepoId, + RepoId, type SignalDeliverFrame, type DrainDeliverFrame, + type SourcesUpdateFrame, type SyncRequestFrame, - type DeployApplyErrorFrame, } from "@intx/types/sidecar"; import { createPackReceiver, createPackSender } from "@intx/pack-transport"; import { base64Decode, base64Encode, hexDecode, hexEncode } from "@intx/types"; +import type { InferenceEvent } from "@intx/types/runtime"; import type { AgentKeyStore } from "../agent-key-store"; -import type { - ConnectorStateSink, - SessionEventSink, - SessionManager, -} from "../session-manager"; +import type { SessionManager } from "../session-manager"; + +/** + * Sink the link exposes for forwarding a spawned child's verified + * InferenceEvents to the hub timeline, keyed by the deploy's session id. + */ +export type SessionEventSink = ( + agentAddress: string, + sessionId: string, + event: InferenceEvent, +) => void; const logger = getLogger(["interchange", "hub-agent", "ws"]); +/** + * Permissive envelope over a raw inbound frame that failed `HubFrame` + * validation. A malformed request/ack frame usually still carries an + * intact discriminator and correlation key -- the malformation is in a + * nested field -- so these top-level fields can be recovered to answer the + * requester. + */ +const MalformedRequestEnvelope = type({ + "type?": "string", + "requestId?": "string", + "agentAddress?": "string", + "transferId?": "string", + // `repoId` is carried as `unknown` and validated only inside the pack + // branch below. Validating it here would fail the whole envelope for a + // non-pack frame that happens to carry a malformed `repoId`-shaped field, + // sinking its recovery through its own correlation key. + "repoId?": "unknown", +}); + +/** + * Inbound request/ack frames the sidecar dispatches that the hub + * correlates by `requestId`, whose failure reply is a `session.error`. + * Only `sources.update` qualifies -- it is the sole frame answered with a + * `session.error`. Frames answered through the other correlation keys live + * in `AGENT_ERROR_REQUEST_TYPES` and `PACK_REJECT_REQUEST_TYPES`; a + * request-shaped frame in none of the three sets has no requester to + * answer and is dropped. + */ +const SESSION_ERROR_REQUEST_TYPES: ReadonlySet = new Set([ + "sources.update", +]); + +/** + * Inbound request/ack frames the hub correlates by `agentAddress` and + * whose failure reply is an `agent.error` -- the frames the hub tracks in + * its per-address pending-deploy / pending-undeploy maps. + */ +const AGENT_ERROR_REQUEST_TYPES: ReadonlySet = new Set([ + "agent.deploy", + "agent.undeploy", +]); + +/** + * Inbound chunked-pack request frames the hub correlates by `transferId` + * and whose failure reply is a `repo.pack.reject`. The hub tracks these in + * its per-transfer pending map with the longest timeout of any request + * frame. + */ +const PACK_REJECT_REQUEST_TYPES: ReadonlySet = new Set([ + "repo.pack.push", + "repo.pack.done", +]); + +/** + * Answer a malformed inbound request/ack control frame with an error reply + * so the hub's request does not hang to its timeout. Two control-frame + * families answer through their correlation key: the `requestId`-correlated + * frame (sources.update) replies `session.error`; the + * `agentAddress`-correlated frames (agent.deploy, agent.undeploy) reply + * `agent.error`. The fire-and-forget frames + * (mail/signal/drain/...) have no requester waiting on a reply, so a + * malformed one is correctly left to be logged and dropped by the caller. + * + * The chunked `repo.pack` streaming transfers (repo.pack.push, + * repo.pack.done) are the third family: correlated by `transferId`, + * rejected by `repo.pack.reject`. A valid reject also carries the frame's + * `agentAddress` and structured `repoId`, so it is answerable only when + * all three survive the malformation; when `repoId` (or the transferId) is + * itself unrecoverable the frame is left to be logged and dropped, because + * a valid `repo.pack.reject` cannot be constructed without them. + * + * Returns `true` when it answered; `false` when no correlation key is + * recoverable (an unknown/absent type, a fire-and-forget frame, or a + * request/ack frame whose key is itself missing) -- the caller then logs + * and drops, because there is nothing to answer. + */ +export function answerMalformedRequestFrame( + raw: unknown, + summary: string, + send: (frame: SessionErrorFrame | AgentErrorFrame | PackRejectFrame) => void, +): boolean { + const envelope = MalformedRequestEnvelope(raw); + if (envelope instanceof type.errors) return false; + const frameType = envelope.type; + if (frameType === undefined) return false; + if ( + SESSION_ERROR_REQUEST_TYPES.has(frameType) && + envelope.requestId !== undefined && + envelope.requestId.length > 0 + ) { + send({ + type: "session.error", + requestId: envelope.requestId, + error: `malformed ${frameType} frame: ${summary}`, + }); + return true; + } + if ( + AGENT_ERROR_REQUEST_TYPES.has(frameType) && + envelope.agentAddress !== undefined && + envelope.agentAddress.length > 0 + ) { + send({ + type: "agent.error", + agentAddress: envelope.agentAddress, + error: `malformed ${frameType} frame: ${summary}`, + }); + return true; + } + if ( + PACK_REJECT_REQUEST_TYPES.has(frameType) && + envelope.transferId !== undefined && + envelope.transferId.length > 0 && + envelope.agentAddress !== undefined && + envelope.agentAddress.length > 0 + ) { + // A valid repo.pack.reject carries the frame's structured repoId, so + // recover it here (kept out of the shared envelope to protect the other + // families). When the repoId is itself malformed there is no valid + // reject to build, so the frame is left to be dropped. The hub + // correlates the reject by transferId alone; "corrupt" is the reason + // for a frame that failed to parse. + const repoId = RepoId(envelope.repoId); + if (repoId instanceof type.errors) return false; + send({ + type: "repo.pack.reject", + agentAddress: envelope.agentAddress, + repoId, + transferId: envelope.transferId, + reason: "corrupt", + }); + return true; + } + return false; +} + const DEFAULT_PING_INTERVAL_MS = 30_000; const DEFAULT_RECONNECT_DELAY_MS = 3_000; @@ -65,10 +206,10 @@ const defaultScheduleReconnect: ReconnectScheduler = (callback, delayMs) => { }; /** - * Result the deploy router returns to the link after the trivial - * branch completes. Carries the values the link folds into the - * outbound `agent.deploy.ack` frame; the link itself stays out of - * the provisioning details. + * Result the deploy router returns to the link once a deploy has + * staged. Carries the values the link folds into the outbound + * `agent.deploy.ack` frame; the link itself stays out of the deploy + * details. */ export type DeployRouterResult = { /** Hex-encoded agent public key the hub records for verification. */ @@ -77,11 +218,11 @@ export type DeployRouterResult = { /** * Single-ingress deploy contract the link routes every `agent.deploy` - * frame through. The workflow-host supervisor is the production - * implementation -- the supervisor decides between the trivial - * (1-step) passthrough and the multi-step IPC-backed branch. The - * shape lives on hub-agent so the package boundary stays one-way - * (`@intx/hub-agent` does not import `@intx/workflow-host`). + * frame through. The sidecar's workflow-run deploy router is the + * production implementation -- it stages every deploy through the + * workflow-run substrate. The shape lives on hub-agent so the package + * boundary stays one-way (`@intx/hub-agent` does not import + * `@intx/workflow-host`). */ export interface DeployRouter { deploy(frame: AgentDeployFrame): Promise; @@ -91,30 +232,26 @@ export interface DeployRouter { * per-deployment registrations the deploy path installed * (`MultistepMailRouter`, `MultistepSignalRouter`, * `MultistepDrainRouter`, `DeploymentAddressRegistry`). Optional - * so test routers and the inline trivial test fixture can omit - * the implementation. + * so test routers can omit the implementation. */ undeploy?: (frame: AgentUndeployFrame) => Promise; } /** * Per-address mail handler registry the link consults on every - * `mail.inbound` frame before falling back to `transport.deliver` / - * `sessions.commitInboundMail`. Production wires this against the - * sidecar's `createMultistepMailRouter` so a multi-step deployment's - * supervisor receives the bytes through its mail-bus subscription. - * Trivial deploys never register a handler; their mail flows through - * the legacy session path unchanged. The shape lives on hub-agent so - * the link does not import the sidecar host's wiring module, and so - * tests can substitute a stub. + * `mail.inbound` frame. Production wires this against the sidecar's + * `createMultistepMailRouter` so a supervised deployment's supervisor + * receives the bytes through its mail-bus subscription. Mail for an + * address with no registered handler has no receiver and is dropped. + * The shape lives on hub-agent so the link does not import the sidecar + * host's wiring module, and so tests can substitute a stub. */ export interface MailInboundRouter { /** * Attempt to dispatch `message` to a handler registered against * `agentAddress`. Returns `true` if a handler claimed the message; - * `false` if no handler is registered. A `true` return causes the - * link to skip the legacy `transport.deliver` / - * `sessions.commitInboundMail` fallback entirely. + * `false` if no handler is registered, in which case the link logs + * and drops the mail. */ tryRoute(agentAddress: string, message: Uint8Array): boolean; } @@ -124,10 +261,10 @@ export interface MailInboundRouter { * every inbound `signal.deliver` frame. Production wires this against * the sidecar's multi-step deploy registry so the frame flows into the * deployment's supervisor (which forwards `signal.deliver` over the - * control IPC to the workflow-process child). Trivial deployments do - * not register a handler; the link logs and drops a frame whose - * `agentAddress` is unknown so the wire surface fails loudly rather - * than silently absorbing a misrouted delivery. + * control IPC to the workflow-process child). The link logs and drops + * a frame whose `agentAddress` matches no registered handler so the + * wire surface fails loudly rather than silently absorbing a misrouted + * delivery. * * The shape lives on hub-agent so the link does not import the sidecar * host's wiring module, and so tests can substitute a stub. @@ -151,10 +288,9 @@ export interface SignalInboundRouter { * the sidecar's multi-step deploy registry so the frame flows into the * deployment's supervisor (which forwards a `drain` control IPC frame * to the workflow-process child and arms one drainTimeout accumulator - * per in-flight run). Trivial deployments do not register a handler; - * the link logs and drops a frame whose `agentAddress` is unknown so - * the wire surface fails loudly rather than silently absorbing a - * misrouted delivery. + * per in-flight run). The link logs and drops a frame whose + * `agentAddress` matches no registered handler so the wire surface + * fails loudly rather than silently absorbing a misrouted delivery. * * The shape lives on hub-agent so the link does not import the sidecar * host's wiring module, and so tests can substitute a stub. @@ -172,6 +308,28 @@ export interface DrainInboundRouter { tryRoute(frame: DrainDeliverFrame): Promise; } +/** + * Per-deployment-address sources-rotation registry the link consults on + * every inbound `sources.update` frame. Unlike signal/drain, `sources.update` + * is a REQUEST/ACK frame, so the link answers `session.ack` / `session.error` + * rather than logging and dropping -- a missing answer hangs the hub's + * request for its full timeout. + * + * The shape lives on hub-agent so the link does not import the sidecar + * host's wiring module, and so tests can substitute a stub. + */ +export interface SourcesInboundRouter { + /** + * Attempt to dispatch `frame` to the supervisor registered against + * `frame.agentAddress`. Resolves `true` when a handler accepted the + * rotation, `false` when no handler is registered (an unrouted address). + * Rejects when the handler is registered but the rotation is invalid or + * the supervisor's `deliverSources` throws; the link turns a rejection + * into a `session.error` carrying the reason. + */ + tryRoute(frame: SourcesUpdateFrame): Promise; +} + export type HubLinkConfig = { hubURL: string; sidecarId: string; @@ -187,47 +345,63 @@ export type HubLinkConfig = { keyStore: AgentKeyStore; /** * Routes every inbound `agent.deploy` frame. Production wiring - * supplies a router that calls `supervisor.deploy(frame)` on a - * freshly-constructed workflow-host supervisor whose - * `trivialLaunch` closes over `SessionManager.provisionAgent`. - * The supervisor owns the trivial vs multi-step decision; the - * link does not re-decide. + * supplies a router that stages each deploy through the workflow-run + * substrate: a provision-step frame primes a per-step repo, and a + * workflow frame spawns the supervised workflow-process child. The + * router owns the routing decision; the link does not re-decide. */ deployRouter: DeployRouter; /** - * Optional pre-fallback mail dispatcher. When present, the link - * consults this router on every inbound `mail.inbound` frame; a - * `true` return takes the bytes off the legacy - * `transport.deliver` + `sessions.commitInboundMail` path entirely. - * Production wires this against the sidecar's multi-step deploy - * registry so a deployment-address inbound flows into the - * supervisor's mail-bus subscription. Trivial deployments (and - * tests that exercise the legacy path) omit it; an absent router - * is treated as "no handler ever claims" so behaviour is unchanged. + * Optional inbound mail dispatcher. When present, the link consults + * this router on every inbound `mail.inbound` frame. Production wires + * this against the sidecar's multi-step deploy registry so a + * deployment-address inbound flows into the supervisor's mail-bus + * subscription. Absent (or a `false` return) means no handler claims + * the mail, so the link logs and drops it. */ mailInboundRouter?: MailInboundRouter; /** - * Optional pre-fallback signal dispatcher. When present, the link - * routes every inbound `signal.deliver` frame through this router. - * Production wires this against the sidecar's multi-step deploy - * registry so a deployment-address signal flows into the - * supervisor's `deliverSignal`. Trivial deployments (and tests that - * do not exercise the signal-delivery surface) omit it; an absent - * router causes inbound signal frames to be logged-and-dropped so a - * misrouted delivery is observable rather than silent. + * Optional inbound signal dispatcher. When present, the link routes + * every inbound `signal.deliver` frame through this router. Production + * wires this against the sidecar's multi-step deploy registry so a + * deployment-address signal flows into the supervisor's + * `deliverSignal`. Absent (or a `false` return) causes inbound signal + * frames to be logged-and-dropped so a misrouted delivery is + * observable rather than silent. */ signalInboundRouter?: SignalInboundRouter; /** - * Optional pre-fallback drain dispatcher. When present, the link - * routes every inbound `drain.deliver` frame through this router. - * Production wires this against the sidecar's multi-step deploy - * registry so a deployment-address drain flows into the supervisor's - * `drain`. Trivial deployments (and tests that do not exercise the - * drain surface) omit it; an absent router causes inbound drain - * frames to be logged-and-dropped so a misrouted delivery is - * observable rather than silent. + * Optional inbound drain dispatcher. When present, the link routes + * every inbound `drain.deliver` frame through this router. Production + * wires this against the sidecar's multi-step deploy registry so a + * deployment-address drain flows into the supervisor's `drain`. Absent + * (or a `false` return) causes inbound drain frames to be + * logged-and-dropped so a misrouted delivery is observable rather than + * silent. */ drainInboundRouter?: DrainInboundRouter; + /** + * Optional inbound sources-rotation dispatcher. When present, the link + * routes every inbound `sources.update` frame through this router and + * answers the request/ack frame: `session.ack` when the router accepted + * the rotation, `session.error` when no deployment is registered, when + * the rotation is invalid, or when delivery throws. Absent means the + * link answers `session.error` for every rotation -- required because a + * request/ack frame with no reply hangs the hub's request. + */ + sourcesInboundRouter?: SourcesInboundRouter; + /** + * Returns the workflow-substrate deployment addresses this sidecar + * currently hosts a live supervisor for. Called on every (re)connect to + * announce them to the hub for routing through the CHALLENGED reconnect + * frame: each deployment carries its own Ed25519 key (minted at deploy, + * acked to the hub), so it proves ownership via challenge/response exactly + * like a launched agent -- there is no keyless routing shortcut. Without + * this announcement the hub drops the deployment's route on a WS reconnect. + * Defaults to none when omitted (tests / deployments with no workflow + * substrate). + */ + getWorkflowAddresses?: () => string[]; pingIntervalMs?: number; reconnectDelayMs?: number; scheduleReconnect?: ReconnectScheduler; @@ -241,16 +415,6 @@ export type HubLink = { connect(): void; close(): void; sendEvent: SessionEventSink; - sendConnectorState: ConnectorStateSink; - /** - * Ship a deploy.apply.error frame to the hub. Caller supplies the - * agentAddress separately so the frame's other fields stay close to - * the loader's failure-site description. - */ - sendDeployApplyError: ( - agentAddress: string, - payload: Omit, - ) => void; /** * Ship a workflow-run pack to the hub. Streams the supplied pack as * `repo.pack.push` chunks followed by a `repo.pack.done`, then @@ -280,6 +444,8 @@ export function createHubLink(config: HubLinkConfig): HubLink { mailInboundRouter, signalInboundRouter, drainInboundRouter, + sourcesInboundRouter, + getWorkflowAddresses = () => [], pingIntervalMs = DEFAULT_PING_INTERVAL_MS, reconnectDelayMs = DEFAULT_RECONNECT_DELAY_MS, scheduleReconnect = defaultScheduleReconnect, @@ -364,36 +530,18 @@ export function createHubLink(config: HubLinkConfig): HubLink { async function handleAgentDeploy(frame: AgentDeployFrame): Promise { try { - // The deploy router (production: workflow-host supervisor) - // owns the agent.deploy framing decision -- trivial vs - // multi-step -- and returns the deploy public key the link - // folds into the outbound ack. The link itself does not - // re-decide; routing lives on the supervisor side of the seam. + // The deploy router (production: the sidecar's workflow-run deploy + // router) stages the deploy through the substrate and returns the + // deploy public key the link folds into the outbound ack. The link + // itself does not re-decide; routing lives on the router side of + // the seam. const result = await deployRouter.deploy(frame); send({ type: "agent.deploy.ack", agentAddress: frame.agentAddress, publicKey: result.publicKey, }); - logger.info`Provisioned agent ${frame.agentAddress}`; - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - send({ - type: "agent.error", - agentAddress: frame.agentAddress, - error: message, - }); - } - } - - async function handleSessionStart(frame: SessionStartFrame): Promise { - try { - await sessions.startSession(frame.agentAddress); - send({ - type: "session.start.ack", - agentAddress: frame.agentAddress, - }); - logger.info`Started session for ${frame.agentAddress}`; + logger.info`Deployed agent ${frame.agentAddress}`; } catch (err) { const message = err instanceof Error ? err.message : String(err); send({ @@ -413,9 +561,8 @@ export function createHubLink(config: HubLinkConfig): HubLink { // the registrations released, any in-flight `signal.deliver` / // `drain.deliver` / `mail.inbound` frame that lands during teardown // is rejected by the router rather than dispatched into a - // soon-to-be-orphaned supervisor handler. Trivial-deploy routers - // (and test stubs) omit the hook; an absent hook means there was - // nothing to release. + // soon-to-be-orphaned supervisor handler. Test stubs omit the hook; + // an absent hook means there was nothing to release. if (deployRouter.undeploy !== undefined) { try { await deployRouter.undeploy(frame); @@ -443,14 +590,6 @@ export function createHubLink(config: HubLinkConfig): HubLink { workflowRunPackBootstrappedByAddress.delete(frame.agentAddress); } - // Stop the harness first. - try { - await sessions.destroySession(frame.agentAddress); - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - logger.warn`Failed to stop session for ${frame.agentAddress}: ${msg}`; - } - // Best-effort state push to the hub before deleting the directory. // statePushed reflects whether we sent the pack frames, not whether // the hub acknowledged them. We intentionally skip waiting for @@ -532,71 +671,13 @@ export function createHubLink(config: HubLinkConfig): HubLink { async function handleChallengeFailed( frame: ChallengeFailedFrame, ): Promise { - // The hub rejected this agent during reconnect — tear it down so - // the address is freed for future deploys. The agent may not have - // an active session (provisioned but never started, or already - // destroyed by a concurrent path), so any error from destroySession - // is logged but does not abort the wire layer. Destroy first so any - // disposer or mail-commit code still has the agent's crypto handle; - // forget the key material only once the session lifecycle methods - // have returned. - try { - await sessions.destroySession(frame.address); - } catch (err: unknown) { - const msg = err instanceof Error ? err.message : String(err); - logger.warn`destroySession during challenge.failed for ${frame.address}: ${msg}`; - } + // The hub rejected this agent during reconnect -- forget its key + // material so the address is freed for future deploys. keyStore.forgetAgent(frame.address); logger.warn`Challenge failed for ${frame.address}, agent torn down: ${frame.reason}`; } - async function handleSessionAbort(frame: SessionAbortFrame): Promise { - try { - await sessions.abortSession(frame.agentAddress, frame.reason); - send({ type: "session.ack", requestId: frame.requestId }); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - send({ - type: "session.error", - requestId: frame.requestId, - error: message, - }); - } - } - - async function handleGrantsUpdate(frame: GrantsUpdateFrame): Promise { - try { - await sessions.updateGrants(frame.agentAddress, frame.grants); - send({ type: "session.ack", requestId: frame.requestId }); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - send({ - type: "session.error", - requestId: frame.requestId, - error: message, - }); - } - } - - async function handleSourcesUpdate(frame: SourcesUpdateFrame): Promise { - try { - await sessions.updateSources( - frame.agentAddress, - frame.sources, - frame.defaultSource, - ); - send({ type: "session.ack", requestId: frame.requestId }); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - send({ - type: "session.error", - requestId: frame.requestId, - error: message, - }); - } - } - function handlePackPush(frame: PackPushFrame): void { const reason = packReceiver.handlePush(frame); if (reason !== null) { @@ -800,6 +881,44 @@ export function createHubLink(config: HubLinkConfig): HubLink { } } + async function handleSourcesUpdate(frame: SourcesUpdateFrame): Promise { + // `sources.update` is request/ack (the hub awaits a reply within its + // request timeout), so every path answers `session.ack` or + // `session.error` -- unlike the fire-and-forget signal/drain frames + // that log and drop. A missing router still answers, or the hub hangs. + if (sourcesInboundRouter === undefined) { + send({ + type: "session.error", + requestId: frame.requestId, + error: "no sourcesInboundRouter is wired", + }); + return; + } + try { + const routed = await sourcesInboundRouter.tryRoute(frame); + if (routed) { + send({ type: "session.ack", requestId: frame.requestId }); + } else { + send({ + type: "session.error", + requestId: frame.requestId, + error: `no deployment registered for ${frame.agentAddress}`, + }); + } + } catch (err) { + // A registered address whose rotation was rejected: an invalid list + // (the router validates before dispatch) or the supervisor's + // `deliverSources` throwing (e.g. a recycling phase). The reason + // rides back verbatim so the hub sees why the rotation failed. + const msg = err instanceof Error ? err.message : String(err); + send({ + type: "session.error", + requestId: frame.requestId, + error: msg, + }); + } + } + async function pushWorkflowRunPack(opts: { agentAddress: string; repoId: RepoId; @@ -883,6 +1002,12 @@ export function createHubLink(config: HubLinkConfig): HubLink { } const validated = HubFrame(raw); if (validated instanceof type.errors) { + // A malformed request/ack frame must still be answered, or the hub's + // request hangs to its timeout. `sources.update` and `agent.deploy` + // usually keep an intact correlation key even when a nested field is + // malformed, so reply with the matching error frame; a fire-and-forget + // frame (or one with no recoverable key) is only logged and dropped. + answerMalformedRequestFrame(raw, validated.summary, send); logger.warn`Invalid hub frame: ${validated.summary}`; return; } @@ -891,17 +1016,13 @@ export function createHubLink(config: HubLinkConfig): HubLink { switch (frame.type) { case "mail.inbound": { const rawBytes = base64Decode(frame.rawMessage); - // Multi-step deployments register the deployment-level mail - // address on `mailInboundRouter` once their supervisor spawns. - // For those addresses the legacy session path is not the right - // receiver -- the transport mailbox is never registered for the - // deployment address (no `startSession` ever runs against it), - // and there is no `sessions` entry to satisfy - // `commitInboundMail`. Routing through the registered handler - // delivers the bytes to the supervisor's mail-bus subscription, - // which is what the workflow-host's `awaitSignal` listens on. - // Trivial deployments do not register a handler; the fallback - // path is the legacy single-agent provisioning surface. + // Supervised deployments register the deployment-level mail + // address on `mailInboundRouter` once their supervisor spawns; + // that handler delivers the bytes to the supervisor's mail-bus + // subscription, which is what the workflow-host's `awaitSignal` + // listens on. Mail for an address with no registered handler has + // no receiver -- the in-process session runtime that once backed + // it is retired -- so it is logged and dropped. // // Guard the router call with try/catch so a throwing handler // does not reject this `handleMessage` promise and wedge the @@ -918,19 +1039,14 @@ export function createHubLink(config: HubLinkConfig): HubLink { logger.warn`mail.inbound router threw for ${frame.agentAddress}: ${msg}`; } } - if (routed) { - break; + if (!routed) { + logger.warn`Dropping mail.inbound for ${frame.agentAddress}: no registered handler`; } - deliverLocalMail(frame.agentAddress, rawBytes); - void sessions.commitInboundMail(frame.agentAddress, rawBytes); break; } case "agent.deploy": await handleAgentDeploy(frame); break; - case "session.start": - await handleSessionStart(frame); - break; case "agent.undeploy": await handleAgentUndeploy(frame); break; @@ -943,15 +1059,6 @@ export function createHubLink(config: HubLinkConfig): HubLink { case "challenge.failed": await handleChallengeFailed(frame); break; - case "session.abort": - await handleSessionAbort(frame); - break; - case "grants.update": - await handleGrantsUpdate(frame); - break; - case "sources.update": - await handleSourcesUpdate(frame); - break; case "repo.pack.push": handlePackPush(frame); break; @@ -967,6 +1074,9 @@ export function createHubLink(config: HubLinkConfig): HubLink { case "drain.deliver": await handleDrainDeliver(frame); break; + case "sources.update": + await handleSourcesUpdate(frame); + break; case "repo.pack.ack": handlePackAck(frame); break; @@ -978,15 +1088,6 @@ export function createHubLink(config: HubLinkConfig): HubLink { } } - function deliverLocalMail(agentAddress: string, message: Uint8Array): void { - try { - transport.deliver(agentAddress, message); - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - logger.warn`Failed to deliver inbound mail to ${agentAddress}: ${msg}`; - } - } - function connect(): void { // Reconnect cancellation in close() is the load-bearing protection // against post-close reconnect attempts. A caller invoking connect() @@ -1017,82 +1118,63 @@ export function createHubLink(config: HubLinkConfig): HubLink { packReceiver.reset(); packSender.cancelAll("Connection lost"); - // Register the connection for routing before the restore loop runs. The - // hub learns of a sidecar only from a register/reconnect frame, and - // connections is the map sendAgentDeploy consults to route a new - // provision. An empty-address register here populates that map - // immediately, so a provision arriving during restore routes instead of - // hitting an empty map and failing with "No sidecar available". The - // address list must stay empty: carrying addresses through register - // writes addressIndex with no challenge and discards disconnect queues, - // so restored sessions enter addressIndex through the challenged - // reconnect frame below instead. + // Announce this sidecar to the hub for routing. The hub learns of a + // sidecar only from a register frame; `connections` is the map + // `sendAgentDeploy` consults to route a deploy. This first-connect + // register carries no addresses -- it only establishes the sidecar in + // that map. Restored deployments are announced through the CHALLENGED + // reconnect frame below. send({ type: "register", sidecarId, token, agentAddresses: [], }); - - void (async () => { - try { - const { restored, failed } = await sessions.restoreSessions(); - - if (restored.length > 0) { + flush(); + + // Re-announce every deployment restored at boot through the reconnect + // frame so the hub proves ownership of each address (Ed25519 + // challenge/response, signed by the deployment's own key via + // `signChallenge`) before it routes mail. Routing a restored address + // through `register`/`workflowAddresses` -- unchallenged -- would let a + // rogue sidecar holding a valid token reclaim a victim's address. + // Restore runs before `connect()`, so `getWorkflowAddresses()` is + // already populated; the only async work is reading each address's + // deploy ref for the hub's deploy-pack freshness check. + const restoredAddresses = getWorkflowAddresses(); + if (restoredAddresses.length > 0) { + void (async () => { + try { const deployRefs: Record = {}; - for (const entry of restored) { - // SessionManager.restoreSessions populated AgentKeyStore's - // in-memory keypair cache via scanKeys. Replay the - // hub-side pairing record here so verifyDeployCommit can - // accept incoming packs without re-running an agent.deploy. - if (entry.hubPublicKey !== undefined) { - keyStore.recordHubKey(entry.address, entry.hubPublicKey); - } - const ref = await sessions.getDeployRef(entry.address); + for (const address of restoredAddresses) { + const ref = await sessions.getDeployRef(address); if (ref !== null) { - deployRefs[entry.address] = ref; + deployRefs[address] = ref; } } send({ type: "reconnect", sidecarId, token, - agentAddresses: restored.map((e) => e.address), - deployRefs, + agentAddresses: restoredAddresses, + ...(Object.keys(deployRefs).length > 0 ? { deployRefs } : {}), }); - if (failed.length > 0) { - logger.warn`Reconnected ${String(restored.length)} agent(s), ${String(failed.length)} failed to restore`; - } else { - logger.info`Sent reconnect with ${String(restored.length)} agent(s)`; - } - } else { - // Reached only when nothing was restored from disk. The - // empty register on open already established routability, so - // skip a second empty frame. Any address present here belongs - // to an agent provisioned during the restore window — - // provisions route on that open register — which the hub - // already placed in addressIndex when it routed the deploy. - // Re-announcing those is consistent with that entry, not an - // unchallenged write of a disk-restored address; those take - // the reconnect branch above. - const addresses = sessions.getAddresses(); - if (addresses.length > 0) { - send({ - type: "register", - sidecarId, - token, - agentAddresses: addresses, - }); - } + flush(); + } catch (err) { + // A failing deploy-ref read (corrupt or unreadable ref state) + // must not silently drop the reconnect. Without this frame the + // hub never re-challenges these addresses, so their routes + // vanish with nothing logged. Surface the failure and close the + // socket to force a clean reconnect retry. No partial reconnect + // frame is sent: the send sits after the loop, so a throw skips + // it. This IIFE runs in the `open` handler, outside the + // `messageQueue` tail-catch, so this catch is its only net. + const msg = err instanceof Error ? err.message : String(err); + logger.error`Deployment re-announce failed, closing connection: ${msg}`; + ws?.close(); } - - flush(); - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - logger.error`Session restore failed, closing connection: ${msg}`; - ws?.close(); - } - })(); + })(); + } }); ws.addEventListener("message", (event) => { @@ -1106,6 +1188,14 @@ export function createHubLink(config: HubLinkConfig): HubLink { // Per-arm guards (mail/signal/drain) are the primary defence; // this catch is the belt-and-braces guarantee that no future // unguarded arm can wedge the link. + // + // This chain also serializes inbound frames: each frame's + // handler runs to completion before the next begins. A downstream + // invariant depends on that ordering -- the workflow + // source-rotation persist rolls back on failure assuming no second + // rotation is in flight, which holds only because sources.update + // frames are processed one at a time here. Parallelizing this + // dispatch would break that rollback. const data = event.data; messageQueue = messageQueue.then(() => handleMessage(data).catch((err: unknown) => { @@ -1167,34 +1257,10 @@ export function createHubLink(config: HubLinkConfig): HubLink { }); }; - const sendConnectorState: ConnectorStateSink = ( - agentAddress, - connectorState, - ) => { - send({ - type: "connector.state.changed", - agentAddress, - connectorState, - }); - }; - - const sendDeployApplyError: HubLink["sendDeployApplyError"] = ( - agentAddress, - payload, - ) => { - send({ - type: "deploy.apply.error", - agentAddress, - ...payload, - }); - }; - return { connect, close, sendEvent, - sendConnectorState, - sendDeployApplyError, pushWorkflowRunPack, }; } diff --git a/packages/hub-api/src/app.test.ts b/packages/hub-api/src/app.test.ts index 08fc0957..b7a6ac71 100644 --- a/packages/hub-api/src/app.test.ts +++ b/packages/hub-api/src/app.test.ts @@ -20,14 +20,24 @@ const OpenAPISpec = type({ const mockDb = {} as unknown as DB["db"]; const sidecarRouter = createSidecarRouter({}); const sessionService: SessionService = { - launchSession(_params) { - throw new Error("mock: sessionService.launchSession not implemented"); + stageWorkflowStep(_params) { + throw new Error("mock: sessionService.stageWorkflowStep not implemented"); + }, + deployInstanceAtHead(_params) { + throw new Error( + "mock: sessionService.deployInstanceAtHead not implemented", + ); }, deployWorkflowDefinition(_params) { throw new Error( "mock: sessionService.deployWorkflowDefinition not implemented", ); }, + deploySingleStepAtHead(_params) { + throw new Error( + "mock: sessionService.deploySingleStepAtHead not implemented", + ); + }, sendUserMessage(_params) { throw new Error("mock: sessionService.sendUserMessage not implemented"); }, diff --git a/packages/hub-api/src/git-http/receive-pack.test.ts b/packages/hub-api/src/git-http/receive-pack.test.ts index 01a97920..d05586a1 100644 --- a/packages/hub-api/src/git-http/receive-pack.test.ts +++ b/packages/hub-api/src/git-http/receive-pack.test.ts @@ -97,6 +97,9 @@ function createStubStore(stub: ReceivePackStub): { writeTreePreservingPrefix: async () => { throw new Error("writeTreePreservingPrefix: not used in this test"); }, + writeTreeDelta: async () => { + throw new Error("writeTreeDelta: not used in this test"); + }, createPack: async () => { throw new Error("createPack: not used in this test"); }, diff --git a/packages/hub-api/src/routes/agents.test.ts b/packages/hub-api/src/routes/agents.test.ts index 340f6a51..4c773432 100644 --- a/packages/hub-api/src/routes/agents.test.ts +++ b/packages/hub-api/src/routes/agents.test.ts @@ -174,11 +174,11 @@ function createMockSidecarRouter(): SidecarRouter { routeMail: () => notImpl("routeMail"), sendAgentDeploy: () => notImpl("sendAgentDeploy"), sendAgentUndeploy: () => notImpl("sendAgentUndeploy"), - sendSessionStart: () => notImpl("sendSessionStart"), - sendSessionAbort: () => notImpl("sendSessionAbort"), - sendGrantsUpdate: () => notImpl("sendGrantsUpdate"), sendSourcesUpdate: () => notImpl("sendSourcesUpdate"), sendPack: () => notImpl("sendPack"), + sendProvisionStep: () => notImpl("sendProvisionStep"), + bindStepRoute: () => notImpl("bindStepRoute"), + unbindStepRoute: () => notImpl("unbindStepRoute"), sendSyncRequest: () => notImpl("sendSyncRequest"), sendSignalDeliver: () => notImpl("sendSignalDeliver"), sendDrain: () => notImpl("sendDrain"), @@ -193,14 +193,24 @@ function createMockSidecarRouter(): SidecarRouter { function createMockSessionService(): SessionService { return { - launchSession: () => { - throw new Error("mock: sessionService.launchSession not implemented"); + stageWorkflowStep: () => { + throw new Error("mock: sessionService.stageWorkflowStep not implemented"); + }, + deployInstanceAtHead: () => { + throw new Error( + "mock: sessionService.deployInstanceAtHead not implemented", + ); }, deployWorkflowDefinition: () => { throw new Error( "mock: sessionService.deployWorkflowDefinition not implemented", ); }, + deploySingleStepAtHead: () => { + throw new Error( + "mock: sessionService.deploySingleStepAtHead not implemented", + ); + }, sendUserMessage: () => { throw new Error("mock: sessionService.sendUserMessage not implemented"); }, diff --git a/packages/hub-api/src/routes/assets.test.ts b/packages/hub-api/src/routes/assets.test.ts index 7414f3e2..901e1f7b 100644 --- a/packages/hub-api/src/routes/assets.test.ts +++ b/packages/hub-api/src/routes/assets.test.ts @@ -290,11 +290,11 @@ function createMockSidecarRouter(): SidecarRouter { routeMail: () => notImpl("routeMail"), sendAgentDeploy: () => notImpl("sendAgentDeploy"), sendAgentUndeploy: () => notImpl("sendAgentUndeploy"), - sendSessionStart: () => notImpl("sendSessionStart"), - sendSessionAbort: () => notImpl("sendSessionAbort"), - sendGrantsUpdate: () => notImpl("sendGrantsUpdate"), sendSourcesUpdate: () => notImpl("sendSourcesUpdate"), sendPack: () => notImpl("sendPack"), + sendProvisionStep: () => notImpl("sendProvisionStep"), + bindStepRoute: () => notImpl("bindStepRoute"), + unbindStepRoute: () => notImpl("unbindStepRoute"), sendSyncRequest: () => notImpl("sendSyncRequest"), sendSignalDeliver: () => notImpl("sendSignalDeliver"), sendDrain: () => notImpl("sendDrain"), @@ -309,14 +309,24 @@ function createMockSidecarRouter(): SidecarRouter { function createMockSessionService(): SessionService { return { - launchSession: () => { - throw new Error("mock: sessionService.launchSession not implemented"); + stageWorkflowStep: () => { + throw new Error("mock: sessionService.stageWorkflowStep not implemented"); + }, + deployInstanceAtHead: () => { + throw new Error( + "mock: sessionService.deployInstanceAtHead not implemented", + ); }, deployWorkflowDefinition: () => { throw new Error( "mock: sessionService.deployWorkflowDefinition not implemented", ); }, + deploySingleStepAtHead: () => { + throw new Error( + "mock: sessionService.deploySingleStepAtHead not implemented", + ); + }, sendUserMessage: () => { throw new Error("mock: sessionService.sendUserMessage not implemented"); }, diff --git a/packages/hub-api/src/routes/catalog-routes.test.ts b/packages/hub-api/src/routes/catalog-routes.test.ts index e32fbdd3..bd8557ce 100644 --- a/packages/hub-api/src/routes/catalog-routes.test.ts +++ b/packages/hub-api/src/routes/catalog-routes.test.ts @@ -188,12 +188,12 @@ function createMockSidecarRouter(): SidecarRouter { routeMail: () => notImpl("routeMail"), sendAgentDeploy: () => notImpl("sendAgentDeploy"), sendAgentUndeploy: () => notImpl("sendAgentUndeploy"), - sendSessionStart: () => notImpl("sendSessionStart"), - sendSessionAbort: () => notImpl("sendSessionAbort"), - sendGrantsUpdate: () => notImpl("sendGrantsUpdate"), // Mutations fire a fire-and-forget source push; resolve so it is a no-op. sendSourcesUpdate: () => Promise.resolve(), sendPack: () => notImpl("sendPack"), + sendProvisionStep: () => notImpl("sendProvisionStep"), + bindStepRoute: () => notImpl("bindStepRoute"), + unbindStepRoute: () => notImpl("unbindStepRoute"), sendSyncRequest: () => notImpl("sendSyncRequest"), sendSignalDeliver: () => notImpl("sendSignalDeliver"), sendDrain: () => notImpl("sendDrain"), @@ -208,14 +208,24 @@ function createMockSidecarRouter(): SidecarRouter { function createMockSessionService(): SessionService { return { - launchSession: () => { - throw new Error("mock: sessionService.launchSession not implemented"); + stageWorkflowStep: () => { + throw new Error("mock: sessionService.stageWorkflowStep not implemented"); + }, + deployInstanceAtHead: () => { + throw new Error( + "mock: sessionService.deployInstanceAtHead not implemented", + ); }, deployWorkflowDefinition: () => { throw new Error( "mock: sessionService.deployWorkflowDefinition not implemented", ); }, + deploySingleStepAtHead: () => { + throw new Error( + "mock: sessionService.deploySingleStepAtHead not implemented", + ); + }, sendUserMessage: () => { throw new Error("mock: sessionService.sendUserMessage not implemented"); }, diff --git a/packages/hub-api/src/routes/git-tokens.test.ts b/packages/hub-api/src/routes/git-tokens.test.ts index 94965876..85445508 100644 --- a/packages/hub-api/src/routes/git-tokens.test.ts +++ b/packages/hub-api/src/routes/git-tokens.test.ts @@ -213,11 +213,11 @@ function createMockSidecarRouter(): SidecarRouter { routeMail: () => notImpl("routeMail"), sendAgentDeploy: () => notImpl("sendAgentDeploy"), sendAgentUndeploy: () => notImpl("sendAgentUndeploy"), - sendSessionStart: () => notImpl("sendSessionStart"), - sendSessionAbort: () => notImpl("sendSessionAbort"), - sendGrantsUpdate: () => notImpl("sendGrantsUpdate"), sendSourcesUpdate: () => notImpl("sendSourcesUpdate"), sendPack: () => notImpl("sendPack"), + sendProvisionStep: () => notImpl("sendProvisionStep"), + bindStepRoute: () => notImpl("bindStepRoute"), + unbindStepRoute: () => notImpl("unbindStepRoute"), sendSyncRequest: () => notImpl("sendSyncRequest"), sendSignalDeliver: () => notImpl("sendSignalDeliver"), sendDrain: () => notImpl("sendDrain"), @@ -232,14 +232,24 @@ function createMockSidecarRouter(): SidecarRouter { function createMockSessionService(): SessionService { return { - launchSession: () => { - throw new Error("mock: sessionService.launchSession not implemented"); + stageWorkflowStep: () => { + throw new Error("mock: sessionService.stageWorkflowStep not implemented"); + }, + deployInstanceAtHead: () => { + throw new Error( + "mock: sessionService.deployInstanceAtHead not implemented", + ); }, deployWorkflowDefinition: () => { throw new Error( "mock: sessionService.deployWorkflowDefinition not implemented", ); }, + deploySingleStepAtHead: () => { + throw new Error( + "mock: sessionService.deploySingleStepAtHead not implemented", + ); + }, sendUserMessage: () => { throw new Error("mock: sessionService.sendUserMessage not implemented"); }, diff --git a/packages/hub-api/src/routes/instances.test.ts b/packages/hub-api/src/routes/instances.test.ts index 4a2367f2..b7e0a8ba 100644 --- a/packages/hub-api/src/routes/instances.test.ts +++ b/packages/hub-api/src/routes/instances.test.ts @@ -240,21 +240,21 @@ function createMockSidecarRouter( sendAgentUndeploy(_addr, _reason) { return notImpl("sendAgentUndeploy"); }, - sendSessionStart(_addr) { - return notImpl("sendSessionStart"); - }, - sendSessionAbort(_addr, _reason) { - return notImpl("sendSessionAbort"); - }, - sendGrantsUpdate(_addr, _grants) { - return notImpl("sendGrantsUpdate"); - }, sendSourcesUpdate(_addr, _sources, _defaultSource) { return notImpl("sendSourcesUpdate"); }, sendPack(_addr, _pack, _ref, _sha) { return notImpl("sendPack"); }, + sendProvisionStep(_agentAddress, _config) { + return notImpl("sendProvisionStep"); + }, + bindStepRoute(_stepAddress) { + notImpl("bindStepRoute"); + }, + unbindStepRoute(_stepAddress) { + notImpl("unbindStepRoute"); + }, sendSyncRequest(_addr) { notImpl("sendSyncRequest"); }, @@ -284,12 +284,18 @@ function createMockSessionService(): SessionService { throw new Error(`mock: sessionService.${name} not implemented`); } return { - launchSession(_params) { - return notImpl("launchSession"); + stageWorkflowStep(_params) { + return notImpl("stageWorkflowStep"); + }, + deployInstanceAtHead(_params) { + return notImpl("deployInstanceAtHead"); }, deployWorkflowDefinition(_params) { return notImpl("deployWorkflowDefinition"); }, + deploySingleStepAtHead(_params) { + return notImpl("deploySingleStepAtHead"); + }, sendUserMessage(_params) { return notImpl("sendUserMessage"); }, @@ -654,12 +660,18 @@ describe("POST /agents/instances/:instanceId/mail", () => { } { const captured: CapturedSendArgs[] = []; const service: SessionService = { - launchSession() { + stageWorkflowStep() { + throw new Error("not implemented"); + }, + deployInstanceAtHead() { throw new Error("not implemented"); }, deployWorkflowDefinition() { throw new Error("not implemented"); }, + deploySingleStepAtHead() { + throw new Error("not implemented"); + }, endSession() { throw new Error("not implemented"); }, @@ -802,12 +814,18 @@ describe("POST /agents/instances/:instanceId/mail attachments", () => { } { const captured: (MessageAttachment[] | undefined)[] = []; const service: SessionService = { - launchSession() { + stageWorkflowStep() { + throw new Error("not implemented"); + }, + deployInstanceAtHead() { throw new Error("not implemented"); }, deployWorkflowDefinition() { throw new Error("not implemented"); }, + deploySingleStepAtHead() { + throw new Error("not implemented"); + }, endSession() { throw new Error("not implemented"); }, @@ -1234,10 +1252,14 @@ describe("POST /agents/instances seeds creator agent-state grant", () => { function createCapturingSessionService(): SessionService { return { - launchSession: async () => undefined, + stageWorkflowStep: async () => undefined, + deployInstanceAtHead: async () => ({ publicKey: "pk-instance-mock" }), deployWorkflowDefinition: () => { throw new Error("mock: deployWorkflowDefinition not implemented"); }, + deploySingleStepAtHead: () => { + throw new Error("mock: deploySingleStepAtHead not implemented"); + }, sendUserMessage: () => { throw new Error("mock: sendUserMessage not implemented"); }, @@ -1442,18 +1464,25 @@ describe("POST /agents/instances seeds creator agent-state grant", () => { expect(JSON.stringify(await res.json())).toContain("wallet_backed"); }); - test("passes catalog-ordered sources to launchSession and persists modelPreferences", async () => { + test("passes catalog-ordered sources to deployInstanceAtHead and persists modelPreferences", async () => { const inserts: TableInsert[] = []; let launchedSources: unknown; let launchedDefaultSource: unknown; const sessionService: SessionService = { - launchSession: async (params) => { + stageWorkflowStep: () => { + throw new Error("mock: stageWorkflowStep not implemented"); + }, + deployInstanceAtHead: async (params) => { launchedSources = params.config.sources; launchedDefaultSource = params.config.defaultSource; + return { publicKey: "pk-instance-mock" }; }, deployWorkflowDefinition: () => { throw new Error("mock: deployWorkflowDefinition not implemented"); }, + deploySingleStepAtHead: () => { + throw new Error("mock: deploySingleStepAtHead not implemented"); + }, sendUserMessage: () => { throw new Error("mock: sendUserMessage not implemented"); }, diff --git a/packages/hub-api/src/routes/instances.ts b/packages/hub-api/src/routes/instances.ts index e9d0b2a5..3aa29695 100644 --- a/packages/hub-api/src/routes/instances.ts +++ b/packages/hub-api/src/routes/instances.ts @@ -72,11 +72,6 @@ const GrantRequirements = GrantRequirement.array(); // rejected here before the JSON parser allocates a giant string. const MAX_MAIL_BODY_BYTES = 44 * 1024 * 1024; -const AbortBody = type({ - "reason?": - "'user_disconnect' | 'wallet_exhaustion' | 'admin_kill' | 'session_timeout' | 'credential_revocation'", -}); - function formatInstance( row: typeof agentInstance.$inferSelect, agentName: string, @@ -517,7 +512,13 @@ export function createInstanceRoutes({ eventCollectors.create(agentAddress, tenant.id, sessionId, instanceId); try { - await sessionService.launchSession({ + // Deploy the instance as a single-step workflow at the head: it runs + // as a supervised workflow-process child. The real `agentId` (row.id) + // is passed so the child resolves the instance's skills and pinned + // tool packages. The returned head public key is surfaced + // separately via the sidecar's `agent.deploy.ack`, so the route + // discards it here. + await sessionService.deployInstanceAtHead({ agentAddress, agentId: row.id, instanceId, @@ -1226,93 +1227,6 @@ export function createInstanceRoutes({ }, ); - app.post( - "/:instanceId/abort", - requireGrant(idResource("instance", "instanceId"), "manage"), - describeRoute({ - tags: ["Instances"], - summary: "Abort current operation", - description: "Aborts the agent's current inference or tool execution.", - responses: { - 204: { - description: "Abort signal sent", - }, - 404: { - description: "Instance not found", - content: { - "application/json": { schema: resolver(ErrorResponse) }, - }, - }, - 409: { - description: "Instance not running", - content: { - "application/json": { schema: resolver(ErrorResponse) }, - }, - }, - 502: { - description: "Sidecar unavailable", - content: { - "application/json": { schema: resolver(ErrorResponse) }, - }, - }, - }, - }), - validator("json", AbortBody), - async (c) => { - const tenantCtx = c.get("tenant"); - const instanceId = c.req.param("instanceId"); - const body = c.req.valid("json"); - - const row = await db.query.agentInstance.findFirst({ - where: and( - eq(agentInstance.id, instanceId), - eq(agentInstance.tenantId, tenantCtx.id), - ), - }); - - if (!row) { - return c.json( - { error: { code: "not_found", message: "Instance not found" } }, - 404, - ); - } - - if (row.status !== "running") { - return c.json( - { - error: { - code: "conflict", - message: `Instance is not running (status: ${row.status})`, - }, - }, - 409, - ); - } - - try { - await sidecarRouter.sendSessionAbort( - row.address, - body.reason ?? "user_disconnect", - ); - } catch (err) { - return c.json( - { - error: { - code: "sidecar_unavailable", - message: - err instanceof Error - ? err.message - : "Failed to reach sidecar for abort", - }, - }, - 502, - ); - } - - return c.body(null, 204); - }, - ); - // Crypto providers for signing outbound messages, keyed by instance ID. // Evicted when an instance is stopped. The cache is per-factory call, // not per-process; two createInstanceRoutes() calls in the same process diff --git a/packages/hub-api/src/routes/workflows.test.ts b/packages/hub-api/src/routes/workflows.test.ts index 31bde7cb..95e4f040 100644 --- a/packages/hub-api/src/routes/workflows.test.ts +++ b/packages/hub-api/src/routes/workflows.test.ts @@ -35,7 +35,7 @@ const DEPLOYMENT_ID = "dep_abc"; // The sidecar's deploy router keys the workflow-run repo by the // sanitized deployment address, NOT the bare deployment id (see -// `deriveTrivialDeploymentId` -> `deriveWorkflowRunRepoId` in +// `deriveDeploymentId` -> `deriveWorkflowRunRepoId` in // apps/sidecar/src/workflow-host-wiring.ts). The read routes must // reconstruct the same id from `(deploymentId, tenantDomain)`; the // run-observe tests build their on-disk repo under this derived id so a @@ -196,11 +196,11 @@ function createMockSidecarRouter( }, sendAgentDeploy: () => notImpl("sendAgentDeploy"), sendAgentUndeploy: () => notImpl("sendAgentUndeploy"), - sendSessionStart: () => notImpl("sendSessionStart"), - sendSessionAbort: () => notImpl("sendSessionAbort"), - sendGrantsUpdate: () => notImpl("sendGrantsUpdate"), sendSourcesUpdate: () => notImpl("sendSourcesUpdate"), sendPack: () => notImpl("sendPack"), + sendProvisionStep: () => notImpl("sendProvisionStep"), + bindStepRoute: () => notImpl("bindStepRoute"), + unbindStepRoute: () => notImpl("unbindStepRoute"), sendSyncRequest: () => notImpl("sendSyncRequest"), sendSignalDeliver: (opts) => { signalCalls.push(opts); @@ -223,7 +223,8 @@ function createMockSessionService( throw new Error(`mock: sessionService.${name} not implemented`); } return { - launchSession: () => notImpl("launchSession"), + stageWorkflowStep: () => notImpl("stageWorkflowStep"), + deployInstanceAtHead: () => notImpl("deployInstanceAtHead"), deployWorkflowDefinition: (params) => { deployCalls.push(params); if (result === undefined) { @@ -231,6 +232,7 @@ function createMockSessionService( } return Promise.resolve(result); }, + deploySingleStepAtHead: () => notImpl("deploySingleStepAtHead"), sendUserMessage: () => notImpl("sendUserMessage"), endSession: () => notImpl("endSession"), }; @@ -267,6 +269,7 @@ function createStubRepoStore(repoDirById?: Map): RepoStore { initRepo: unused, writeTree: unused, writeTreePreservingPrefix: unused, + writeTreeDelta: unused, receivePack: unused, createPack: unused, resolveRef: unused, diff --git a/packages/hub-sessions/README.md b/packages/hub-sessions/README.md index 1bca7946..dcd0734f 100644 --- a/packages/hub-sessions/README.md +++ b/packages/hub-sessions/README.md @@ -16,7 +16,7 @@ feed agent events back to the HTTP layer for observability. `assetService` paired with a `db` handle for the asset manifest inserts. `createHubSessionOrchestrator` takes a `HubSessionOrchestratorDeps` of `events`, `router`, `db`, -`eventCollectors`, `grantStore`, and `agentRepoStore`. See the +`eventCollectors`, and `agentRepoStore`. See the exported types in `src/session-service.ts` and `src/hub-session-orchestrator.ts` for the authoritative shapes; `@intx/hub-api` is the in-tree consumer that wires these @@ -37,8 +37,8 @@ for the authoritative type shapes. provisions and tears down agent sessions on the connected sidecar. - `createHubSessionOrchestrator` / `HubSessionOrchestrator`, `HubSessionOrchestratorDeps`, `HubSessionRouterFacade` — the - higher-level orchestrator that wires the router, event collectors, - and grant store together. + higher-level orchestrator that wires the router and event + collectors together. - `createHubSessionLookups` / `HubSessionLookupsDeps` — builds the lookup callbacks the sidecar router needs to resolve sessions. diff --git a/packages/hub-sessions/src/hub-session-lookups.ts b/packages/hub-sessions/src/hub-session-lookups.ts index e87f0297..acfb9f8c 100644 --- a/packages/hub-sessions/src/hub-session-lookups.ts +++ b/packages/hub-sessions/src/hub-session-lookups.ts @@ -8,9 +8,14 @@ import { eq, and, isNull } from "drizzle-orm"; import type { DB } from "@intx/db"; -import { agentInstance, sessionMail } from "@intx/db/schema"; +import { + agentInstance, + sessionMail, + workflowDeployment, +} from "@intx/db/schema"; import { getLogger } from "@intx/log"; import { parseAgentAddress } from "@intx/types"; +import { isWorkflowDerivedAddress } from "@intx/workflow-deploy"; import type { AgentRepoStore } from "./agent-repo"; import { generateId } from "@intx/hub-common"; @@ -30,6 +35,33 @@ export function createHubSessionLookups( return { async lookupPublicKey(agentAddress) { + // Route by address space, not a blind two-table fallback: a + // workflow-derived address's key lives on its workflow_deployment + // row, a launched agent's on its agent_instance row, and the two + // spaces are disjoint. Routing (rather than falling back) means a + // launched agent that is missing its instance row returns null and + // fails its challenge visibly, instead of silently resolving against + // the wrong table. + if (isWorkflowDerivedAddress(agentAddress)) { + // Filter to a live ("deployed") deployment so a torn-down + // deployment's key can no longer satisfy a challenge. A null + // publicKey (deployed but not yet acked, or pre-migration) or an + // absent row returns null -- the challenge fails closed and the + // address stays unrouted rather than routing without ownership + // proof. + const row = await db + .select({ publicKey: workflowDeployment.publicKey }) + .from(workflowDeployment) + .where( + and( + eq(workflowDeployment.address, agentAddress), + eq(workflowDeployment.status, "deployed"), + ), + ) + .limit(1) + .then((rows) => rows[0]); + return row?.publicKey ?? null; + } const row = await db .select({ publicKey: agentInstance.publicKey }) .from(agentInstance) @@ -41,10 +73,7 @@ export function createHubSessionLookups( ) .limit(1) .then((rows) => rows[0]); - if (!row) { - return null; - } - return row.publicKey; + return row?.publicKey ?? null; }, async lookupDeployRef(agentAddress) { diff --git a/packages/hub-sessions/src/hub-session-orchestrator.test.ts b/packages/hub-sessions/src/hub-session-orchestrator.test.ts index e838bb68..3538bfdb 100644 --- a/packages/hub-sessions/src/hub-session-orchestrator.test.ts +++ b/packages/hub-sessions/src/hub-session-orchestrator.test.ts @@ -1,8 +1,7 @@ import { describe, test, expect, beforeEach } from "bun:test"; import type { DB } from "@intx/db"; -import type { GrantRule, GrantStore } from "@intx/types/authz"; -import type { InferenceEvent, InferenceSource } from "@intx/types/runtime"; +import type { InferenceEvent } from "@intx/types/runtime"; import type { AgentRepoStore, DeployContent } from "./agent-repo"; import type { RepoStore } from "./repo-store"; @@ -119,13 +118,6 @@ function createMockDB(opts: MockDBOpts) { } type RouterCall = - | { kind: "sendGrantsUpdate"; addr: string; grants: GrantRule[] } - | { - kind: "sendSourcesUpdate"; - addr: string; - sources: InferenceSource[]; - defaultSource: string; - } | { kind: "sendPack"; addr: string; @@ -143,12 +135,6 @@ function createRouterFacade(): { return { calls, facade: { - async sendGrantsUpdate(addr, grants) { - calls.push({ kind: "sendGrantsUpdate", addr, grants }); - }, - async sendSourcesUpdate(addr, sources, defaultSource) { - calls.push({ kind: "sendSourcesUpdate", addr, sources, defaultSource }); - }, async sendPack(addr, pack, ref, sha) { calls.push({ kind: "sendPack", addr, pack, ref, sha }); }, @@ -197,12 +183,6 @@ function createCollectorRegistry( }; } -function createGrantStoreStub(grants: GrantRule[] = []): GrantStore { - return { - collectGrants: async () => grants, - }; -} - type RepoCall = { kind: "createDeployPack"; agentId: string }; function createRepoStoreStub(): { @@ -251,6 +231,7 @@ function unusedRepoStore(): RepoStore { initRepo: unused, writeTree: unused, writeTreePreservingPrefix: unused, + writeTreeDelta: unused, receivePack: unused, createPack: unused, resolveRef: unused, @@ -274,21 +255,19 @@ type Harness = { dispose: () => void; }; -function setup(opts: MockDBOpts & { grants?: GrantRule[] } = {}): Harness { +function setup(opts: MockDBOpts = {}): Harness { const updates: UpdateCall[] = []; const db = createMockDB({ ...opts, recordUpdates: updates }); const events = createSidecarEmitter(); const router = createRouterFacade(); const collectors = createCollectorRegistry(); const repo = createRepoStoreStub(); - const grantStore = createGrantStoreStub(opts.grants); const orchestrator = createHubSessionOrchestrator({ events, router: router.facade, db, eventCollectors: collectors.registry, - grantStore, agentRepoStore: repo.store, }); @@ -345,7 +324,7 @@ describe("createHubSessionOrchestrator", () => { describe("sidecar.disconnect", () => { test("abandons every collector for the closed connection", () => { harness.events.emit("sidecar.disconnect", { - agentAddresses: ["a@x", "b@x", "c@x"], + ownedAddresses: ["a@x", "b@x", "c@x"], }); const abandonedAddrs = harness.collectors.calls .filter((c) => c.kind === "abandon") @@ -364,13 +343,18 @@ describe("createHubSessionOrchestrator", () => { expect(harness.updates[0]?.set).toEqual({ publicKey: "deadbeef" }); }); - test("no-ops for a derived workflow-step address with no instance row", async () => { + test("persists the public key for a workflow-derived deployment address", async () => { harness = setup({ instance: undefined }); await harness.events.emitAndAwait("agent.deploy.ack", { - agentAddress: "ins_dep_abc-step1@workflow.interchange", + agentAddress: "ins_dep_abc@workflow.interchange", publicKey: "deadbeef", }); - expect(harness.updates).toHaveLength(0); + // A workflow-derived deployment address has no agent_instance row; its + // key is persisted on the workflow_deployment projection row (keyed by + // address) so the reconnect challenge can verify it. Previously this + // no-oped, which is what left these addresses un-verifiable. + expect(harness.updates).toHaveLength(1); + expect(harness.updates[0]?.set).toEqual({ publicKey: "deadbeef" }); }); test("throws when a launched-agent instance address has no row", async () => { @@ -385,33 +369,17 @@ describe("createHubSessionOrchestrator", () => { }); describe("agent.reconnected", () => { - test("refreshes grants and skips status update when already running", async () => { - const grant: GrantRule = { - id: "grant-1", - resource: "x:y", - action: "read", - effect: "allow", - origin: "system", - conditions: null, - expiresAt: null, - roleId: null, - principalId: PRINCIPAL_ID, - }; - harness = setup({ instance: makeInstance(), grants: [grant] }); + test("skips the status update when already running", async () => { + harness = setup({ instance: makeInstance() }); await harness.events.emitAndAwait("agent.reconnected", { agentAddress: AGENT_ADDRESS, }); - const grantsCall = harness.router.calls.find( - (c) => c.kind === "sendGrantsUpdate", - ); - expect(grantsCall).toBeDefined(); - if (grantsCall?.kind === "sendGrantsUpdate") { - expect(grantsCall.grants).toEqual([grant]); - } - - // status was already "running", no update should fire + // A supervised deployment refreshes grants/sources over the + // supervisor IPC snapshot, not a reconnect wire push, so no router + // call fires here. status was already "running", so no update fires. + expect(harness.router.calls).toHaveLength(0); expect(harness.updates).toHaveLength(0); }); @@ -521,7 +489,7 @@ describe("createHubSessionOrchestrator", () => { test("removes all subscriptions so later emits are inert", () => { harness.dispose(); harness.events.emit("sidecar.disconnect", { - agentAddresses: [AGENT_ADDRESS], + ownedAddresses: [AGENT_ADDRESS], }); const abandoned = harness.collectors.calls.find( (c) => c.kind === "abandon", diff --git a/packages/hub-sessions/src/hub-session-orchestrator.ts b/packages/hub-sessions/src/hub-session-orchestrator.ts index 0bda152c..eadfb5df 100644 --- a/packages/hub-sessions/src/hub-session-orchestrator.ts +++ b/packages/hub-sessions/src/hub-session-orchestrator.ts @@ -2,9 +2,9 @@ // // Subscribes to the sidecar router's events and runs the host-side // reactions: dispatching inference events to the event collector, -// refreshing grants and credentials on reconnect, ingesting state -// packs, re-deploying stale agents, persisting mail, and forwarding -// mail.delivered notifications back to subscribers. +// restoring the event collector on reconnect, re-deploying stale +// agents, and forwarding mail.delivered notifications back to +// subscribers. // // The orchestrator depends on a narrow `HubSessionRouterFacade` rather // than the full SidecarRouter, so tests can drive subscriber behavior @@ -13,12 +13,9 @@ import { eq } from "drizzle-orm"; import { type } from "arktype"; import type { DB } from "@intx/db"; -import { agentInstance } from "@intx/db/schema"; -import { pushInstanceSourceUpdate } from "./credential-push"; +import { agentInstance, workflowDeployment } from "@intx/db/schema"; import { parseMailToEmail } from "@intx/mime"; import { parseInferenceEvent } from "@intx/types/runtime"; -import type { GrantRule, GrantStore } from "@intx/types/authz"; -import type { InferenceSource } from "@intx/types/runtime"; import { getLogger } from "@intx/log"; import { isWorkflowDerivedAddress } from "@intx/workflow-deploy"; @@ -37,12 +34,6 @@ const log = getLogger(["hub", "orchestrator"]); * narrow surface keeps tests honest and decouples the orchestrator * from the rest of the router API. */ export type HubSessionRouterFacade = { - sendGrantsUpdate(agentAddress: string, grants: GrantRule[]): Promise; - sendSourcesUpdate( - agentAddress: string, - sources: InferenceSource[], - defaultSource: string, - ): Promise; sendPack( agentAddress: string, pack: Uint8Array, @@ -57,7 +48,6 @@ export type HubSessionOrchestratorDeps = { router: HubSessionRouterFacade; db: DB["db"]; eventCollectors: EventCollectorRegistry; - grantStore: GrantStore; agentRepoStore: AgentRepoStore; }; @@ -71,8 +61,7 @@ export type HubSessionOrchestrator = { export function createHubSessionOrchestrator( deps: HubSessionOrchestratorDeps, ): HubSessionOrchestrator { - const { events, router, db, eventCollectors, grantStore, agentRepoStore } = - deps; + const { events, router, db, eventCollectors, agentRepoStore } = deps; const unsubscribers: (() => void)[] = []; @@ -91,8 +80,8 @@ export function createHubSessionOrchestrator( ); unsubscribers.push( - events.on("sidecar.disconnect", ({ agentAddresses }) => { - for (const addr of agentAddresses) { + events.on("sidecar.disconnect", ({ ownedAddresses }) => { + for (const addr of ownedAddresses) { eventCollectors.abandon(addr); } }), @@ -111,21 +100,26 @@ export function createHubSessionOrchestrator( unsubscribers.push( events.on("agent.deploy.ack", async ({ agentAddress, publicKey }) => { + // Workflow-derived addresses (the deployment-level + // `ins_@` and the per-step + // `ins_-@`) have no agent_instance row; + // their public key lives on the workflow_deployment projection row, + // keyed by address. Persist it there so the reconnect ownership + // challenge can verify the deployment address. Only the + // deployment-level address has a row, so a stray per-step ack updates + // nothing. (This was previously a no-op, which is what left + // workflow-deployment addresses un-verifiable on reconnect.) + if (isWorkflowDerivedAddress(agentAddress)) { + await db + .update(workflowDeployment) + .set({ publicKey }) + .where(eq(workflowDeployment.address, agentAddress)); + return; + } + // A launched agent has an agent_instance row; a missing one is a bug + // to surface, not to drop silently. const instance = await findInstance(db, agentAddress); if (instance === undefined) { - // Workflow agents are provisioned at derived addresses — the - // deployment-level `ins_@` and the per-step - // `ins_-@` — with no agent_instance - // row, and nothing reads back their public key. The only reader - // of `agentInstance.publicKey` is the instance-keyed reconnect - // challenge, which workflow-derived addresses never take. So a - // missing row for a workflow-derived address is a correct no-op. - // A missing row for any other address is a launched agent whose - // instance should exist; surface that rather than silently - // dropping it. - if (isWorkflowDerivedAddress(agentAddress)) { - return; - } throw new Error( `No active instance found for deploy ack on address "${agentAddress}"`, ); @@ -148,27 +142,10 @@ export function createHubSessionOrchestrator( } const sessionId = instance.sessionId; - // Refresh grants before creating any local state. If this - // fails, the address is rejected and nothing needs cleanup. - const grants = await grantStore.collectGrants( - instance.principalId, - instance.tenantId, - ); - await router.sendGrantsUpdate(agentAddress, grants); - - // Re-resolve and push sources so the agent picks up any catalog or - // credential changes that happened while the sidecar was - // disconnected. Fail-open: a stale source causes runtime 401s, not a - // security escalation, so we log rather than reject the reconnect. - try { - await pushInstanceSourceUpdate(db, router, instance); - } catch (err: unknown) { - const msg = err instanceof Error ? err.message : String(err); - log.warn( - "Failed to push sources on reconnect for {agentAddress}: {msg}", - { agentAddress, msg }, - ); - } + // A supervised deployment carries its grants and sources in the + // deploy pack and refreshes them over the supervisor's IPC + // credentials snapshot at spawn and recycle, so reconnect does not + // re-push them over the wire. const now = new Date(); if (instance.status !== "running") { diff --git a/packages/hub-sessions/src/index.ts b/packages/hub-sessions/src/index.ts index 6b48c71e..a2273a0d 100644 --- a/packages/hub-sessions/src/index.ts +++ b/packages/hub-sessions/src/index.ts @@ -6,6 +6,7 @@ export { export { createSessionService, SessionLaunchError, + bridgeOrchestratorDeployContent, type SessionService, type DeployWorkflowDefinitionParams, type DeployWorkflowDefinitionResult, diff --git a/packages/hub-sessions/src/repo-store/delta-adversarial.test.ts b/packages/hub-sessions/src/repo-store/delta-adversarial.test.ts new file mode 100644 index 00000000..9b0e8edc --- /dev/null +++ b/packages/hub-sessions/src/repo-store/delta-adversarial.test.ts @@ -0,0 +1,340 @@ +// Adversarial correctness tests for writeTreeDelta / assembleTree. The +// oracle here is isomorphic-git's INDEX-based writeTree (git.add + +// git.commit), which is genuinely independent of assembleTree — unlike +// write-tree-delta.test.ts, whose full-replace comparison also runs +// through assembleTree, so a shared bug would pass both sides. + +import { test, expect, afterAll, beforeAll } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import git from "isomorphic-git"; +import { generateKeyPair } from "@intx/crypto"; +import type { KeyPair } from "@intx/types/runtime"; +import { createRepoStore } from "./store"; +import type { AuthorizeFn, Principal, RepoId } from "./types"; + +const tempDirs: string[] = []; +let signingKey: KeyPair; + +beforeAll(async () => { + signingKey = await generateKeyPair(); +}); + +afterAll(async () => { + for (const d of tempDirs.splice(0)) { + await fs.promises + .rm(d, { recursive: true, force: true }) + .catch(() => undefined); + } +}); + +const allowAll: AuthorizeFn = () => ({ allowed: true }); +const principal: Principal = { kind: "test" }; +const REF = "refs/heads/events"; +const repoId: RepoId = { kind: "agent-state", id: "subject" }; + +function makeStore(dataDir: string) { + return createRepoStore({ + dataDir, + signingKey, + handlers: { + "agent-state": { + kind: "agent-state", + directoryPrefix: "repos-under-test", + validatePush: () => ({ ok: true }), + onRefUpdated: () => undefined, + }, + }, + authorize: allowAll, + }); +} + +async function freshStore(label: string) { + const dataDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), label)); + tempDirs.push(dataDir); + return makeStore(dataDir); +} + +async function committedTreeOid( + store: ReturnType, +): Promise { + const dir = store.getRepoDir(repoId); + const sha = await git.resolveRef({ fs, dir, ref: REF }); + const { commit } = await git.readCommit({ fs, dir, oid: sha }); + return commit.tree; +} + +// Independent oracle: materialize `files` in a scratch repo, stage each +// via git.add, and let isomorphic-git's index build the tree. This does +// not touch assembleTree at all. +async function canonicalTreeOid( + label: string, + files: Record, +): Promise { + const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), label)); + tempDirs.push(dir); + await git.init({ fs, dir, defaultBranch: "main" }); + for (const [rel, content] of Object.entries(files)) { + const full = path.join(dir, rel); + await fs.promises.mkdir(path.dirname(full), { recursive: true }); + await fs.promises.writeFile(full, content); + await git.add({ fs, dir, filepath: rel }); + } + const sha = await git.commit({ + fs, + dir, + message: "oracle", + author: { name: "o", email: "o@o" }, + }); + const { commit } = await git.readCommit({ fs, dir, oid: sha }); + return commit.tree; +} + +async function readTreePaths( + store: ReturnType, +): Promise> { + const dir = store.getRepoDir(repoId); + const sha = await git.resolveRef({ fs, dir, ref: REF }); + const { commit } = await git.readCommit({ fs, dir, oid: sha }); + const out = new Set(); + const walk = async (oid: string, prefix: string): Promise => { + const { tree } = await git.readTree({ fs, dir, oid }); + for (const e of tree) { + const p = prefix === "" ? e.path : `${prefix}/${e.path}`; + if (e.type === "tree") await walk(e.oid, p); + else out.add(p); + } + }; + await walk(commit.tree, ""); + return out; +} + +async function seed( + store: ReturnType, + base: Record, +) { + await store.initRepo(repoId); + await store.writeTree(principal, repoId, REF, { + files: base, + message: "seed", + }); +} + +test("dequeue-shaped delta removes inbox entry and adds processing, OID == canonical git", async () => { + const A = "addresses/a/"; + const base = { + [`${A}inbox/10-m3.json`]: `{"messageId":"m3"}`, + [`${A}consumed/m1.json`]: `{"receivedAt":1}`, + [`${A}consumed/m2.json`]: `{"receivedAt":2}`, + [`${A}watermark.json`]: `{"watermark":0}`, + }; + const store = await freshStore("adv-dequeue-"); + await seed(store, base); + await store.writeTreeDelta(principal, repoId, REF, { + changedPathPrefixes: new Set([A]), + message: "dequeue", + computeDelta: async () => ({ + puts: { [`${A}processing/10-m3.json`]: `{"messageId":"m3"}` }, + deletes: [`${A}inbox/10-m3.json`], + }), + }); + const paths = await readTreePaths(store); + expect(paths.has(`${A}inbox/10-m3.json`)).toBe(false); // removed, not carried + expect(paths.has(`${A}processing/10-m3.json`)).toBe(true); + expect(paths.has(`${A}consumed/m1.json`)).toBe(true); + expect(paths.has(`${A}consumed/m2.json`)).toBe(true); + + const expected = { + [`${A}processing/10-m3.json`]: `{"messageId":"m3"}`, + [`${A}consumed/m1.json`]: `{"receivedAt":1}`, + [`${A}consumed/m2.json`]: `{"receivedAt":2}`, + [`${A}watermark.json`]: `{"watermark":0}`, + }; + expect(await committedTreeOid(store)).toBe( + await canonicalTreeOid("adv-dequeue-oracle-", expected), + ); +}); + +test("cascade delete empties processing/ and the whole address subtree, OID == canonical git", async () => { + const A = "addresses/lonely/"; + const other = "addresses/keep/inbox/9-k1.json"; + const base = { + [`${A}processing/10-m3.json`]: `{"messageId":"m3"}`, + [other]: `{"messageId":"k1"}`, + }; + const store = await freshStore("adv-cascade-"); + await seed(store, base); + await store.writeTreeDelta(principal, repoId, REF, { + changedPathPrefixes: new Set([A]), + message: "prune-processing", + computeDelta: async () => ({ + puts: {}, + deletes: [`${A}processing/10-m3.json`], + }), + }); + const paths = await readTreePaths(store); + expect([...paths].some((p) => p.startsWith(A))).toBe(false); + expect(paths.has(other)).toBe(true); + + const expected = { [other]: `{"messageId":"k1"}` }; + expect(await committedTreeOid(store)).toBe( + await canonicalTreeOid("adv-cascade-oracle-", expected), + ); +}); + +test("put and delete at same exact path is rejected (delta_ambiguous)", async () => { + const A = "addresses/a/"; + const base = { + [`${A}inbox/10-m3.json`]: `{"old":true}`, + [`${A}consumed/m1.json`]: `{"receivedAt":1}`, + }; + const store = await freshStore("adv-h2a-"); + await seed(store, base); + await expect( + store.writeTreeDelta(principal, repoId, REF, { + changedPathPrefixes: new Set([A]), + message: "collide", + computeDelta: async () => ({ + puts: { [`${A}inbox/10-m3.json`]: `{"new":true}` }, + deletes: [`${A}inbox/10-m3.json`], + }), + }), + ).rejects.toThrow(/delta_ambiguous/); +}); + +test("put under a deleted subtree is rejected (delta_ambiguous)", async () => { + const A = "addresses/a/"; + const base = { + [`${A}inbox/10-m3.json`]: `{"old":true}`, + [`${A}inbox/11-m4.json`]: `{"sibling":true}`, + [`${A}consumed/m1.json`]: `{"receivedAt":1}`, + }; + const store = await freshStore("adv-h2b-"); + await seed(store, base); + await expect( + store.writeTreeDelta(principal, repoId, REF, { + changedPathPrefixes: new Set([A]), + message: "delete-subtree-with-put", + computeDelta: async () => ({ + puts: { [`${A}inbox/99-m9.json`]: `{"new":true}` }, + deletes: [`${A}inbox/`], + }), + }), + ).rejects.toThrow(/delta_ambiguous/); +}); + +test("cascade prune emptying dir and parent == canonical git", async () => { + const A = "addresses/solo/"; + const base = { + [`${A}consumed/m1.json`]: `{"receivedAt":1}`, + "addresses/other/consumed/m9.json": `{"receivedAt":9}`, + }; + const store = await freshStore("adv-cascade2-"); + await seed(store, base); + await store.writeTreeDelta(principal, repoId, REF, { + changedPathPrefixes: new Set([A]), + message: "prune-all", + computeDelta: async () => ({ + puts: {}, + deletes: [`${A}consumed/m1.json`], + }), + }); + const expected = { "addresses/other/consumed/m9.json": `{"receivedAt":9}` }; + expect(await committedTreeOid(store)).toBe( + await canonicalTreeOid("adv-cascade2-oracle-", expected), + ); +}); + +test("no-slash delete of a directory is rejected (delete_type_mismatch)", async () => { + const base = { + "foo/a.json": `{"in":"dir"}`, + "keep.json": `{"keep":true}`, + }; + const store = await freshStore("adv-del-dir-"); + await seed(store, base); + await expect( + store.writeTreeDelta(principal, repoId, REF, { + changedPathPrefixes: new Set(["foo"]), + message: "delete-dir-as-file", + computeDelta: async () => ({ + puts: {}, + deletes: ["foo"], + }), + }), + ).rejects.toThrow(/delete_type_mismatch/); +}); + +test("trailing-slash delete descending into a base blob is rejected (delete_type_mismatch)", async () => { + const base = { + foo: `{"is":"blob"}`, + "keep.json": `{"keep":true}`, + }; + const store = await freshStore("adv-del-blob-"); + await seed(store, base); + await expect( + store.writeTreeDelta(principal, repoId, REF, { + changedPathPrefixes: new Set(["foo/bar/"]), + message: "delete-blob-as-subtree", + computeDelta: async () => ({ + puts: {}, + deletes: ["foo/bar/"], + }), + }), + ).rejects.toThrow(/delete_type_mismatch/); +}); + +test("put descending into a base blob is not mislabeled a delete mismatch", async () => { + const base = { + foo: `{"is":"blob"}`, + "keep.json": `{"keep":true}`, + }; + const store = await freshStore("adv-file2dir-"); + await seed(store, base); + const run = store.writeTreeDelta(principal, repoId, REF, { + changedPathPrefixes: new Set(["foo/"]), + message: "put-under-base-blob", + computeDelta: async () => ({ + puts: { "foo/bar.json": `{"now":"dir"}` }, + deletes: [], + }), + }); + // The carve-out holds: a put-driven descent into a base blob is a + // file-to-directory replacement, not a delete against the wrong base + // type, so it must NOT raise delete_type_mismatch. The store cannot + // complete the swap end-to-end today -- working-tree materialization + // mkdir's over the base file and EEXISTs -- a separate, unreachable + // limitation. The load-bearing assertion is the absence of + // delete_type_mismatch; the EEXIST pins where the store gives out. + await expect(run).rejects.toThrow(); + await run.catch((e: unknown) => { + expect(String(e)).not.toContain("delete_type_mismatch"); + expect(String(e)).toContain("EEXIST"); + }); +}); + +test("trailing-slash delete clearing a base directory still succeeds, OID == canonical git", async () => { + const base = { + "deploy/x.json": `{"x":1}`, + "deploy/y.json": `{"y":2}`, + "keep.json": `{"keep":true}`, + }; + const store = await freshStore("adv-clear-dir-"); + await seed(store, base); + await store.writeTreeDelta(principal, repoId, REF, { + changedPathPrefixes: new Set(["deploy/"]), + message: "clear-deploy", + computeDelta: async () => ({ + puts: {}, + deletes: ["deploy/"], + }), + }); + const paths = await readTreePaths(store); + expect([...paths].some((p) => p.startsWith("deploy/"))).toBe(false); + expect(paths.has("keep.json")).toBe(true); + + const expected = { "keep.json": `{"keep":true}` }; + expect(await committedTreeOid(store)).toBe( + await canonicalTreeOid("adv-clear-dir-oracle-", expected), + ); +}); diff --git a/packages/hub-sessions/src/repo-store/guard-probes.test.ts b/packages/hub-sessions/src/repo-store/guard-probes.test.ts new file mode 100644 index 00000000..a845635b --- /dev/null +++ b/packages/hub-sessions/src/repo-store/guard-probes.test.ts @@ -0,0 +1,103 @@ +// Recursion coverage for the delete_type_mismatch guards in assembleTree. +// delta-adversarial.test.ts exercises the two mismatch shapes only at the +// tree root; these two cover paths it misses: +// +// 1. A NESTED no-slash delete of a directory -- proves the guard is +// carried through assembleTree's recursion and the thrown path +// string reflects the full nested prefix, not just the leaf name. +// 2. A single-trailing-slash delete naming a base BLOB directly +// ("foo/"), as opposed to the two-level descent into a blob +// ("foo/bar/"). This is the minimal trailing-slash-over-blob shape +// and hits the child-extraction branch at the root. + +import { test, expect, beforeAll, afterAll } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { generateKeyPair } from "@intx/crypto"; +import type { KeyPair } from "@intx/types/runtime"; +import { createRepoStore } from "./store"; +import type { AuthorizeFn, Principal, RepoId } from "./types"; + +const tempDirs: string[] = []; +let signingKey: KeyPair; +const allowAll: AuthorizeFn = () => ({ allowed: true }); +const principal: Principal = { kind: "test" }; +const REF = "refs/heads/events"; +const repoId: RepoId = { kind: "agent-state", id: "subject" }; + +beforeAll(async () => { + signingKey = await generateKeyPair(); +}); +afterAll(async () => { + for (const d of tempDirs.splice(0)) { + await fs.promises + .rm(d, { recursive: true, force: true }) + .catch(() => undefined); + } +}); + +function makeStore(dataDir: string) { + return createRepoStore({ + dataDir, + signingKey, + handlers: { + "agent-state": { + kind: "agent-state", + directoryPrefix: "repos-under-test", + validatePush: () => ({ ok: true }), + onRefUpdated: () => undefined, + }, + }, + authorize: allowAll, + }); +} +async function freshStore(label: string) { + const dataDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), label)); + tempDirs.push(dataDir); + return makeStore(dataDir); +} +async function seed( + store: ReturnType, + base: Record, +) { + await store.initRepo(repoId); + await store.writeTree(principal, repoId, REF, { + files: base, + message: "seed", + }); +} + +test("nested no-slash delete of a directory is rejected with the full path", async () => { + const base = { + "top/sub/a.json": `{"a":1}`, + "top/keep.json": `{"k":1}`, + }; + const store = await freshStore("guard-nested-deldir-"); + await seed(store, base); + let caught: unknown; + await store + .writeTreeDelta(principal, repoId, REF, { + changedPathPrefixes: new Set(["top/"]), + message: "delete-nested-dir-as-file", + computeDelta: async () => ({ puts: {}, deletes: ["top/sub"] }), + }) + .catch((e: unknown) => (caught = e)); + expect(String(caught)).toContain("delete_type_mismatch"); + expect(String(caught)).toContain("top/sub"); +}); + +test("single trailing-slash delete naming a base blob directly is rejected", async () => { + const base = { foo: `{"is":"blob"}`, "keep.json": `{"keep":true}` }; + const store = await freshStore("guard-trailblob-"); + await seed(store, base); + let caught: unknown; + await store + .writeTreeDelta(principal, repoId, REF, { + changedPathPrefixes: new Set(["foo/"]), + message: "trailing-del-blob", + computeDelta: async () => ({ puts: {}, deletes: ["foo/"] }), + }) + .catch((e: unknown) => (caught = e)); + expect(String(caught)).toContain("delete_type_mismatch"); +}); diff --git a/packages/hub-sessions/src/repo-store/index.ts b/packages/hub-sessions/src/repo-store/index.ts index 89eb4c54..73aeb873 100644 --- a/packages/hub-sessions/src/repo-store/index.ts +++ b/packages/hub-sessions/src/repo-store/index.ts @@ -3,6 +3,7 @@ export type { InitRepoOpts, KindHandler, NewlyTerminalRun, + PriorDeltaReads, Principal, RefEntry, RepoAction, diff --git a/packages/hub-sessions/src/repo-store/store.ts b/packages/hub-sessions/src/repo-store/store.ts index ac9e04b7..2d50b84f 100644 --- a/packages/hub-sessions/src/repo-store/store.ts +++ b/packages/hub-sessions/src/repo-store/store.ts @@ -1,6 +1,6 @@ import fs from "node:fs"; import path from "node:path"; -import git from "isomorphic-git"; +import git, { type TreeEntry } from "isomorphic-git"; import { createSSHSignature } from "@intx/crypto"; import { initRepo as storageInitRepo, @@ -26,6 +26,7 @@ import type { RepoStoreSubscribeEvent, TreeContent, WriteResult, + WriteTreeDeltaArgs, WriteTreePreservingPrefixArgs, } from "./types"; import { SAFE_REPO_ID } from "./types"; @@ -126,6 +127,64 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { const { dataDir, signingKey, handlers, authorize, signingCallback, gc } = config; + // Per-repo-directory isomorphic-git memoization cache. The store + // threads that dir's cache through every index-touching and + // object-reading git.* call, so isomorphic-git reuses the parsed + // on-disk index across the repo's serialized write sequence instead + // of re-reading and re-parsing it on every git.add / remove / + // updateIndex / commit / listFiles, and reuses parsed packfile + // indexes across object reads. The index-free tree assembly + // (git.writeTree / git.writeBlob) uses no index and threads no cache. + // The store is the single writer under withRepoLock, so a long-lived + // per-repo cache never races a concurrent mutator. + // + // The cache stays a pure accelerator, never a second source of truth, + // because the on-disk repo remains authoritative on every axis: + // isomorphic-git persists the index to disk on each dirty mutation and + // re-reads it on a stat mismatch; the object cache is OID-keyed + // (content-addressed, so it is never stale for a committed object); + // object reads enumerate pack files from disk on every call (so a GC + // repack's now-pruned packs are simply never consulted again); and + // refs are not cached at all — resolveRef and friends take no cache — + // so the cache can never serve a stale ref tip. Those same properties + // make it safe to drop a dir's cache at any instant: the next call + // just re-reads from disk. + // + // Two bounds keep a long-lived store from retaining parsed packfiles + // (and their pack bytes) without limit: a dir's cache is rebuilt once + // it has served GIT_CACHE_MAX_OPS calls, and the store holds at most + // GIT_CACHE_MAX_REPOS dir caches, evicting the least-recently-used. + // `invalidateGitCache` additionally drops a dir's cache after an + // out-of-band mutation that bypasses it — a received pack writes + // objects and advances a ref without threading this cache — so the + // next read rebuilds from the mutated repo. + const GIT_CACHE_MAX_OPS = 8192; + const GIT_CACHE_MAX_REPOS = 256; + type RepoGitCache = { cache: object; ops: number }; + const gitCaches = new Map(); + function cacheFor(dir: string): object { + let entry = gitCaches.get(dir); + if (entry === undefined) { + entry = { cache: {}, ops: 0 }; + } else { + // Re-insert so this dir ranks most-recently-used for the LRU + // eviction below; rebuild once it has spent its op budget. + gitCaches.delete(dir); + if (entry.ops >= GIT_CACHE_MAX_OPS) entry = { cache: {}, ops: 0 }; + } + entry.ops += 1; + gitCaches.set(dir, entry); + while (gitCaches.size > GIT_CACHE_MAX_REPOS) { + const lru = gitCaches.keys().next().value; + if (lru === undefined) break; + gitCaches.delete(lru); + } + return entry.cache; + } + function invalidateGitCache(dir: string): void { + gitCaches.delete(dir); + } + // Per-(repoId, ref) seq cache. Value is the seq of the ref's // current tip; the next commit on the ref gets `cached + 1`. The // cache is populated lazily — on each ref update we either bump @@ -139,14 +198,8 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { // subscribers do not interfere with each other. const subscribers = new Map>(); - // Per-repoId cache of "the ref whose tree currently matches the - // on-disk index". When the next writeTreeUnderLock call targets - // the same `(repoId, ref)` pair as the last successful commit, the - // index already mirrors that ref's tip and the resetIndexToRef - // pass is a no-op — skip it. The cache is invalidated on any - // commit, ref update, or pack receive that advances or replaces - // the ref. - const indexRefCache = new Map(); + // Repo-scoped cache key shared by the per-repoId caches below + // (existing-commit set, and any future per-repo bookkeeping). function indexCacheKey(repoId: RepoId): string { return `${repoId.kind}/${repoId.id}`; } @@ -202,7 +255,7 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { // tip's seq is `count - 1`). async function countCommits(dir: string, ref: string): Promise { try { - const entries = await git.log({ fs, dir, ref }); + const entries = await git.log({ fs, dir, cache: cacheFor(dir), ref }); return entries.length; } catch (err) { if (hasCode(err) && err.code === "NotFoundError") return 0; @@ -220,7 +273,7 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { ): Promise { let entries: Awaited>; try { - entries = await git.log({ fs, dir, ref }); + entries = await git.log({ fs, dir, cache: cacheFor(dir), ref }); } catch (err) { if (hasCode(err) && err.code === "NotFoundError") return []; throw err; @@ -354,6 +407,81 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { } } + // Validate a delta write path. A put is always a file (no trailing + // slash); a delete is either an exact file or a subtree prefix + // (trailing slash allowed). Both reject empties, absolute paths, and + // any `..` traversal segment. + function validateDeltaPath(p: string, isDelete: boolean): void { + const malformed = + p.length === 0 || + p.startsWith("/") || + p.split("/").includes("..") || + (!isDelete && p.endsWith("/")); + if (malformed) { + throw new Error(`delta_path_invalid: ${JSON.stringify(p)}`); + } + } + + // Reject an ambiguous delta rather than silently pick a winner: a path + // that is both put and deleted, or a put that lands under a + // subtree-prefix delete. assembleTree would let the put win in both + // cases; making the contradiction loud keeps a caller from committing a + // tree that does not match its stated intent. + function assertDeltaUnambiguous( + puts: Record, + deletes: readonly string[], + ): void { + const deleteSet = new Set(deletes); + const deletePrefixes = deletes.filter((d) => d.endsWith("/")); + for (const p of Object.keys(puts)) { + if (deleteSet.has(p)) { + throw new Error( + `delta_ambiguous: path ${JSON.stringify(p)} is in both puts and deletes`, + ); + } + for (const dp of deletePrefixes) { + if (p.startsWith(dp)) { + throw new Error( + `delta_ambiguous: put ${JSON.stringify(p)} lands under deleted subtree ${JSON.stringify(dp)}`, + ); + } + } + } + } + + // When a delta declares a change scope, every put and delete must fall + // under it — otherwise a handler that scopes validation to + // `changedPathPrefixes` would skip a region the write actually mutated, + // an exactly-once hole. An undefined scope means validate-all, so there + // is nothing to under-cover. + function assertDeltaScoped( + puts: Record, + deletes: readonly string[], + changedPathPrefixes: ReadonlySet | undefined, + ): void { + if (changedPathPrefixes === undefined) return; + const covered = (p: string): boolean => { + for (const prefix of changedPathPrefixes) { + if (p === prefix || p.startsWith(prefix)) return true; + } + return false; + }; + for (const p of Object.keys(puts)) { + if (!covered(p)) { + throw new Error( + `delta_out_of_scope: put ${JSON.stringify(p)} is not under any changedPathPrefix`, + ); + } + } + for (const d of deletes) { + if (!covered(d)) { + throw new Error( + `delta_out_of_scope: delete ${JSON.stringify(d)} is not under any changedPathPrefix`, + ); + } + } + } + function storageOptsFor( repoId: RepoId, opts: InitRepoOpts | undefined, @@ -443,171 +571,31 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { } } - async function clearIndexPrefix(dir: string, prefix: string): Promise { - // `filepaths: [prefix]` scopes statusMatrix to the cleared subtree - // instead of walking every tracked file in the repo and discarding - // the non-prefix rows. The narrowed call returns status rows only - // for files under `prefix` -- exactly the set this loop then - // removes -- so the removal set is identical while the per-commit - // walk becomes O(prefix) rather than O(repo). A prefix with no - // tracked entries (a first write) yields an empty matrix and clears - // nothing. - const matrix = await git.statusMatrix({ fs, dir, filepaths: [prefix] }); - for (const row of matrix) { - const filepath = row[0]; - if (filepath.startsWith(prefix)) { - await git.remove({ fs, dir, filepath }); - } - } - } - - async function writeFileEntry( + // Walk from a tree object's root to the tree-or-blob entry at + // `relPath`. Returns `null` when any path segment is missing, or when + // the final entry does not match `expectedType`. `relPath === ""` + // resolves to the root tree itself, and only when a tree is expected. + async function resolveTreeOid( dir: string, - relPath: string, - contents: string | Uint8Array, - ): Promise { - const fullPath = path.join(dir, relPath); - await fs.promises.mkdir(path.dirname(fullPath), { recursive: true }); - await fs.promises.writeFile(fullPath, contents); - await git.add({ fs, dir, filepath: relPath }); - } - - // Read the set of paths present in the tree at the given ref so a - // post-validation rollback can distinguish genuinely-new staged paths - // (which the rollback must unlink) from paths that existed at the - // ref's tip and were merely overwritten by the staging pass (which - // the rollback must restore, not delete). - async function refPathSet(dir: string, ref: string): Promise> { - let sha: string; - try { - sha = await git.resolveRef({ fs, dir, ref }); - } catch (err) { - if (hasCode(err) && err.code === "NotFoundError") { - return new Set(); - } - throw err; - } - const entries = await git.listFiles({ fs, dir, ref: sha }); - return new Set(entries); - } - - // Undo the staged writes from a failed writeTreeUnderLock pass. For - // each newly-staged file: if the push target ref held that path, - // restore it from the ref's tip so the working tree and index match - // the ref bit-for-bit; otherwise drop the file from disk and the - // index because it had no prior presence to restore. Then re-checkout - // any prefix that the clearPrefix step removed wholesale. The - // combination leaves the working tree and index bit-identical to the - // ref for every path the failed pass touched, so a subsequent - // legitimate push does not pick up half-applied state and does not - // lose ref content that the rejected push happened to overwrite - // outside `clearedPrefix`. - async function resetWorkingTreeToRef( - dir: string, - ref: string, - args: { - stagedFiles: string[]; - clearedPrefix: string | undefined; - refPaths: ReadonlySet; - }, - ): Promise { - const toRestore: string[] = []; - for (const relPath of args.stagedFiles) { - if (args.refPaths.has(relPath)) { - toRestore.push(relPath); - continue; - } - try { - await git.remove({ fs, dir, filepath: relPath }); - } catch (err) { - if (!hasCode(err) || err.code !== "NotFoundError") throw err; - } - try { - await fs.promises.unlink(path.join(dir, relPath)); - } catch (err) { - if (!hasCode(err) || err.code !== "ENOENT") throw err; - } - } - if (toRestore.length > 0) { - // Read each ref-existing path's blob directly and rewrite the - // working-tree file plus stage it. `git.checkout` with the - // `filepaths` narrowing does not consistently re-materialize a - // working-tree file that the rollback flow has already touched - // (isomorphic-git treats the narrowed checkout as a subtree-walk - // hint and leaves the working tree untouched for paths already - // present in the index), so the rollback restores each path - // explicitly from the ref's tree. - let refSha: string | undefined; - try { - refSha = await git.resolveRef({ fs, dir, ref }); - } catch (err) { - if (!hasCode(err) || err.code !== "NotFoundError") throw err; - } - if (refSha !== undefined) { - for (const relPath of toRestore) { - let blob: Uint8Array; - try { - const result = await git.readBlob({ - fs, - dir, - oid: refSha, - filepath: relPath, - }); - blob = result.blob; - } catch (err) { - if (hasCode(err) && err.code === "NotFoundError") continue; - throw err; - } - const fullPath = path.join(dir, relPath); - await fs.promises.mkdir(path.dirname(fullPath), { recursive: true }); - await fs.promises.writeFile(fullPath, blob); - await git.add({ fs, dir, filepath: relPath }); - } - } - } - if (args.clearedPrefix !== undefined) { - // Re-checkout the prefix from HEAD so the files the clearPrefix - // step removed reappear in the working tree and the index. - // `force: true` overwrites any leftover writes from the failed - // pass; the filepaths array narrows the checkout to just the - // prefix so we do not perturb unrelated paths. - try { - await git.checkout({ - fs, - dir, - ref: "HEAD", - force: true, - filepaths: [args.clearedPrefix], - }); - } catch (err) { - if (!hasCode(err) || err.code !== "NotFoundError") throw err; - } - } - } - - // Walk from a commit's root tree to the tree-or-blob entry at - // `relPath`. Returns `null` when any path segment is missing, or - // when the final entry does not match `expectedType`. Used to back - // both `priorReadBlob` and `priorListDir` so a handler can inspect - // the parent commit's tree from inside validatePush without each - // call duplicating the tree-walk. - async function resolveTreeEntry( - dir: string, - commitSha: string, + rootTreeOid: string, relPath: string, expectedType: "blob" | "tree", ): Promise { - const { commit } = await git.readCommit({ fs, dir, oid: commitSha }); if (relPath === "") { - return expectedType === "tree" ? commit.tree : null; + return expectedType === "tree" ? rootTreeOid : null; } const segments = relPath.split("/").filter((s) => s !== ""); - let currentOid = commit.tree; + let currentOid = rootTreeOid; for (let i = 0; i < segments.length; i++) { const segment = segments[i]; if (segment === undefined) throw new Error("unreachable"); const isLast = i === segments.length - 1; - const { tree } = await git.readTree({ fs, dir, oid: currentOid }); + const { tree } = await git.readTree({ + fs, + dir, + cache: cacheFor(dir), + oid: currentOid, + }); const entry = tree.find((e) => e.path === segment); if (entry === undefined) return null; if (isLast) { @@ -620,6 +608,25 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { return currentOid; } + // Walk from a commit's root tree to the tree-or-blob entry at + // `relPath`. Used to back both `priorReadBlob` and `priorListDir` so a + // handler can inspect the parent commit's tree from inside validatePush + // without each call duplicating the tree-walk. + async function resolveTreeEntry( + dir: string, + commitSha: string, + relPath: string, + expectedType: "blob" | "tree", + ): Promise { + const { commit } = await git.readCommit({ + fs, + dir, + cache: cacheFor(dir), + oid: commitSha, + }); + return resolveTreeOid(dir, commit.tree, relPath, expectedType); + } + // Build the `(priorReadBlob, priorListDir)` pair fed into a kind // handler's validatePush. When `commitSha` is `null`, both closures // surface the "ref had no prior commit" state — readBlob returns @@ -633,11 +640,30 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { ): { priorReadBlob: (path: string) => Promise; priorListDir: (path: string) => Promise; + priorListDirOids: ( + path: string, + ) => Promise<{ name: string; oid: string }[]>; + readBlobByOid: (oid: string) => Promise; } { + // Read any blob by its object id, cache-backed. Object ids are + // content addresses, so this is independent of `commitSha` and is + // available even on the no-prior-commit branch (where callers have + // no ids to read). + const readBlobByOid = async (oid: string): Promise => { + const { blob } = await git.readBlob({ + fs, + dir, + cache: cacheFor(dir), + oid, + }); + return blob; + }; if (commitSha === null) { return { priorReadBlob: async () => null, priorListDir: async () => [], + priorListDirOids: async () => [], + readBlobByOid, }; } const priorReadBlob = async ( @@ -645,16 +671,43 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { ): Promise => { const oid = await resolveTreeEntry(dir, commitSha, relPath, "blob"); if (oid === null) return null; - const { blob } = await git.readBlob({ fs, dir, oid }); + const { blob } = await git.readBlob({ + fs, + dir, + cache: cacheFor(dir), + oid, + }); return blob; }; const priorListDir = async (relPath: string): Promise => { const oid = await resolveTreeEntry(dir, commitSha, relPath, "tree"); if (oid === null) return []; - const { tree } = await git.readTree({ fs, dir, oid }); + const { tree } = await git.readTree({ + fs, + dir, + cache: cacheFor(dir), + oid, + }); return tree.map((e) => e.path); }; - return { priorReadBlob, priorListDir }; + // Same walk as `priorListDir` but carries each child's git object id + // out of the tree listing. A kind handler validating a large subtree + // by its per-commit delta uses the OID to prove a retained entry is + // byte-unchanged without re-reading the blob. + const priorListDirOids = async ( + relPath: string, + ): Promise<{ name: string; oid: string }[]> => { + const oid = await resolveTreeEntry(dir, commitSha, relPath, "tree"); + if (oid === null) return []; + const { tree } = await git.readTree({ + fs, + dir, + cache: cacheFor(dir), + oid, + }); + return tree.map((e) => ({ name: e.path, oid: e.oid })); + }; + return { priorReadBlob, priorListDir, priorListDirOids, readBlobByOid }; } // Build the `(readBlob, listDir, topLevelTreePaths)` triple a kind @@ -670,10 +723,16 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { readBlob: (path: string) => Promise; listDir: (path: string) => Promise; }> { - const { commit } = await git.readCommit({ fs, dir, oid: commitSha }); + const { commit } = await git.readCommit({ + fs, + dir, + cache: cacheFor(dir), + oid: commitSha, + }); const { tree: rootTree } = await git.readTree({ fs, dir, + cache: cacheFor(dir), oid: commit.tree, }); const topLevelTreePaths = rootTree.map((e) => e.path); @@ -684,7 +743,12 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { `readBlob: path ${relPath} not found in commit ${commitSha} tree`, ); } - const { blob } = await git.readBlob({ fs, dir, oid }); + const { blob } = await git.readBlob({ + fs, + dir, + cache: cacheFor(dir), + oid, + }); return blob; }; const listDir = async (relPath: string): Promise => { @@ -699,7 +763,12 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { // `path_violation` reason instead of a raw substrate throw. A real // read fault inside `git.readTree` below still bubbles. if (oid === null) return []; - const { tree } = await git.readTree({ fs, dir, oid }); + const { tree } = await git.readTree({ + fs, + dir, + cache: cacheFor(dir), + oid, + }); return tree.map((e) => e.path); }; return { topLevelTreePaths, readBlob, listDir }; @@ -716,10 +785,17 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { ): Promise | null> { const oid = relPath === "" - ? (await git.readCommit({ fs, dir, oid: commitSha })).commit.tree + ? ( + await git.readCommit({ + fs, + dir, + cache: cacheFor(dir), + oid: commitSha, + }) + ).commit.tree : await resolveTreeEntry(dir, commitSha, relPath, "tree"); if (oid === null) return null; - const { tree } = await git.readTree({ fs, dir, oid }); + const { tree } = await git.readTree({ fs, dir, cache: cacheFor(dir), oid }); const out = new Map(); for (const e of tree) out.set(e.path, e.oid); return out; @@ -787,7 +863,7 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { if (out.has(oid)) continue; let parsed: Awaited>; try { - parsed = await git.readCommit({ fs, dir, oid }); + parsed = await git.readCommit({ fs, dir, cache: cacheFor(dir), oid }); } catch (err) { // A previously-received single-commit pack may leave the // tip's `parent` field pointing at a SHA the receiver has @@ -849,7 +925,12 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { if (existingCommits.has(current)) break; let parsed: Awaited>; try { - parsed = await git.readCommit({ fs, dir, oid: current }); + parsed = await git.readCommit({ + fs, + dir, + cache: cacheFor(dir), + oid: current, + }); } catch (err) { if (hasCode(err) && err.code === "NotFoundError") break; throw err; @@ -869,83 +950,270 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { return chain.reverse(); } - // Reset the index so it bit-matches the tree at `ref`'s tip - // without re-materialising the working tree. The substrate uses - // the index — not the working tree — when it builds the next - // commit's tree via `git.commit({ref})`, so re-pointing index - // entries at the ref-tree's oids is correct and substantially - // cheaper than walking disk for every path. When the ref does not - // yet exist the index is cleared so the next staging pass starts - // from an empty repo state. + // Assemble a new root tree by splicing `puts` (repo-root-relative path + // -> blob oid) and `deletes` onto the parent's root tree, reusing every + // unchanged entry by oid. The result is committed directly via + // `git.commit({ tree })`, so the on-disk index is never touched. // - // isomorphic-git's index is a single repo-global structure shared - // across refs, and `git.add` mutates it in place; without this - // reset, every commit's tree is the union of `ref`'s tree and - // whatever was last staged for a different ref. The reset reads - // `ref` (the push target) rather than HEAD because HEAD may point - // at a different branch the repo was initialised on, so seeding - // from HEAD would mis-construct trees that target a named ref. - async function resetIndexToRef(dir: string, ref: string): Promise { - let refSha: string | null; - try { - refSha = await git.resolveRef({ fs, dir, ref }); - } catch (err) { - if (hasCode(err) && err.code === "NotFoundError") { - refSha = null; - } else { - throw err; + // `deletes` is a set of repo-root-relative paths, and removal must + // match the base entry's type. A no-slash entry names an exact path + // and clears the blob there; deleting a path that is a directory in + // the base is rejected with `delete_type_mismatch` (there is no + // git-rm-style recursive drop). A trailing-slash entry names a + // subtree prefix and clears it (the clear-prefix shape); a + // trailing-slash delete whose leading segment is a base blob is + // rejected with `delete_type_mismatch`, unless a `put` drives the + // same descent -- that is a legitimate file-to-directory replacement + // and is accepted (the put wins). A `put` at a path overrides a + // delete of the same path. A delete of a path absent from the parent + // tree is an idempotent no-op (the entry is simply never emitted), + // matching `git rm --ignore-unmatch` and the working-tree `rm` with + // `force: true`. + // + // Recursion is scoped to the touched subtrees: a level is read and + // rewritten only when a put lands under it or a delete removes within + // it. Every entry the write does not touch — sibling subtrees AND + // sibling blobs inside a subtree being touched — is carried forward by + // its existing oid without a re-hash or a walk, so the per-commit cost + // tracks the size of the change rather than the repo. Returns the new + // tree oid, or `null` when the subtree ends up empty (the caller writes + // an empty root tree when the whole repo empties). + async function assembleTree( + dir: string, + baseTreeOid: string | null, + prefix: string, + puts: ReadonlyMap, + deletes: ReadonlySet, + ): Promise { + // A subtree-delete naming exactly this node drops the base wholesale; + // only puts under it survive. + const cleared = prefix !== "" && deletes.has(prefix); + const baseEntries = new Map< + string, + { mode: string; oid: string; type: TreeEntry["type"] } + >(); + if (baseTreeOid !== null && !cleared) { + const { tree } = await git.readTree({ + fs, + dir, + cache: cacheFor(dir), + oid: baseTreeOid, + }); + for (const e of tree) { + baseEntries.set(e.path, { mode: e.mode, oid: e.oid, type: e.type }); } } - const indexFiles = new Set(await git.listFiles({ fs, dir })); - if (refSha === null) { - for (const filepath of indexFiles) { - try { - await git.remove({ fs, dir, filepath }); - } catch (err) { - if (!hasCode(err) || err.code !== "NotFoundError") throw err; - } + + // Classify what changes at this level: direct blob puts, exact-file + // deletes here, and subtrees a put or a delete descends into. + const blobPutsHere = new Set(); + const subtreeNames = new Set(); + const subtreePutNames = new Set(); + const fileDeletesHere = new Set(); + for (const full of puts.keys()) { + if (prefix !== "" && !full.startsWith(prefix)) continue; + const rest = full.slice(prefix.length); + if (rest.length === 0) continue; + const slash = rest.indexOf("/"); + if (slash === -1) blobPutsHere.add(rest); + else { + const child = rest.slice(0, slash); + subtreeNames.add(child); + subtreePutNames.add(child); } - return; } - const refBlobs = await walkRefBlobs(dir, refSha); - for (const filepath of indexFiles) { - if (refBlobs.has(filepath)) continue; - try { - await git.remove({ fs, dir, filepath }); - } catch (err) { - if (!hasCode(err) || err.code !== "NotFoundError") throw err; + for (const del of deletes) { + if (prefix !== "" && !del.startsWith(prefix)) continue; + const rest = del.slice(prefix.length); + if (rest.length === 0) continue; // del === prefix, handled by `cleared` + const slash = rest.indexOf("/"); + if (slash === -1) fileDeletesHere.add(rest); + else subtreeNames.add(rest.slice(0, slash)); + } + + // A name written both as a direct blob and as a directory (a put/base + // `foo` plus a put or delete under `foo/`) is contradictory. Left + // implicit, the blob-put branch below wins and the subtree side is + // silently dropped; reject it loudly instead. This covers every + // caller — writeTree's `content.files`, writeTreePreservingPrefix's + // merge output, and writeTreeDelta's puts/deletes — since all funnel + // into this same classification. + for (const name of blobPutsHere) { + if (subtreeNames.has(name)) { + throw new Error( + `tree_name_collision: ${JSON.stringify( + prefix + name, + )} is written both as a file and as a directory`, + ); } } - for (const [filepath, oid] of refBlobs) { - await git.updateIndex({ fs, dir, filepath, oid, add: true }); + + const names = new Set([ + ...baseEntries.keys(), + ...blobPutsHere, + ...subtreeNames, + ]); + const entries: TreeEntry[] = []; + for (const name of names) { + const full = prefix + name; + const putOid = puts.get(full); + if (putOid !== undefined) { + // A put overrides whatever the base held and any delete of the + // same path. Puts are always regular files (mode 100644). If a + // put ever overwrites a base entry with a different mode (an + // executable 100755 blob, a symlink 120000, or a submodule), this + // downgrades it to 100644 — the store only ever writes 100644 + // content blobs today, so no caller hits it, but a future caller + // that needs to preserve an executable bit would have to carry + // the mode through `puts` rather than assume 100644. + entries.push({ + mode: "100644", + path: name, + oid: putOid, + type: "blob", + }); + continue; + } + if (fileDeletesHere.has(name)) { + const base = baseEntries.get(name); + if (base !== undefined && base.type === "tree") { + throw new Error( + `delete_type_mismatch: ${JSON.stringify( + prefix + name, + )} is deleted as a file but is a directory in the base tree`, + ); + } + continue; // file removed + } + if (subtreeNames.has(name)) { + const base = baseEntries.get(name); + // A trailing-slash delete descending into a base blob + // contradicts the base type -- reject it. The + // `!subtreePutNames` carve-out lets a put-driven descent + // through: a `put` under `name/` is a file-to-directory + // replacement, not a delete mismatch, so it must not raise + // delete_type_mismatch. That replacement still dead-ends at + // working-tree materialization in writeTreeUnderLock (mkdir + // over the base file EEXISTs) -- a separate limitation the + // claim-check callers never reach, which must not be masked by + // mislabeling a put as a delete mismatch. + if ( + base !== undefined && + base.type === "blob" && + !subtreePutNames.has(name) + ) { + throw new Error( + `delete_type_mismatch: ${JSON.stringify( + prefix + name, + )} is deleted as a subtree but is a file in the base tree`, + ); + } + const baseChildOid = + base !== undefined && base.type === "tree" ? base.oid : null; + const childOid = await assembleTree( + dir, + baseChildOid, + `${full}/`, + puts, + deletes, + ); + if (childOid !== null) { + entries.push({ + mode: "040000", + path: name, + oid: childOid, + type: "tree", + }); + } + continue; + } + const base = baseEntries.get(name); + if (base === undefined) continue; + entries.push({ + mode: base.mode, + path: name, + oid: base.oid, + type: base.type, + }); } + if (entries.length === 0) return null; + return await git.writeTree({ fs, dir, tree: entries }); } - // Walk every blob entry reachable from a commit's tree, returning - // a (filepath -> blob oid) map. Avoids the O(n^2) cost of pairing - // `git.listFiles` with per-file `resolveTreeEntry` calls — both of - // which walk the tree top-to-bottom — by doing a single recursive - // walk and emitting blob entries as we encounter them. - async function walkRefBlobs( + // Prospective-side validatePush closures sourced from the assembled + // tree oid, so a handler sees exactly the tree the commit will carry. + // The blobs and trees the assembly wrote are unreferenced objects + // until the commit lands, so reading them here advances nothing. + function buildTreeReadClosures( dir: string, - commitSha: string, - ): Promise> { - const out = new Map(); - const { commit } = await git.readCommit({ fs, dir, oid: commitSha }); - const recurse = async (treeOid: string, prefix: string): Promise => { - const { tree } = await git.readTree({ fs, dir, oid: treeOid }); - for (const entry of tree) { - const childPath = - prefix === "" ? entry.path : `${prefix}/${entry.path}`; - if (entry.type === "blob") { - out.set(childPath, entry.oid); - } else if (entry.type === "tree") { - await recurse(entry.oid, childPath); - } + rootTreeOid: string, + ): { + topLevelTreePaths: () => Promise; + readBlob: (relPath: string) => Promise; + listDir: (relPath: string) => Promise; + listDirOids: (relPath: string) => Promise<{ name: string; oid: string }[]>; + } { + const topLevelTreePaths = async (): Promise => { + const { tree } = await git.readTree({ + fs, + dir, + cache: cacheFor(dir), + oid: rootTreeOid, + }); + return tree.map((e) => e.path); + }; + const readBlob = async (relPath: string): Promise => { + const oid = await resolveTreeOid(dir, rootTreeOid, relPath, "blob"); + if (oid === null) { + throw new Error( + `readBlob: path ${relPath} not present in prospective tree`, + ); } + const { blob } = await git.readBlob({ + fs, + dir, + cache: cacheFor(dir), + oid, + }); + return blob; }; - await recurse(commit.tree, ""); - return out; + const listDir = async (relPath: string): Promise => { + const oid = + relPath === "" + ? rootTreeOid + : await resolveTreeOid(dir, rootTreeOid, relPath, "tree"); + if (oid === null) return []; + const { tree } = await git.readTree({ + fs, + dir, + cache: cacheFor(dir), + oid, + }); + return tree.map((e) => e.path); + }; + // Same walk as `listDir` but carries each child's git object id out of + // the assembled tree's listing, mirroring the prior side's + // `priorListDirOids`. A handler validating a large retained subtree by + // its per-commit delta (workflow-run's consumed dedup index) reads the + // prospective OID straight from here instead of re-reading and hashing + // every retained blob. + const listDirOids = async ( + relPath: string, + ): Promise<{ name: string; oid: string }[]> => { + const oid = + relPath === "" + ? rootTreeOid + : await resolveTreeOid(dir, rootTreeOid, relPath, "tree"); + if (oid === null) return []; + const { tree } = await git.readTree({ + fs, + dir, + cache: cacheFor(dir), + oid, + }); + return tree.map((e) => ({ name: e.path, oid: e.oid })); + }; + return { topLevelTreePaths, readBlob, listDir, listDirOids }; } // Unlocked body of writeTree. The caller is responsible for @@ -957,196 +1225,145 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { principal: Principal, repoId: RepoId, ref: string, - content: TreeContent, + w: { + files: Record; + deletes: ReadonlySet; + changedPathPrefixes: ReadonlySet | undefined; + message: string; + }, + // When present, the parent tip already resolved under this lock. The + // delta path pins it once and hands the SAME oid to both computeDelta + // (its dedup reads) and this assembly, so the dedup snapshot, the + // committed tree, and validation are provably one pre-image. Absent, + // the write resolves its own (the writeTree / preserving-prefix path, + // which has no prior read to keep consistent with). `{ sha }` wraps + // the value so a genuinely-null pin is distinct from "not provided". + pinnedParent?: { sha: string | null }, ): Promise { const dir = repoDir(repoId); await storageInitRepo(dir, storageOptsFor(repoId, undefined)); const handler = handlerFor(repoId); - // Reset the working tree and index to the target ref's tip so a - // commit to ref A does not pick up index state left behind by a - // prior commit to ref B. isomorphic-git's index is a single - // repo-global structure shared across refs, and `git.add` / - // `writeFileEntry` mutate it in place; without this reset, every - // commit's tree is the union of `ref`'s tree and whatever was last - // staged for a different ref. The reset reads `ref` (the push - // target) rather than HEAD: writes target a named ref, and HEAD - // may point at a different branch the repo was initialized on, so - // checking out HEAD would seed the new commit from the wrong tip. - // - // Skip the reset when the index cache says the on-disk index - // already mirrors `ref` — i.e. the previous commit on this repo - // also targeted this ref. The cache is invalidated on every - // successful commit so a write to a different ref always pays the - // reset cost on the next return. - const indexKey = indexCacheKey(repoId); - if (indexRefCache.get(indexKey) !== ref) { - await resetIndexToRef(dir, ref); - indexRefCache.set(indexKey, ref); - } - - // Snapshot the target ref's path set before the clearPrefix + - // writeFileEntry pass mutates the index. A post-validation rollback - // uses this set to decide whether each staged path needs to be - // restored from the ref's tip (it existed there and was - // overwritten) or unlinked (it had no prior presence). Captured - // here, not inside resetWorkingTreeToHead, because the rollback - // runs after the index has been mutated. The snapshot reads `ref` - // (the push target) rather than HEAD: writes target a named ref - // and HEAD may point at a different branch the repo was - // initialized on, so reading HEAD would mis-classify every staged - // path as new and fall back to the unlink path. - const refPaths = await refPathSet(dir, ref); - - if (content.clearPrefix !== undefined) { - validateClearPrefix(content.clearPrefix); - await clearIndexPrefix(dir, content.clearPrefix); - await fs.promises.rm(path.join(dir, content.clearPrefix), { - recursive: true, - force: true, + // Assemble the commit's tree directly and commit that tree oid, + // never staging into the on-disk index. The index is a single + // repo-global structure shared across refs; routing writes through + // it forced a full index rebuild on every events<->workflow-run ref + // flip and re-serialized the whole index once per staged blob. + // Splicing the tree from the parent's root tree — applying `w.files` + // as puts and `w.deletes` as removals, reusing every untouched entry + // by oid — keeps the per-commit cost tracking the change rather than + // the accumulated history. + + // Pin the parent under the lock: the new commit's parent is the + // ref's tip, and the splice runs against THAT commit's root tree, + // read under the same lock, so the pre-image is race-free. A ref + // that does not yet exist has no base tree (the splice starts from + // empty) and parents on HEAD, matching the prior index-reset path + // which seeded an empty index for a missing ref. + const parentCommitSha = + pinnedParent !== undefined + ? pinnedParent.sha + : await resolveRefSha(dir, ref); + let baseRootTreeOid: string | null = null; + if (parentCommitSha !== null) { + const { commit } = await git.readCommit({ + fs, + dir, + cache: cacheFor(dir), + oid: parentCommitSha, }); + baseRootTreeOid = commit.tree; } - for (const [relPath, contents] of Object.entries(content.files)) { - await writeFileEntry(dir, relPath, contents); + // Write the put blobs, then splice them and `w.deletes` onto the + // parent root tree. writeBlob and writeTree emit unreferenced + // objects; nothing the ref can reach moves until the commit lands. + const puts = new Map(); + for (const [relPath, contents] of Object.entries(w.files)) { + const bytes = + typeof contents === "string" + ? new TextEncoder().encode(contents) + : contents; + const oid = await git.writeBlob({ fs, dir, blob: bytes }); + puts.set(relPath, oid); } - - // The prior tree is whatever the ref points at the moment - // validatePush runs — i.e. the parent of the commit we are about - // to produce. `refPaths` above was snapshotted from the same - // commit, so both reads observe the same pre-image. - const priorCommitSha = await resolveRefSha(dir, ref); - const { priorReadBlob, priorListDir } = buildPriorTreeClosures( + const assembled = await assembleTree( dir, - priorCommitSha, - ); - - // The prospective tree is the actual commit `git.commit` will - // produce: the prior tree, minus paths cleared by `clearPrefix`, - // unioned with paths newly staged via `writeFileEntry`. The - // closures handed to validatePush must reflect that union — not - // just `content.files` — so a kind handler that walks the prior - // tree (workflow-run's append-only event check) can see every - // path the commit will carry. Building the closures from - // `content.files` alone would advertise only the writer's - // explicitly-staged paths, hiding prior-tree paths the writer - // did not touch and tripping prior-vs-prospective comparisons - // that the substrate does not actually delete. - const clearedPrefix = content.clearPrefix; - const survivingPriorPaths: string[] = []; - for (const p of refPaths) { - if (clearedPrefix !== undefined && p.startsWith(clearedPrefix)) continue; - if (Object.prototype.hasOwnProperty.call(content.files, p)) continue; - survivingPriorPaths.push(p); - } - const prospectivePaths = new Set([ - ...Object.keys(content.files), - ...survivingPriorPaths, - ]); - const survivingPriorPathSet = new Set(survivingPriorPaths); - const topLevelTreePaths = Array.from( - new Set( - Array.from(prospectivePaths, (p) => { - const slash = p.indexOf("/"); - return slash === -1 ? p : p.substring(0, slash); - }), - ), + baseRootTreeOid, + "", + puts, + w.deletes, ); - const readBlob = async (relPath: string): Promise => { - const entry = content.files[relPath]; - if (entry !== undefined) { - if (typeof entry === "string") { - return new TextEncoder().encode(entry); - } - return entry; - } - if (survivingPriorPathSet.has(relPath)) { - const blob = await priorReadBlob(relPath); - if (blob === null) { - throw new Error( - `readBlob: path ${relPath} listed in prior tree but unreadable from prior commit ${String(priorCommitSha)}`, - ); - } - return blob; - } - throw new Error( - `readBlob: path ${relPath} not present in prospective tree`, - ); - }; - const listDir = async (dirPath: string): Promise => { - const prefix = dirPath === "" ? "" : `${dirPath}/`; - const names = new Set(); - for (const p of prospectivePaths) { - if (prefix !== "" && !p.startsWith(prefix)) continue; - const rest = p.slice(prefix.length); - if (rest.length === 0) continue; - const slash = rest.indexOf("/"); - names.add(slash === -1 ? rest : rest.substring(0, slash)); - } - return Array.from(names); - }; - // A writeTree carrying a `clearPrefix` mutates only paths under that - // prefix: the clearPrefix step removes the prefix's prior entries and - // `writeFileEntry` restages the writer's files, while every other - // prior path is carried forward verbatim in `survivingPriorPaths` - // above. So the change set the handler may scope to is exactly - // `{clearPrefix}`. Without a clearPrefix the write is purely additive - // over an arbitrary set of paths the substrate does not summarise, so - // leave the change set unbounded (validate-all). - const changedPathPrefixes = - clearedPrefix === undefined ? undefined : new Set([clearedPrefix]); + const newRootTreeOid = + assembled ?? (await git.writeTree({ fs, dir, tree: [] })); + + // validatePush sees the full prospective tree via closures over the + // assembled tree oid; the prior-side closures read the parent commit + // (`parentCommitSha`) exactly as before. `w.changedPathPrefixes` is + // the handler's scoping hint — the prefixes this write may have + // touched — and stays undefined for an unbounded (validate-all) write. + const { priorReadBlob, priorListDir, priorListDirOids } = + buildPriorTreeClosures(dir, parentCommitSha); + const prospective = buildTreeReadClosures(dir, newRootTreeOid); + const changedPathPrefixes = w.changedPathPrefixes; const validation = await handler.validatePush({ repoId, ref, principal, - topLevelTreePaths, - readBlob, - listDir, + topLevelTreePaths: await prospective.topLevelTreePaths(), + readBlob: prospective.readBlob, + listDir: prospective.listDir, + listDirOids: prospective.listDirOids, priorReadBlob, priorListDir, + priorListDirOids, changedPathPrefixes, }); if (!validation.ok) { - // Roll the working tree and index back to HEAD before - // surfacing the validation refusal. Otherwise the staged - // writeFileEntry / clearPrefix steps above leave files and - // index entries on disk that the next legitimate push would - // either include silently (statusMatrix re-reports them) or - // trip a second validation refusal against. The reset is - // best-effort but logged loudly: a failure here means the - // repo is in a half-applied state the operator needs to fix - // by hand, which is strictly better than the silent-leak - // alternative. - try { - await resetWorkingTreeToRef(dir, ref, { - stagedFiles: Object.keys(content.files), - clearedPrefix: content.clearPrefix, - refPaths, - }); - } catch (resetErr) { - logger.warn`repo-store rollback after validation refusal failed for ${repoId.kind}/${repoId.id}; the repo is in a half-applied state and may need manual cleanup — ${resetErr instanceof Error ? resetErr.message : String(resetErr)}`; - } + // Nothing was staged and no ref advanced: the assembly wrote only + // unreferenced blob and tree objects, which the next GC reclaims. + // There is no half-applied index or working-tree state to roll + // back, so the refusal just surfaces. throw new Error(`path_violation: ${validation.reason}`); } - // Resolve the previous ref value for the post-update hook (precise: - // null only when the ref truly doesn't exist) and the parent SHA for - // the new commit (best-effort: any error falls back to HEAD, so a - // first write on a never-touched ref produces a commit parented on - // the repo's initial commit instead of failing). - const oldSha = await resolveRefSha(dir, ref); - let parentSha: string; - try { - parentSha = await git.resolveRef({ fs, dir, ref }); - } catch { - parentSha = await git.resolveRef({ fs, dir, ref: "HEAD" }); + // Materialize the working tree for the paths this write touched: + // remove each deleted path, then write each put file. The store's own + // reads resolve through the object store, but some consumers (the + // workflow-run claim-check processing scan) read these files straight + // from disk, so the working tree must mirror the committed change. + // Only the deleted paths and the put paths are touched, so this is + // O(change), not O(repo). It runs only after validation passes, so a + // rejected push leaves the working tree untouched — the failure + // atomicity the old index rollback provided, now without any + // rollback. `rm` with `force` no-ops a missing path and `recursive` + // covers both a file delete and a subtree-prefix delete. + for (const del of w.deletes) { + await fs.promises.rm(path.join(dir, del), { + recursive: true, + force: true, + }); + } + for (const [relPath, contents] of Object.entries(w.files)) { + const fullPath = path.join(dir, relPath); + await fs.promises.mkdir(path.dirname(fullPath), { recursive: true }); + await fs.promises.writeFile(fullPath, contents); } + // The parent is the pinned tip, or HEAD for a never-written ref so a + // first write parents on the repo's initial commit. `oldSha` is + // precise: null only when the ref truly does not exist. + const oldSha = parentCommitSha; + const parentSha = + parentCommitSha ?? (await git.resolveRef({ fs, dir, ref: "HEAD" })); + const commitSha = await git.commit({ fs, dir, - message: content.message, + cache: cacheFor(dir), + tree: newRootTreeOid, + message: w.message, author: AUTHOR, parent: [parentSha], ref, @@ -1170,6 +1387,33 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { return { commitSha, newlyTerminalRuns: validation.newlyTerminalRuns ?? [] }; } + // Normalize a `TreeContent` (files + optional clearPrefix) into the + // puts/deletes/scope shape writeTreeUnderLock consumes. A clearPrefix + // becomes a single subtree-delete and the handler's change scope; its + // absence is a purely-additive write validated in full. + function normalizeTreeContent(content: TreeContent): { + files: Record; + deletes: ReadonlySet; + changedPathPrefixes: ReadonlySet | undefined; + message: string; + } { + if (content.clearPrefix !== undefined) { + validateClearPrefix(content.clearPrefix); + return { + files: content.files, + deletes: new Set([content.clearPrefix]), + changedPathPrefixes: new Set([content.clearPrefix]), + message: content.message, + }; + } + return { + files: content.files, + deletes: new Set(), + changedPathPrefixes: undefined, + message: content.message, + }; + } + async function writeTree( principal: Principal, repoId: RepoId, @@ -1178,12 +1422,12 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { ): Promise { gateAccess(principal, repoId, ref, "writeTree"); - // The lock spans the entire substrate body of writeTree: index - // mutation, validatePush, commit, and the onRefUpdated hook. Holding + // The lock spans the entire substrate body of writeTree: tree + // assembly, validatePush, commit, and the onRefUpdated hook. Holding // the lock through onRefUpdated keeps post-update consumers // serialized against the same ref's next writer. return withRepoLock(repoId, () => - writeTreeUnderLock(principal, repoId, ref, content), + writeTreeUnderLock(principal, repoId, ref, normalizeTreeContent(content)), ); } @@ -1206,21 +1450,36 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { if (!repoExists) return out; const commitSha = await resolveRefSha(dir, ref); if (commitSha === null) return out; - const { commit } = await git.readCommit({ fs, dir, oid: commitSha }); + const { commit } = await git.readCommit({ + fs, + dir, + cache: cacheFor(dir), + oid: commitSha, + }); let currentOid = commit.tree; const segments = prefix .replace(/\/$/, "") .split("/") .filter((s) => s !== ""); for (const segment of segments) { - const { tree } = await git.readTree({ fs, dir, oid: currentOid }); + const { tree } = await git.readTree({ + fs, + dir, + cache: cacheFor(dir), + oid: currentOid, + }); const entry = tree.find((e) => e.path === segment); if (entry === undefined || entry.type !== "tree") { return out; } currentOid = entry.oid; } - const { tree } = await git.readTree({ fs, dir, oid: currentOid }); + const { tree } = await git.readTree({ + fs, + dir, + cache: cacheFor(dir), + oid: currentOid, + }); // N+1 isomorphic-git round-trips: one tree read plus one // readBlob per blob entry. Acceptable at the current scale // (single-digit tarballs per registry); when a registry grows @@ -1232,6 +1491,7 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { const { blob } = await git.readBlob({ fs, dir, + cache: cacheFor(dir), oid: commitSha, filepath: `${prefix}${entry.path}`, }); @@ -1258,12 +1518,66 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { const files = await args.merge(existing); return writeTreeUnderLock(principal, repoId, ref, { files, - clearPrefix: args.preservePrefix, + deletes: new Set([args.preservePrefix]), + changedPathPrefixes: new Set([args.preservePrefix]), message: args.message, }); }); } + // Commit a targeted delta: `computeDelta` runs under the per-repo lock + // against the pinned parent tip and returns the exact files to put and + // paths to delete; everything else is carried forward by oid. Unlike + // writeTreePreservingPrefix — which clears and rebuilds a whole prefix + // from a full merge output — a delta touches only the entries it names, + // so a caller that mutates one file in a large directory (a claim-check + // move that adds one entry and deletes another) does not re-hash or + // re-materialize the untouched siblings. `changedPathPrefixes` is the + // handler's scoping hint for the touched region; the caller supplies it + // because the delta has no single clear-prefix to derive it from. + async function writeTreeDelta( + principal: Principal, + repoId: RepoId, + ref: string, + args: WriteTreeDeltaArgs, + ): Promise { + gateAccess(principal, repoId, ref, "writeTree"); + return withRepoLock(repoId, async () => { + const dir = repoDir(repoId); + await storageInitRepo(dir, storageOptsFor(repoId, undefined)); + // Pin the parent tip ONCE under the lock and hand the same oid to + // computeDelta's dedup reads and to the assembly below, so the + // pre-image the delta is computed against is exactly the pre-image + // it is committed against — no lost-update window, no second + // resolve that could (in principle) observe a different tip. + const parentCommitSha = await resolveRefSha(dir, ref); + const { priorListDirOids, readBlobByOid } = buildPriorTreeClosures( + dir, + parentCommitSha, + ); + const delta = await args.computeDelta(parentCommitSha, { + readBlobByOid, + listDirOids: priorListDirOids, + }); + for (const p of Object.keys(delta.puts)) validateDeltaPath(p, false); + for (const d of delta.deletes) validateDeltaPath(d, true); + assertDeltaUnambiguous(delta.puts, delta.deletes); + assertDeltaScoped(delta.puts, delta.deletes, args.changedPathPrefixes); + return writeTreeUnderLock( + principal, + repoId, + ref, + { + files: delta.puts, + deletes: new Set(delta.deletes), + changedPathPrefixes: args.changedPathPrefixes, + message: args.message, + }, + { sha: parentCommitSha }, + ); + }); + } + async function receivePack( principal: Principal, repoId: RepoId, @@ -1325,6 +1639,7 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { const { commit: parsed } = await git.readCommit({ fs, dir, + cache: cacheFor(dir), oid: newCommit, }); const parents = parsed.parent; @@ -1353,7 +1668,12 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { let parentSha: string | null = null; if (declaredParent !== null) { try { - await git.readCommit({ fs, dir, oid: declaredParent }); + await git.readCommit({ + fs, + dir, + cache: cacheFor(dir), + oid: declaredParent, + }); parentSha = declaredParent; } catch (err) { if (!hasCode(err) || err.code !== "NotFoundError") throw err; @@ -1364,10 +1684,8 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { } } } - const { priorReadBlob, priorListDir } = buildPriorTreeClosures( - dir, - parentSha, - ); + const { priorReadBlob, priorListDir, priorListDirOids } = + buildPriorTreeClosures(dir, parentSha); const { topLevelTreePaths, readBlob, listDir } = await buildCommitTreeClosures(dir, newCommit); const changedPathPrefixes = await computeChangedPathPrefixes( @@ -1384,6 +1702,7 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { listDir, priorReadBlob, priorListDir, + priorListDirOids, changedPathPrefixes, }); if (!result.ok) { @@ -1395,12 +1714,11 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { }, ); - // receivePack does not touch the index — it goes straight from - // pack bytes to ref pointer — so the index now points at - // whatever was last committed (possibly a different ref). The - // index cache must be cleared so the next writeTreeUnderLock - // resets the index against the chosen target ref. - indexRefCache.delete(indexCacheKey(repoId)); + // receivePackObjects wrote new objects and advanced the ref + // straight to disk without threading the memoization cache, so + // drop the dir's cache; the next read rebuilds against the packed + // objects and the new tip. + invalidateGitCache(dir); for (const sha of newCommitsFromPack) existingCommits.add(sha); await handler.onRefUpdated({ repoId, ref, oldSha, newSha: commitSha }); await emitRefUpdate(repoId, ref, oldSha, commitSha); @@ -1433,6 +1751,7 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { const result = await git.packObjects({ fs, dir, + cache: cacheFor(dir), oids, write: false, }); @@ -1475,7 +1794,12 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { for (const o of perCommit) seen.add(o); let parsed: Awaited>; try { - parsed = await git.readCommit({ fs, dir, oid: current }); + parsed = await git.readCommit({ + fs, + dir, + cache: cacheFor(dir), + oid: current, + }); } catch (err) { if (hasCode(err) && err.code === "NotFoundError") break; throw err; @@ -1664,6 +1988,7 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { initRepo, writeTree, writeTreePreservingPrefix, + writeTreeDelta, receivePack, createPack, resolveRef, diff --git a/packages/hub-sessions/src/repo-store/types.ts b/packages/hub-sessions/src/repo-store/types.ts index cbc35ff5..0382faaf 100644 --- a/packages/hub-sessions/src/repo-store/types.ts +++ b/packages/hub-sessions/src/repo-store/types.ts @@ -184,6 +184,44 @@ export type WriteTreePreservingPrefixArgs = { message: string; }; +/** + * Per-call options for `RepoStore.writeTreeDelta`. `computeDelta` runs + * under the per-repo lock against the pinned parent tip (`parentCommitSha`, + * null when the ref does not yet exist) and returns the exact files to + * write and paths to delete; the substrate carries every other entry + * forward by its object id. A delete ending in `/` clears that subtree; + * any other delete clears the single file at that path. A delete whose + * base entry is the wrong type -- a no-slash delete naming a directory, + * or a trailing-slash delete descending into a file -- is rejected. + * + * `changedPathPrefixes` is the validation scoping hint for the touched + * region (e.g. `addresses//`), supplied by the caller because a + * delta has no single clear-prefix to derive it from; `undefined` means + * validate the whole tree. + * + * The `prior` argument exposes cache-backed reads of that pinned parent + * tree -- `listDirOids` for a directory's `{name, oid}` children and + * `readBlobByOid` for a blob by its object id -- so the callback reads + * through the store's per-repo object cache under the same lock rather + * than re-opening the repo. + */ +export type PriorDeltaReads = { + readBlobByOid: (oid: string) => Promise; + listDirOids: (path: string) => Promise<{ name: string; oid: string }[]>; +}; + +export type WriteTreeDeltaArgs = { + computeDelta: ( + parentCommitSha: string | null, + prior: PriorDeltaReads, + ) => Promise<{ + puts: Record; + deletes: readonly string[]; + }>; + changedPathPrefixes: ReadonlySet | undefined; + message: string; +}; + export interface KindHandler { kind: RepoKind; /** @@ -241,6 +279,22 @@ export interface KindHandler { * log) uses this to skip re-validating subtrees the commit provably * did not touch; a handler with no such structure ignores it and * validates unconditionally. + * + * `priorListDirOids` mirrors `priorListDir` but returns each child + * entry's git object id alongside its name, read straight from the + * prior commit's tree (git trees are content-addressed, so the listing + * already carries the OID). A handler that validates a large retained + * subtree by its per-commit delta uses it to prove a retained entry is + * byte-unchanged by OID equality instead of re-reading the blob. It is + * `undefined` when no prior commit exists. + * + * `listDirOids` is the prospective-side mirror of `priorListDirOids`: + * each child entry's OID read straight from the prospective tree's + * listing, so a handler comparing a retained subtree by OID gets the + * prospective OID without re-reading and hashing every entry's bytes. + * It is `undefined` on paths that do not surface it (a hand-built + * validatePush in a test), in which case the handler falls back to + * hashing the prospective bytes. */ validatePush: (args: { repoId: RepoId; @@ -249,8 +303,12 @@ export interface KindHandler { topLevelTreePaths: string[]; readBlob: (path: string) => Promise; listDir: (path: string) => Promise; + listDirOids?: (path: string) => Promise<{ name: string; oid: string }[]>; priorReadBlob: (path: string) => Promise; priorListDir: (path: string) => Promise; + priorListDirOids?: ( + path: string, + ) => Promise<{ name: string; oid: string }[]>; changedPathPrefixes?: ReadonlySet | undefined; }) => Promise | ValidatePushResult; /** @@ -304,6 +362,22 @@ export interface RepoStore { ref: string, args: WriteTreePreservingPrefixArgs, ): Promise; + /** + * Delta variant for mutating a few named entries in a large subtree + * without re-materializing the untouched siblings. `computeDelta` runs + * under the per-repo lock against the pinned parent tip and returns the + * files to put and the paths to delete; everything else is carried + * forward by object id. Use this over `writeTreePreservingPrefix` when + * the untouched remainder of the prefix is large (e.g. a claim-check + * move that adds one entry and deletes another while the consumed dedup + * index carries forward unchanged). + */ + writeTreeDelta( + principal: Principal, + repoId: RepoId, + ref: string, + args: WriteTreeDeltaArgs, + ): Promise; /** * Receive a packfile and advance `ref` to `commitSha`. * diff --git a/packages/hub-sessions/src/repo-store/write-tree-delta.test.ts b/packages/hub-sessions/src/repo-store/write-tree-delta.test.ts new file mode 100644 index 00000000..cd26116a --- /dev/null +++ b/packages/hub-sessions/src/repo-store/write-tree-delta.test.ts @@ -0,0 +1,266 @@ +// OID-equivalence guard for the delta write path (writeTreeDelta) added +// alongside the claim-check narrowing. A delta ({puts, deletes} carried +// against the parent tree, reusing untouched entries by oid) MUST commit +// the byte-identical tree that a full-replace writeTree of the same +// logical final state produces. SHA-1 tree hashing is deterministic +// given identical content and structure, so equal commit-tree oids prove +// the delta path never diverges from canonical git across the move +// shapes: enqueue-add, dequeue-move (delete+put), markConsumed-move, +// empty->first-entry, prune-to-empty, a standalone exact-file-delete, and +// a cascade-delete that empties a directory and its parent. It also pins +// the ambiguity (put and delete of the same path, or a put under a +// deleted subtree) and out-of-scope guards that fail loud. + +import { test, expect, afterAll, beforeAll } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import git from "isomorphic-git"; +import { generateKeyPair } from "@intx/crypto"; +import type { KeyPair } from "@intx/types/runtime"; +import { createRepoStore } from "./store"; +import type { AuthorizeFn, Principal, RepoId } from "./types"; + +const tempDirs: string[] = []; +let signingKey: KeyPair; + +beforeAll(async () => { + signingKey = await generateKeyPair(); +}); + +afterAll(async () => { + for (const d of tempDirs.splice(0)) { + await fs.promises + .rm(d, { recursive: true, force: true }) + .catch(() => undefined); + } +}); + +const allowAll: AuthorizeFn = () => ({ allowed: true }); +const principal: Principal = { kind: "test" }; +const REF = "refs/heads/events"; +const SCOPE = "addresses/a/"; + +// Permissive store: the delta path is what is under test, so the handler +// accepts every prospective tree. +function makeStore(dataDir: string) { + return createRepoStore({ + dataDir, + signingKey, + handlers: { + "agent-state": { + kind: "agent-state", + directoryPrefix: "repos-under-test", + validatePush: () => ({ ok: true }), + onRefUpdated: () => undefined, + }, + }, + authorize: allowAll, + }); +} + +async function freshStore(label: string) { + const dataDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), label)); + tempDirs.push(dataDir); + return makeStore(dataDir); +} + +const repoId: RepoId = { kind: "agent-state", id: "subject" }; + +async function commitTreeOid( + store: ReturnType, +): Promise { + const dir = store.getRepoDir(repoId); + const sha = await git.resolveRef({ fs, dir, ref: REF }); + const { commit } = await git.readCommit({ fs, dir, oid: sha }); + return commit.tree; +} + +// Apply a delta to a flat path->content base, mirroring assembleTree's +// delete semantics (trailing-slash prefix delete vs exact file delete). +function applyDelta( + base: Record, + puts: Record, + deletes: readonly string[], +): Record { + const exactDeletes = new Set(deletes.filter((d) => !d.endsWith("/"))); + const prefixDeletes = deletes.filter((d) => d.endsWith("/")); + const out: Record = {}; + for (const [k, v] of Object.entries(base)) { + if (exactDeletes.has(k)) continue; + if (prefixDeletes.some((d) => k.startsWith(d))) continue; + out[k] = v; + } + for (const [k, v] of Object.entries(puts)) out[k] = v; + return out; +} + +// Seed a repo's SCOPE subtree from `base`, then produce the final tree +// two ways and assert byte-identical commit trees: +// delta — writeTreeDelta({ puts, deletes }) +// full-replace — writeTree clearing SCOPE and re-supplying the whole +// post-delta SCOPE content (the pre-narrowing shape) +async function assertDeltaMatchesFullReplace( + label: string, + base: Record, + puts: Record, + deletes: readonly string[], +) { + const finalState = applyDelta(base, puts, deletes); + const finalScopeFiles: Record = {}; + for (const [k, v] of Object.entries(finalState)) { + if (k.startsWith(SCOPE)) finalScopeFiles[k] = v; + } + + const deltaStore = await freshStore(`wtd-delta-${label}-`); + await deltaStore.initRepo(repoId); + if (Object.keys(base).length > 0) { + await deltaStore.writeTree(principal, repoId, REF, { + files: base, + message: "seed", + }); + } + await deltaStore.writeTreeDelta(principal, repoId, REF, { + changedPathPrefixes: new Set([SCOPE]), + message: "delta", + computeDelta: async () => ({ puts, deletes }), + }); + const deltaTree = await commitTreeOid(deltaStore); + + const fullStore = await freshStore(`wtd-full-${label}-`); + await fullStore.initRepo(repoId); + if (Object.keys(base).length > 0) { + await fullStore.writeTree(principal, repoId, REF, { + files: base, + message: "seed", + }); + } + await fullStore.writeTree(principal, repoId, REF, { + files: finalScopeFiles, + clearPrefix: SCOPE, + message: "full-replace", + }); + const fullTree = await commitTreeOid(fullStore); + + expect(deltaTree).toBe(fullTree); +} + +const consumedBase = { + [`${SCOPE}consumed/m1.json`]: `{"receivedAt":1}`, + [`${SCOPE}consumed/m2.json`]: `{"receivedAt":2}`, + [`${SCOPE}watermark.json`]: `{"watermark":0}`, +}; + +test("delta enqueue-add matches full-replace", async () => { + await assertDeltaMatchesFullReplace( + "enqueue", + consumedBase, + { [`${SCOPE}inbox/10-m3.json`]: `{"messageId":"m3"}` }, + [], + ); +}); + +test("delta dequeue-move matches full-replace", async () => { + await assertDeltaMatchesFullReplace( + "dequeue", + { ...consumedBase, [`${SCOPE}inbox/10-m3.json`]: `{"messageId":"m3"}` }, + { [`${SCOPE}processing/10-m3.json`]: `{"messageId":"m3"}` }, + [`${SCOPE}inbox/10-m3.json`], + ); +}); + +test("delta markConsumed-move matches full-replace", async () => { + await assertDeltaMatchesFullReplace( + "consume", + { + ...consumedBase, + [`${SCOPE}processing/10-m3.json`]: `{"messageId":"m3"}`, + }, + { + [`${SCOPE}consumed/m3.json`]: `{"receivedAt":10}`, + [`${SCOPE}watermark.json`]: `{"watermark":5}`, + }, + [`${SCOPE}processing/10-m3.json`], + ); +}); + +test("delta empty->first-entry matches full-replace", async () => { + await assertDeltaMatchesFullReplace( + "first", + {}, + { [`${SCOPE}inbox/10-m1.json`]: `{"messageId":"m1"}` }, + [], + ); +}); + +test("delta prune-to-empty matches full-replace", async () => { + await assertDeltaMatchesFullReplace( + "prune", + consumedBase, + { [`${SCOPE}watermark.json`]: `{"watermark":9}` }, + [`${SCOPE}consumed/m1.json`, `${SCOPE}consumed/m2.json`], + ); +}); + +test("delta exact-file-delete matches full-replace", async () => { + await assertDeltaMatchesFullReplace("exactdel", consumedBase, {}, [ + `${SCOPE}consumed/m1.json`, + ]); +}); + +test("delta cascade-delete (empties dir and parent) matches full-replace", async () => { + // The only entry under SCOPE is one inbox blob; deleting it empties + // inbox/, then addresses/a/, then addresses/, so all three subtrees + // must vanish exactly as a full clearPrefix of an emptied scope does. + await assertDeltaMatchesFullReplace( + "cascade", + { [`${SCOPE}inbox/10-m1.json`]: `{"messageId":"m1"}` }, + {}, + [`${SCOPE}inbox/10-m1.json`], + ); +}); + +test("writeTreeDelta rejects a path in both puts and deletes", async () => { + const store = await freshStore("wtd-ambig-both-"); + await store.initRepo(repoId); + await expect( + store.writeTreeDelta(principal, repoId, REF, { + changedPathPrefixes: new Set([SCOPE]), + message: "ambiguous", + computeDelta: async () => ({ + puts: { [`${SCOPE}inbox/x.json`]: "v" }, + deletes: [`${SCOPE}inbox/x.json`], + }), + }), + ).rejects.toThrow(/delta_ambiguous/); +}); + +test("writeTreeDelta rejects a put under a deleted subtree", async () => { + const store = await freshStore("wtd-ambig-under-"); + await store.initRepo(repoId); + await expect( + store.writeTreeDelta(principal, repoId, REF, { + changedPathPrefixes: new Set([SCOPE]), + message: "ambiguous", + computeDelta: async () => ({ + puts: { [`${SCOPE}inbox/x.json`]: "v" }, + deletes: [`${SCOPE}inbox/`], + }), + }), + ).rejects.toThrow(/delta_ambiguous/); +}); + +test("writeTreeDelta rejects a delta path outside changedPathPrefixes", async () => { + const store = await freshStore("wtd-scope-"); + await store.initRepo(repoId); + await expect( + store.writeTreeDelta(principal, repoId, REF, { + changedPathPrefixes: new Set([SCOPE]), + message: "out of scope", + computeDelta: async () => ({ + puts: { "addresses/b/inbox/x.json": "v" }, + deletes: [], + }), + }), + ).rejects.toThrow(/delta_out_of_scope/); +}); diff --git a/packages/hub-sessions/src/session-service.test.ts b/packages/hub-sessions/src/session-service.test.ts index a1d02a62..623efafd 100644 --- a/packages/hub-sessions/src/session-service.test.ts +++ b/packages/hub-sessions/src/session-service.test.ts @@ -70,15 +70,6 @@ function createMockRouter(): SidecarRouter & { sendAgentUndeploy: track( "sendAgentUndeploy", ) as SidecarRouter["sendAgentUndeploy"], - sendSessionStart: track( - "sendSessionStart", - ) as SidecarRouter["sendSessionStart"], - sendSessionAbort: track( - "sendSessionAbort", - ) as SidecarRouter["sendSessionAbort"], - sendGrantsUpdate: track( - "sendGrantsUpdate", - ) as SidecarRouter["sendGrantsUpdate"], sendSourcesUpdate: track( "sendSourcesUpdate", ) as SidecarRouter["sendSourcesUpdate"], @@ -95,6 +86,15 @@ function createMockRouter(): SidecarRouter & { }); return Promise.resolve(); }) as SidecarRouter["sendPack"], + sendProvisionStep: track( + "sendProvisionStep", + ) as SidecarRouter["sendProvisionStep"], + bindStepRoute(stepAddress: string) { + calls.push({ method: "bindStepRoute", args: [stepAddress] }); + }, + unbindStepRoute(stepAddress: string) { + calls.push({ method: "unbindStepRoute", args: [stepAddress] }); + }, sendSyncRequest: track( "sendSyncRequest", ) as SidecarRouter["sendSyncRequest"], @@ -167,6 +167,7 @@ function unusedRepoStore(): RepoStore { initRepo: unused, writeTree: unused, writeTreePreservingPrefix: unused, + writeTreeDelta: unused, receivePack: unused, createPack: unused, resolveRef: unused, @@ -201,6 +202,7 @@ function createFakeRepoStore( initRepo: unused, writeTree: unused, writeTreePreservingPrefix: unused, + writeTreeDelta: unused, receivePack: unused, listRefs: unused, resolveHead: unused, @@ -321,13 +323,13 @@ describe("SessionService", () => { repoStore = createMockRepoStore(); }); - test("launchSession calls steps in order", async () => { + test("stageWorkflowStep stages without a warm harness", async () => { const service = createSessionService({ sidecarRouter: router, agentRepoStore: repoStore, }); - await service.launchSession({ + await service.stageWorkflowStep({ agentAddress: AGENT_ADDRESS, agentId: AGENT_ID, instanceId: INSTANCE_ID, @@ -340,16 +342,22 @@ describe("SessionService", () => { ...router.calls.map((c) => c.method), ]; + // A stage-only per-step deploy binds a transient route, fires the + // no-spawn provision frame, delivers the deploy pack, and unbinds the + // route -- and NEVER provisions a warm harness (`sendAgentDeploy` with + // no workflow frame). expect(methods).toEqual([ "writeDeployTree", "createDeployPack", - "sendAgentDeploy", + "bindStepRoute", + "sendProvisionStep", "sendPack", - "sendSessionStart", + "unbindStepRoute", ]); + expect(methods).not.toContain("sendAgentDeploy"); }); - test("launchSession cleans up on pack failure", async () => { + test("stageWorkflowStep unbinds the route even when the pack fails", async () => { router.sendPack = () => Promise.reject(new Error("pack failed")); const service = createSessionService({ @@ -357,8 +365,8 @@ describe("SessionService", () => { agentRepoStore: repoStore, }); - const err = await service - .launchSession({ + await service + .stageWorkflowStep({ agentAddress: AGENT_ADDRESS, agentId: AGENT_ID, instanceId: INSTANCE_ID, @@ -367,72 +375,12 @@ describe("SessionService", () => { }) .catch((e: unknown) => e); - expect(err).toBeInstanceOf(SessionLaunchError); - if (!(err instanceof SessionLaunchError)) throw new Error("unreachable"); - expect(err.phase).toBe("pack"); - expect(err.leakedAgent).toBe(false); - const routerMethods = router.calls.map((c) => c.method); - expect(routerMethods).toContain("sendAgentUndeploy"); - }); - - test("launchSession cleans up on session start failure", async () => { - router.sendSessionStart = () => Promise.reject(new Error("start failed")); - - const service = createSessionService({ - sidecarRouter: router, - agentRepoStore: repoStore, - }); - - const err = await service - .launchSession({ - agentAddress: AGENT_ADDRESS, - agentId: AGENT_ID, - instanceId: INSTANCE_ID, - config: MOCK_CONFIG, - deployContent: MOCK_CONTENT, - }) - .catch((e: unknown) => e); - - expect(err).toBeInstanceOf(SessionLaunchError); - if (!(err instanceof SessionLaunchError)) throw new Error("unreachable"); - expect(err.phase).toBe("start"); - expect(err.leakedAgent).toBe(false); - - const routerMethods = router.calls.map((c) => c.method); - expect(routerMethods).toContain("sendAgentUndeploy"); - }); - - test("launchSession reports leaked agent when cleanup fails", async () => { - router.sendPack = () => Promise.reject(new Error("pack failed")); - router.sendAgentUndeploy = () => - Promise.reject(new Error("cleanup failed")); - - const service = createSessionService({ - sidecarRouter: router, - agentRepoStore: repoStore, - }); - - const err = await service - .launchSession({ - agentAddress: AGENT_ADDRESS, - agentId: AGENT_ID, - instanceId: INSTANCE_ID, - config: MOCK_CONFIG, - deployContent: MOCK_CONTENT, - }) - .catch((e: unknown) => e); - - expect(err).toBeInstanceOf(SessionLaunchError); - if (!(err instanceof SessionLaunchError)) throw new Error("unreachable"); - expect(err.leakedAgent).toBe(true); - expect(err.phase).toBe("pack"); - - // The original error (pack failure) must be preserved as the cause, - // not the cleanup failure. - expect(err.cause).toBeInstanceOf(Error); - if (!(err.cause instanceof Error)) throw new Error("unreachable"); - expect(err.cause.message).toBe("pack failed"); + // The transient route is dropped in the `finally`, even on failure, so no + // stale per-step route leaks. A stage-only step has no warm harness to + // tear down, so it never undeploys. + expect(routerMethods).toContain("unbindStepRoute"); + expect(routerMethods).not.toContain("sendAgentUndeploy"); }); test("launchSession does not provision on write failure", async () => { @@ -444,7 +392,7 @@ describe("SessionService", () => { }); const err = await service - .launchSession({ + .stageWorkflowStep({ agentAddress: AGENT_ADDRESS, agentId: AGENT_ID, instanceId: INSTANCE_ID, @@ -461,7 +409,7 @@ describe("SessionService", () => { }); test("launchSession does not send pack on provision failure", async () => { - router.sendAgentDeploy = () => + router.sendProvisionStep = () => Promise.reject(new Error("provision failed")); const service = createSessionService({ @@ -470,7 +418,7 @@ describe("SessionService", () => { }); const err = await service - .launchSession({ + .stageWorkflowStep({ agentAddress: AGENT_ADDRESS, agentId: AGENT_ID, instanceId: INSTANCE_ID, @@ -732,7 +680,7 @@ describe("SessionService", () => { >, }); - await service.launchSession({ + await service.stageWorkflowStep({ agentAddress: AGENT_ADDRESS, agentId: AGENT_ID, instanceId: INSTANCE_ID, @@ -889,7 +837,7 @@ describe("SessionService", () => { >, }); - await service.launchSession({ + await service.stageWorkflowStep({ agentAddress: AGENT_ADDRESS, agentId: AGENT_ID, instanceId: INSTANCE_ID, @@ -940,7 +888,7 @@ describe("SessionService", () => { agentRepoStore: repoStore, }); - await service.launchSession({ + await service.stageWorkflowStep({ agentAddress: AGENT_ADDRESS, agentId: AGENT_ID, instanceId: INSTANCE_ID, @@ -965,38 +913,6 @@ describe("SessionService", () => { expect(content.systemPrompt).not.toContain(""); }); - test("launchSession without assetService leaves the deploy-only flow unchanged", async () => { - const service = createSessionService({ - sidecarRouter: router, - agentRepoStore: repoStore, - }); - - await service.launchSession({ - agentAddress: AGENT_ADDRESS, - agentId: AGENT_ID, - instanceId: INSTANCE_ID, - config: MOCK_CONFIG, - deployContent: MOCK_CONTENT, - }); - - const methods = [ - ...repoStore.calls.map((c) => c.method), - ...router.calls.map((c) => c.method), - ]; - expect(methods).toEqual([ - "writeDeployTree", - "createDeployPack", - "sendAgentDeploy", - "sendPack", - "sendSessionStart", - ]); - - // No mountPath options on the single sendPack call. - const packCall = router.calls.find((c) => c.method === "sendPack"); - if (packCall === undefined) throw new Error("sendPack not called"); - expect(packCall.args[4]).toBeUndefined(); - }); - test("launchSession writes a resolved-source session_asset row for resolver-derived packs", async () => { // Build a single-tarball asset registry, fake the DB query path // the session service walks (`listAssetsForTenant` walks @@ -1126,7 +1042,7 @@ describe("SessionService", () => { }, }); - await service.launchSession({ + await service.stageWorkflowStep({ agentAddress: AGENT_ADDRESS, agentId: AGENT_ID, instanceId: INSTANCE_ID, @@ -1241,7 +1157,7 @@ describe("SessionService", () => { let err: unknown; try { - await service.launchSession({ + await service.stageWorkflowStep({ agentAddress: AGENT_ADDRESS, agentId: AGENT_ID, instanceId: INSTANCE_ID, @@ -1412,7 +1328,7 @@ describe("SessionService", () => { let caught: unknown; try { - await service.launchSession({ + await service.stageWorkflowStep({ agentAddress: AGENT_ADDRESS, agentId: AGENT_ID, instanceId: INSTANCE_ID, @@ -1591,7 +1507,7 @@ describe("deployWorkflowDefinition", () => { // Multi-step branch: each step is provisioned via a per-step // sendAgentDeploy without a workflow field, then a single // deployment-level agent.deploy frame carries the workflow - // projection. The trivial branch never emits a workflow field at all. + // projection. const workflowFrames = sentWorkflows.filter( (w): w is NonNullable => w !== undefined, ); @@ -1677,21 +1593,26 @@ describe("sendMultiStepDeployFrame", () => { execute: step({ agent: stubAgent, after: ["plan"] }), }, }); + // Each step's value is its ordered inference-source failover chain. const sources = { - plan: { - id: "src-plan", - provider: "anthropic", - baseURL: "https://api.example/anthropic", - apiKey: "secret-plan", - model: "mock-model", - }, - execute: { - id: "src-execute", - provider: "anthropic", - baseURL: "https://api.example/anthropic", - apiKey: "secret-execute", - model: "mock-model", - }, + plan: [ + { + id: "src-plan", + provider: "anthropic", + baseURL: "https://api.example/anthropic", + apiKey: "secret-plan", + model: "mock-model", + }, + ], + execute: [ + { + id: "src-execute", + provider: "anthropic", + baseURL: "https://api.example/anthropic", + apiKey: "secret-execute", + model: "mock-model", + }, + ], }; const config: HarnessConfig = { sessionId: "ses-multi", @@ -1702,7 +1623,7 @@ describe("sendMultiStepDeployFrame", () => { systemPrompt: "deployment-level", tools: [], grants: [], - sources: Object.values(sources), + sources: Object.values(sources).flat(), defaultSource: "src-plan", }; @@ -1729,3 +1650,95 @@ describe("sendMultiStepDeployFrame", () => { expect(sent.sources).toEqual(sources); }); }); + +describe("deployInstanceAtHead inference-source pinning", () => { + const MULTI_SOURCE_CONFIG: HarnessConfig = { + ...MOCK_CONFIG, + sources: [ + { + id: "primary", + provider: "anthropic", + baseURL: "https://api.example/anthropic", + apiKey: "sk-primary", + model: "claude-mock", + }, + { + id: "failover", + provider: "openai", + baseURL: "https://api.example/openai", + apiKey: "sk-failover", + model: "gpt-mock", + }, + ], + defaultSource: "primary", + }; + + test("pins the instance's full ordered source chain to the sole step", async () => { + const router = createMockRouter(); + // Capture the workflow frame off `sendAgentDeploy` with its typed + // signature rather than casting the tracker's `unknown[]` args. + const sentWorkflows: Parameters[2][] = []; + router.sendAgentDeploy = (( + _agentAddress: string, + _config: HarnessConfig, + workflow?: Parameters[2], + ) => { + sentWorkflows.push(workflow); + return Promise.resolve({ publicKey: "mock-public-key" }); + }) as SidecarRouter["sendAgentDeploy"]; + const repoStore = createMockRepoStore(); + const service = createSessionService({ + sidecarRouter: router, + agentRepoStore: repoStore, + }); + + await service.deployInstanceAtHead({ + agentAddress: AGENT_ADDRESS, + agentId: AGENT_ID, + instanceId: INSTANCE_ID, + config: MULTI_SOURCE_CONFIG, + deployContent: MOCK_CONTENT, + }); + + const workflow = sentWorkflows[0]; + if (workflow === undefined) { + throw new Error("sendAgentDeploy was not called with a workflow frame"); + } + const chains = Object.values(workflow.sources); + // A single-step-at-head deploy has exactly one step, so one chain. + expect(chains).toHaveLength(1); + // The reactor fails over forward across the whole chain, so the head + // pins `config.sources` verbatim rather than only the default source. + expect(chains[0]).toEqual(MULTI_SOURCE_CONFIG.sources); + }); + + test("rejects a config whose first source is not the default source", async () => { + const router = createMockRouter(); + const repoStore = createMockRepoStore(); + const service = createSessionService({ + sidecarRouter: router, + agentRepoStore: repoStore, + }); + + // The reactor resolves its initial source by `defaultSource` and fails + // over forward with no wrap; a default that is not element 0 would leave + // the head unreachable, so the deploy must fail loudly. + const misordered: HarnessConfig = { + ...MULTI_SOURCE_CONFIG, + defaultSource: "failover", + }; + await expect( + service.deployInstanceAtHead({ + agentAddress: AGENT_ADDRESS, + agentId: AGENT_ID, + instanceId: INSTANCE_ID, + config: misordered, + deployContent: MOCK_CONTENT, + }), + ).rejects.toThrow(/must be the default source/); + + expect(router.calls.some((c) => c.method === "sendAgentDeploy")).toBe( + false, + ); + }); +}); diff --git a/packages/hub-sessions/src/session-service.ts b/packages/hub-sessions/src/session-service.ts index 38c215ad..074d8d1c 100644 --- a/packages/hub-sessions/src/session-service.ts +++ b/packages/hub-sessions/src/session-service.ts @@ -3,7 +3,7 @@ import { and, eq } from "drizzle-orm"; import { createDefaultDirectorRegistry, - defaultDirectorFactory, + type DirectorRegistry, } from "@intx/agent"; import { getLogger } from "@intx/log"; import { @@ -50,9 +50,12 @@ import { createWorkflowDeployOrchestrator, deriveDeploymentAddress, walkCapabilities, - wrapHarnessAsTrivialAgent, + wrapHarnessAsSingleStepWorkflow, type ApprovalSet, type DeployContent as OrchestratorDeployContent, + type DeployWorkflowArgs, + type DeployWorkflowResult, + type DeploySingleStepFn, type LaunchSessionFn, type SendMultiStepDeployFn, type WorkflowRepoWriter, @@ -93,44 +96,58 @@ export class SessionLaunchError extends Error { export type SessionService = { /** - * Orchestrate the full deploy lifecycle: - * 1. Write deploy tree to hub repo and produce packfile - * 2. Provision agent on sidecar (sendAgentDeploy) - * 3. Deliver deploy packfile (sendPack) - * 4. Fan-out attached asset packs: for each row returned by - * `assetService.listAgentAssets(agentId)`, resolve the - * mountPath, resolve the ref to a source commit SHA, build a - * pack, insert a `session_asset` row, and send the pack to the - * sidecar with `mountPath` set on the `repo.pack.done` frame. - * The manifest insert MUST precede the pack send. - * 5. Start session (sendSessionStart) - * - * On partial failure after provision, attempts cleanup via - * sendAgentUndeploy before re-throwing. + * Stage one step of a multi-step workflow deploy: bind a transient route + * for the step address, fire a no-spawn provision frame (init the step's + * agent-state repo and record the hub key), deliver the deploy + asset + * packs, and unbind the route -- no warm harness. The multi-step branch + * stages every step this way before firing the deployment-level workflow + * frame that spawns the supervised child; the child reads each staged step + * tree from disk and runs the step itself. */ - launchSession(params: { + stageWorkflowStep(params: { agentAddress: string; agentId: string; instanceId: string; config: HarnessConfig; deployContent: DeployContent; - /** - * The agent definition's pinned tool packages. When non-empty, the - * service uses its injected `toolPackageResolver` to compute the - * full pinned closure and ships the resulting manifest as part of - * the deploy tree. An empty array (or absent value) means no - * tool-package manifest is materialized; the sidecar's loader is - * a no-op for this deploy. - */ toolPackagePins?: readonly ToolPackagePin[]; }): Promise; + /** + * Deploy a single-agent instance through the single-step-at-head path, + * wrapping the harness as a one-step workflow and routing it through the + * deploy core with the instance's real identity. Replaces `launchSession` + * as the production instance-deploy entry point: the instance runs as a + * supervised workflow-process child. Writes no `workflow_deployment` + * row. Returns the head's agent-key ack (the key the head signs its + * reconnect challenges with). + */ + deployInstanceAtHead(params: { + agentAddress: string; + agentId: string; + instanceId: string; + config: HarnessConfig; + deployContent: DeployContent; + toolPackagePins?: readonly ToolPackagePin[]; + }): Promise<{ publicKey: string }>; + + /** + * Deploy a one-step workflow once at the head through the deploy core, + * without the DB-backed `workflow_deployment` projection row. Stages the + * head's deploy tree (deploy-tree write, pack, asset fan-out), fires the + * deployment `agent.deploy` frame carrying the workflow definition + + * source pin (the sidecar initializes the head repo and spawns the + * workflow-process child), then delivers the pack to the head. Returns + * the sidecar supervisor's principal public key. See `DeploySingleStepFn`. + */ + deploySingleStepAtHead: DeploySingleStepFn; + /** * Deploy a multi-step `WorkflowDefinition` through the workflow-deploy * orchestrator's multi-step branch. This is the general workflow - * deploy entry point: it carries no `trivialBindings` and is not - * coupled to a single agent's credential/session model the way - * `launchSession` is. The orchestrator derives every per-step address + * deploy entry point: it is not coupled to a single agent's + * credential/session model the way `launchSession` is. The + * orchestrator derives every per-step address * from `deploymentId` + `deploymentDomain`, provisions each step's * agent-state repo via the shared per-agent deploy phases, writes the * workflow repo, and fires the deployment-level `agent.deploy` frame. @@ -223,11 +240,11 @@ export type SessionServiceDeps = { sidecarRouter: SidecarRouter; agentRepoStore: AgentRepoStore; /** - * Optional asset attachment integration. When set, `launchSession` + * Optional asset attachment integration. When set, the deploy flow * fans out per-attachment packs after the deploy pack lands and * inserts a `session_asset` row per attachment. When unset, only - * the deploy pack is sent — the legacy single-pack path is - * preserved bit-for-bit. + * the deploy pack is sent — the single-pack path is preserved + * bit-for-bit. */ assetService?: AssetService; /** DB handle used for `session_asset` manifest inserts. Required @@ -402,45 +419,6 @@ function resolveMountPath(row: AgentAssetWithAsset): string { } } -/** - * Compute the operator-approval set the workflow-deploy orchestrator - * needs for a trivial-wrap deploy. The wrapped agent carries no - * capability list and no director ref (see `wrapHarnessAsTrivialAgent`), - * but `HarnessConfig.tools` projects onto the synthesized agent's - * `toolFactories` so the walk emits `tool:` grants the gate - * checks. The approval set mirrors that projection: every tool the - * `HarnessConfig` already names gets the matching `tool:` approval, - * the per-source inference grants land alongside the default-director - * grant, and the trigger-derived mail grants close the set out. - * - * The legacy agent-deploy path has already authorized the deployment - * in toto -- the harness's `tools` array is the operator-supplied - * surface the hub ships to the sidecar -- and re-running deploy - * through the workflow surface must not synthesize a fresh approval - * prompt for grants the legacy path implicitly approved. The shape - * here keeps the gate honest (an unapproved tool fails the deploy) - * while the legacy passthrough remains bit-for-bit on the wire. - */ -function buildTrivialApprovalSet(args: { - agentAddress: string; - config: HarnessConfig; -}): ApprovalSet { - const approvals = new Set(); - for (const tool of args.config.tools) { - approvals.add(`tool:${tool.name}`); - } - for (const source of args.config.sources) { - approvals.add(`inference.source:${source.provider}:${source.model}`); - } - approvals.add(`director:${defaultDirectorFactory.id}`); - approvals.add(`mail.address:${args.agentAddress}`); - const at = args.agentAddress.lastIndexOf("@"); - if (at >= 0 && at < args.agentAddress.length - 1) { - approvals.add(`mail.send:${args.agentAddress.slice(at + 1)}`); - } - return approvals; -} - /** * Translate the orchestrator's structural `DeployContent` (which types * `toolPackageManifest` as `unknown`) back into the hub-sessions @@ -448,8 +426,12 @@ function buildTrivialApprovalSet(args: { * caller supplied, but the surface type widens `toolPackageManifest` to * `unknown`; the validator narrows it back to the canonical shape * `agentRepoStore.writeDeployTree` consumes. + * + * Exported so a test fixture that forwards orchestrator-shaped deploy + * content into `launchSession` narrows it the same validated way the + * production multi-step callback does, rather than casting `unknown`. */ -function bridgeOrchestratorDeployContent( +export function bridgeOrchestratorDeployContent( content: OrchestratorDeployContent, ): DeployContent { const bridged: DeployContent = { systemPrompt: content.systemPrompt }; @@ -472,9 +454,9 @@ function bridgeOrchestratorDeployContent( * Wire the workflow-deploy orchestrator's `sendMultiStepDeploy` * dependency against `SidecarRouter.sendAgentDeploy`. The router * accepts an optional `workflow` projection on the deploy frame; the - * sidecar's deploy router uses field presence to discriminate the - * multi-step branch from the trivial branch. The supervisor public key - * returned by the sidecar's `agent.deploy.ack` is threaded back as the + * sidecar's deploy router uses field presence to route the frame to + * the workflow deploy path. The supervisor public key returned by the + * sidecar's `agent.deploy.ack` is threaded back as the * `MultiStepDeployResult.publicKey`. * * Exported so the co-located caller-site test can assert that the @@ -487,7 +469,7 @@ export async function sendMultiStepDeployFrame(args: { agentAddress: string; config: HarnessConfig; definition: WorkflowDefinition; - sources: Record; + sources: Record; }): Promise<{ publicKey: string }> { // The wire validator's projection types `stepOrder` and `triggers` // as mutable arrays while `WorkflowDefinition` declares them as @@ -515,22 +497,6 @@ export async function sendMultiStepDeployFrame(args: { }); } -/** - * No-op `WorkflowRepoWriter` used by the legacy `launchSession` - * delegate. The workflow-deploy orchestrator writes a workflow repo - * tree before per-step launch; the legacy agent-deploy path never - * materialized a workflow repo and the integration-test surface does - * not exercise one, so the trivial wrap skips the write rather than - * inventing a phantom repo. - */ -function createNoopWorkflowRepoWriter(): WorkflowRepoWriter { - return { - async writeWorkflowRepo(_args) { - return; - }, - }; -} - /** * `WorkflowRepoWriter` backed by the hub's repo substrate. Writes the * orchestrator-produced workflow tree (`workflow.json`, @@ -580,12 +546,18 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { } /** - * Drive the per-agent deploy + session-start phases. Factored out of - * `launchSession` so the workflow-deploy orchestrator's trivial branch - * can call back into the exact phases the legacy agent-deploy path - * owns. The body here is the legacy `launchSession` body verbatim; - * `launchSession` itself now wraps the call in a single-step workflow - * and routes through the orchestrator. + * Stage a deploy on the sidecar: resolve assets and tool packages, write + * the deploy tree, provision the agent, and deliver the deploy + asset + * packs (Phases 0-2b). Phase 1's provision has two shapes: + * - `workflowFrame` set: the single-step head hand-off fires the + * deployment `agent.deploy` frame that spawns the workflow-process + * child. Returns the supervisor public key. + * - `stageOnly` set: a multi-step per-step stage binds a transient route + * for the step address, fires a no-spawn provision frame (init repo + + * record hub key), and unbinds the route once the packs land. No + * child. + * A call with neither is rejected -- the legacy warm-harness path + * is gone. */ async function executeLaunchPhases(params: { agentAddress: string; @@ -594,9 +566,40 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { config: HarnessConfig; deployContent: DeployContent; toolPackagePins?: readonly ToolPackagePin[]; - }): Promise { + /** + * Single-step workflow deploy. When present, Phase 1 fires the + * deployment `agent.deploy` frame carrying the workflow definition + + * source pins (the sidecar initializes the head repo on receipt and + * spawns the workflow-process child) instead of the plain provision + * frame. The returned supervisor public key comes from that frame's + * ack. + * + * Mutually exclusive with `stageOnly`. + */ + workflowFrame?: { + definition: WorkflowDefinition; + sources: Record; + }; + /** + * Multi-step per-step stage. When true, Phase 1 binds a transient route + * for the step address, fires a no-spawn provision frame (the sidecar + * inits the step's agent-state repo and records the hub key), delivers + * the deploy + asset packs, and unbinds the route -- no provision of a + * warm harness and no child. The deployment-level workflow frame, sent + * once after every step is staged, spawns the child. Returns no ack. + * Mutually exclusive with `workflowFrame`. + */ + stageOnly?: boolean; + }): Promise<{ publicKey: string } | undefined> { const { agentAddress, agentId, instanceId, config, deployContent } = params; const toolPackagePins = params.toolPackagePins ?? []; + const stageOnly = params.stageOnly ?? false; + if (params.workflowFrame !== undefined && stageOnly) { + throw new Error( + "executeLaunchPhases: workflowFrame and stageOnly are mutually exclusive", + ); + } + const workflowFrame = params.workflowFrame; // Phase 0: Resolve attached assets first so the skill index is in // hand before the deploy tree is written. The `` @@ -746,113 +749,179 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { throw new SessionLaunchError("write", err, false); } - // Phase 1: Provision on sidecar. - try { - await sidecarRouter.sendAgentDeploy(agentAddress, config); - } catch (err) { - throw new SessionLaunchError("provision", err, false); + // A stage-only per-step deploy binds a transient route for the step + // address so the packs below route to the deployment's sidecar; the + // route is held only for the pack window and dropped in the `finally`. + if (stageOnly) { + try { + sidecarRouter.bindStepRoute(agentAddress); + } catch (err) { + throw new SessionLaunchError("provision", err, false); + } } - - // Phases 2-3: Pack delivery and session start. If either fails, - // attempt cleanup so the sidecar doesn't retain a zombie agent. try { - await sidecarRouter.sendPack(agentAddress, pack, ref, commitSha); - } catch (err) { - await attemptCleanup(agentAddress, "pack", err); - throw new SessionLaunchError("pack", err, false); - } + // Phase 1: Provision on sidecar. A single-step workflow deploy sends + // the deployment `agent.deploy` frame carrying the workflow definition + // + source pins: the sidecar's deploy router initializes the head repo + // on receipt (so the Phase 2 pack has a repo to apply into) and spawns + // the workflow-process child. A stage-only per-step deploy sends a + // no-spawn provision frame: the sidecar inits the step's agent-state + // repo and records the hub key, but spawns nothing. Firing the frame + // before the Phase 2 pack is the ordering barrier -- the repo must + // exist before the pack applies. A workflow frame's ack surfaces the + // supervisor public key to the caller. + let deployAckPublicKey: string | undefined; + try { + if (workflowFrame !== undefined) { + const ack = await sendMultiStepDeployFrame({ + sidecarRouter, + agentAddress, + config, + definition: workflowFrame.definition, + sources: workflowFrame.sources, + }); + deployAckPublicKey = ack.publicKey; + } else if (stageOnly) { + await sidecarRouter.sendProvisionStep(agentAddress, config); + } else { + // Every caller supplies `workflowFrame` (single-step head) or + // `stageOnly` (multi-step per-step). A deploy with neither has no + // provisioning shape -- the legacy warm-harness path is gone -- so + // fail loud rather than ship a deploy pack the sidecar never + // provisioned a repo for. + throw new Error( + "executeLaunchPhases: a deploy requires either workflowFrame or stageOnly", + ); + } + } catch (err) { + throw new SessionLaunchError("provision", err, false); + } - // Phase 2b: Asset-pack fan-out. For each attached asset, build a - // pack, insert the manifest row, then send the pack. The manifest - // insert MUST happen before the pack send: if the sidecar acks - // but the row is missing, the session has materialization without - // a recorded manifest. If the row insert fails, the pack send - // must not happen. - // - // The fan-out covers two sources: the agent's direct attachments - // (skills, today) and the package-registry assets the tool-package - // resolver picked from. The latter live behind tenant inheritance - // rather than a per-agent attachment row, so the session service - // synthesizes the attachment view in `manifestAssetAttachments`. - // - // Both sources can name the same `package-registry` asset — a - // direct attachment and a resolver pin would each compute - // `mountPath = "package-registries//"` and collide on - // the `(instanceId, mountPath)` PK in `session_asset`. Dedup by - // asset id BEFORE the inserts and let the direct attachment win: - // it is an explicit operator action and carries an `agentAssetId` - // the audit query joins against. The resolver-derived row would - // produce the same materialized contents, so dropping it is - // semantically lossless. - const fanOut: ResolvedAttachment[] = dedupAttachmentsByAssetId({ - direct: attachments, - resolved: manifestAssetAttachments, - }); - if (assetService !== undefined && fanOut.length > 0) { - // Track every successfully committed attachment so a later - // fan-out failure can roll back the earlier rows in lockstep - // with the sidecar undeploy. Without this, fan-out[0] succeeds, - // fan-out[1] fails, attemptCleanup tears down the sidecar — but - // fan-out[0]'s session_asset row survives and a future - // materialization query reads a manifest the sidecar no longer - // honors. - const committed: ResolvedAttachment[] = []; - for (const att of fanOut) { - try { - await sendAttachmentPack(instanceId, agentAddress, att); - committed.push(att); - } catch (err) { - await rollbackCommittedAttachments(instanceId, committed); - await attemptCleanup(agentAddress, "pack", err); - throw new SessionLaunchError("pack", err, false); + // Phase 2: Pack delivery. On failure, the warm/workflow paths tear the + // sidecar deployment down; a stage-only step has no supervisor to + // undeploy, so it only drops its transient route (in the `finally`). + // The step's inited agent-state repo is left on the sidecar: the + // orchestrator aborts the whole deploy before the deployment frame is + // sent, so there is nothing to undeploy, and a redeploy of the same + // deployment overwrites the orphaned repo. This is an acceptable minor + // leak on the exceptional staging-failure path, not a live-path cost. + try { + await sidecarRouter.sendPack(agentAddress, pack, ref, commitSha); + } catch (err) { + if (!stageOnly) await attemptCleanup(agentAddress, "pack", err); + throw new SessionLaunchError("pack", err, false); + } + + // Phase 2b: Asset-pack fan-out. For each attached asset, build a + // pack, insert the manifest row, then send the pack. The manifest + // insert MUST happen before the pack send: if the sidecar acks + // but the row is missing, the session has materialization without + // a recorded manifest. If the row insert fails, the pack send + // must not happen. + // + // The fan-out covers two sources: the agent's direct attachments + // (skills, today) and the package-registry assets the tool-package + // resolver picked from. The latter live behind tenant inheritance + // rather than a per-agent attachment row, so the session service + // synthesizes the attachment view in `manifestAssetAttachments`. + // + // Both sources can name the same `package-registry` asset — a + // direct attachment and a resolver pin would each compute + // `mountPath = "package-registries//"` and collide on + // the `(instanceId, mountPath)` PK in `session_asset`. Dedup by + // asset id BEFORE the inserts and let the direct attachment win: + // it is an explicit operator action and carries an `agentAssetId` + // the audit query joins against. The resolver-derived row would + // produce the same materialized contents, so dropping it is + // semantically lossless. + const fanOut: ResolvedAttachment[] = dedupAttachmentsByAssetId({ + direct: attachments, + resolved: manifestAssetAttachments, + }); + if (assetService !== undefined && fanOut.length > 0) { + // Track every successfully committed attachment so a later + // fan-out failure can roll back the earlier rows in lockstep + // with the sidecar undeploy. Without this, fan-out[0] succeeds, + // fan-out[1] fails, attemptCleanup tears down the sidecar — but + // fan-out[0]'s session_asset row survives and a future + // materialization query reads a manifest the sidecar no longer + // honors. + const committed: ResolvedAttachment[] = []; + for (const att of fanOut) { + try { + await sendAttachmentPack(instanceId, agentAddress, att); + committed.push(att); + } catch (err) { + await rollbackCommittedAttachments(instanceId, committed); + if (!stageOnly) await attemptCleanup(agentAddress, "pack", err); + throw new SessionLaunchError("pack", err, false); + } } } - } - try { - await sidecarRouter.sendSessionStart(agentAddress); - } catch (err) { - await attemptCleanup(agentAddress, "start", err); - throw new SessionLaunchError("start", err, false); + return deployAckPublicKey === undefined + ? undefined + : { publicKey: deployAckPublicKey }; + } finally { + if (stageOnly) { + sidecarRouter.unbindStepRoute(agentAddress); + } } } /** - * Legacy agent-deploy entry point preserved bit-for-bit at its wire - * shape. The body now constructs a single-step trivial workflow from - * the deploy's `HarnessConfig` + `DeployContent`, synthesizes the - * matching operator-approval set, and delegates to the workflow-deploy - * orchestrator. The orchestrator's trivial branch round-trips back - * into `executeLaunchPhases` with the original `trivialBindings`, - * which preserves every on-disk and wire-level surface the legacy - * agent-deploy path exposed. + * Deploy a one-step workflow once at the head. Reuses the full + * launch-phase machinery (deploy-tree write, pack, asset fan-out) via + * `executeLaunchPhases`, swapping the Phase 1 provision frame for the + * workflow frame. The workflow frame makes the sidecar initialize the + * head repo and spawn the workflow-process child; the follow-up pack + * lands the head's deploy tree. Returns the supervisor's principal + * public key from the frame's ack. A workflow-frame launch always + * yields a deploy-ack key; its absence is a wiring bug, not a + * tolerable case. */ - async function launchSession(params: { - agentAddress: string; - agentId: string; - instanceId: string; - config: HarnessConfig; - deployContent: DeployContent; - toolPackagePins?: readonly ToolPackagePin[]; - }): Promise { - const { agentAddress, agentId, instanceId, config, deployContent } = params; - - const trivialAgent = wrapHarnessAsTrivialAgent({ - config, - deployContent, - }); - const workflow = defineWorkflow({ - id: `wf_${agentId}`, - agent: trivialAgent, - trigger: { type: "mail", to: agentAddress }, - }); - const operatorApprovals = buildTrivialApprovalSet({ - agentAddress, - config, + const deploySingleStepAtHead: DeploySingleStepFn = async (deployParams) => { + const result = await executeLaunchPhases({ + agentAddress: deployParams.agentAddress, + agentId: deployParams.agentId, + instanceId: deployParams.instanceId, + config: deployParams.config, + deployContent: bridgeOrchestratorDeployContent( + deployParams.deployContent, + ), + workflowFrame: { + definition: deployParams.definition, + sources: deployParams.sources, + }, + ...(deployParams.toolPackagePins !== undefined + ? { toolPackagePins: deployParams.toolPackagePins } + : {}), }); + if (result === undefined) { + throw new Error( + "single-step deploy at head: executeLaunchPhases returned no deploy-ack public key for a workflow-frame deploy", + ); + } + return result; + }; - const launchSessionCallback: LaunchSessionFn = async (orchestratorParams) => - executeLaunchPhases({ + /** + * Build the workflow-deploy orchestrator (with its launch-session and + * multi-step callbacks) and run one deploy. Shared by `launchSession` + * and `deployWorkflowDefinition`, which differ only in the workflow + * repo writer, the director registry, and the deploy args. + */ + async function runWorkflowDeploy(args: { + workflowRepo: WorkflowRepoWriter; + directorRegistry: DirectorRegistry; + deployArgs: DeployWorkflowArgs; + }): Promise { + // The per-step launcher: stage each step's deploy tree WITHOUT a warm + // harness (the supervised child runs the step), with the orchestrator's + // structural `DeployContent` narrowed back to the hub-sessions shape + // first. + const launchSessionCallback: LaunchSessionFn = (orchestratorParams) => + stageWorkflowStep({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, @@ -865,31 +934,136 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { : {}), }); - const sendMultiStepDeployCallback: SendMultiStepDeployFn = (params) => + const sendMultiStepDeployCallback: SendMultiStepDeployFn = (deployParams) => sendMultiStepDeployFrame({ sidecarRouter, - agentAddress: params.agentAddress, - config: params.config, - definition: params.definition, - sources: params.sources, + agentAddress: deployParams.agentAddress, + config: deployParams.config, + definition: deployParams.definition, + sources: deployParams.sources, }); const orchestrator = createWorkflowDeployOrchestrator({ - directorRegistry: createDefaultDirectorRegistry(), - workflowRepo: createNoopWorkflowRepoWriter(), + directorRegistry: args.directorRegistry, + workflowRepo: args.workflowRepo, launchSession: launchSessionCallback, sendMultiStepDeploy: sendMultiStepDeployCallback, + deploySingleStepAtHead, + }); + + return orchestrator.deployWorkflow(args.deployArgs); + } + + /** + * Stage one step of a multi-step workflow deploy: bind a transient route + * for the step address, fire a no-spawn provision frame (the sidecar inits + * the step's agent-state repo and records the hub key), deliver the deploy + * and asset packs, and unbind the route -- no warm harness. The multi-step + * branch stages every step this way, then fires ONE deployment-level + * workflow frame that writes the step grants and spawns the supervised + * workflow-process child; the child reads each step's staged deploy tree + * from disk and runs the step itself. + */ + async function stageWorkflowStep(params: { + agentAddress: string; + agentId: string; + instanceId: string; + config: HarnessConfig; + deployContent: DeployContent; + toolPackagePins?: readonly ToolPackagePin[]; + }): Promise { + await executeLaunchPhases({ + agentAddress: params.agentAddress, + agentId: params.agentId, + instanceId: params.instanceId, + config: params.config, + deployContent: params.deployContent, + stageOnly: true, + ...(params.toolPackagePins !== undefined + ? { toolPackagePins: params.toolPackagePins } + : {}), + }); + } + + /** + * Deploy a single-agent instance through the single-step-at-head path: wrap + * the harness as a one-step workflow (the same wrap `launchSession` uses) and + * route it through `deploySingleStepAtHead` with the instance's REAL identity + * -- so the head address IS the instance address and the deploy runs as a + * supervised workflow-process child. + * + * Unlike the orchestrator's `runSingleStepAtHead`, this calls + * `deploySingleStepAtHead` directly with the route's real `agentId` + * (`row.id`), NOT a `deriveDeploymentAgentId(deploymentId)` -- the child + * resolves skills, deploy tree, and tool-package pins by `agentId`, so + * collapsing it to the deployment id would strip the instance's attachments. + * It writes no `workflow_deployment` row (a plain instance has no workflow + * asset). Returns the head's agent-key ack. + */ + async function deployInstanceAtHead(params: { + agentAddress: string; + agentId: string; + instanceId: string; + config: HarnessConfig; + deployContent: DeployContent; + toolPackagePins?: readonly ToolPackagePin[]; + }): Promise<{ publicKey: string }> { + const { agentAddress, agentId, instanceId, config, deployContent } = params; + + const singleStepAgent = wrapHarnessAsSingleStepWorkflow({ + config, + deployContent, + }); + const workflow = defineWorkflow({ + id: `wf_${agentId}`, + agent: singleStepAgent, + trigger: { type: "mail", to: agentAddress }, }); - await orchestrator.deployWorkflow({ - workflow, - trivialBindings: { agentAddress, agentId, instanceId }, + // The sole step's id, read off the built definition. + const stepId = workflow.stepOrder[0]; + if (stepId === undefined) { + throw new Error( + `instance deploy for ${agentAddress}: the wrapped single-step workflow has an empty stepOrder`, + ); + } + + // Pin the step's inference sources to the instance's FULL ordered source + // chain so the workflow-process child's reactor fails over across it at + // runtime. The route already resolved and authorized `config.sources` + // against the tenant catalog, so it is pinned directly rather than re-run + // through the orchestrator's operator-approval gate. + // + // Fail loud on the invariant the reactor depends on: the reactor resolves + // its initial source by id (`defaultSource`) and fails over FORWARD-ONLY + // with no wrap, so the default must be element 0 or part of the chain is + // unreachable -- and if the default were last, failover would silently + // no-op. The route guarantees `config.sources[0].id === config.defaultSource` + // (head = active); assert it here so a future reordering fails loudly + // rather than silently disabling failover. + if (config.sources.length === 0) { + throw new Error( + `instance deploy for ${agentAddress}: config.sources is empty; at least the default source is required`, + ); + } + if (config.sources[0]?.id !== config.defaultSource) { + throw new Error( + `instance deploy for ${agentAddress}: config.sources[0] (${JSON.stringify(config.sources[0]?.id)}) must be the default source ${JSON.stringify(config.defaultSource)}; the reactor fails over forward from the default and would otherwise skip the head`, + ); + } + + return deploySingleStepAtHead({ + agentAddress, + agentId, + instanceId, config, deployContent, + definition: workflow, + sources: { [stepId]: config.sources }, + hubPublicKey: hexEncode(agentRepoStore.getSigningPublicKey()), ...(params.toolPackagePins !== undefined ? { toolPackagePins: params.toolPackagePins } : {}), - operatorApprovals, }); } @@ -919,58 +1093,22 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { ]), ); - const launchSessionCallback: LaunchSessionFn = async (orchestratorParams) => - executeLaunchPhases({ - agentAddress: orchestratorParams.agentAddress, - agentId: orchestratorParams.agentId, - instanceId: orchestratorParams.instanceId, - config: orchestratorParams.config, - deployContent: bridgeOrchestratorDeployContent( - orchestratorParams.deployContent, - ), - ...(orchestratorParams.toolPackagePins !== undefined - ? { toolPackagePins: orchestratorParams.toolPackagePins } - : {}), - }); - - const sendMultiStepDeployCallback: SendMultiStepDeployFn = (deployParams) => - sendMultiStepDeployFrame({ - sidecarRouter, - agentAddress: deployParams.agentAddress, - config: deployParams.config, - definition: deployParams.definition, - sources: deployParams.sources, - }); - - const orchestrator = createWorkflowDeployOrchestrator({ - directorRegistry, + const result = await runWorkflowDeploy({ workflowRepo: createHubWorkflowRepoWriter(agentRepoStore), - launchSession: launchSessionCallback, - sendMultiStepDeploy: sendMultiStepDeployCallback, - }); - - const result = await orchestrator.deployWorkflow({ - workflow: definition, - deploymentId, - deploymentDomain, - config, - deployContent, - operatorApprovals, - hubPublicKey: hexEncode(agentRepoStore.getSigningPublicKey()), - ...(params.toolPackagePins !== undefined - ? { toolPackagePins: params.toolPackagePins } - : {}), + directorRegistry, + deployArgs: { + workflow: definition, + deploymentId, + deploymentDomain, + config, + deployContent, + operatorApprovals, + hubPublicKey: hexEncode(agentRepoStore.getSigningPublicKey()), + ...(params.toolPackagePins !== undefined + ? { toolPackagePins: params.toolPackagePins } + : {}), + }, }); - if (result.kind !== "multi-step") { - // The general deploy path always omits trivialBindings, so the - // orchestrator must take the multi-step branch. A trivial result - // here means the orchestrator's branch decision diverged from this - // method's contract; surface it loudly rather than recording a - // projection row for a deployment that never reached the sidecar. - throw new Error( - `deployWorkflowDefinition expected a multi-step deploy for ${deploymentId} but the orchestrator returned a ${result.kind} result`, - ); - } if (db === undefined) { throw new Error( @@ -983,6 +1121,9 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { id: deploymentId, tenantId, definitionAssetId, + address: deriveDeploymentAddress({ deploymentId, deploymentDomain }), + // publicKey is left null here; the sidecar's deploy-ack persists the + // deployment's minted key once the child has spawned. status: "deployed", createdAt: now, }); @@ -1400,7 +1541,9 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { } return { - launchSession, + stageWorkflowStep, + deployInstanceAtHead, + deploySingleStepAtHead, deployWorkflowDefinition, sendUserMessage, endSession, diff --git a/packages/hub-sessions/src/workflow-run-kind.test.ts b/packages/hub-sessions/src/workflow-run-kind.test.ts index cec80235..3d8f13b7 100644 --- a/packages/hub-sessions/src/workflow-run-kind.test.ts +++ b/packages/hub-sessions/src/workflow-run-kind.test.ts @@ -2134,6 +2134,25 @@ describe("claim-check API — enqueueInbox per-messageId atomicity in inbox", () const entries = await fs.promises.readdir(inboxDir); expect(entries.sort()).toEqual(["100-msg-X.json"]); }); + + test("rejects a duplicate enqueue for the same messageId at the same receivedAt", async () => { + const { store, repoId, principal } = + await makeClaimCheckStore("cc-enq-dup-key-"); + await enqueueInbox(store, principal, repoId, { + address: ADDRESS, + messageId: "msg-D", + receivedAt: 100, + mailAuditRef: { store: "audit", path: "mail/D" }, + }); + await expect( + enqueueInbox(store, principal, repoId, { + address: ADDRESS, + messageId: "msg-D", + receivedAt: 100, + mailAuditRef: { store: "audit", path: "mail/D" }, + }), + ).rejects.toThrow(/claim_check_duplicate_inbox/); + }); }); // Regression at the validatePush layer for the same intra-state @@ -2154,6 +2173,78 @@ describe("workflowRunKindHandler.validatePush — claim-check intra-state atomic }); }); +describe("claim-check API — replayProcessingToInbox collision guard", () => { + test("rejects when a processing entry collides with an existing inbox entry", async () => { + // The real handler's atomicity check forbids the same key in both + // inbox/ and processing/, so seed that (impossible-in-practice) state + // through a permissive handler and confirm replay refuses rather than + // clobbering the inbox entry. + const dataDir = await makeClaimCheckTempDir("cc-replay-collide-"); + const store = createRepoStore({ + dataDir, + signingKey: claimCheckSigningKey, + handlers: { + "workflow-run": { + kind: "workflow-run", + directoryPrefix: "workflow-runs", + validatePush: () => ({ ok: true }), + onRefUpdated: () => undefined, + }, + }, + authorize: () => ({ allowed: true }), + }); + const repoId: RepoId = { kind: "workflow-run", id: "dep-replaycollide" }; + await store.initRepo(repoId); + const body = inboxBody("msg-1", 100); + await store.writeTree(HUB_PRINCIPAL, repoId, "refs/heads/events", { + files: { + [WORKFLOW_RUN_GITIGNORE_PATH]: "", + [inboxPathFor(ADDRESS_SEG, 100, "msg-1")]: body, + [processingPathFor(ADDRESS_SEG, 100, "msg-1")]: body, + }, + message: "seed colliding inbox+processing state", + }); + await expect( + replayProcessingToInbox(store, HUB_PRINCIPAL, repoId, ADDRESS), + ).rejects.toThrow(/claim_check_replay_collision/); + }); +}); + +describe("claim-check API — dequeueToProcessing filename guard", () => { + test("rejects an inbox entry whose filename is not -.json", async () => { + // Seed a malformed inbox filename via a permissive handler (the real + // handler's shape check would reject it), then confirm the FIFO + // dequeue refuses it rather than mis-parsing. + const dataDir = await makeClaimCheckTempDir("cc-bad-inbox-fname-"); + const store = createRepoStore({ + dataDir, + signingKey: claimCheckSigningKey, + handlers: { + "workflow-run": { + kind: "workflow-run", + directoryPrefix: "workflow-runs", + validatePush: () => ({ ok: true }), + onRefUpdated: () => undefined, + }, + }, + authorize: () => ({ allowed: true }), + }); + const repoId: RepoId = { kind: "workflow-run", id: "dep-badinboxfname" }; + await store.initRepo(repoId); + await store.writeTree(HUB_PRINCIPAL, repoId, "refs/heads/events", { + files: { + [WORKFLOW_RUN_GITIGNORE_PATH]: "", + [`${WORKFLOW_RUN_ADDRESSES_PREFIX}/${ADDRESS_SEG}/${WORKFLOW_RUN_INBOX_DIR}/bad.json`]: + "{}", + }, + message: "seed malformed inbox filename", + }); + await expect( + dequeueToProcessing(store, HUB_PRINCIPAL, repoId, ADDRESS), + ).rejects.toThrow(/claim_check_invalid_inbox_filename/); + }); +}); + // Regression for per-commit-walk pack validation. A single pack // carrying [enqueue, dequeue] commits — produced by the supervisor's // first-mail bootstrap — must validate cleanly against a fresh target @@ -2902,9 +2993,9 @@ describe("workflow-run substrate — pack-path per-run scope completeness", () = // markConsumed commit advances it and prunes consumed entries below it // (the oldest tail only); enqueueInbox refuses any inbound below it as // definitively-stale. These tests prove the gate items: the structural -// contract relaxation (validate-level: suffix-only prune, monotonic -// watermark, retained-floor) and the end-to-end exactly-once + bounded -// behaviour against a real on-disk store. +// contract relaxation (validate-level: below-watermark prune, +// monotonic watermark, retained-floor) and the end-to-end +// exactly-once + bounded behaviour against a real on-disk store. describe("workflowRunKindHandler.validatePush — retention watermark contract", () => { // Gate 3: a watermark regression is rejected. @@ -2936,12 +3027,14 @@ describe("workflowRunKindHandler.validatePush — retention watermark contract", expect(r.reason).toMatch(/watermark regressed/); }); - // Gate 2: a non-suffix deletion (drop a recent consumed entry while - // keeping an older one) is rejected. Dropping msg-recent (receivedAt - // 200) while retaining msg-old (receivedAt 100) is not a suffix of - // the age-ordered set -- the suffix guard fires (max dropped 200 > - // min retained 100), regardless of where the watermark sits. - test("rejects dropping a recent consumed entry while keeping an older one", async () => { + // A non-suffix deletion (drop a recent consumed entry while keeping + // an older one) is ACCEPTED when both entries sit below the + // watermark. The suffix relation is not enforced; a retained entry + // below the watermark only adds dedup (every resubmit in that region + // is stale-rejected at enqueue), so the prune opens no reprocess. The + // production markConsumed writer still prunes only the oldest tail, + // so a non-suffix tree never arises in practice. + test("accepts a non-suffix consumed prune below the watermark and locks the boundary", async () => { const prior = { [watermarkPathFor(ADDRESS_SEG)]: watermarkBody(0), [consumedPathFor(ADDRESS_SEG, "msg-old")]: consumedBody( @@ -2958,10 +3051,9 @@ describe("workflowRunKindHandler.validatePush — retention watermark contract", ), }; // Drop msg-recent (the younger one), keep msg-old (the older one). - // To "permit" the drop the writer must claim a watermark above 200 - // (so the dropped 200 is below it); the retained 100 is then newer - // than nothing dropped above it, but the dropped 200 is newer than - // the retained 100 -- a non-suffix prune. + // The writer claims a watermark above 200 so the dropped 200 is + // below it; the retained 100 is below it too. Both sit below the + // watermark, so every resubmit in that region is stale-rejected. const r = await validate( { [WORKFLOW_RUN_GITIGNORE_PATH]: "", @@ -2975,9 +3067,71 @@ describe("workflowRunKindHandler.validatePush — retention watermark contract", }, { priorFiles: prior }, ); - expect(r.ok).toBe(false); - if (r.ok) throw new Error("unreachable"); - expect(r.reason).toMatch(/prune is not a suffix/); + // The consumed walk does NOT enforce the suffix relation: computing + // the min receivedAt over all retained entries would require + // reading every retained consumed blob, the O(retained) work the + // delta walk exists to avoid. Both the dropped (receivedAt 200) and + // retained (receivedAt 100) entries are strictly below the stored + // watermark (201), so `claim_check_stale_enqueue` refuses every + // resubmit in that region at the enqueue boundary regardless of the + // consumed/ shape -- the retained entry only adds dedup, so a + // non-suffix prune below the watermark cannot open a reprocess. + expect(r.ok).toBe(true); + + // Boundary lock. The removed-check rejects a dropped entry at + // receivedAt >= watermark (strict, mirroring the strict + // `receivedAt < watermark` stale-reject so the entry AT the + // watermark is both retained and not stale-rejected -- no gap). + // Assert BOTH sides of the boundary so a later refactor cannot + // silently widen `>=` to `>` and drop the entry sitting exactly at + // the watermark, which would let a resubmit at that receivedAt miss + // dedup. + // (a) strictly above the watermark -> rejected. + const droppedAbove = await validate( + { + [WORKFLOW_RUN_GITIGNORE_PATH]: "", + [watermarkPathFor(ADDRESS_SEG)]: watermarkBody(200), + }, + { + priorFiles: { + [watermarkPathFor(ADDRESS_SEG)]: watermarkBody(200), + [consumedPathFor(ADDRESS_SEG, "msg-above")]: consumedBody( + "msg-above", + 300, + "run-above", + 350, + ), + }, + }, + ); + expect(droppedAbove.ok).toBe(false); + if (droppedAbove.ok) throw new Error("unreachable"); + expect(droppedAbove.reason).toMatch(/not below the retention watermark/); + + // (b) exactly equal to the watermark -> rejected (the off-by-one + // that widening `>=` to `>` would open). + const droppedAtBoundary = await validate( + { + [WORKFLOW_RUN_GITIGNORE_PATH]: "", + [watermarkPathFor(ADDRESS_SEG)]: watermarkBody(200), + }, + { + priorFiles: { + [watermarkPathFor(ADDRESS_SEG)]: watermarkBody(200), + [consumedPathFor(ADDRESS_SEG, "msg-at")]: consumedBody( + "msg-at", + 200, + "run-at", + 250, + ), + }, + }, + ); + expect(droppedAtBoundary.ok).toBe(false); + if (droppedAtBoundary.ok) throw new Error("unreachable"); + expect(droppedAtBoundary.reason).toMatch( + /not below the retention watermark/, + ); }); // A dropped entry that the watermark has NOT passed is rejected (you diff --git a/packages/hub-sessions/src/workflow-run-kind.ts b/packages/hub-sessions/src/workflow-run-kind.ts index 0336d02e..40eeb416 100644 --- a/packages/hub-sessions/src/workflow-run-kind.ts +++ b/packages/hub-sessions/src/workflow-run-kind.ts @@ -122,23 +122,19 @@ // DELETED only as a watermark-consistent retention prune (see the // watermark invariants below); any other deletion is rejected. // - Retention prune (the bounded-`consumed/` contract): the consumed -// dedup index may shrink only by a watermark-passed SUFFIX prune. -// A consumed entry present in the prior tree may be absent from the +// dedup index may shrink only by a watermark-passed prune. A +// consumed entry present in the prior tree may be absent from the // prospective tree only when (a) its `receivedAt` is strictly // below the prospective `watermark.json` value (you may prune only -// what the watermark passed), (b) the watermark did not regress -// (`prospective watermark >= prior watermark`), and (c) it is older -// than every retained prior consumed entry (`max receivedAt of -// dropped <= min receivedAt of retained`) -- the suffix relation, -// so a writer can trim only the oldest age-ordered tail and can -// never selectively delete a recent messageId to sneak a reprocess -// past dedup. A RETAINED entry is NOT required to sit at or above -// the watermark: a message consumed long after receipt (or one -// replayed back in-flight after a crash) may legitimately carry a -// below-watermark `receivedAt` and survive until a later commit -// prunes it. Retaining it gives only EXTRA dedup -- a re-submission -// at or above the watermark still hits the entry, one below it is -// stale-rejected at enqueue -- so it never weakens exactly-once. +// what the watermark passed) and (b) the watermark did not regress +// (`prospective watermark >= prior watermark`). A RETAINED entry is +// NOT required to sit at or above the watermark: a message consumed +// long after receipt (or one replayed back in-flight after a crash) +// may legitimately carry a below-watermark `receivedAt` and survive +// until a later commit prunes it. Retaining it gives only EXTRA +// dedup -- a re-submission at or above the watermark still hits the +// entry, one below it is stale-rejected at enqueue -- so it never +// weakens exactly-once. // - Inbox→processing transition: a processing entry that is newly // added (not present in the prior tree) must be backed by a // matching inbox entry in the prior tree at the same @@ -176,6 +172,7 @@ import { type AuthorizeFn, type KindHandler, type NewlyTerminalRun, + type PriorDeltaReads, type Principal, type RepoId, type RepoStore, @@ -1033,6 +1030,16 @@ type ClaimCheckBlob = { */ messageIdFromFilename: string; blobPath: string; + /** + * Git blob object id of the entry, resolved for `consumed` entries + * when the enumeration is given an OID resolver. Two entries at the + * same path whose OIDs match are byte-identical (git trees are + * content-addressed), so the consumed immutability check compares + * OIDs instead of re-reading both blobs for retained entries. Left + * `undefined` on inbox/processing entries, which the resolver does + * not cover. + */ + oid?: string; }; /** @@ -1075,6 +1082,7 @@ type ClaimCheckAddressBucket = { async function enumerateClaimCheckBlobs( listDir: (path: string) => Promise, + resolveConsumedOid?: (blobPath: string) => Promise, ): Promise< | { ok: true; perAddress: Map } | { ok: false; reason: string } @@ -1154,15 +1162,20 @@ async function enumerateClaimCheckBlobs( reason: `${WORKFLOW_RUN_CONSUMED_DIR} filename ${dirPath}/${filename} produced no message-id capture`, }; } - bucket.consumed.push({ + const consumedBlobPath = `${dirPath}/${filename}`; + const consumedEntry: ClaimCheckBlob = { kind: "consumed", addressSegment: segment, decodedAddress: roundTrip.decoded, filename, receivedAtFromFilename: null, messageIdFromFilename: messageId, - blobPath: `${dirPath}/${filename}`, - }); + blobPath: consumedBlobPath, + }; + if (resolveConsumedOid !== undefined) { + consumedEntry.oid = await resolveConsumedOid(consumedBlobPath); + } + bucket.consumed.push(consumedEntry); } } } @@ -1352,49 +1365,126 @@ async function parseQueueBlob( return { ok: true, body: validated }; } -async function checkPriorBytesImmutable( - blobPath: string, - readBlob: (path: string) => Promise, +/** + * Compute the git blob OID of a consumed entry from a byte reader, + * used only when the delta-scoped path lacks a substrate-provided prior + * OID listing (e.g. a hand-built test validatePush). `git.hashBlob` + * reproduces the same content-addressed OID a `git.readTree` listing + * carries, so the delta path's intersection compare is identical + * whether the OID came from the tree listing or from hashing the bytes. + */ +async function hashConsumedBlobOid(bytes: Uint8Array): Promise { + const { oid } = await git.hashBlob({ object: bytes }); + return oid; +} + +/** + * Resolve each consumed entry's git blob OID for the delta-scoped path. + * When the substrate supplies a directory OID listing (`listDirOids`) the + * OID comes straight from the tree — one `readTree` per consumed + * directory, cached — so that side is not re-read blob-by-blob. When the + * listing is absent (a hand-built validatePush in a unit test) each OID + * falls back to hashing the entry's bytes, which preserves identical + * semantics at O(retained) cost. Both the prior and prospective sides use + * this; `sideLabel` distinguishes them in the missing-OID error. + */ +function makeListingOidResolver( + sideLabel: string, + listDirOids: + | ((path: string) => Promise<{ name: string; oid: string }[]>) + | undefined, + hashFallback: (blobPath: string) => Promise, +): (blobPath: string) => Promise { + const dirOidCache = new Map>(); + return async (blobPath) => { + if (listDirOids !== undefined) { + const slash = blobPath.lastIndexOf("/"); + const dir = blobPath.slice(0, slash); + const name = blobPath.slice(slash + 1); + let byName = dirOidCache.get(dir); + if (byName === undefined) { + byName = new Map(); + for (const entry of await listDirOids(dir)) { + byName.set(entry.name, entry.oid); + } + dirOidCache.set(dir, byName); + } + const oid = byName.get(name); + if (oid === undefined) { + throw new Error( + `delta claim-check: ${sideLabel} tree listing has no OID for enumerated consumed entry ${blobPath}`, + ); + } + return oid; + } + return hashFallback(blobPath); + }; +} + +function makePriorConsumedOidResolver( priorReadBlob: (path: string) => Promise, - label: string, -): Promise { - const prior = await priorReadBlob(blobPath); - if (prior === null) return { ok: true }; - const prospective = await readBlob(blobPath); - if (prior.byteLength !== prospective.byteLength) { - return { - ok: false, - reason: `${label} ${blobPath} bytes diverge from the prior tree (lengths ${String(prior.byteLength)} vs ${String(prospective.byteLength)}); ${label} entries are immutable once written`, - }; - } - for (let i = 0; i < prior.byteLength; i++) { - if (prior[i] !== prospective[i]) { - return { - ok: false, - reason: `${label} ${blobPath} bytes diverge from the prior tree at offset ${String(i)}; ${label} entries are immutable once written`, - }; + priorListDirOids: + | ((path: string) => Promise<{ name: string; oid: string }[]>) + | undefined, +): (blobPath: string) => Promise { + return makeListingOidResolver("prior", priorListDirOids, async (blobPath) => { + const bytes = await priorReadBlob(blobPath); + if (bytes === null) { + throw new Error( + `delta claim-check: consumed entry ${blobPath} was enumerated in the prior tree but its bytes could not be read`, + ); } - } - return { ok: true }; + return hashConsumedBlobOid(bytes); + }); } /** * Validate the `addresses//{inbox,processing,consumed}` * subtree as a whole. The walk enforces filename shape, JSON envelope * structure, address round-trip, per-messageId atomicity across the - * three queue states, consumed-blob immutability via prior-bytes - * equality, and the inbox→processing / processing→consumed - * transition invariants against the prior tree. + * three queue states, consumed-blob immutability, and the + * inbox→processing / processing→consumed transition invariants against + * the prior tree. + * + * The consumed dedup index is validated by its per-commit DELTA against + * the prior tree rather than by re-walking the whole retained set: + * retained entries (same filename, same blob OID) are skipped as + * already-validated-and-immutable, added entries are parsed and + * validated, and removed entries are checked against the retention + * watermark. `priorListDirOids` and `listDirOids`, when supplied by the + * substrate, surface the prior and prospective consumed OIDs straight + * from their tree listings so neither side is re-read blob-by-blob. */ async function validateClaimCheckSubtree( listDir: (path: string) => Promise, readBlob: (path: string) => Promise, priorReadBlob: (path: string) => Promise, priorListDir: (path: string) => Promise, + priorListDirOids?: (path: string) => Promise<{ name: string; oid: string }[]>, + listDirOids?: (path: string) => Promise<{ name: string; oid: string }[]>, ): Promise { - const enumerated = await enumerateClaimCheckBlobs(listDir); + // Surface each consumed entry's git blob OID during enumeration + // straight from the tree listing on both sides when the substrate + // provides it, falling back to hashing the bytes otherwise. + const prospectiveConsumedOid = makeListingOidResolver( + "prospective", + listDirOids, + async (blobPath) => hashConsumedBlobOid(await readBlob(blobPath)), + ); + const priorConsumedOid = makePriorConsumedOidResolver( + priorReadBlob, + priorListDirOids, + ); + + const enumerated = await enumerateClaimCheckBlobs( + listDir, + prospectiveConsumedOid, + ); if (!enumerated.ok) return enumerated; - const priorEnumerated = await enumerateClaimCheckBlobs(priorListDir); + const priorEnumerated = await enumerateClaimCheckBlobs( + priorListDir, + priorConsumedOid, + ); if (!priorEnumerated.ok) { // The prior tree is the committed state — if its claim-check // shape is already broken, surface it with a distinct rejection @@ -1452,8 +1542,12 @@ async function validateClaimCheckSubtree( messageIdToLocations.set(entry.messageIdFromFilename, list); } for (const entry of bucket.consumed) { - const parsed = await parseConsumedBlob(entry, readBlob); - if (!parsed.ok) return parsed; + // Cross-state atomicity needs each consumed messageId in the map; + // the messageId is the filename stem, so this needs no blob read. + // Retained consumed entries are not re-parsed (their envelope was + // validated when first written and their bytes are proven + // immutable by the OID compare below); added consumed entries are + // parsed and validated by the transition check further down. const list = messageIdToLocations.get(entry.messageIdFromFilename) ?? []; list.push({ kind: entry.kind, filename: entry.filename }); messageIdToLocations.set(entry.messageIdFromFilename, list); @@ -1482,16 +1576,38 @@ async function validateClaimCheckSubtree( } } - // Consumed entries are immutable: enforce byte-equality against - // the prior tree for every consumed blob that already existed. + // Consumed entries are immutable. Compare the git blob OID the + // enumeration surfaced: a consumed entry present in the prior tree + // at the same path must carry the same OID (git trees are + // content-addressed, so equal OID proves byte-equality without + // reading either blob). A diverging OID is an immutability + // violation. Immutability is load-bearing for exactly-once: a + // mutated `receivedAt` on a retained consumed entry could fake it + // below the watermark, get it pruned, and let a re-submission miss + // dedup -- so this compare is not optional. + const priorConsumedOidByPath = new Map(); + for (const e of priorBucket?.consumed ?? []) { + if (e.oid === undefined) { + throw new Error( + `delta claim-check: prior consumed entry ${e.blobPath} was enumerated without an OID`, + ); + } + priorConsumedOidByPath.set(e.blobPath, e.oid); + } for (const entry of bucket.consumed) { - const immutability = await checkPriorBytesImmutable( - entry.blobPath, - readBlob, - priorReadBlob, - "consumed", - ); - if (!immutability.ok) return immutability; + const priorOid = priorConsumedOidByPath.get(entry.blobPath); + if (priorOid === undefined) continue; // newly added; validated below + if (entry.oid === undefined) { + throw new Error( + `delta claim-check: prospective consumed entry ${entry.blobPath} was enumerated without an OID`, + ); + } + if (entry.oid !== priorOid) { + return { + ok: false, + reason: `consumed ${entry.blobPath} bytes diverge from the prior tree (blob OID ${entry.oid} vs ${priorOid}); consumed entries are immutable once written`, + }; + } } const prospectiveConsumedPaths = new Set( @@ -1547,51 +1663,40 @@ async function validateClaimCheckSubtree( } if (priorBucket !== undefined) { - // The consumed dedup index may shrink ONLY by a watermark-passed - // SUFFIX prune: the entries dropped from the prior tree must be - // the oldest age-ordered tail, every one of them strictly below - // the prospective watermark, and never younger than any entry - // that survives. Expressed as two structural facts the handler - // can check without wall-clock policy: - // (1) every DROPPED prior consumed entry has receivedAt < - // watermark (you may prune only what the watermark passed); - // (2) every dropped entry is older than every RETAINED prior - // entry (max receivedAt of dropped <= min receivedAt of - // retained) -- the suffix relation, so a writer cannot - // selectively delete a recent messageId while keeping an - // older one to sneak a reprocess past dedup. - // A RETAINED entry is NOT required to sit at or above the - // watermark: a message consumed long after receipt (or replayed - // back in-flight) may legitimately carry a below-watermark - // receivedAt and survive until a later commit prunes it. Holding - // it gives only EXTRA dedup (a re-submission at or above the - // watermark still hits the retained entry; one below is - // stale-rejected at enqueue), so it never weakens exactly-once. - // The receivedAt lives in the body; read it from the prior tree + // The consumed dedup index may shrink only by a watermark-passed + // prune: a consumed entry dropped from the prior tree must have a + // receivedAt strictly below the prospective watermark (you may + // prune only what the watermark passed). Combined with the + // already-verified watermark monotonicity, this is the whole of + // the exactly-once retention contract: pruning is bound to the + // watermark and the watermark only advances. + // + // The suffix relation (dropped entries older than every retained + // entry) is deliberately NOT enforced. A RETAINED entry is NOT + // required to sit at or above the watermark: a message consumed + // long after receipt (or replayed back in-flight) may + // legitimately carry a below-watermark receivedAt and survive + // until a later commit prunes it. Holding it gives EXTRA dedup (a + // re-submission at or above the watermark still hits the retained + // entry; one below is stale-rejected at enqueue), so a hole left + // by an out-of-order prune weakens nothing. + // + // Only the dropped entries are read. A retained entry (present in + // both trees) is proven byte-identical by the OID compare above, + // so its receivedAt is unchanged and need not be read. The + // receivedAt lives in the body; read it from the prior tree // (retained bytes are immutable, so prior and prospective agree). - let maxDroppedReceivedAt = Number.NEGATIVE_INFINITY; - let minRetainedReceivedAt = Number.POSITIVE_INFINITY; for (const e of priorBucket.consumed) { + if (prospectiveConsumedPaths.has(e.blobPath)) continue; const priorParsed = await parseConsumedBlobFrom(e, priorReadBlob); if (!priorParsed.ok) return priorParsed; const receivedAt = priorParsed.body.receivedAt; - if (prospectiveConsumedPaths.has(e.blobPath)) { - minRetainedReceivedAt = Math.min(minRetainedReceivedAt, receivedAt); - continue; - } if (receivedAt >= prospectiveWatermark) { return { ok: false, reason: `consumed ${e.blobPath} present in the prior tree is missing from the prospective tree but its receivedAt ${String(receivedAt)} is not below the retention watermark ${String(prospectiveWatermark)}; consumed entries may be pruned only once the watermark has passed them`, }; } - maxDroppedReceivedAt = Math.max(maxDroppedReceivedAt, receivedAt); - } - if (maxDroppedReceivedAt > minRetainedReceivedAt) { - return { - ok: false, - reason: `address ${JSON.stringify(decodedAddress)} consumed prune is not a suffix: a dropped entry (receivedAt ${String(maxDroppedReceivedAt)}) is newer than a retained entry (receivedAt ${String(minRetainedReceivedAt)}); only the oldest age-ordered tail may be pruned`, - }; } for (const e of priorBucket.processing) { if (prospectiveProcessingPaths.has(e.blobPath)) continue; @@ -1832,8 +1937,10 @@ export const workflowRunKindHandler: KindHandler = { topLevelTreePaths, readBlob, listDir, + listDirOids, priorReadBlob, priorListDir, + priorListDirOids, changedPathPrefixes, }): Promise { // Bound the per-run event/blob walks to the runs this commit could @@ -1902,6 +2009,8 @@ export const workflowRunKindHandler: KindHandler = { readBlob, priorReadBlob, priorListDir, + priorListDirOids, + listDirOids, ); if (!claimCheck.ok) { logger.debug`workflow-run validatePush rejected ${repoId.kind}/${repoId.id} on ${ref}: ${claimCheck.reason}`; @@ -2291,8 +2400,8 @@ export const workflowRunAuthorize: AuthorizeFn = ( // --------------------------------------------------------------------- // Claim-check API. // -// Four operations layer on top of `RepoStore.writeTreePreservingPrefix` -// to give the workflow runtime a FIFO claim-check queue per address: +// Four operations layer on top of `RepoStore.writeTreeDelta` to give +// the workflow runtime a FIFO claim-check queue per address: // // enqueueInbox — append a new inbox entry for an inbound // message. @@ -2308,16 +2417,18 @@ export const workflowRunAuthorize: AuthorizeFn = ( // its `-` filename // key so FIFO ordering survives a crash. // -// All four route through `writeTreePreservingPrefix` with the -// per-address subtree as `preservePrefix`. The substrate serializes -// concurrent claim-check operations on the per-repo lock; the merge -// callback reads the prior address subtree directly via -// `isomorphic-git` (the substrate's `existing` parameter is a -// direct-children-only view, which does not see entries nested under -// inbox/processing/consumed), computes the new state, and returns the -// full set of files. The substrate's `clearPrefix` semantics replace -// the address subtree wholesale with the returned set, which is the -// atomic-commit guarantee these operations require. +// All four route through `writeTreeDelta`, scoped to the per-address +// subtree via `changedPathPrefixes`. The substrate serializes concurrent +// claim-check operations on the per-repo lock and invokes each +// operation's `computeDelta` callback with a `prior` view of the +// committed tree. The callback reads only what it needs directly -- +// `prior.listDirOids` for a directory's names and OIDs, and +// `prior.readBlobByOid` for a specific entry's bytes -- and returns a +// TARGETED delta (the `puts` and `deletes` for the paths that change), +// not the full subtree. The substrate applies that delta atomically over +// the prior tree, carrying every untouched entry forward by OID and +// landing the whole delta in a single commit, which is the atomic-commit +// guarantee these operations require. function claimCheckCommitRef(): string { // Every claim-check operation targets the same canonical ref used by @@ -2333,6 +2444,10 @@ function addressSegmentFor(address: string): string { return encodeURIComponent(address); } +function addressPrefix(addressSegment: string): string { + return `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/`; +} + function inboxPath(addressSegment: string, key: string): string { return `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/${WORKFLOW_RUN_INBOX_DIR}/${key}.json`; } @@ -2341,94 +2456,71 @@ function processingPath(addressSegment: string, key: string): string { return `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/${WORKFLOW_RUN_PROCESSING_DIR}/${key}.json`; } +function consumedPath(addressSegment: string, messageId: string): string { + return `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/${WORKFLOW_RUN_CONSUMED_DIR}/${messageId}.json`; +} + +function watermarkPath(addressSegment: string): string { + return `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/${WORKFLOW_RUN_WATERMARK_FILE}`; +} + function filenameKey(receivedAt: number, messageId: string): string { return `${String(receivedAt)}-${messageId}`; } +type ClaimCheckEntry = { name: string; oid: string }; + +type AddressListing = { + inbox: ClaimCheckEntry[]; + processing: ClaimCheckEntry[]; + consumed: ClaimCheckEntry[]; + watermark: number; +}; + /** - * Read the full state of one address subtree from the ref's tip via - * `isomorphic-git`. The substrate's `writeTreePreservingPrefix` only - * surfaces direct-child blobs; the address subtree's blobs live one - * level deeper, so the merge callback uses this helper to assemble - * the full pre-image inside the per-repo lock. - * - * Returns the bytes of every entry under - * `addresses//{inbox,processing,consumed}/` keyed by - * repo-root-relative path. An empty map covers the cases where the - * repo, ref, or address subtree do not yet exist — all legitimate - * first-write states for a brand-new claim-check operation. + * Read one address's claim-check listing from the parent commit: the + * filenames and blob OIDs directly under + * `addresses//{inbox,processing,consumed}/` (NOT their + * bytes), plus the retention watermark. The bytes of the single entry a + * leg actually moves are read separately by OID via + * `prior.readBlobByOid`, so the unbounded consumed/ dedup index is + * enumerated (one `listDirOids` per bucket, names and OIDs only) but + * never read blob-by-blob. Every read goes through the store's + * cache-backed `prior` closures under the write lock. An empty listing + * covers the repo/ref/address-absent first-write states -- all + * legitimate for a brand-new operation. */ -async function readAddressSubtree( - repoDir: string, - ref: string, +async function readAddressListing( + prior: PriorDeltaReads, addressSegment: string, -): Promise> { - const out = new Map(); - let commitSha: string; - try { - commitSha = await git.resolveRef({ fs, dir: repoDir, ref }); - } catch { - return out; - } - const commit = await git.readCommit({ fs, dir: repoDir, oid: commitSha }); - const addrTreeOid = await resolveSubtreeOid(repoDir, commit.commit.tree, [ - WORKFLOW_RUN_ADDRESSES_PREFIX, - addressSegment, - ]); - if (addrTreeOid === null) return out; - const { tree: addrChildren } = await git.readTree({ - fs, - dir: repoDir, - oid: addrTreeOid, - }); - for (const child of addrChildren) { - if (child.type === "blob" && child.path === WORKFLOW_RUN_WATERMARK_FILE) { - const { blob } = await git.readBlob({ - fs, - dir: repoDir, - oid: child.oid, - }); - out.set( - `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/${WORKFLOW_RUN_WATERMARK_FILE}`, - blob, - ); +): Promise { + const listing: AddressListing = { + inbox: [], + processing: [], + consumed: [], + watermark: 0, + }; + const addrDir = `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}`; + for (const child of await prior.listDirOids(addrDir)) { + if (child.name === WORKFLOW_RUN_WATERMARK_FILE) { + const blob = await prior.readBlobByOid(child.oid); + listing.watermark = parseWatermark(blob, watermarkPath(addressSegment)); continue; } - if (child.type !== "tree") continue; - if (!CLAIM_CHECK_SUBDIRS.has(child.path)) continue; - const { tree: children } = await git.readTree({ - fs, - dir: repoDir, - oid: child.oid, - }); - for (const blobEntry of children) { - if (blobEntry.type !== "blob") continue; - const { blob } = await git.readBlob({ - fs, - dir: repoDir, - oid: blobEntry.oid, - }); - const blobPath = `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/${child.path}/${blobEntry.path}`; - out.set(blobPath, blob); + const bucket = + child.name === WORKFLOW_RUN_INBOX_DIR + ? listing.inbox + : child.name === WORKFLOW_RUN_PROCESSING_DIR + ? listing.processing + : child.name === WORKFLOW_RUN_CONSUMED_DIR + ? listing.consumed + : null; + if (bucket === null) continue; + for (const entry of await prior.listDirOids(`${addrDir}/${child.name}`)) { + bucket.push({ name: entry.name, oid: entry.oid }); } } - return out; -} - -async function resolveSubtreeOid( - repoDir: string, - rootTreeOid: string, - segments: readonly string[], -): Promise { - let current = rootTreeOid; - for (const segment of segments) { - const { tree } = await git.readTree({ fs, dir: repoDir, oid: current }); - const entry = tree.find((e) => e.path === segment); - if (entry === undefined) return null; - if (entry.type !== "tree") return null; - current = entry.oid; - } - return current; + return listing; } function utf8(s: string): Uint8Array { @@ -2474,17 +2566,11 @@ function decodeConsumedReceivedAtOrThrow( } /** - * Read the per-address watermark from a claim-check subtree map keyed - * by repo-root-relative path. An absent watermark blob means the - * address has never pruned; treat it as watermark 0 (nothing pruned, - * nothing refused). + * Decode the per-address retention watermark from its blob bytes. The + * caller treats an absent watermark blob as 0 (the address has never + * pruned; nothing refused). */ -function readWatermarkFromMap( - existing: Map, - watermarkFull: string, -): number { - const bytes = existing.get(watermarkFull); - if (bytes === undefined) return 0; +function parseWatermark(bytes: Uint8Array, watermarkFull: string): number { let parsed: unknown; try { parsed = JSON.parse(new TextDecoder().decode(bytes)); @@ -2542,7 +2628,6 @@ export async function enqueueInbox( ): Promise { const addressSegment = addressSegmentFor(args.address); const ref = claimCheckCommitRef(); - const repoDir = store.getRepoDir(repoId); const inboxKey = filenameKey(args.receivedAt, args.messageId); const envelope: ClaimCheckEnvelope = { messageId: args.messageId, @@ -2552,81 +2637,60 @@ export async function enqueueInbox( ...(args.rawMessage !== undefined ? { rawMessage: args.rawMessage } : {}), }; const newInboxPath = inboxPath(addressSegment, inboxKey); - const { commitSha } = await store.writeTreePreservingPrefix( - principal, - repoId, - ref, - { - preservePrefix: `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/`, - merge: async () => { - const existing = await readAddressSubtree(repoDir, ref, addressSegment); - const inboxPrefix = `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/${WORKFLOW_RUN_INBOX_DIR}/`; - const processingPrefix = `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/${WORKFLOW_RUN_PROCESSING_DIR}/`; - const consumedPrefix = `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/${WORKFLOW_RUN_CONSUMED_DIR}/`; - const watermarkFull = `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/${WORKFLOW_RUN_WATERMARK_FILE}`; - const messageIdSuffix = `-${args.messageId}.json`; - // Refuse a definitively-stale enqueue: a message whose - // receivedAt is strictly below the retention watermark could - // have had its consumed/ dedup entry pruned, so a duplicate can - // no longer be ruled out. Reject it LOUDLY rather than risk - // reprocessing. This is the second half of the exactly-once - // guarantee: above the watermark the consumed/ index is - // authoritative; below it, refuse. - const watermark = readWatermarkFromMap(existing, watermarkFull); - if (args.receivedAt < watermark) { - throw new Error( - `claim_check_stale_enqueue: address ${args.address} message ${args.messageId} receivedAt ${String(args.receivedAt)} is below the retention watermark ${String(watermark)}; its dedup entry may have been pruned, so it is refused as definitively-stale`, - ); - } - for (const [blobPath] of existing) { - if (blobPath === watermarkFull) continue; - if (blobPath === newInboxPath) { - throw new Error( - `claim_check_duplicate_inbox: ${newInboxPath} already exists`, - ); - } - if (blobPath.startsWith(consumedPrefix)) { - const fname = blobPath.slice(consumedPrefix.length); - if (fname === `${args.messageId}.json`) { - throw new Error( - `claim_check_already_consumed: address ${args.address} message ${args.messageId} is already in the consumed dedup index`, - ); - } - } - if (blobPath.startsWith(processingPrefix)) { - const fname = blobPath.slice(processingPrefix.length); - if (fname.endsWith(messageIdSuffix)) { - throw new Error( - `claim_check_already_processing: address ${args.address} message ${args.messageId} is currently in processing`, - ); - } - } - // Reject a second inbox entry for the same messageId at a - // different receivedAt. The validatePush atomicity check - // also catches this on the commit path, but the inbox - // scan here surfaces the rejection at the API boundary so - // the caller sees a precise error (rather than a generic - // tree-validation failure) and the bad tree never reaches - // the substrate. - if (blobPath.startsWith(inboxPrefix)) { - const fname = blobPath.slice(inboxPrefix.length); - if (fname.endsWith(messageIdSuffix)) { - throw new Error( - `claim_check_already_inbox: address ${args.address} message ${args.messageId} is already in the inbox at ${blobPath}`, - ); - } - } - } - const files: Record = {}; - for (const [blobPath, bytes] of existing) { - files[blobPath] = bytes; - } - files[newInboxPath] = utf8(JSON.stringify(envelope)); - return files; - }, - message: `enqueue inbox ${args.address} ${args.messageId}`, + const inboxFname = `${inboxKey}.json`; + const consumedFname = `${args.messageId}.json`; + const messageIdSuffix = `-${args.messageId}.json`; + const { commitSha } = await store.writeTreeDelta(principal, repoId, ref, { + changedPathPrefixes: new Set([addressPrefix(addressSegment)]), + message: `enqueue inbox ${args.address} ${args.messageId}`, + computeDelta: async (_parentCommitSha, prior) => { + const listing = await readAddressListing(prior, addressSegment); + // Refuse a definitively-stale enqueue: a message whose receivedAt + // is strictly below the retention watermark could have had its + // consumed/ dedup entry pruned, so a duplicate can no longer be + // ruled out. Reject it LOUDLY rather than risk reprocessing. This + // is the second half of the exactly-once guarantee: above the + // watermark the consumed/ index is authoritative; below it, refuse. + if (args.receivedAt < listing.watermark) { + throw new Error( + `claim_check_stale_enqueue: address ${args.address} message ${args.messageId} receivedAt ${String(args.receivedAt)} is below the retention watermark ${String(listing.watermark)}; its dedup entry may have been pruned, so it is refused as definitively-stale`, + ); + } + if (listing.inbox.some((e) => e.name === inboxFname)) { + throw new Error( + `claim_check_duplicate_inbox: ${newInboxPath} already exists`, + ); + } + // consumed/ is keyed by messageId alone, so this is an exact + // filename lookup against the dedup index. + if (listing.consumed.some((e) => e.name === consumedFname)) { + throw new Error( + `claim_check_already_consumed: address ${args.address} message ${args.messageId} is already in the consumed dedup index`, + ); + } + if (listing.processing.some((e) => e.name.endsWith(messageIdSuffix))) { + throw new Error( + `claim_check_already_processing: address ${args.address} message ${args.messageId} is currently in processing`, + ); + } + // Reject a second inbox entry for the same messageId at a + // different receivedAt. The validatePush atomicity check also + // catches this on the commit path, but surfacing it here gives the + // caller a precise error and keeps the bad tree off the substrate. + const inboxDup = listing.inbox.find((e) => + e.name.endsWith(messageIdSuffix), + ); + if (inboxDup !== undefined) { + throw new Error( + `claim_check_already_inbox: address ${args.address} message ${args.messageId} is already in the inbox at ${inboxPath(addressSegment, inboxDup.name.slice(0, -".json".length))}`, + ); + } + return { + puts: { [newInboxPath]: utf8(JSON.stringify(envelope)) }, + deletes: [], + }; }, - ); + }); return { commitSha, inboxKey, envelope }; } @@ -2655,70 +2719,59 @@ export async function dequeueToProcessing( ): Promise { const addressSegment = addressSegmentFor(address); const ref = claimCheckCommitRef(); - const repoDir = store.getRepoDir(repoId); let dequeued: { key: string; envelope: ClaimCheckEnvelope } | null = null; - const { commitSha } = await store.writeTreePreservingPrefix( - principal, - repoId, - ref, - { - preservePrefix: `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/`, - merge: async () => { - const existing = await readAddressSubtree(repoDir, ref, addressSegment); - const inboxPrefix = `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/${WORKFLOW_RUN_INBOX_DIR}/`; - // Parse each inbox path's filename and sort by numeric - // receivedAt with a messageId tiebreak. A raw string sort - // would not agree with chronological order when receivedAt - // values have non-uniform digit widths. - type InboxCandidate = { - path: string; - receivedAt: number; - messageId: string; - }; - const candidates: InboxCandidate[] = []; - for (const p of existing.keys()) { - if (!p.startsWith(inboxPrefix)) continue; - const fname = p.slice(inboxPrefix.length); - const m = QUEUE_FILENAME_RE.exec(fname); - if (m === null || m[1] === undefined || m[2] === undefined) { - throw new Error(`claim_check_invalid_inbox_filename: ${p}`); - } - candidates.push({ - path: p, - receivedAt: Number.parseInt(m[1], 10), - messageId: m[2], - }); + const { commitSha } = await store.writeTreeDelta(principal, repoId, ref, { + changedPathPrefixes: new Set([addressPrefix(addressSegment)]), + message: `dequeue ${address}`, + computeDelta: async (_parentCommitSha, prior) => { + const listing = await readAddressListing(prior, addressSegment); + const inboxDir = `${addressPrefix(addressSegment)}${WORKFLOW_RUN_INBOX_DIR}/`; + // Sort by numeric receivedAt with a messageId tiebreak. A raw + // string sort would not agree with chronological order when + // receivedAt values have non-uniform digit widths. + type InboxCandidate = { + entry: ClaimCheckEntry; + receivedAt: number; + messageId: string; + }; + const candidates: InboxCandidate[] = []; + for (const entry of listing.inbox) { + const m = QUEUE_FILENAME_RE.exec(entry.name); + if (m === null || m[1] === undefined || m[2] === undefined) { + throw new Error( + `claim_check_invalid_inbox_filename: ${inboxDir}${entry.name}`, + ); } - candidates.sort((a, b) => { - if (a.receivedAt !== b.receivedAt) return a.receivedAt - b.receivedAt; - if (a.messageId < b.messageId) return -1; - if (a.messageId > b.messageId) return 1; - return 0; + candidates.push({ + entry, + receivedAt: Number.parseInt(m[1], 10), + messageId: m[2], }); - if (candidates.length === 0) { - dequeued = null; - return Object.fromEntries(existing); - } - const first = candidates[0]; - if (first === undefined) throw new Error("unreachable"); - const firstPath = first.path; - const bytes = existing.get(firstPath); - if (bytes === undefined) throw new Error("unreachable"); - const fname = firstPath.slice(inboxPrefix.length); - const key = fname.slice(0, -".json".length); - const envelope = decodeQueueEnvelopeOrThrow(bytes, firstPath); - dequeued = { key, envelope }; - const files: Record = {}; - for (const [blobPath, blobBytes] of existing) { - if (blobPath === firstPath) continue; - files[blobPath] = blobBytes; - } - files[processingPath(addressSegment, key)] = bytes; - return files; - }, - message: `dequeue ${address}`, + } + candidates.sort((a, b) => { + if (a.receivedAt !== b.receivedAt) return a.receivedAt - b.receivedAt; + if (a.messageId < b.messageId) return -1; + if (a.messageId > b.messageId) return 1; + return 0; + }); + const first = candidates[0]; + if (first === undefined) { + // Empty inbox: nothing to move. The commit is a no-op rewrite of + // the same tree; the caller reads `dequeued === null`. + dequeued = null; + return { puts: {}, deletes: [] }; + } + const firstPath = `${inboxDir}${first.entry.name}`; + const key = first.entry.name.slice(0, -".json".length); + const bytes = await prior.readBlobByOid(first.entry.oid); + const envelope = decodeQueueEnvelopeOrThrow(bytes, firstPath); + dequeued = { key, envelope }; + return { + puts: { [processingPath(addressSegment, key)]: bytes }, + deletes: [firstPath], + }; }, - ); + }); if (dequeued === null) return null; const captured: { key: string; envelope: ClaimCheckEnvelope } = dequeued; return { commitSha, key: captured.key, envelope: captured.envelope }; @@ -2742,12 +2795,12 @@ export type ReadProcessingEntryResult = { * input. * * The read is a flat working-tree read of - * `addresses//processing/`. The substrate materializes every - * claim-check commit into the repo's working tree (each - * `writeTreePreservingPrefix` resets the working tree to the ref tip), - * so a read issued after `dequeueToProcessing` committed -- which is - * exactly when the supervisor forwards `trigger.fired` -- observes the - * processing entry. Reading the working tree (rather than walking the + * `addresses//processing/`. The substrate materializes each + * claim-check commit's touched paths into the repo's working tree (the + * delta write removes each deleted path and writes each put after + * validation passes), so a read issued after `dequeueToProcessing` + * committed -- which is exactly when the supervisor forwards + * `trigger.fired` -- observes the processing entry. Reading the working tree (rather than walking the * committed git tree) matches the workflow-process child's sibling * reads of `workflow.json` and `runs//events/`. Because the * read issues no commit it cannot race the supervisor's `markConsumed` @@ -2846,103 +2899,92 @@ export async function markConsumed( ): Promise { const addressSegment = addressSegmentFor(args.address); const ref = claimCheckCommitRef(); - const repoDir = store.getRepoDir(repoId); const retentionHorizonMs = args.retentionHorizonMs ?? DEFAULT_CONSUMED_RETENTION_MS; let consumedEnvelope: ConsumedEnvelope | null = null; let advancedWatermark = 0; const prunedMessageIds: string[] = []; - const { commitSha } = await store.writeTreePreservingPrefix( - principal, - repoId, - ref, - { - preservePrefix: `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/`, - merge: async () => { - const existing = await readAddressSubtree(repoDir, ref, addressSegment); - const processingPrefix = `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/${WORKFLOW_RUN_PROCESSING_DIR}/`; - const consumedPrefix = `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/${WORKFLOW_RUN_CONSUMED_DIR}/`; - const watermarkFull = `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/${WORKFLOW_RUN_WATERMARK_FILE}`; - const consumedFname = `${args.messageId}.json`; - const consumedFull = `${consumedPrefix}${consumedFname}`; - if (existing.has(consumedFull)) { - throw new Error( - `claim_check_already_consumed: ${consumedFull} already in the dedup index`, - ); - } - let processingFull: string | null = null; - let processingBytes: Uint8Array | null = null; - for (const [blobPath, bytes] of existing) { - if (!blobPath.startsWith(processingPrefix)) continue; - const fname = blobPath.slice(processingPrefix.length); - if (fname.endsWith(`-${args.messageId}.json`)) { - processingFull = blobPath; - processingBytes = bytes; - break; - } - } - if (processingFull === null || processingBytes === null) { - throw new Error( - `claim_check_processing_not_found: address ${args.address} message ${args.messageId} has no processing entry`, - ); - } - const processingEnvelope = decodeQueueEnvelopeOrThrow( - processingBytes, - processingFull, + const { commitSha } = await store.writeTreeDelta(principal, repoId, ref, { + changedPathPrefixes: new Set([addressPrefix(addressSegment)]), + message: `consume ${args.address} ${args.messageId}`, + computeDelta: async (_parentCommitSha, prior) => { + const listing = await readAddressListing(prior, addressSegment); + const consumedFull = consumedPath(addressSegment, args.messageId); + const consumedFname = `${args.messageId}.json`; + if (listing.consumed.some((e) => e.name === consumedFname)) { + throw new Error( + `claim_check_already_consumed: ${consumedFull} already in the dedup index`, ); - const envelope: ConsumedEnvelope = { - messageId: args.messageId, - receivedAt: processingEnvelope.receivedAt, - address: args.address, - runId: args.runId, - consumedAt: args.consumedAt, - mailAuditRef: processingEnvelope.mailAuditRef, - }; - consumedEnvelope = envelope; - - const priorWatermark = readWatermarkFromMap(existing, watermarkFull); - // The watermark may only advance, and never past the entry - // this commit writes (so the new entry is always retained -- - // a message consumed long after receipt may legitimately sit - // below `consumedAt - horizon`, and it is pruned on a later - // commit once the watermark passes ITS receivedAt). - const horizonBoundary = args.consumedAt - retentionHorizonMs; - const newWatermark = Math.max( - priorWatermark, - Math.min(horizonBoundary, envelope.receivedAt), + } + const processingDir = `${addressPrefix(addressSegment)}${WORKFLOW_RUN_PROCESSING_DIR}/`; + const processingEntry = listing.processing.find((e) => + e.name.endsWith(`-${args.messageId}.json`), + ); + if (processingEntry === undefined) { + throw new Error( + `claim_check_processing_not_found: address ${args.address} message ${args.messageId} has no processing entry`, ); - advancedWatermark = newWatermark; - - const files: Record = {}; - for (const [blobPath, blobBytes] of existing) { - if (blobPath === processingFull) continue; - if (blobPath === watermarkFull) continue; - if (blobPath.startsWith(consumedPrefix)) { - // Prune the oldest consumed tail: drop any retained entry - // whose receivedAt has fallen strictly below the new - // watermark. The new entry (added below) is never below - // the watermark by construction. - const consumedReceivedAt = decodeConsumedReceivedAtOrThrow( - blobBytes, - blobPath, - ); - if (consumedReceivedAt < newWatermark) { - const prunedFname = blobPath.slice(consumedPrefix.length); - prunedMessageIds.push(prunedFname.slice(0, -".json".length)); - continue; - } - } - files[blobPath] = blobBytes; - } - files[consumedFull] = utf8(JSON.stringify(envelope)); - files[watermarkFull] = utf8( - JSON.stringify({ watermark: newWatermark }), + } + const processingFull = `${processingDir}${processingEntry.name}`; + const processingBytes = await prior.readBlobByOid(processingEntry.oid); + const processingEnvelope = decodeQueueEnvelopeOrThrow( + processingBytes, + processingFull, + ); + const envelope: ConsumedEnvelope = { + messageId: args.messageId, + receivedAt: processingEnvelope.receivedAt, + address: args.address, + runId: args.runId, + consumedAt: args.consumedAt, + mailAuditRef: processingEnvelope.mailAuditRef, + }; + consumedEnvelope = envelope; + + // The watermark may only advance, and never past the entry this + // commit writes (so the new entry is always retained -- a message + // consumed long after receipt may legitimately sit below + // `consumedAt - horizon`, and it is pruned on a later commit once + // the watermark passes ITS receivedAt). + const horizonBoundary = args.consumedAt - retentionHorizonMs; + const newWatermark = Math.max( + listing.watermark, + Math.min(horizonBoundary, envelope.receivedAt), + ); + advancedWatermark = newWatermark; + + // Prune the oldest consumed tail: read each retained consumed + // entry's receivedAt and drop any that has fallen strictly below + // the new watermark. This is the one leg that must scan the + // consumed index — its filenames carry only the messageId, so the + // receivedAt lives in the bytes — and is the residual the + // consumed-shard lever removes. The new entry (added via puts) is + // never below the watermark by construction. + const consumedDir = `${addressPrefix(addressSegment)}${WORKFLOW_RUN_CONSUMED_DIR}/`; + const deletes: string[] = [processingFull]; + for (const entry of listing.consumed) { + const blobPath = `${consumedDir}${entry.name}`; + const bytes = await prior.readBlobByOid(entry.oid); + const consumedReceivedAt = decodeConsumedReceivedAtOrThrow( + bytes, + blobPath, ); - return files; - }, - message: `consume ${args.address} ${args.messageId}`, + if (consumedReceivedAt < newWatermark) { + prunedMessageIds.push(entry.name.slice(0, -".json".length)); + deletes.push(blobPath); + } + } + return { + puts: { + [consumedFull]: utf8(JSON.stringify(envelope)), + [watermarkPath(addressSegment)]: utf8( + JSON.stringify({ watermark: newWatermark }), + ), + }, + deletes, + }; }, - ); + }); if (consumedEnvelope === null) throw new Error("unreachable"); const captured: ConsumedEnvelope = consumedEnvelope; return { @@ -2989,45 +3031,36 @@ export async function replayProcessingToInbox( ): Promise { const addressSegment = addressSegmentFor(address); const ref = claimCheckCommitRef(); - const repoDir = store.getRepoDir(repoId); const replayedKeys: string[] = []; - const { commitSha } = await store.writeTreePreservingPrefix( - principal, - repoId, - ref, - { - preservePrefix: `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/`, - merge: async () => { - const existing = await readAddressSubtree(repoDir, ref, addressSegment); - const processingPrefix = `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/${WORKFLOW_RUN_PROCESSING_DIR}/`; - const inboxPrefix = `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/${WORKFLOW_RUN_INBOX_DIR}/`; - const files: Record = {}; - for (const [blobPath, bytes] of existing) { - if (blobPath.startsWith(processingPrefix)) { - const fname = blobPath.slice(processingPrefix.length); - const key = fname.slice(0, -".json".length); - const inboxFull = `${inboxPrefix}${fname}`; - if (existing.has(inboxFull)) { - throw new Error( - `claim_check_replay_collision: ${inboxFull} already exists; cannot replay processing entry`, - ); - } - // Re-admit the in-flight entry WITHOUT the watermark - // stale-reject enqueueInbox applies: it was already past - // dedup, so a below-watermark receivedAt is no reason to - // refuse it. Applying the stale-check here would lose a - // legitimately in-flight message after a crash. Do not - // tighten this. - files[inboxFull] = bytes; - replayedKeys.push(key); - continue; - } - files[blobPath] = bytes; + const { commitSha } = await store.writeTreeDelta(principal, repoId, ref, { + changedPathPrefixes: new Set([addressPrefix(addressSegment)]), + message: `replay processing ${address}`, + computeDelta: async (_parentCommitSha, prior) => { + const listing = await readAddressListing(prior, addressSegment); + const processingDir = `${addressPrefix(addressSegment)}${WORKFLOW_RUN_PROCESSING_DIR}/`; + const inboxDir = `${addressPrefix(addressSegment)}${WORKFLOW_RUN_INBOX_DIR}/`; + const inboxNames = new Set(listing.inbox.map((e) => e.name)); + const puts: Record = {}; + const deletes: string[] = []; + for (const entry of listing.processing) { + const inboxFull = `${inboxDir}${entry.name}`; + if (inboxNames.has(entry.name)) { + throw new Error( + `claim_check_replay_collision: ${inboxFull} already exists; cannot replay processing entry`, + ); } - return files; - }, - message: `replay processing ${address}`, + // Re-admit the in-flight entry WITHOUT the watermark stale-reject + // enqueueInbox applies: it was already past dedup, so a + // below-watermark receivedAt is no reason to refuse it. Applying + // the stale-check here would lose a legitimately in-flight + // message after a crash. Do not tighten this. + const bytes = await prior.readBlobByOid(entry.oid); + puts[inboxFull] = bytes; + deletes.push(`${processingDir}${entry.name}`); + replayedKeys.push(entry.name.slice(0, -".json".length)); + } + return { puts, deletes }; }, - ); + }); return { commitSha, replayedKeys }; } diff --git a/packages/hub-sessions/src/workflow-run-reader.test.ts b/packages/hub-sessions/src/workflow-run-reader.test.ts index 951bd596..fddc8026 100644 --- a/packages/hub-sessions/src/workflow-run-reader.test.ts +++ b/packages/hub-sessions/src/workflow-run-reader.test.ts @@ -20,6 +20,7 @@ function repoStoreFor(dirById: Map): RepoStore { initRepo: unused, writeTree: unused, writeTreePreservingPrefix: unused, + writeTreeDelta: unused, receivePack: unused, createPack: unused, resolveRef: unused, diff --git a/packages/hub-sessions/src/ws/sidecar-events.test.ts b/packages/hub-sessions/src/ws/sidecar-events.test.ts index 54388a77..d2714afd 100644 --- a/packages/hub-sessions/src/ws/sidecar-events.test.ts +++ b/packages/hub-sessions/src/ws/sidecar-events.test.ts @@ -28,7 +28,7 @@ describe("createSidecarEmitter", () => { order.push("b"); }); - emitter.emit("sidecar.disconnect", { agentAddresses: [] }); + emitter.emit("sidecar.disconnect", { ownedAddresses: [] }); expect(order).toEqual(["a", "b"]); }); @@ -40,9 +40,9 @@ describe("createSidecarEmitter", () => { count++; }); - emitter.emit("sidecar.disconnect", { agentAddresses: [] }); + emitter.emit("sidecar.disconnect", { ownedAddresses: [] }); unsubscribe(); - emitter.emit("sidecar.disconnect", { agentAddresses: [] }); + emitter.emit("sidecar.disconnect", { ownedAddresses: [] }); expect(count).toBe(1); }); @@ -57,7 +57,7 @@ describe("createSidecarEmitter", () => { seen.push("ran"); }); - emitter.emit("sidecar.disconnect", { agentAddresses: [] }); + emitter.emit("sidecar.disconnect", { ownedAddresses: [] }); expect(seen).toEqual(["ran"]); }); diff --git a/packages/hub-sessions/src/ws/sidecar-events.ts b/packages/hub-sessions/src/ws/sidecar-events.ts index 450b4dd7..4b4bab6d 100644 --- a/packages/hub-sessions/src/ws/sidecar-events.ts +++ b/packages/hub-sessions/src/ws/sidecar-events.ts @@ -19,11 +19,7 @@ // wire layer already has both behaviors, and pretending otherwise // would silently change failure handling. -import type { - DeployApplyErrorCategory, - PackRejectReason, - RepoId, -} from "@intx/types/sidecar"; +import type { PackRejectReason, RepoId } from "@intx/types/sidecar"; import type { ConnectorThreadState } from "@intx/types/runtime"; import { getLogger } from "@intx/log"; @@ -53,10 +49,11 @@ export type SidecarEventMap = { }; /** Notification. Emitted once when a sidecar's connection closes, - * carrying the set of agent addresses that were registered on that - * connection. */ + * carrying every address the connection owned -- challenged session + * addresses and hub-minted workflow-substrate deployment addresses + * alike -- so lifecycle teardown covers both. */ "sidecar.disconnect": { - agentAddresses: string[]; + ownedAddresses: string[]; }; /** Notification. Emitted when a mail.outbound frame from a sidecar @@ -111,26 +108,6 @@ export type SidecarEventMap = { "deploy.ref.stale": { agentAddress: string; }; - - /** Notification. Emitted when the sidecar reports that its - * tool-package apply pipeline rejected a deploy. `category` matches - * the closed `DeployApplyErrorCategory` enum; `package` is set - * when the failure implicates a specific manifest entry. - * `previousDeployId` is the atomicity contract — it is always the - * deploy id the instance was running before the rejected attempt - * (the instance keeps running that deploy untouched). Listeners - * are responsible for surfacing the failure to operators and/or - * the deploy lifecycle subsystem; the wire layer only delivers. - */ - "deploy.apply.error": { - agentAddress: string; - attemptId: string; - previousDeployId: string; - category: DeployApplyErrorCategory; - message: string; - package?: { name: string; version: string }; - occurredAt: string; - }; }; export type SidecarEventType = keyof SidecarEventMap; @@ -164,7 +141,6 @@ export function createSidecarEmitter(): SidecarEventEmitter { "agent.deploy.ack": new Set(), "agent.reconnected": new Set(), "deploy.ref.stale": new Set(), - "deploy.apply.error": new Set(), "connector.state.changed": new Set(), }; diff --git a/packages/hub-sessions/src/ws/sidecar-handler.test.ts b/packages/hub-sessions/src/ws/sidecar-handler.test.ts index 7854df0a..e8094fd3 100644 --- a/packages/hub-sessions/src/ws/sidecar-handler.test.ts +++ b/packages/hub-sessions/src/ws/sidecar-handler.test.ts @@ -1,4 +1,5 @@ import { describe, test, expect, beforeEach } from "bun:test"; +import { configureSync, getConfig, resetSync } from "@intx/log"; import { generateKeyPair, signEd25519 } from "@intx/crypto"; import { hexDecode, hexEncode, parseAgentAddress } from "@intx/types"; import { chunkPack } from "@intx/pack-transport"; @@ -38,20 +39,48 @@ function lastSent(ws: ReturnType) { return JSON.parse(last); } +// Let the async register key-existence gate settle. A `register` frame now +// routes on a microtask (handleRegister awaits `lookupPublicKey` per address), +// so a test that reads the routing table right after sending one must await +// this first. +async function tick(): Promise { + await new Promise((resolve) => setTimeout(resolve, 0)); +} + describe("SidecarRouter", () => { let router: ReturnType; const TEST_HUB_KEY = "a".repeat(64); + // A minimal valid source list for exercising the requestId-correlated + // sendRequest path (sendSourcesUpdate). The routing/reconnect invariants + // under test are independent of the source content. + const TEST_SOURCES = [ + { + id: "anthropic:claude-sonnet-4-20250514", + provider: "anthropic", + baseURL: "https://api.anthropic.com", + apiKey: "sk-test", + model: "claude-sonnet-4-20250514", + }, + ]; + const TEST_DEFAULT_SOURCE = "anthropic:claude-sonnet-4-20250514"; + beforeEach(() => { router = createSidecarRouter({ requestTimeoutMs: 500, hubPublicKey: TEST_HUB_KEY, + // Always-null lookup: no address has a stored key, so the register + // key-existence gate classifies every registered address as a keyless + // first-deploy and routes it. This makes the dependency explicit for + // the pure-routing tests (the gate is fail-closed without it) while + // preserving their "register routes the address" behavior. + lookups: { lookupPublicKey: async () => null }, }); }); describe("registration", () => { - test("register frame populates routing table", () => { + test("register frame populates routing table", async () => { const ws = createMockWs(); router.handleOpen(ws); router.handleMessage( @@ -63,6 +92,7 @@ describe("SidecarRouter", () => { agentAddresses: ["agent-a@local", "agent-b@local"], }), ); + await tick(); expect(router.getConnectedSidecars()).toEqual(["sc-1"]); expect(router.getRoutableAddresses().sort()).toEqual([ @@ -71,7 +101,10 @@ describe("SidecarRouter", () => { ]); }); - test("re-registration updates addresses", () => { + test("re-registration adds addresses without dropping owned routes", async () => { + // Additive re-register: a second register ADDS its addresses without + // dropping those from the first. Removal happens via undeploy/disconnect, + // not register-omission (the frame no longer carries the full live set). const ws = createMockWs(); router.handleOpen(ws); router.handleMessage( @@ -83,6 +116,7 @@ describe("SidecarRouter", () => { agentAddresses: ["agent-a@local"], }), ); + await tick(); router.handleMessage( ws, @@ -93,8 +127,12 @@ describe("SidecarRouter", () => { agentAddresses: ["agent-c@local"], }), ); + await tick(); - expect(router.getRoutableAddresses()).toEqual(["agent-c@local"]); + expect(router.getRoutableAddresses().sort()).toEqual([ + "agent-a@local", + "agent-c@local", + ]); }); test("disconnect cleans up routing table", () => { @@ -115,7 +153,7 @@ describe("SidecarRouter", () => { expect(router.getRoutableAddresses()).toEqual([]); }); - test("invalid token closes connection", () => { + test("invalid token closes connection", async () => { const router = createSidecarRouter({ validateToken: () => false, }); @@ -126,53 +164,797 @@ describe("SidecarRouter", () => { JSON.stringify({ type: "register", sidecarId: "sc-1", - token: "bad", - agentAddresses: [], + token: "bad", + agentAddresses: [], + }), + ); + await tick(); + + expect(ws.closed).toBe(true); + expect(router.getConnectedSidecars()).toEqual([]); + }); + + test("re-registration by another sidecar cleans ghost from old connection", async () => { + const ws1 = createMockWs(); + router.handleOpen(ws1); + router.handleMessage( + ws1, + JSON.stringify({ + type: "register", + sidecarId: "sc-1", + token: "tok", + agentAddresses: ["agent@local", "other@local"], + }), + ); + await tick(); + + const ws2 = createMockWs(); + router.handleOpen(ws2); + router.handleMessage( + ws2, + JSON.stringify({ + type: "register", + sidecarId: "sc-2", + token: "tok", + agentAddresses: ["agent@local"], + }), + ); + await tick(); + + // ws2 now owns agent@local. Closing ws1 should only remove + // other@local (which ws1 still owns), not agent@local. + router.handleClose(ws1); + + expect(router.getRoutableAddresses()).toContain("agent@local"); + expect(router.getRoutableAddresses()).not.toContain("other@local"); + expect(router.routeMail("agent@local", "hello")).toBe(true); + expect(ws2.sent).toHaveLength(1); + }); + }); + + describe("register key-existence gate", () => { + // A register frame is token-authenticated but proves no per-address + // ownership, so it may route only a KEYLESS first-deploy address. An + // address that already has a stored key must prove ownership through the + // challenged reconnect path; register must neither route it to the caller + // nor disturb its existing owner. + const KEYED = "victim@local"; + + function gatedRouter(publicKeyHex: string | null) { + return createSidecarRouter({ + requestTimeoutMs: 5000, + hubPublicKey: TEST_HUB_KEY, + lookups: { + lookupPublicKey: async (addr) => + addr === KEYED ? publicKeyHex : null, + }, + }); + } + + // Bring KEYED up as owned by `ws` through the challenged reconnect path. + async function reconnectVerify( + r: ReturnType, + ws: ReturnType, + privateKey: Uint8Array, + ) { + r.handleOpen(ws); + r.handleMessage( + ws, + JSON.stringify({ + type: "reconnect", + sidecarId: "owner", + token: "tok", + agentAddresses: [KEYED], + }), + ); + await new Promise((res) => setTimeout(res, 50)); + const challenge = ws.sent + .map((s) => JSON.parse(s)) + .find((f: { type: string }) => f.type === "challenge"); + r.handleMessage( + ws, + JSON.stringify({ + type: "challenge.response", + responses: [ + { + address: KEYED, + signature: await signChallenge( + challenge.challenges[0].nonce, + KEYED, + privateKey, + ), + }, + ], + }), + ); + await new Promise((res) => setTimeout(res, 50)); + } + + test("routes a keyless first-deploy address", async () => { + const r = gatedRouter(null); + const ws = createMockWs(); + r.handleOpen(ws); + r.handleMessage( + ws, + JSON.stringify({ + type: "register", + sidecarId: "sc-1", + token: "tok", + agentAddresses: ["fresh@local"], + }), + ); + await tick(); + + expect(r.getRoutableAddresses()).toContain("fresh@local"); + expect(r.routeMail("fresh@local", "dGVzdA==")).toBe(true); + }); + + test("refuses a keyed address and leaves its owner's route untouched", async () => { + const kp = await generateKeyPair(); + const r = gatedRouter(hexEncode(kp.publicKey)); + + // The true owner establishes KEYED via the challenged reconnect. + const ownerWs = createMockWs(); + await reconnectVerify(r, ownerWs, kp.privateKey); + expect(r.getRoutableAddresses()).toContain(KEYED); + + // A rogue sidecar with a valid token names KEYED in a register frame. + const rogueWs = createMockWs(); + r.handleOpen(rogueWs); + r.handleMessage( + rogueWs, + JSON.stringify({ + type: "register", + sidecarId: "rogue", + token: "tok", + agentAddresses: [KEYED], + }), + ); + await tick(); + + // KEYED is still routed to the owner, not the rogue: the gate refused to + // route it AND -- because it runs before the ghost-cleanup -- never + // evicted the owner. No hijack, and no downgrade to a denial of service. + expect(r.getRoutableAddresses()).toContain(KEYED); + ownerWs.sent.length = 0; + rogueWs.sent.length = 0; + expect(r.routeMail(KEYED, "dGVzdA==")).toBe(true); + expect(ownerWs.sent).toHaveLength(1); + expect(rogueWs.sent).toHaveLength(0); + }); + + test("a re-register does not drop an address already verified via reconnect", async () => { + // A connection that proved a keyed address via challenged reconnect keeps + // that route across a later register, even though the gate would refuse + // to re-add a keyed address: additive re-register inherits the owned set + // rather than replacing it. Removal is via undeploy/disconnect, not + // register-omission. + const kp = await generateKeyPair(); + const r = gatedRouter(hexEncode(kp.publicKey)); + const ws = createMockWs(); + await reconnectVerify(r, ws, kp.privateKey); + expect(r.getRoutableAddresses()).toContain(KEYED); + + // The same ws registers a keyless first-deploy. The keyed + // reconnect-verified route must survive alongside the new one. + r.handleMessage( + ws, + JSON.stringify({ + type: "register", + sidecarId: "owner", + token: "tok", + agentAddresses: ["fresh@local"], + }), + ); + await tick(); + + expect(r.getRoutableAddresses().sort()).toEqual( + [KEYED, "fresh@local"].sort(), + ); + expect(r.routeMail(KEYED, "dGVzdA==")).toBe(true); + }); + + test("fails closed and surfaces an error when no lookup is configured", async () => { + const captured: { level: string; message: string }[] = []; + const savedConfig = getConfig(); + configureSync({ + reset: true, + sinks: { + capture: (record) => { + const message = Array.isArray(record.message) + ? record.message + .map((part) => + typeof part === "string" ? part : JSON.stringify(part), + ) + .join("") + : String(record.message); + captured.push({ level: record.level, message }); + }, + }, + loggers: [{ category: [], lowestLevel: "debug", sinks: ["capture"] }], + }); + try { + // No lookups configured: the gate cannot distinguish a keyed address + // from a first-deploy, so it must route nothing rather than permit. + const r = createSidecarRouter({ + requestTimeoutMs: 500, + hubPublicKey: TEST_HUB_KEY, + }); + const ws = createMockWs(); + r.handleOpen(ws); + r.handleMessage( + ws, + JSON.stringify({ + type: "register", + sidecarId: "sc-1", + token: "tok", + agentAddresses: ["x@local"], + }), + ); + await tick(); + + expect(r.getRoutableAddresses()).toEqual([]); + expect(r.routeMail("x@local", "dGVzdA==")).toBe(false); + // The missing dependency surfaces as an error, not a silent permit. + expect( + captured.some( + (l) => + l.level === "error" && + l.message.includes("lookupPublicKey is not configured"), + ), + ).toBe(true); + } finally { + if (savedConfig) { + configureSync({ reset: true, ...savedConfig }); + } else { + resetSync(); + } + } + }); + + test("fails closed on a key-lookup error instead of crashing", async () => { + // A rejecting lookup (e.g. a transient DB failure) must be caught and + // surfaced, not floated out of the void-dispatched handler as an + // unhandled rejection that could take down the hub. Fail closed: route + // nothing and log an error. (The test completing rather than hanging on + // an unhandled rejection is itself part of the assertion.) + const captured: { level: string; message: string }[] = []; + const savedConfig = getConfig(); + configureSync({ + reset: true, + sinks: { + capture: (record) => { + const message = Array.isArray(record.message) + ? record.message + .map((part) => + typeof part === "string" ? part : JSON.stringify(part), + ) + .join("") + : String(record.message); + captured.push({ level: record.level, message }); + }, + }, + loggers: [{ category: [], lowestLevel: "debug", sinks: ["capture"] }], + }); + try { + const r = createSidecarRouter({ + requestTimeoutMs: 500, + hubPublicKey: TEST_HUB_KEY, + lookups: { + lookupPublicKey: async () => { + throw new Error("db unavailable"); + }, + }, + }); + const ws = createMockWs(); + r.handleOpen(ws); + r.handleMessage( + ws, + JSON.stringify({ + type: "register", + sidecarId: "sc-1", + token: "tok", + agentAddresses: ["x@local"], + }), + ); + await tick(); + + expect(r.getRoutableAddresses()).toEqual([]); + expect( + captured.some( + (l) => + l.level === "error" && l.message.includes("Key lookup failed"), + ), + ).toBe(true); + } finally { + if (savedConfig) { + configureSync({ reset: true, ...savedConfig }); + } else { + resetSync(); + } + } + }); + }); + + describe("frame dispatch serialization (F1)", () => { + const STATE = { + threadRoot: "", + lastMessageId: "", + replyTo: "user@example.com", + cc: [], + }; + + async function driveReconnect( + r: ReturnType, + ws: ReturnType, + addr: string, + privateKey: Uint8Array, + ) { + r.handleMessage( + ws, + JSON.stringify({ + type: "reconnect", + sidecarId: "sc", + token: "tok", + agentAddresses: [addr], + }), + ); + await new Promise((res) => setTimeout(res, 50)); + const challenge = ws.sent + .map((s) => JSON.parse(s)) + .find((f: { type: string }) => f.type === "challenge"); + r.handleMessage( + ws, + JSON.stringify({ + type: "challenge.response", + responses: [ + { + address: addr, + signature: await signChallenge( + challenge.challenges[0].nonce, + addr, + privateKey, + ), + }, + ], + }), + ); + await new Promise((res) => setTimeout(res, 50)); + } + + test("a dependent frame after a non-empty register is processed, not dropped", async () => { + // The register key-existence gate awaits lookupPublicKey, so routing lands + // asynchronously. Per-ws serialization makes a following dependent frame + // (connector.state.changed for the just-registered keyless address) wait + // for that routing rather than be dropped because addressIndex was empty. + const r = createSidecarRouter({ + requestTimeoutMs: 500, + hubPublicKey: TEST_HUB_KEY, + lookups: { lookupPublicKey: async () => null }, + }); + const ws = createMockWs(); + r.handleOpen(ws); + // Non-empty register immediately followed by a dependent frame, with NO + // await between -- the follower must still observe the route. + r.handleMessage( + ws, + JSON.stringify({ + type: "register", + sidecarId: "sc", + token: "tok", + agentAddresses: ["fresh@local"], + }), + ); + r.handleMessage( + ws, + JSON.stringify({ + type: "connector.state.changed", + agentAddress: "fresh@local", + connectorState: STATE, + }), + ); + await tick(); + + expect(r.getConnectorState("fresh@local")).toEqual(STATE); + }); + + test("a rogue's dependent frame for a keyed victim is dropped, not poisoned", async () => { + // Serialization must NOT reintroduce the poisoning it exists to prevent: a + // rogue's non-empty register naming a KEYED victim is refused by the gate, + // so a connector.state.changed from the rogue in the same window still + // sees addressIndex.get(victim) !== rogueWs and is dropped; the victim's + // cached state is untouched. + const kp = await generateKeyPair(); + const victim = "victim@local"; + const r = createSidecarRouter({ + requestTimeoutMs: 5000, + hubPublicKey: TEST_HUB_KEY, + lookups: { + lookupPublicKey: async (addr) => + addr === victim ? hexEncode(kp.publicKey) : null, + }, + }); + const ownerWs = createMockWs(); + r.handleOpen(ownerWs); + await driveReconnect(r, ownerWs, victim, kp.privateKey); + r.handleMessage( + ownerWs, + JSON.stringify({ + type: "connector.state.changed", + agentAddress: victim, + connectorState: STATE, + }), + ); + await tick(); + expect(r.getConnectorState(victim)).toEqual(STATE); + + // Rogue: register naming the keyed victim + a poisoning frame, no await. + const rogueWs = createMockWs(); + r.handleOpen(rogueWs); + r.handleMessage( + rogueWs, + JSON.stringify({ + type: "register", + sidecarId: "rogue", + token: "tok", + agentAddresses: [victim], + }), + ); + r.handleMessage( + rogueWs, + JSON.stringify({ + type: "connector.state.changed", + agentAddress: victim, + connectorState: { ...STATE, replyTo: "attacker@evil" }, + }), + ); + await tick(); + + // Not poisoned: still the owner's state. + expect(r.getConnectorState(victim)).toEqual(STATE); + }); + + test("a reconnect whose reaction awaits a later session.ack does not deadlock", async () => { + // handleChallengeResponse awaits the agent.reconnected reaction, whose + // subscriber issues sendSourcesUpdate -> awaits a LATER session.ack frame. + // session.ack must BYPASS the per-ws chain, or the reconnect wedges behind + // its own in-flight handler. (The test completing rather than timing out + // is the assertion.) + const kp = await generateKeyPair(); + const addr = "agent@local"; + const r = createSidecarRouter({ + requestTimeoutMs: 5000, + hubPublicKey: TEST_HUB_KEY, + lookups: { + lookupPublicKey: async (a) => + a === addr ? hexEncode(kp.publicKey) : null, + }, + }); + let reactionDone = false; + r.events.on("agent.reconnected", async () => { + await r.sendSourcesUpdate(addr, TEST_SOURCES, TEST_DEFAULT_SOURCE); + reactionDone = true; + }); + const ws = createMockWs(); + r.handleOpen(ws); + await driveReconnect(r, ws, addr, kp.privateKey); + + // The reaction has sent a sources.update request and is awaiting its ack. + const sourcesFrame = ws.sent + .map((s) => JSON.parse(s)) + .find((f: { type: string }) => f.type === "sources.update"); + expect(sourcesFrame).toBeDefined(); + + // Answer with a session.ack (a later inbound frame). If it queued behind + // the still-in-flight handleChallengeResponse it would never resolve. + r.handleMessage( + ws, + JSON.stringify({ + type: "session.ack", + requestId: sourcesFrame.requestId, + }), + ); + await new Promise((res) => setTimeout(res, 50)); + + expect(reactionDone).toBe(true); + expect(r.getRoutableAddresses()).toContain(addr); + }); + }); + + describe("workflow-address reconnect (challenged, at parity with launched agents)", () => { + // A workflow-substrate deployment address (ins_dep_...) proves ownership + // through the SAME Ed25519 challenge as a launched agent: the hub resolves + // the deployment's public key, issues a nonce, and routes the address only + // after a valid signature. These tests pin that parity -- there is no + // keyless register-field shortcut for a workflow address anymore, so a + // token-holding sidecar cannot reclaim a deployment's route without the + // deployment's own key. + const WF_ADDR = "ins_dep_abc@local"; + + // Router whose key lookup resolves WF_ADDR to `publicKeyHex` (a live + // deployment) or to null (unknown / torn-down), mirroring the + // launched-agent reconnect tests. + function workflowReconnectRouter(publicKeyHex: string | null) { + return createSidecarRouter({ + requestTimeoutMs: 5000, + hubPublicKey: TEST_HUB_KEY, + lookups: { + lookupPublicKey: async (addr) => + addr === WF_ADDR ? publicKeyHex : null, + }, + }); + } + + function findFrame(ws: ReturnType, type: string) { + return ws.sent.map((s) => JSON.parse(s)).find((f) => f.type === type); + } + + async function reconnect( + r: ReturnType, + ws: ReturnType, + ) { + r.handleOpen(ws); + r.handleMessage( + ws, + JSON.stringify({ + type: "reconnect", + sidecarId: "sc-1", + token: "tok", + agentAddresses: [WF_ADDR], + }), + ); + await new Promise((res) => setTimeout(res, 50)); + } + + async function respondToChallenge( + r: ReturnType, + ws: ReturnType, + privateKey: Uint8Array, + ) { + const { address, nonce } = findFrame(ws, "challenge").challenges[0]; + r.handleMessage( + ws, + JSON.stringify({ + type: "challenge.response", + responses: [ + { + address, + signature: await signChallenge(nonce, address, privateKey), + }, + ], + }), + ); + await new Promise((res) => setTimeout(res, 50)); + } + + test("routes a workflow address only after a passed challenge", async () => { + const kp = await generateKeyPair(); + const r = workflowReconnectRouter(hexEncode(kp.publicKey)); + const ws = createMockWs(); + await reconnect(r, ws); + + // The reconnect frame alone earns nothing: the address is not routable + // until the challenge is answered. + expect(r.getRoutableAddresses()).not.toContain(WF_ADDR); + const challenge = findFrame(ws, "challenge"); + expect(challenge).toBeDefined(); + expect(challenge.challenges[0].address).toBe(WF_ADDR); + + await respondToChallenge(r, ws, kp.privateKey); + + expect(r.getRoutableAddresses()).toContain(WF_ADDR); + expect(r.routeMail(WF_ADDR, "dGVzdA==")).toBe(true); + }); + + test("rejects a workflow address signed with the wrong key (no hijack)", async () => { + const kp = await generateKeyPair(); + const wrongKp = await generateKeyPair(); + const r = workflowReconnectRouter(hexEncode(kp.publicKey)); + const ws = createMockWs(); + await reconnect(r, ws); + + await respondToChallenge(r, ws, wrongKp.privateKey); + + expect(r.getRoutableAddresses()).not.toContain(WF_ADDR); + expect(r.routeMail(WF_ADDR, "dGVzdA==")).toBe(false); + expect(findFrame(ws, "challenge.failed").address).toBe(WF_ADDR); + }); + + test("fails closed when the deployment has no live key", async () => { + // lookupPublicKey resolves null -- an unknown or torn-down deployment. + // No challenge is issued and the address never routes. + const r = workflowReconnectRouter(null); + const ws = createMockWs(); + await reconnect(r, ws); + + expect(r.getRoutableAddresses()).not.toContain(WF_ADDR); + expect(findFrame(ws, "challenge")).toBeUndefined(); + expect(findFrame(ws, "challenge.failed").address).toBe(WF_ADDR); + }); + + test("disconnect removes a verified workflow address from routing", async () => { + const kp = await generateKeyPair(); + const r = workflowReconnectRouter(hexEncode(kp.publicKey)); + const ws = createMockWs(); + await reconnect(r, ws); + await respondToChallenge(r, ws, kp.privateKey); + expect(r.routeMail(WF_ADDR, "dGVzdA==")).toBe(true); + + r.handleClose(ws); + + expect(r.getRoutableAddresses()).toEqual([]); + expect(r.routeMail(WF_ADDR, "dGVzdA==")).toBe(false); + }); + + test("a fresh ws reclaims a workflow address only by passing its own challenge", async () => { + const kp = await generateKeyPair(); + const r = workflowReconnectRouter(hexEncode(kp.publicKey)); + + const oldWs = createMockWs(); + await reconnect(r, oldWs); + await respondToChallenge(r, oldWs, kp.privateKey); + expect(r.routeMail(WF_ADDR, "dGVzdA==")).toBe(true); + + const newWs = createMockWs(); + await reconnect(r, newWs); + await respondToChallenge(r, newWs, kp.privateKey); + + // The stale ws's later close must not clobber the new owner (the + // ownership guard in handleClose). + r.handleClose(oldWs); + + expect(r.getRoutableAddresses()).toContain(WF_ADDR); + expect(r.routeMail(WF_ADDR, "dGVzdA==")).toBe(true); + }); + }); + + describe("per-step route bind/unbind", () => { + const DEPLOYMENT_ADDR = "ins_dep_multi@local"; + const STEP_ADDR = "ins_dep_multi-step1@local"; + const ZERO_SHA = "0".repeat(40); + + async function registerDeploymentSidecar(): Promise< + ReturnType + > { + const ws = createMockWs(); + router.handleOpen(ws); + router.handleMessage( + ws, + JSON.stringify({ + type: "register", + sidecarId: "sc-1", + token: "tok", + agentAddresses: [DEPLOYMENT_ADDR], }), ); + // Register routing is async (the key-existence gate awaits the lookup); + // settle it before assertions read the routing table. + await new Promise((res) => setTimeout(res, 0)); + return ws; + } - expect(ws.closed).toBe(true); - expect(router.getConnectedSidecars()).toEqual([]); + test("bind makes a step address routable; unbind removes it", async () => { + await registerDeploymentSidecar(); + + // An unbound step address has no route: `sendPack` rejects before + // touching the wire. + await expect( + router.sendPack( + STEP_ADDR, + new Uint8Array([1]), + "refs/heads/main", + ZERO_SHA, + ), + ).rejects.toThrow(/No sidecar connected/); + + router.bindStepRoute(STEP_ADDR); + expect(router.getRoutableAddresses()).toContain(STEP_ADDR); + // Routable: `routeMail` (and `sendPack`) now resolve the sidecar. + expect(router.routeMail(STEP_ADDR, "dGVzdA==")).toBe(true); + + router.unbindStepRoute(STEP_ADDR); + expect(router.getRoutableAddresses()).not.toContain(STEP_ADDR); + await expect( + router.sendPack( + STEP_ADDR, + new Uint8Array([1]), + "refs/heads/main", + ZERO_SHA, + ), + ).rejects.toThrow(/No sidecar connected/); }); - test("re-registration by another sidecar cleans ghost from old connection", () => { - const ws1 = createMockWs(); - router.handleOpen(ws1); - router.handleMessage( - ws1, - JSON.stringify({ - type: "register", - sidecarId: "sc-1", - token: "tok", - agentAddresses: ["agent@local", "other@local"], - }), - ); + test("a reconnect after unbind does not resurrect the transient route", async () => { + // The deployment address is re-announced on reconnect and proves + // ownership through the challenge; the transient per-step route + // (bind/unbind) is never persisted into that announcement, so a + // reconnect must not bring it back. + const kp = await generateKeyPair(); + const reconnectRouter = createSidecarRouter({ + requestTimeoutMs: 5000, + hubPublicKey: TEST_HUB_KEY, + lookups: { + lookupPublicKey: async (addr) => + addr === DEPLOYMENT_ADDR ? hexEncode(kp.publicKey) : null, + }, + }); + const ws = createMockWs(); + reconnectRouter.handleOpen(ws); - const ws2 = createMockWs(); - router.handleOpen(ws2); - router.handleMessage( - ws2, - JSON.stringify({ - type: "register", - sidecarId: "sc-2", - token: "tok", - agentAddresses: ["agent@local"], - }), - ); + // Bring the deployment address up through the challenged reconnect path. + async function reconnectDeployment() { + reconnectRouter.handleMessage( + ws, + JSON.stringify({ + type: "reconnect", + sidecarId: "sc-1", + token: "tok", + agentAddresses: [DEPLOYMENT_ADDR], + }), + ); + await new Promise((r) => setTimeout(r, 50)); + const challenges = ws.sent + .map((s) => JSON.parse(s)) + .filter((f) => f.type === "challenge"); + const { address, nonce } = + challenges[challenges.length - 1].challenges[0]; + reconnectRouter.handleMessage( + ws, + JSON.stringify({ + type: "challenge.response", + responses: [ + { + address, + signature: await signChallenge(nonce, address, kp.privateKey), + }, + ], + }), + ); + await new Promise((r) => setTimeout(r, 50)); + } - // ws2 now owns agent@local. Closing ws1 should only remove - // other@local (which ws1 still owns), not agent@local. - router.handleClose(ws1); + await reconnectDeployment(); + expect(reconnectRouter.getRoutableAddresses()).toContain(DEPLOYMENT_ADDR); - expect(router.getRoutableAddresses()).toContain("agent@local"); - expect(router.getRoutableAddresses()).not.toContain("other@local"); - expect(router.routeMail("agent@local", "hello")).toBe(true); - expect(ws2.sent).toHaveLength(1); + reconnectRouter.bindStepRoute(STEP_ADDR); + reconnectRouter.unbindStepRoute(STEP_ADDR); + + // The sidecar reconnects announcing only the deployment address -- never + // the transient per-step address. + await reconnectDeployment(); + + // The reconnect actually ran (it did not bail and close the socket). + expect(ws.closed).toBe(false); + // The deployment address is routable again; the transient step route + // stays gone. + expect(reconnectRouter.getRoutableAddresses()).toContain(DEPLOYMENT_ADDR); + expect(reconnectRouter.getRoutableAddresses()).not.toContain(STEP_ADDR); + }); + + test("bind throws when no sidecar is connected", () => { + expect(() => router.bindStepRoute(STEP_ADDR)).toThrow(/No sidecar/); + }); + + test("unbind is a no-op for an address that was never bound", async () => { + await registerDeploymentSidecar(); + expect(() => router.unbindStepRoute(STEP_ADDR)).not.toThrow(); + expect(router.getRoutableAddresses()).not.toContain(STEP_ADDR); + }); + + test("handleClose reclaims a still-bound step route", async () => { + const ws = await registerDeploymentSidecar(); + router.bindStepRoute(STEP_ADDR); + expect(router.getRoutableAddresses()).toContain(STEP_ADDR); + + // A sidecar drop mid-stage tears the transient binding down with the + // rest of the connection's addresses -- no stale route survives. + router.handleClose(ws); + expect(router.getRoutableAddresses()).not.toContain(STEP_ADDR); }); }); describe("mail routing", () => { - test("routes mail between two sidecars", () => { + test("routes mail between two sidecars", async () => { const ws1 = createMockWs(); const ws2 = createMockWs(); router.handleOpen(ws1); @@ -196,6 +978,7 @@ describe("SidecarRouter", () => { agentAddresses: ["receiver@local"], }), ); + await tick(); router.handleMessage( ws1, @@ -205,6 +988,7 @@ describe("SidecarRouter", () => { recipients: ["receiver@local"], }), ); + await tick(); const delivered = lastSent(ws2); expect(delivered.type).toBe("mail.inbound"); @@ -216,7 +1000,7 @@ describe("SidecarRouter", () => { expect(router.routeMail("nobody@local", "dGVzdA==")).toBe(false); }); - test("routeMail returns true for routable address", () => { + test("routeMail returns true for routable address", async () => { const ws = createMockWs(); router.handleOpen(ws); router.handleMessage( @@ -228,15 +1012,18 @@ describe("SidecarRouter", () => { agentAddresses: ["agent@local"], }), ); + await tick(); expect(router.routeMail("agent@local", "dGVzdA==")).toBe(true); const delivered = lastSent(ws); expect(delivered.type).toBe("mail.inbound"); }); - test("unroutable mail emits mail.outbound.undelivered", () => { + test("unroutable mail emits mail.outbound.undelivered", async () => { const outbound: { rawMessage: string; recipients: string[] }[] = []; - const router = createSidecarRouter({}); + const router = createSidecarRouter({ + lookups: { lookupPublicKey: async () => null }, + }); router.events.on("mail.outbound.undelivered", (event) => { outbound.push({ rawMessage: event.rawMessage, @@ -264,6 +1051,7 @@ describe("SidecarRouter", () => { recipients: ["external@remote"], }), ); + await tick(); expect(outbound).toHaveLength(1); expect(outbound[0]?.recipients).toEqual(["external@remote"]); @@ -277,6 +1065,7 @@ describe("SidecarRouter", () => { const persisted: { id: string; address: string }[] = []; const router = createSidecarRouter({ lookups: { + lookupPublicKey: async () => null, persistMail: async ({ senderAddress, recipients }) => { // Mirrors the fixed persistMail: always create the // outbound record for the sender, and only create @@ -355,6 +1144,7 @@ describe("SidecarRouter", () => { const persisted: { id: string; address: string }[] = []; const router = createSidecarRouter({ lookups: { + lookupPublicKey: async () => null, persistMail: async ({ senderAddress, recipients }) => { return [ { @@ -426,6 +1216,7 @@ describe("SidecarRouter", () => { agentAddresses: [], }), ); + await tick(); const config = { sessionId: "ses_test", @@ -488,6 +1279,7 @@ describe("SidecarRouter", () => { agentAddresses: [], }), ); + await tick(); const config = { sessionId: "ses_test", @@ -547,6 +1339,7 @@ describe("SidecarRouter", () => { agentAddresses: [], }), ); + await tick(); const config = { sessionId: "ses_test", @@ -596,6 +1389,7 @@ describe("SidecarRouter", () => { agentAddresses: [], }), ); + await tick(); const config = { sessionId: "ses_test", @@ -645,6 +1439,7 @@ describe("SidecarRouter", () => { agentAddresses: ["agent@local"], }), ); + await tick(); const promise = router.sendAgentUndeploy("agent@local", "session_ended"); const frame = lastSent(ws); @@ -680,6 +1475,7 @@ describe("SidecarRouter", () => { agentAddresses: [], }), ); + await tick(); const config = { sessionId: "ses_test", @@ -725,6 +1521,7 @@ describe("SidecarRouter", () => { agentAddresses: ["undeploy-dc@local"], }), ); + await tick(); // Start an undeploy but disconnect before the ack arrives. const promise = router.sendAgentUndeploy("undeploy-dc@local", "teardown"); @@ -747,6 +1544,7 @@ describe("SidecarRouter", () => { agentAddresses: [], }), ); + await tick(); const config = { sessionId: "ses_test", @@ -775,7 +1573,7 @@ describe("SidecarRouter", () => { await expect(promise).rejects.toThrow(/disconnected/); }); - test("abort preserves routing when address re-registered during await", async () => { + test("preserves routing when address re-registered during a request await", async () => { const ws1 = createMockWs(); router.handleOpen(ws1); router.handleMessage( @@ -787,8 +1585,13 @@ describe("SidecarRouter", () => { agentAddresses: ["agent@local"], }), ); + await tick(); - const promise = router.sendSessionAbort("agent@local", "user_disconnect"); + const promise = router.sendSourcesUpdate( + "agent@local", + TEST_SOURCES, + TEST_DEFAULT_SOURCE, + ); const frame = lastSent(ws1); const ws2 = createMockWs(); @@ -812,7 +1615,7 @@ describe("SidecarRouter", () => { expect(router.getRoutableAddresses()).toContain("agent@local"); }); - test("closing stale sidecar after reconnect-during-abort does not evict address", async () => { + test("closing stale sidecar after reconnect-during-request does not evict address", async () => { const ws1 = createMockWs(); router.handleOpen(ws1); router.handleMessage( @@ -824,8 +1627,13 @@ describe("SidecarRouter", () => { agentAddresses: ["agent@local"], }), ); + await tick(); - const promise = router.sendSessionAbort("agent@local", "user_disconnect"); + const promise = router.sendSourcesUpdate( + "agent@local", + TEST_SOURCES, + TEST_DEFAULT_SOURCE, + ); const frame = lastSent(ws1); const ws2 = createMockWs(); @@ -853,9 +1661,11 @@ describe("SidecarRouter", () => { }); describe("agent events", () => { - test("agent.event frames are forwarded to subscribers", () => { + test("agent.event frames are forwarded to subscribers", async () => { const events: { addr: string; sid: string; event: unknown }[] = []; - const router = createSidecarRouter({}); + const router = createSidecarRouter({ + lookups: { lookupPublicKey: async () => null }, + }); router.events.on("agent.event", ({ agentAddress, sessionId, event }) => { events.push({ addr: agentAddress, sid: sessionId, event }); }); @@ -881,6 +1691,7 @@ describe("SidecarRouter", () => { event: { type: "reactor.start", seq: 0, data: {} }, }), ); + await tick(); expect(events).toHaveLength(1); expect(events[0]?.addr).toBe("agent@local"); @@ -891,9 +1702,11 @@ describe("SidecarRouter", () => { }); }); - test("agent.event frames are emitted on router.events", () => { + test("agent.event frames are emitted on router.events", async () => { const seen: { addr: string; sid: string }[] = []; - const router = createSidecarRouter({}); + const router = createSidecarRouter({ + lookups: { lookupPublicKey: async () => null }, + }); router.events.on("agent.event", ({ agentAddress, sessionId }) => { seen.push({ addr: agentAddress, sid: sessionId }); }); @@ -918,15 +1731,18 @@ describe("SidecarRouter", () => { event: { type: "reactor.start", seq: 0, data: {} }, }), ); + await tick(); expect(seen).toEqual([{ addr: "agent@local", sid: "sess-1" }]); }); - test("sidecar.disconnect is emitted on router.events", () => { - const router = createSidecarRouter({}); + test("sidecar.disconnect is emitted on router.events", async () => { + const router = createSidecarRouter({ + lookups: { lookupPublicKey: async () => null }, + }); const seen: string[][] = []; - router.events.on("sidecar.disconnect", ({ agentAddresses }) => { - seen.push(agentAddresses); + router.events.on("sidecar.disconnect", ({ ownedAddresses }) => { + seen.push(ownedAddresses); }); const ws = createMockWs(); @@ -940,13 +1756,16 @@ describe("SidecarRouter", () => { agentAddresses: ["agent@local"], }), ); + await tick(); router.handleClose(ws); expect(seen).toEqual([["agent@local"]]); }); - test("connector.state.changed populates the cache and is readable via getConnectorState", () => { - const router = createSidecarRouter({}); + test("connector.state.changed populates the cache and is readable via getConnectorState", async () => { + const router = createSidecarRouter({ + lookups: { lookupPublicKey: async () => null }, + }); const ws = createMockWs(); router.handleOpen(ws); router.handleMessage( @@ -958,6 +1777,7 @@ describe("SidecarRouter", () => { agentAddresses: ["agent@local"], }), ); + await tick(); // Before any state frame, the cache is absent → null. expect(router.getConnectorState("agent@local")).toBeNull(); @@ -976,6 +1796,7 @@ describe("SidecarRouter", () => { connectorState: state, }), ); + await tick(); expect(router.getConnectorState("agent@local")).toEqual(state); @@ -988,12 +1809,15 @@ describe("SidecarRouter", () => { connectorState: null, }), ); + await tick(); expect(router.getConnectorState("agent@local")).toBeNull(); }); - test("connector.state.changed is emitted on router.events", () => { - const router = createSidecarRouter({}); + test("connector.state.changed is emitted on router.events", async () => { + const router = createSidecarRouter({ + lookups: { lookupPublicKey: async () => null }, + }); const seen: { addr: string; state: unknown }[] = []; router.events.on( "connector.state.changed", @@ -1013,6 +1837,7 @@ describe("SidecarRouter", () => { agentAddresses: ["agent@local"], }), ); + await tick(); const state = { threadRoot: "", @@ -1029,12 +1854,15 @@ describe("SidecarRouter", () => { connectorState: state, }), ); + await tick(); expect(seen).toEqual([{ addr: "agent@local", state }]); }); - test("live takeover via register evicts the prior owner's cached connector state", () => { - const router = createSidecarRouter({}); + test("live takeover via register evicts the prior owner's cached connector state", async () => { + const router = createSidecarRouter({ + lookups: { lookupPublicKey: async () => null }, + }); // First sidecar registers and reports connector state. const ws1 = createMockWs(); @@ -1048,6 +1876,7 @@ describe("SidecarRouter", () => { agentAddresses: ["agent@local"], }), ); + await tick(); router.handleMessage( ws1, JSON.stringify({ @@ -1061,6 +1890,7 @@ describe("SidecarRouter", () => { }, }), ); + await tick(); expect(router.getConnectorState("agent@local")).not.toBeNull(); // A second sidecar registers claiming the same address without @@ -1078,12 +1908,15 @@ describe("SidecarRouter", () => { agentAddresses: ["agent@local"], }), ); + await tick(); expect(router.getConnectorState("agent@local")).toBeNull(); }); - test("disconnect clears cached connector state for affected agents", () => { - const router = createSidecarRouter({}); + test("disconnect clears cached connector state for affected agents", async () => { + const router = createSidecarRouter({ + lookups: { lookupPublicKey: async () => null }, + }); const ws = createMockWs(); router.handleOpen(ws); router.handleMessage( @@ -1095,6 +1928,7 @@ describe("SidecarRouter", () => { agentAddresses: ["agent@local"], }), ); + await tick(); router.handleMessage( ws, @@ -1109,6 +1943,7 @@ describe("SidecarRouter", () => { }, }), ); + await tick(); expect(router.getConnectorState("agent@local")).not.toBeNull(); @@ -1119,7 +1954,7 @@ describe("SidecarRouter", () => { }); describe("session subscriptions", () => { - test("subscriber receives events for its session", () => { + test("subscriber receives events for its session", async () => { const received: unknown[] = []; const ws = createMockWs(); router.handleOpen(ws); @@ -1144,12 +1979,13 @@ describe("SidecarRouter", () => { event: { type: "reactor.start", seq: 0, data: {} }, }), ); + await tick(); expect(received).toHaveLength(1); expect(received[0]).toEqual({ type: "reactor.start", seq: 0, data: {} }); }); - test("subscriber does not receive events for other agents", () => { + test("subscriber does not receive events for other agents", async () => { const received: unknown[] = []; const ws = createMockWs(); router.handleOpen(ws); @@ -1174,11 +2010,12 @@ describe("SidecarRouter", () => { event: { type: "reactor.start", seq: 0, data: {} }, }), ); + await tick(); expect(received).toHaveLength(0); }); - test("unsubscribe stops delivery", () => { + test("unsubscribe stops delivery", async () => { const received: unknown[] = []; const ws = createMockWs(); router.handleOpen(ws); @@ -1205,6 +2042,7 @@ describe("SidecarRouter", () => { event: { type: "reactor.start", seq: 0, data: {} }, }), ); + await tick(); unsub(); @@ -1217,12 +2055,13 @@ describe("SidecarRouter", () => { event: { type: "reactor.end", seq: 1, data: {} }, }), ); + await tick(); expect(received).toHaveLength(1); expect(received[0]).toEqual({ type: "reactor.start", seq: 0, data: {} }); }); - test("multiple subscribers receive the same event", () => { + test("multiple subscribers receive the same event", async () => { const received1: unknown[] = []; const received2: unknown[] = []; const ws = createMockWs(); @@ -1249,12 +2088,13 @@ describe("SidecarRouter", () => { event: { type: "reactor.start", seq: 0, data: {} }, }), ); + await tick(); expect(received1).toHaveLength(1); expect(received2).toHaveLength(1); }); - test("stale unsubscribe does not evict a later subscriber", () => { + test("stale unsubscribe does not evict a later subscriber", async () => { const ws = createMockWs(); router.handleOpen(ws); router.handleMessage( @@ -1286,11 +2126,12 @@ describe("SidecarRouter", () => { event: { type: "reactor.start", seq: 0, data: {} }, }), ); + await tick(); expect(received).toHaveLength(1); }); - test("subscriber that unsubscribes mid-dispatch does not drop later subscribers", () => { + test("subscriber that unsubscribes mid-dispatch does not drop later subscribers", async () => { const received: unknown[] = []; const ws = createMockWs(); router.handleOpen(ws); @@ -1319,6 +2160,7 @@ describe("SidecarRouter", () => { event: { type: "reactor.start", seq: 0, data: {} }, }), ); + await tick(); expect(received).toHaveLength(1); expect(received[0]).toEqual({ type: "reactor.start", seq: 0, data: {} }); @@ -1374,6 +2216,7 @@ describe("SidecarRouter", () => { agentAddresses: [], }), ); + await tick(); // A fresh provision routes to this sidecar during the restore window. const deployPromise = router.sendAgentDeploy( @@ -1455,6 +2298,7 @@ describe("SidecarRouter", () => { agentAddresses: [], }), ); + await tick(); const deployPromise = router.sendAgentDeploy( "window-agent@local", @@ -1485,6 +2329,7 @@ describe("SidecarRouter", () => { connectorState, }), ); + await tick(); expect(router.getConnectorState("window-agent@local")).toEqual( connectorState, ); @@ -1645,6 +2490,74 @@ describe("SidecarRouter", () => { expect(failedFrame.reason).toBe("Unknown agent address"); }); + test("fails closed on a key-lookup error during reconnect instead of crashing", async () => { + // A rejecting lookup (e.g. a transient DB failure) on the production + // reconnect path must be caught and surfaced, not floated out of the + // void-dispatched handler as an unhandled rejection that could take down + // the hub. Fail closed: the address is treated as unverifiable, fails + // its challenge, and stays unrouted. (The test completing rather than + // hanging on an unhandled rejection is itself part of the assertion.) + const captured: { level: string; message: string }[] = []; + const savedConfig = getConfig(); + configureSync({ + reset: true, + sinks: { + capture: (record) => { + const message = Array.isArray(record.message) + ? record.message + .map((part) => + typeof part === "string" ? part : JSON.stringify(part), + ) + .join("") + : String(record.message); + captured.push({ level: record.level, message }); + }, + }, + loggers: [{ category: [], lowestLevel: "debug", sinks: ["capture"] }], + }); + try { + const router = createSidecarRouter({ + requestTimeoutMs: 5000, + lookups: { + lookupPublicKey: async () => { + throw new Error("db unavailable"); + }, + }, + }); + const ws = createMockWs(); + router.handleOpen(ws); + router.handleMessage( + ws, + JSON.stringify({ + type: "reconnect", + sidecarId: "sc-1", + token: "tok", + agentAddresses: ["agent@local"], + }), + ); + + await new Promise((r) => setTimeout(r, 50)); + + expect(router.getRoutableAddresses()).not.toContain("agent@local"); + const failedFrame = ws.sent + .map((s) => JSON.parse(s)) + .find((f: { type: string }) => f.type === "challenge.failed"); + expect(failedFrame?.address).toBe("agent@local"); + expect( + captured.some( + (l) => + l.level === "error" && l.message.includes("Key lookup failed"), + ), + ).toBe(true); + } finally { + if (savedConfig) { + configureSync({ reset: true, ...savedConfig }); + } else { + resetSync(); + } + } + }); + test("partial success routes verified addresses only", async () => { const kp1 = await generateKeyPair(); const kp2 = await generateKeyPair(); @@ -1949,6 +2862,46 @@ describe("SidecarRouter", () => { }); describe("disconnect message queuing", () => { + // Establish an initial route for a KEYED address through the challenged + // reconnect path. A keyed address cannot be routed by a plain register + // (the key-existence gate rejects it), so the disconnect-queue tests -- + // whose agents have a stored key so the reconnect challenge can verify -- + // bring the address up the same way production does. + async function connectAgentViaChallenge( + r: ReturnType, + ws: ReturnType, + addr: string, + privateKey: Uint8Array, + ) { + r.handleOpen(ws); + r.handleMessage( + ws, + JSON.stringify({ + type: "reconnect", + sidecarId: "sc-1", + token: "tok", + agentAddresses: [addr], + }), + ); + await new Promise((res) => setTimeout(res, 50)); + const challengeFrame = ws.sent + .map((s) => JSON.parse(s)) + .find((f: { type: string }) => f.type === "challenge"); + const responses = await Promise.all( + challengeFrame.challenges.map( + async (c: { address: string; nonce: string }) => ({ + address: c.address, + signature: await signChallenge(c.nonce, c.address, privateKey), + }), + ), + ); + r.handleMessage( + ws, + JSON.stringify({ type: "challenge.response", responses }), + ); + await new Promise((res) => setTimeout(res, 50)); + } + test("mail queued during disconnect is flushed on reconnect", async () => { const kp = await generateKeyPair(); const router = createSidecarRouter({ @@ -1960,18 +2913,10 @@ describe("SidecarRouter", () => { }, }); - // Initial connection with one agent. + // Initial connection with one agent, via the challenged reconnect path + // (a keyed address cannot be routed by a plain register). const ws1 = createMockWs(); - router.handleOpen(ws1); - router.handleMessage( - ws1, - JSON.stringify({ - type: "register", - sidecarId: "sc-1", - token: "tok", - agentAddresses: ["agent@local"], - }), - ); + await connectAgentViaChallenge(router, ws1, "agent@local", kp.privateKey); // Disconnect — creates a queue entry. router.handleClose(ws1); @@ -2040,16 +2985,7 @@ describe("SidecarRouter", () => { }); const ws = createMockWs(); - router.handleOpen(ws); - router.handleMessage( - ws, - JSON.stringify({ - type: "register", - sidecarId: "sc-1", - token: "tok", - agentAddresses: ["agent@local"], - }), - ); + await connectAgentViaChallenge(router, ws, "agent@local", kp.privateKey); router.handleClose(ws); // Queue 3 messages with max size 2 — oldest should be evicted. @@ -2102,7 +3038,7 @@ describe("SidecarRouter", () => { expect(flushed[1].rawMessage).toBe("msg-2"); }); - test("sendSessionAbort rejects when agent is disconnected", async () => { + test("sendSourcesUpdate rejects when agent is disconnected", async () => { const ws = createMockWs(); router.handleOpen(ws); router.handleMessage( @@ -2117,7 +3053,11 @@ describe("SidecarRouter", () => { router.handleClose(ws); await expect( - router.sendSessionAbort("agent@local", "user_disconnect"), + router.sendSourcesUpdate( + "agent@local", + TEST_SOURCES, + TEST_DEFAULT_SOURCE, + ), ).rejects.toThrow("No sidecar connected"); }); }); @@ -2148,6 +3088,7 @@ describe("SidecarRouter", () => { const router = createSidecarRouter({ requestTimeoutMs: 500, pingTimeoutMs: 100, + lookups: { lookupPublicKey: async () => null }, }); const ws = createMockWs(); @@ -2444,6 +3385,10 @@ describe("SidecarRouter", () => { requestTimeoutMs: 500, hubPublicKey: TEST_HUB_KEY, lookups: { + // Keyless lookup: the pack tests register the deployment address as a + // first-deploy so the key-existence gate routes it (the gate is + // fail-closed without a lookup configured). + lookupPublicKey: async () => null, async receiveAgentStatePack(repoId, pack, ref, commitSha) { calls.push({ method: "receiveAgentStatePack", @@ -2469,7 +3414,7 @@ describe("SidecarRouter", () => { return { router: packRouter, calls }; } - function registerAddr( + async function registerAddr( r: ReturnType, ws: ReturnType, sidecarId: string, @@ -2485,6 +3430,9 @@ describe("SidecarRouter", () => { agentAddresses: [addr], }), ); + // Register routing is async (the key-existence gate awaits the lookup); + // settle it before assertions read the routing table. + await new Promise((res) => setTimeout(res, 0)); } function pushPack( @@ -2529,7 +3477,7 @@ describe("SidecarRouter", () => { const { router: r, calls } = buildPackRouter(); const ws = createMockWs(); const addr = "agent-wfr@local"; - registerAddr(r, ws, "sc-wfr", addr); + await registerAddr(r, ws, "sc-wfr", addr); const transferId = "t-wfr-1"; const repoId: RepoId = { kind: "workflow-run", id: "dep-wfr-1" }; @@ -2564,11 +3512,201 @@ describe("SidecarRouter", () => { expect(ack.repoId).toEqual(repoId); }); + test("a workflow deployment owned by its connection can push workflow-run packs", async () => { + // Pack-push authorization keys on `connOwnsAddress`, which unions the + // connection's launched-agent and workflow-substrate address sets. A + // workflow deployment address owned by the connection must authorize a + // workflow-run pack -- otherwise the hub's workflow-run observation + // mirror silently stops updating (mail routing resumes, but pack push + // rejects the address as unrouted). The challenged-reconnect ownership + // path itself is covered by the "workflow-address reconnect" block; here + // the address is registered directly to keep the test on pack routing. + const { router: r, calls } = buildPackRouter(); + const ws = createMockWs(); + const addr = "ins_dep_wfr@local"; + await registerAddr(r, ws, "sc-wfr", addr); + + const repoId: RepoId = { kind: "workflow-run", id: "dep-wfr-2" }; + pushPack(r, ws, { + agentAddress: addr, + repoId, + transferId: "t-wfr-2", + pack: new Uint8Array([9, 8, 7]), + ref: "refs/heads/events", + commitSha: "e".repeat(40), + }); + + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(calls.length).toBe(1); + expect(calls[0]?.method).toBe("receiveWorkflowRunPack"); + const ack = lastSent(ws); + expect(ack.type).toBe("repo.pack.ack"); + }); + + test("a superseded connection's close does not cancel the new owner's workflow-run transfer", async () => { + // Abrupt restart: a fresh ws reclaims a workflow address a still-open + // stale ws owns, then starts a pack transfer. When the stale ws finally + // closes, its teardown must not cancel the new owner's in-flight + // transfer (cancelByAgent is keyed by address, not connection). The + // ghost cleanup on the fresh register evicts the address from the stale + // connection so its close leaves the new owner alone. + const { router: r, calls } = buildPackRouter(); + const addr = "ins_dep_reclaim@local"; + const repoId: RepoId = { kind: "workflow-run", id: "dep-reclaim" }; + const pack = new Uint8Array([4, 5, 6, 7]); + const transferId = "t-reclaim"; + + const oldWs = createMockWs(); + await registerAddr(r, oldWs, "sc", addr); + + const newWs = createMockWs(); + await registerAddr(r, newWs, "sc", addr); + + // The new owner starts (but does not finish) a workflow-run transfer. + for (const chunk of chunkPack(pack)) { + r.handleMessage( + newWs, + JSON.stringify({ + type: "repo.pack.push", + agentAddress: addr, + repoId, + transferId, + seq: chunk.seq, + data: chunk.data, + }), + ); + } + + // The superseded connection closes mid-transfer. + r.handleClose(oldWs); + + // The new owner completes the transfer; it must not have been cancelled. + r.handleMessage( + newWs, + JSON.stringify({ + type: "repo.pack.done", + agentAddress: addr, + repoId, + transferId, + ref: "refs/heads/events", + commitSha: "a".repeat(40), + }), + ); + + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(calls.length).toBe(1); + expect(calls[0]?.method).toBe("receiveWorkflowRunPack"); + }); + + test("a workflow address reclaimed via challenged reconnect does not cancel the new owner's transfer", async () => { + // The register-reclaim variant above covers the handleRegister ghost + // cleanup. This covers the CHALLENGED RECONNECT reclaim path, where the + // deployment lives on the connection's workflow set: the verified reclaim + // must evict the address from the superseded connection, or that + // connection's close runs cancelByAgent and kills the new owner's + // in-flight transfer. + const kp = await generateKeyPair(); + const calls: { method: string }[] = []; + const r = createSidecarRouter({ + requestTimeoutMs: 5000, + hubPublicKey: TEST_HUB_KEY, + lookups: { + lookupPublicKey: async () => hexEncode(kp.publicKey), + async receiveWorkflowRunPack() { + calls.push({ method: "receiveWorkflowRunPack" }); + return { accepted: true }; + }, + }, + }); + const addr = "ins_dep_reclaim_rc@local"; + const repoId: RepoId = { kind: "workflow-run", id: "dep-reclaim-rc" }; + const pack = new Uint8Array([4, 5, 6, 7]); + const transferId = "t-reclaim-rc"; + + async function reconnectVerify(ws: ReturnType) { + r.handleOpen(ws); + r.handleMessage( + ws, + JSON.stringify({ + type: "reconnect", + sidecarId: "sc", + token: "tok", + agentAddresses: [addr], + }), + ); + await new Promise((resolve) => setTimeout(resolve, 50)); + const challenge = ws.sent + .map((s) => JSON.parse(s)) + .find((f: { type: string }) => f.type === "challenge"); + r.handleMessage( + ws, + JSON.stringify({ + type: "challenge.response", + responses: [ + { + address: addr, + signature: await signChallenge( + challenge.challenges[0].nonce, + addr, + kp.privateKey, + ), + }, + ], + }), + ); + await new Promise((resolve) => setTimeout(resolve, 50)); + } + + const oldWs = createMockWs(); + await reconnectVerify(oldWs); + + const newWs = createMockWs(); + await reconnectVerify(newWs); + + // The new owner starts (but does not finish) a workflow-run transfer. + for (const chunk of chunkPack(pack)) { + r.handleMessage( + newWs, + JSON.stringify({ + type: "repo.pack.push", + agentAddress: addr, + repoId, + transferId, + seq: chunk.seq, + data: chunk.data, + }), + ); + } + + // The superseded connection closes mid-transfer. + r.handleClose(oldWs); + + // The new owner completes the transfer; it must not have been cancelled. + r.handleMessage( + newWs, + JSON.stringify({ + type: "repo.pack.done", + agentAddress: addr, + repoId, + transferId, + ref: "refs/heads/events", + commitSha: "a".repeat(40), + }), + ); + + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(calls.length).toBe(1); + expect(calls[0]?.method).toBe("receiveWorkflowRunPack"); + }); + test("agent-state and workflow-run packs use independent receivers (concurrent transferIds)", async () => { const { router: r, calls } = buildPackRouter(); const ws = createMockWs(); const addr = "agent-mix@local"; - registerAddr(r, ws, "sc-mix", addr); + await registerAddr(r, ws, "sc-mix", addr); // Reuse the same transferId across kinds. The two receivers' // in-flight state must be independent, so this must NOT collide. @@ -2662,7 +3800,7 @@ describe("SidecarRouter", () => { }); const ws = createMockWs(); const addr = "agent-wfr-rej@local"; - registerAddr(r, ws, "sc-wfr-rej", addr); + await registerAddr(r, ws, "sc-wfr-rej", addr); const transferId = "t-wfr-rej"; const repoId: RepoId = { kind: "workflow-run", id: "dep-wfr-rej" }; @@ -2686,7 +3824,7 @@ describe("SidecarRouter", () => { }); describe("sendDrain", () => { - test("ships a drain.deliver frame to the sidecar holding the deployment", () => { + test("ships a drain.deliver frame to the sidecar holding the deployment", async () => { const ws = createMockWs(); router.handleOpen(ws); router.handleMessage( @@ -2698,6 +3836,7 @@ describe("SidecarRouter", () => { agentAddresses: ["dep@integration.interchange"], }), ); + await tick(); ws.sent.length = 0; router.sendDrain({ diff --git a/packages/hub-sessions/src/ws/sidecar-handler.ts b/packages/hub-sessions/src/ws/sidecar-handler.ts index 1f5bee0c..44a51316 100644 --- a/packages/hub-sessions/src/ws/sidecar-handler.ts +++ b/packages/hub-sessions/src/ws/sidecar-handler.ts @@ -8,6 +8,7 @@ import { getLogger } from "@intx/log"; import { verifyEd25519 } from "@intx/crypto"; import { chunkPack, createPackReceiver } from "@intx/pack-transport"; import { hexDecode, hexEncode } from "@intx/types"; +import { isWorkflowDerivedAddress } from "@intx/workflow-deploy"; import { type } from "arktype"; import { SidecarFrame, @@ -18,12 +19,10 @@ import { type RepoId, } from "@intx/types/sidecar"; import type { - AbortReason, ConnectorThreadState, HarnessConfig, InferenceSource, } from "@intx/types/runtime"; -import type { GrantRule } from "@intx/types/authz"; import { createSidecarEmitter, type SidecarEventEmitter, @@ -36,9 +35,35 @@ const logger = getLogger(["hub", "ws", "sidecar"]); export type SidecarConnection = { sidecarId: string; agentAddresses: Set; + // Workflow-substrate deployment addresses (ins_dep_...) this connection + // hosts. Kept separate from `agentAddresses`: these are hub-minted and + // registered for routing WITHOUT the per-address challenge, so they must + // not be dragged through the challenge/re-add dance the session addresses + // take. `handleClose` cleans both sets out of `addressIndex`. + workflowAddresses: Set; send(frame: HubFrame): void; }; +/** + * Whether this connection owns `address` for routing/lifecycle purposes -- + * as a challenged session address OR a hub-minted workflow-substrate address. + * The two sets are kept physically distinct (they differ on the challenge / + * re-add path), but ownership readers -- pack-transfer authorization, + * in-flight cancellation, disconnect teardown -- must see the union, or a + * reconnected workflow deployment (which lives only in `workflowAddresses`) + * is silently treated as unowned even though its mail routes. + */ +function connOwnsAddress(conn: SidecarConnection, address: string): boolean { + return ( + conn.agentAddresses.has(address) || conn.workflowAddresses.has(address) + ); +} + +/** The deduped set of every address this connection owns (session + workflow). */ +function ownedAddresses(conn: SidecarConnection): Set { + return new Set([...conn.agentAddresses, ...conn.workflowAddresses]); +} + type PendingRequest = { requestId: string; ws: WsHandle; @@ -85,9 +110,9 @@ export type SidecarRouter = { * Send an `agent.deploy` frame to the sidecar. When `workflow` is * supplied, the frame carries the multi-step deploy projection * (workflow definition plus per-step source pins); the sidecar's - * deploy router uses the field's presence to discriminate the - * multi-step branch from the trivial branch. Absent on every legacy - * agent-deploy. + * deploy router routes it to the workflow deploy path. The sole + * caller supplies `workflow` on every deploy; per-step provisioning + * uses `sendProvisionStep`. * * The returned promise resolves with the supervisor's principal * public key (hex-encoded Ed25519) carried on `agent.deploy.ack`. @@ -100,9 +125,6 @@ export type SidecarRouter = { workflow?: AgentDeployFrame["workflow"], ): Promise<{ publicKey: string }>; sendAgentUndeploy(agentAddress: string, reason: string): Promise; - sendSessionStart(agentAddress: string): Promise; - sendSessionAbort(agentAddress: string, reason: AbortReason): Promise; - sendGrantsUpdate(agentAddress: string, grants: GrantRule[]): Promise; sendSourcesUpdate( agentAddress: string, sources: InferenceSource[], @@ -115,6 +137,28 @@ export type SidecarRouter = { commitSha: string, options?: SendPackOptions, ): Promise; + /** + * Bind a per-step workflow-substrate address to a sidecar for the staging + * window of a multi-step deploy, so `sendPack` can route the step's deploy + * and asset packs before the deployment-level frame spawns the child. The + * address enters the keyless `workflowAddresses` routing set; call + * `unbindStepRoute` once the step's packs land. Throws if no sidecar is + * available. + */ + bindStepRoute(stepAddress: string): void; + /** + * Remove a per-step route bound by `bindStepRoute`. Idempotent: an unbound + * address is a no-op. + */ + unbindStepRoute(stepAddress: string): void; + /** + * Provision one step of a multi-step deploy on the sidecar WITHOUT + * spawning: the sidecar initializes the step's agent-state repo and + * records the hub key so the follow-up deploy pack applies and verifies. + * The step address must already be bound via `bindStepRoute`. Resolves + * once the sidecar acks, so the caller can then deliver the deploy pack. + */ + sendProvisionStep(agentAddress: string, config: HarnessConfig): Promise; sendSyncRequest(agentAddress: string): void; /** * Deliver a workflow-run signal to the sidecar that hosts the named @@ -260,6 +304,16 @@ export function createSidecarRouter( // ws handle → liveness timer (reset on each ping from the sidecar) const livenessTimers = new Map>(); + // Per-ws serialization chain for QUEUE-class frames (see `frameBypassesQueue` + // for the split and the invariant behind it). A frame that establishes or + // reads routing waits for earlier queued frames on the same ws to complete, + // so it observes their finished effects -- most importantly an async + // register's routing write, which would otherwise land after a following + // connector.state.changed / mail / pack frame and silently drop it. It holds + // a single in-flight promise per ws (replaced each queued frame), cleared on + // close. + const messageChains = new Map>(); + // transferId → pending pack transfer (resolved by repo.pack.ack, rejected by repo.pack.reject) type PendingPack = { transferId: string; @@ -271,16 +325,6 @@ export function createSidecarRouter( const pendingPacks = new Map(); let packCounter = 0; - // agentAddress → pending session start (resolved by session.start.ack) - type PendingSessionStart = { - agentAddress: string; - ws: WsHandle; - resolve(): void; - reject(error: string): void; - timer: ReturnType; - }; - const pendingSessionStarts = new Map(); - // agentAddress → pending undeploy (resolved by agent.undeploy.ack) type PendingUndeploy = { agentAddress: string; @@ -377,57 +421,130 @@ export function createSidecarRouter( } const frame = validated; + // Bypass frames (liveness + terminal responses to outbound requests) + // dispatch immediately: they resolve the very promises a queued handler + // may be blocked on, so queuing them would deadlock the round-trip. + if (frameBypassesQueue(frame)) { + // Guard so a bypass handler's failure -- a synchronous throw or an async + // ack handler's rejection -- is logged rather than floating out of the + // immediate dispatch. The async wrapper turns a synchronous throw into a + // rejection too, matching the queue path's .then/.catch coverage. + void (async () => dispatchFrame(ws, frame))().catch((err: unknown) => { + logger.warn`Frame handler failed for ${frame.type}: ${err instanceof Error ? err.message : String(err)}`; + }); + return; + } + // Everything else serializes per ws so a frame that establishes or reads + // routing observes earlier queued frames' completed effects. + const prev = messageChains.get(ws) ?? Promise.resolve(); + const next = prev + .then(() => dispatchFrame(ws, frame)) + .catch((err: unknown) => { + logger.warn`Frame handler failed for ${frame.type}: ${err instanceof Error ? err.message : String(err)}`; + }); + messageChains.set(ws, next); + } + + function assertNever(x: never): never { + throw new Error(`Unclassified sidecar frame type: ${JSON.stringify(x)}`); + } + + // Whether `frame` bypasses the per-ws serialization chain. Invariant: a frame + // bypasses IFF it is liveness (ping) OR a terminal response to an + // already-issued outbound request -- correlated purely by + // requestId/transferId/agentAddress in the pending maps, touching no routing + // state. Such a frame has no ordering obligation against new inbound frames + // (a response cannot resolve "too early" for a request that already went + // out), and it is exactly what in-flight queued handlers block on, so it MUST + // run out of band or the challenge round-trip deadlocks (challenge.response + // -> agent.reconnected reaction -> sendSourcesUpdate awaits a later + // session.ack). Every other frame establishes or reads routing, or carries an + // inbound payload whose order matters, so it queues. The exhaustive switch + + // assertNever makes adding a SidecarFrame variant without classifying it a + // compile error, not a latent deadlock or a silent bypass hole. + function frameBypassesQueue(frame: SidecarFrame): boolean { switch (frame.type) { + case "ping": + case "session.ack": + case "session.error": + case "agent.deploy.ack": + case "agent.error": + case "agent.undeploy.ack": + case "repo.pack.ack": + case "repo.pack.reject": + return true; case "register": - handleRegister(ws, frame.sidecarId, frame.token, frame.agentAddresses); - break; case "reconnect": - void handleReconnect( + case "challenge.response": + case "mail.outbound": + case "agent.event": + case "connector.state.changed": + case "repo.pack.push": + case "repo.pack.done": + return false; + default: + return assertNever(frame); + } + } + + // Runs one frame's handler. Returns the handler's promise for async handlers + // so the per-ws chain can await bounded completion; sync handlers return + // void. Never awaits a promise that resolves on a LATER same-ws frame -- the + // only such await (the challenge round-trip's session.ack) is reached via a + // bypass frame, which does not queue. + function dispatchFrame( + ws: WsHandle, + frame: SidecarFrame, + ): void | Promise { + switch (frame.type) { + case "register": + return handleRegister( + ws, + frame.sidecarId, + frame.token, + frame.agentAddresses, + ); + case "reconnect": + return handleReconnect( ws, frame.sidecarId, frame.token, frame.agentAddresses, frame.deployRefs ?? {}, ); - break; case "challenge.response": - void handleChallengeResponse(ws, frame.responses); - break; + return handleChallengeResponse(ws, frame.responses); case "agent.deploy.ack": - void handleDeployAck(frame.agentAddress, frame.publicKey); - break; + return handleDeployAck(frame.agentAddress, frame.publicKey); case "agent.error": rejectDeployPending(frame.agentAddress, frame.error); - rejectSessionStartPending(frame.agentAddress, frame.error); rejectUndeployPending(frame.agentAddress, frame.error); - break; - case "session.start.ack": - resolveSessionStartPending(frame.agentAddress); - break; + return; case "agent.undeploy.ack": resolveUndeployPending(frame.agentAddress); - break; + return; case "ping": handlePing(ws); - break; + return; case "mail.outbound": if (frame.delivered !== true) { handleMailOutbound(frame.rawMessage, frame.recipients); - } else if (lookups.persistMail && frame.senderAddress) { - void handleMailPersist( + return; + } + if (lookups.persistMail && frame.senderAddress) { + return handleMailPersist( lookups.persistMail, frame.rawMessage, frame.senderAddress, frame.recipients, ); - } else if (frame.delivered === true) { - if (!frame.senderAddress) { - logger.warn`Dropping delivered mail.outbound frame with no senderAddress`; - } else { - logger.warn`Dropping delivered mail.outbound frame: no persistMail lookup configured`; - } } - break; + if (!frame.senderAddress) { + logger.warn`Dropping delivered mail.outbound frame with no senderAddress`; + } else { + logger.warn`Dropping delivered mail.outbound frame: no persistMail lookup configured`; + } + return; case "agent.event": events.emit("agent.event", { agentAddress: frame.agentAddress, @@ -435,97 +552,113 @@ export function createSidecarRouter( event: frame.event, }); dispatchToSubscribers(frame.agentAddress, frame.event); - break; + return; case "connector.state.changed": // Gate the cache write on the sending sidecar actually owning // the named agent. A misbehaving sidecar that knows another // agent's address could otherwise poison the cached state. if (addressIndex.get(frame.agentAddress) !== ws) { logger.warn`Dropping connector.state.changed for ${frame.agentAddress}: not registered to this sidecar`; - break; + return; } connectorStates.set(frame.agentAddress, frame.connectorState); events.emit("connector.state.changed", { agentAddress: frame.agentAddress, connectorState: frame.connectorState, }); - break; + return; case "session.ack": resolvePending(frame.requestId); - break; + return; case "session.error": rejectPending(frame.requestId, frame.error); - break; + return; case "repo.pack.ack": resolvePackPending(frame.transferId); - break; + return; case "repo.pack.reject": rejectPackPending(frame.transferId, frame.reason); - break; + return; case "repo.pack.push": handlePackPush(ws, frame); - break; + return; case "repo.pack.done": - void handlePackDone(ws, frame); - break; - case "deploy.apply.error": - // Gate the failure emit on the sending sidecar actually - // owning the named agent. A misbehaving sidecar that knows - // another agent's address could otherwise drive the hub to - // record an apply failure against a deploy the other agent's - // sidecar still considers live, contaminating audit trails - // and any failure-driven rollback logic the hub runs. - if (addressIndex.get(frame.agentAddress) !== ws) { - logger.warn`Dropping deploy.apply.error for ${frame.agentAddress}: not registered to this sidecar`; - break; - } - events.emit("deploy.apply.error", { - agentAddress: frame.agentAddress, - attemptId: frame.attemptId, - previousDeployId: frame.previousDeployId, - category: frame.category, - message: frame.message, - ...(frame.package !== undefined ? { package: frame.package } : {}), - occurredAt: frame.occurredAt, - }); - break; + return handlePackDone(ws, frame); default: - logger.warn`Unknown frame type from sidecar: ${(frame as { type: string }).type}`; + return assertNever(frame); } } - function handleRegister( + async function handleRegister( ws: WsHandle, sidecarId: string, token: string, agentAddresses: string[], - ): void { + ): Promise { if (validateToken !== undefined && !validateToken(sidecarId, token)) { logger.warn`Rejected registration from sidecar ${sidecarId}: invalid token`; ws.close(); return; } - // If this same ws is re-registering, drop its addressIndex entries so - // the new register/reconnect rebuilds the owned set (handleReconnect - // and the loop below re-add the addresses that remain owned). Do not - // drop connectorStates here: this branch is reached only on a same-ws - // re-register, where the harness is live and its connector state is - // current. A genuinely restarting harness opens a new ws (so existing - // is undefined) and its stale state is cleared by the cross-ws ghost - // loop below and by handleClose. - const existing = connections.get(ws); - if (existing !== undefined) { - for (const addr of existing.agentAddresses) { - addressIndex.delete(addr); + // Key-existence gate. A register frame is token-authenticated but carries + // no per-address ownership proof, so it may route an address ONLY if that + // address has no stored key yet -- a genuine keyless first-deploy (the + // token-bounded first-deploy trust model). An address that already has a + // key must prove ownership through the challenged reconnect path; routing + // it here on token auth alone is the register-frame sibling of the + // reconnect hijack. The keyless-only set is computed up front, BEFORE the + // ghost-cleanup and every routing mutation below, so a rejected address + // touches nothing: no eviction of a live owner, hence no downgrade from + // hijack to denial-of-service on the victim. + const lookupKey = lookups.lookupPublicKey; + const routableAddresses: string[] = []; + for (const addr of agentAddresses) { + if (lookupKey === undefined) { + // Fail closed: without the ownership lookup a keyed address cannot be + // told apart from a first-deploy, so route nothing and surface the + // misconfiguration. Empty first-connect registers never reach here. + logger.error`Cannot gate register routing for ${addr}: lookupPublicKey is not configured; refusing to route (challenged reconnect required)`; + continue; } - } - - const addrSet = new Set(agentAddresses); - - // Clean up ghost entries from other connections that previously - // owned addresses this sidecar is now claiming. - for (const addr of addrSet) { + let existingKey: string | null; + try { + existingKey = await lookupKey(addr); + } catch (err) { + // Fail closed on a lookup error (e.g. a transient DB failure): route + // nothing for this address and surface the failure, rather than let + // the rejection float out of this void-dispatched handler and take + // down the hub. + logger.error`Key lookup failed for ${addr} during register: ${err instanceof Error ? err.message : String(err)}; failing closed (challenged reconnect required)`; + continue; + } + if (existingKey !== null) { + logger.warn`Refusing to route ${addr} via register: address already has a stored key; ownership must be proven via challenged reconnect`; + continue; + } + routableAddresses.push(addr); + } + + // Additive re-register: inherit every address this ws already owns and ADD + // the frame's keyless first-deploys. Register never drops an owned route -- + // an address proved via challenged reconnect stays routed, and removal + // happens via undeploy/disconnect, not register-omission. + // + // (Under the retired full-set model a register frame carried the sidecar's + // complete live set, so an omitted address meant "removed". Now the frame + // carries only keyless first-deploys the sidecar is adding -- keyed + // addresses arrive via reconnect -- so omission is meaningless, and a + // replace-on-register would wrongly drop an earlier first-deploy when a + // later one is registered, as well as any reconnect-verified keyed route.) + const existing = connections.get(ws); + const inheritedAgent = new Set(existing?.agentAddresses); + const inheritedWorkflow = new Set(existing?.workflowAddresses); + + // Clean up ghost entries from OTHER connections for the newly-claimed + // addresses only. An inherited address is already owned by this ws + // (prevWs === ws), so it needs no eviction and its in-flight deploy must + // not be cancelled. + for (const addr of routableAddresses) { const prevWs = addressIndex.get(addr); if (prevWs !== undefined && prevWs !== ws) { const prevConn = connections.get(prevWs); @@ -550,16 +683,28 @@ export function createSidecarRouter( } } + // New keyless first-deploys join the inherited session set; the workflow + // set is inherited unchanged (register never adds to it -- a + // workflow-derived address routes only through the challenged reconnect). + for (const addr of routableAddresses) { + inheritedAgent.add(addr); + } + const addrSet = inheritedAgent; + const workflowSet = inheritedWorkflow; + const conn: SidecarConnection = { sidecarId, agentAddresses: addrSet, + workflowAddresses: workflowSet, send(frame: HubFrame) { ws.send(JSON.stringify(frame)); }, }; connections.set(ws, conn); - for (const addr of addrSet) { + // Only the newly-claimed addresses need a routing write + queue discard; + // inherited addresses already point at this ws in addressIndex. + for (const addr of routableAddresses) { addressIndex.set(addr, ws); // Discard any disconnect queue — register has no identity // verification, so flushing to an unverified connection is unsafe. @@ -573,8 +718,14 @@ export function createSidecarRouter( disconnectedAgents.delete(addr); } } + // Only keyless first-deploy addresses reach `addressIndex` here; an + // address that already has a stored key was filtered out by the gate + // above and must re-enter routing through the challenged reconnect path. + // Because the gate runs before the ghost-cleanup, a rejected (keyed) + // address never evicts its prior owner -- register cannot reclaim or + // disrupt a victim's route on token auth alone. - logger.info`Sidecar ${sidecarId} registered with ${String(agentAddresses.length)} agents`; + logger.info`Sidecar ${sidecarId} registered; routed ${String(addrSet.size)} of ${String(agentAddresses.length)} address(es) (keyless first-deploy only)`; } async function handleReconnect( @@ -614,9 +765,12 @@ export function createSidecarRouter( // freshly-deployed agent from routing. const previouslyOwned = new Set(connections.get(ws)?.agentAddresses); - // Register the sidecar connection immediately (with no addresses) - // so it can receive frames while the challenge is pending. - handleRegister(ws, sidecarId, token, []); + // Register the sidecar connection immediately (with no addresses) so it + // can receive frames while the ownership challenge is pending. Every + // reconnect address -- session and workflow-derived alike -- enters + // routing only through the verified path below, never unchallenged here. + // Empty address list, so the key-existence gate has nothing to await. + await handleRegister(ws, sidecarId, token, []); const conn = connections.get(ws); if (conn === undefined) return; @@ -631,12 +785,20 @@ export function createSidecarRouter( addressIndex.set(addr, ws); } - // Look up stored public keys for all claimed addresses. + // Look up stored public keys for all claimed addresses. Fail closed on a + // lookup error (e.g. a transient DB failure): treat the address as + // unverifiable so it fails its challenge and stays unrouted, rather than + // letting the rejection float out of this void-dispatched handler as an + // unhandled rejection that could take down the hub. const keyLookups = await Promise.all( - agentAddresses.map(async (addr) => ({ - address: addr, - publicKeyHex: await lookupKey(addr), - })), + agentAddresses.map(async (addr) => { + try { + return { address: addr, publicKeyHex: await lookupKey(addr) }; + } catch (err) { + logger.error`Key lookup failed for ${addr} during reconnect: ${err instanceof Error ? err.message : String(err)}; failing closed`; + return { address: addr, publicKeyHex: null }; + } + }), ); // If the connection was closed or superseded while we were awaiting @@ -769,7 +931,7 @@ export function createSidecarRouter( // Add verified addresses to the routing table immediately so that // `agent.reconnected` subscribers can use sendRequest-based methods - // (e.g. sendGrantsUpdate). Addresses that fail governance are + // (e.g. sendSourcesUpdate). Addresses that fail governance are // rolled back from the routing table afterward. for (const addr of verified) { // If a different ws still owns this address (live takeover via @@ -779,8 +941,31 @@ export function createSidecarRouter( const prevWs = addressIndex.get(addr); if (prevWs !== undefined && prevWs !== ws) { connectorStates.delete(addr); + // Evict the reclaimed address from the superseded connection's owned + // set. handleClose's cancelByAgent sweep iterates a connection's owned + // union WITHOUT an ownership guard, so if the stale connection still + // listed this address it would cancel THIS connection's in-flight pack + // transfer for it when it finally closes. Delete from both sets: a + // workflow-derived address lives on the workflow set, a launched agent + // on the session set, and delete is a no-op for the absent one. + const prevConn = connections.get(prevWs); + if (prevConn !== undefined) { + prevConn.workflowAddresses.delete(addr); + prevConn.agentAddresses.delete(addr); + } + } + // Track the address on the set that matches its lifecycle so + // handleClose reclaims it correctly: a workflow-derived deployment + // address goes on the workflow set (no disconnect queue -- its + // in-flight state is reconstructed sidecar-locally on the next + // reconnect), a launched agent on the session set (queued for + // reconnect). The routing pointer is the same either way; only now + // it is written behind a passed challenge. + if (isWorkflowDerivedAddress(addr)) { + conn.workflowAddresses.add(addr); + } else { + conn.agentAddresses.add(addr); } - conn.agentAddresses.add(addr); addressIndex.set(addr, ws); } @@ -788,6 +973,17 @@ export function createSidecarRouter( const failed: string[] = []; for (const addr of verified) { + // The `agent.reconnected` reaction is session lifecycle -- instance + // status flip, event-collector restore -- owned by the agent_instance + // concept. A workflow-derived deployment address has no agent_instance + // row, so the reaction's `requireInstance` would throw and roll the + // just-verified address back out of routing. It needs routing + queue + // flush only, which the passed challenge has now made safe; skip the + // session reaction for it. + if (isWorkflowDerivedAddress(addr)) { + ready.push(addr); + continue; + } if (events.listenerCount("agent.reconnected") === 0) { ready.push(addr); continue; @@ -810,7 +1006,10 @@ export function createSidecarRouter( return; } - // Roll back failed addresses from the routing table. + // Roll back failed addresses from the routing table. Only the session set + // is touched: a workflow-derived address can never be in `failed` -- it + // early-`continue`s to `ready` above, before the reaction that populates + // `failed` -- so it is never on the workflow set at this point. for (const addr of failed) { conn.agentAddresses.delete(addr); addressIndex.delete(addr); @@ -828,6 +1027,11 @@ export function createSidecarRouter( const checkDeployRef = lookups.lookupDeployRef; if (checkDeployRef !== undefined) { for (const addr of ready) { + // Deploy-pack freshness (createDeployPack / parseAgentId) is an + // agent-repo path scoped to launched agents; it is not validated for + // workflow-derived deployment addresses, and the current register + // path never ran it for them. Skip it here to match that. + if (isWorkflowDerivedAddress(addr)) continue; void (async () => { try { const hubRef = await checkDeployRef(addr); @@ -959,10 +1163,9 @@ export function createSidecarRouter( } // Create a queue entry so messages can accumulate while the - // sidecar is disconnected. Skip if the agent is being undeployed - // or has a pending session start — there is no point queuing - // messages for an agent being torn down or one that never started. - if (!pendingUndeploys.has(addr) && !pendingSessionStarts.has(addr)) { + // sidecar is disconnected. Skip if the agent is being undeployed -- + // there is no point queuing messages for an agent being torn down. + if (!pendingUndeploys.has(addr)) { const timer = setTimeout(() => { disconnectedAgents.delete(addr); }, disconnectQueueTTLMs); @@ -970,6 +1173,17 @@ export function createSidecarRouter( } } } + // Remove this connection's workflow-substrate routes. No disconnect queue + // is created: these addresses re-register (with the complete live set) + // when the sidecar reconnects, and their in-flight run state is + // reconstructed sidecar-locally, not from a hub-side queue. The ownership + // guard mirrors the session loop above so a takeover by a newer ws is not + // clobbered by the prior owner's close. + for (const addr of conn.workflowAddresses) { + if (addressIndex.get(addr) === ws) { + addressIndex.delete(addr); + } + } connections.delete(ws); // Cancel the liveness timer for this connection. @@ -979,6 +1193,9 @@ export function createSidecarRouter( livenessTimers.delete(ws); } + // Drop the per-ws serialization chain; no more frames will queue on it. + messageChains.delete(ws); + // Cancel any pending challenge for this connection. const challengeReq = pendingChallenges.get(ws); if (challengeReq !== undefined) { @@ -1002,14 +1219,6 @@ export function createSidecarRouter( pack.reject(`Sidecar ${conn.sidecarId} disconnected`); } - // Reject any in-flight session starts for this sidecar. - for (const [addr, req] of pendingSessionStarts) { - if (req.ws !== ws) continue; - clearTimeout(req.timer); - pendingSessionStarts.delete(addr); - req.reject(`Sidecar ${conn.sidecarId} disconnected`); - } - // Reject any in-flight undeploys for this sidecar. for (const [addr, req] of pendingUndeploys) { if (req.ws !== ws) continue; @@ -1022,14 +1231,21 @@ export function createSidecarRouter( // across both receivers. The two receivers track their own in- // flight transferIds, so a pending workflow-run transfer for an // agent that just disconnected won't outlive the connection just - // because the agent-state receiver has nothing to cancel. - for (const addr of conn.agentAddresses) { + // because the agent-state receiver has nothing to cancel. Iterate the + // owned union so a reconnected workflow deployment's transfer is + // cancelled too; the deduped set avoids a double-cancel for an address + // that is in both sets. A reclaimed address is not present here -- the + // verified reconnect path that took it over evicts it from this + // (superseded) connection's owned set -- so a stale close does not cancel + // the new owner's work. + const owned = ownedAddresses(conn); + for (const addr of owned) { agentStatePackReceiver.cancelByAgent(addr); workflowRunPackReceiver.cancelByAgent(addr); } events.emit("sidecar.disconnect", { - agentAddresses: [...conn.agentAddresses], + ownedAddresses: [...owned], }); logger.info`Sidecar ${conn.sidecarId} disconnected`; @@ -1115,28 +1331,6 @@ export function createSidecarRouter( entry.reject(reason); } - function resolveSessionStartPending(agentAddress: string): void { - const req = pendingSessionStarts.get(agentAddress); - if (req === undefined) { - logger.warn`Received session.start.ack for "${agentAddress}" with no pending start`; - return; - } - clearTimeout(req.timer); - pendingSessionStarts.delete(agentAddress); - req.resolve(); - } - - function rejectSessionStartPending( - agentAddress: string, - error: string, - ): void { - const req = pendingSessionStarts.get(agentAddress); - if (req === undefined) return; - clearTimeout(req.timer); - pendingSessionStarts.delete(agentAddress); - req.reject(error); - } - function resolveUndeployPending(agentAddress: string): void { const req = pendingUndeploys.get(agentAddress); if (req === undefined) { @@ -1192,7 +1386,7 @@ export function createSidecarRouter( function handlePackPush(ws: WsHandle, frame: PackPushFrame): void { const conn = connections.get(ws); if (conn === undefined) return; - if (!conn.agentAddresses.has(frame.agentAddress)) { + if (!connOwnsAddress(conn, frame.agentAddress)) { logger.warn`Received repo.pack.push for unrouted agent ${frame.agentAddress}`; return; } @@ -1228,7 +1422,7 @@ export function createSidecarRouter( ): Promise { const conn = connections.get(ws); if (conn === undefined) return; - if (!conn.agentAddresses.has(frame.agentAddress)) { + if (!connOwnsAddress(conn, frame.agentAddress)) { logger.warn`Received repo.pack.done for unrouted agent ${frame.agentAddress}`; return; } @@ -1298,6 +1492,53 @@ export function createSidecarRouter( } } + /** + * Bind a per-step workflow-substrate address to a sidecar for the staging + * window of a multi-step deploy, so `sendPack` can route the step's deploy + * and asset packs before the deployment-level frame spawns the child. + * + * The address is hub-minted and workflow-derived (no per-address key), so + * it enters the keyless `workflowAddresses` set -- never the challenged + * `agentAddresses` set -- and is torn down by `unbindStepRoute` once the + * step's packs land. `handleClose` reclaims it if the sidecar drops + * mid-stage. Per-step addresses are not runtime-routed (mail, signals, and + * drains use the deployment address), so the binding is transient: it is + * never persisted into the reconnect set and never resurrected on + * reconnect. + */ + function bindStepRoute(stepAddress: string): void { + const ws = + addressIndex.get(stepAddress) ?? findSidecarForNewAgent(stepAddress); + if (ws === undefined) { + throw new Error( + `No sidecar available to stage workflow step "${stepAddress}"`, + ); + } + const conn = connections.get(ws); + if (conn === undefined) { + throw new Error( + `No sidecar connected to stage workflow step "${stepAddress}"`, + ); + } + conn.workflowAddresses.add(stepAddress); + addressIndex.set(stepAddress, ws); + } + + /** + * Remove a per-step route bound by `bindStepRoute` once the step's packs + * have landed. Idempotent: an address that was never bound (or already + * unbound, e.g. by a mid-stage `handleClose`) is a no-op. + */ + function unbindStepRoute(stepAddress: string): void { + const ws = addressIndex.get(stepAddress); + if (ws === undefined) return; + const conn = connections.get(ws); + if (conn !== undefined) { + conn.workflowAddresses.delete(stepAddress); + } + addressIndex.delete(stepAddress); + } + // Pack transfers may take longer than session requests due to data volume. const PACK_TIMEOUT_MS = requestTimeoutMs * 4; @@ -1533,63 +1774,88 @@ export function createSidecarRouter( }); } - function findSidecarForNewAgent(_agentAddress: string): WsHandle | undefined { - const first = connections.entries().next(); - if (first.done) return undefined; - return first.value[0]; - } - - function sendAgentUndeploy( + /** + * Provision one step of a multi-step deploy on the sidecar WITHOUT + * spawning: the sidecar initializes the step's agent-state repo and + * records the hub key, so the follow-up deploy pack applies into a repo + * and verifies against the recorded key -- but no supervisor or child is + * constructed. The deployment-level workflow frame, sent once after every + * step is provisioned, spawns the child. + * + * The step address must already be bound via `bindStepRoute`, which + * resolves and records the sidecar; this reuses that route rather than + * touching `agentAddresses`. Waits for the sidecar's `agent.deploy.ack` + * so the caller can safely deliver the deploy pack afterward. On failure + * the caller owns tearing the route down via `unbindStepRoute`. + */ + function sendProvisionStep( agentAddress: string, - reason: string, + harnessConfig: HarnessConfig, ): Promise { + if (hubPublicKeyHex === undefined) { + throw new Error("Hub signing key is required for step provisioning"); + } + if (pendingDeploys.has(agentAddress)) { + throw new Error(`Deploy already in progress for agent "${agentAddress}"`); + } const ws = addressIndex.get(agentAddress); if (ws === undefined) { - return Promise.reject( - new Error(`No sidecar connected for agent "${agentAddress}"`), + throw new Error( + `Step route for "${agentAddress}" is not bound; call bindStepRoute before provisioning`, ); } const conn = connections.get(ws); if (conn === undefined) { - return Promise.reject( - new Error(`No sidecar connected for agent "${agentAddress}"`), - ); + throw new Error(`No sidecar connected for agent "${agentAddress}"`); } + const hubKey = hubPublicKeyHex; return new Promise((resolve, reject) => { const timer = setTimeout(() => { - pendingUndeploys.delete(agentAddress); - removeAgentAddress(ws, agentAddress); + pendingDeploys.delete(agentAddress); reject( new Error( - `Undeploy of "${agentAddress}" timed out after ${requestTimeoutMs}ms`, + `Step provision of "${agentAddress}" timed out after ${requestTimeoutMs}ms`, ), ); }, requestTimeoutMs); - - pendingUndeploys.set(agentAddress, { + // The sidecar's `agent.deploy.ack` resolves this through + // `resolveDeployPending` -> `req.resolve()`. The per-step address is + // workflow-derived and records no hub-side key, so the ack's public + // key is not needed and this resolves void. + pendingDeploys.set(agentAddress, { agentAddress, ws, resolve() { - removeAgentAddress(ws, agentAddress); resolve(); }, reject(error: string) { - removeAgentAddress(ws, agentAddress); reject(new Error(error)); }, timer, }); conn.send({ - type: "agent.undeploy", + type: "agent.deploy", agentAddress, - reason, + agentId: harnessConfig.agentId, + config: harnessConfig, + hubPublicKey: hubKey, + provisionStep: true, }); }); } - function sendSessionStart(agentAddress: string): Promise { + function findSidecarForNewAgent(_agentAddress: string): WsHandle | undefined { + const first = connections.entries().next(); + if (first.done) return undefined; + return first.value[0]; + } + + function sendAgentUndeploy( + agentAddress: string, + reason: string, + ): Promise { const ws = addressIndex.get(agentAddress); if (ws === undefined) { return Promise.reject( @@ -1605,19 +1871,20 @@ export function createSidecarRouter( return new Promise((resolve, reject) => { const timer = setTimeout(() => { - pendingSessionStarts.delete(agentAddress); + pendingUndeploys.delete(agentAddress); removeAgentAddress(ws, agentAddress); reject( new Error( - `Session start for "${agentAddress}" timed out after ${requestTimeoutMs}ms`, + `Undeploy of "${agentAddress}" timed out after ${requestTimeoutMs}ms`, ), ); }, requestTimeoutMs); - pendingSessionStarts.set(agentAddress, { + pendingUndeploys.set(agentAddress, { agentAddress, ws, resolve() { + removeAgentAddress(ws, agentAddress); resolve(); }, reject(error: string) { @@ -1628,24 +1895,13 @@ export function createSidecarRouter( }); conn.send({ - type: "session.start", + type: "agent.undeploy", agentAddress, + reason, }); }); } - async function sendSessionAbort( - agentAddress: string, - reason: AbortReason, - ): Promise { - await sendRequest(agentAddress, (requestId) => ({ - type: "session.abort", - requestId, - agentAddress, - reason, - })); - } - function removeAgentAddress(ws: WsHandle, agentAddress: string): void { addressIndex.delete(agentAddress); const conn = connections.get(ws); @@ -1700,18 +1956,6 @@ export function createSidecarRouter( return connectorStates.get(agentAddress) ?? null; } - async function sendGrantsUpdate( - agentAddress: string, - grants: GrantRule[], - ): Promise { - await sendRequest(agentAddress, (requestId) => ({ - type: "grants.update", - requestId, - agentAddress, - grants, - })); - } - async function sendSourcesUpdate( agentAddress: string, sources: InferenceSource[], @@ -1800,11 +2044,11 @@ export function createSidecarRouter( routeMail, sendAgentDeploy, sendAgentUndeploy, - sendSessionStart, - sendSessionAbort, - sendGrantsUpdate, sendSourcesUpdate, sendPack, + bindStepRoute, + unbindStepRoute, + sendProvisionStep, sendSyncRequest, sendSignalDeliver, sendDrain, diff --git a/packages/mail-memory/src/index.test.ts b/packages/mail-memory/src/index.test.ts index 57e98c38..0ae4f83c 100644 --- a/packages/mail-memory/src/index.test.ts +++ b/packages/mail-memory/src/index.test.ts @@ -735,6 +735,21 @@ describe("deliver", () => { expect(headers.from).toBe("sender@remote"); }); + test("does NOT dedup by Message-ID: redelivery appends a second copy", async () => { + // The federation inbound path performs NO Message-ID dedup: + // appendToMailbox appends unconditionally, so delivering the same + // Message-ID twice yields two INBOX messages. This pins the baseline + // the supervisor FIFO + markConsumed dedup improves on. + const { transport } = await createTestTransport(); + const alphaTransport = transport.getTransportFor("alpha@test.interchange"); + + transport.deliver("alpha@test.interchange", VALID_MESSAGE); + transport.deliver("alpha@test.interchange", VALID_MESSAGE); + + const refs = await alphaTransport.search("INBOX", {}); + expect(refs).toHaveLength(2); + }); + test("fires watch callback on delivery", async () => { const { transport } = await createTestTransport(); const alphaTransport = transport.getTransportFor("alpha@test.interchange"); diff --git a/packages/storage-isogit/src/gc-cache-transparency.test.ts b/packages/storage-isogit/src/gc-cache-transparency.test.ts new file mode 100644 index 00000000..321f2673 --- /dev/null +++ b/packages/storage-isogit/src/gc-cache-transparency.test.ts @@ -0,0 +1,193 @@ +// The store threads one long-lived isomorphic-git `cache` object per repo +// through every git.* call. GC repacks the object store — consolidating +// loose objects into a fresh, differently-named pack and pruning the old +// one — behind that cache's back. These tests pin that the cache stays a +// pure accelerator across a repack: an object that migrates from a pruned +// pack into the new one still reads correctly under the SAME warm cache +// (because reads enumerate .idx files from disk, so a stranded parse of a +// pruned pack is never consulted), and an object GC genuinely drops +// surfaces NotFound rather than stale bytes. + +import { describe, test, expect, afterEach } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import git from "isomorphic-git"; +import { initAgentRepo } from "./init"; +import { runGC } from "./gc"; + +const author = { name: "Test", email: "test@test.dev" }; +const tempDirs: string[] = []; + +async function tempDir(): Promise { + const d = await fs.promises.mkdtemp(path.join(os.tmpdir(), "gc-cache-")); + tempDirs.push(d); + return d; +} + +afterEach(async () => { + const dirs = tempDirs.splice(0); + await Promise.all( + dirs.map((d) => fs.promises.rm(d, { recursive: true, force: true })), + ); +}); + +function listIdx(dir: string): string[] { + const p = path.join(dir, ".git", "objects", "pack"); + if (!fs.existsSync(p)) return []; + return fs.readdirSync(p).filter((x) => x.endsWith(".idx")); +} + +function countLoose(dir: string): number { + const base = path.join(dir, ".git", "objects"); + if (!fs.existsSync(base)) return 0; + let n = 0; + for (const d of fs.readdirSync(base)) { + if (!/^[0-9a-f]{2}$/.test(d)) continue; + n += fs.readdirSync(path.join(base, d)).length; + } + return n; +} + +async function commitFile( + dir: string, + name: string, + content: string, +): Promise { + const full = path.join(dir, name); + await fs.promises.mkdir(path.dirname(full), { recursive: true }); + await fs.promises.writeFile(full, content); + await git.add({ fs, dir, filepath: name }); + return git.commit({ fs, dir, message: `add ${name}`, author }); +} + +async function commitFileOnRef( + dir: string, + branch: string, + name: string, + content: string, +): Promise { + await git.checkout({ fs, dir, ref: branch }); + const oid = await commitFile(dir, name, content); + await git.checkout({ fs, dir, ref: "main" }); + return oid; +} + +function packfileCacheKeys(cache: object): string[] { + const packSym = Object.getOwnPropertySymbols(cache).find( + (s) => s.toString() === "Symbol(PackfileCache)", + ); + if (packSym === undefined) return []; + const map: unknown = Reflect.get(cache, packSym); + if (!(map instanceof Map)) return []; + const keys: string[] = []; + for (const k of map.keys()) { + if (typeof k === "string") keys.push(k); + } + return keys; +} + +describe("shared-cache repack transparency", () => { + test("object in a pre-GC pack reads correctly after GC prunes that pack, under the SAME warm cache", async () => { + const dir = await tempDir(); + await initAgentRepo(dir); + + const targetContent = "the-target-blob-payload-v1"; + let tip = await commitFile(dir, "state/target.txt", targetContent); + const targetOid = await git.writeBlob({ + fs, + dir, + blob: Buffer.from(targetContent), + }); + for (let i = 0; i < 5; i++) { + tip = await commitFile(dir, `state/f${i}.txt`, `payload ${i}`); + } + + // First GC consolidates all loose objects into pack #1 and prunes the + // loose copies. The target now lives only in pack #1. + await runGC(dir, { retention: "keep-history" }); + const idxBefore = listIdx(dir); + const firstIdx = idxBefore[0]; + if (firstIdx === undefined) throw new Error("expected a pack .idx"); + expect(countLoose(dir)).toBe(0); + + // One shared, long-lived cache object — the exact shape the store + // threads through every git.* call. + const cache: object = {}; + + // Warm the cache: read the target from the pre-GC pack, populating + // isomorphic-git's PackfileCache keyed by the pre-GC .idx filename. + const warm = await git.readBlob({ fs, dir, oid: targetOid, cache }); + expect(Buffer.from(warm.blob).toString()).toBe(targetContent); + expect(packfileCacheKeys(cache).some((k) => k.includes(firstIdx))).toBe( + true, + ); + + // More commits, then a second GC: repacks into a new pack #2 that + // supersedes and prunes pack #1. The target migrates to pack #2. + for (let i = 0; i < 4; i++) { + tip = await commitFile(dir, `state/g${i}.txt`, `more ${i}`); + } + await runGC(dir, { retention: "keep-history" }); + + const idxAfter = listIdx(dir); + // The pre-GC pack is gone from disk, but its parsed index is still + // stranded in the warm cache map. + expect(idxAfter).not.toContain(firstIdx); + expect(packfileCacheKeys(cache).some((k) => k.includes(firstIdx))).toBe( + true, + ); + + // The test: read the target again under the SAME warm cache, after the + // pack the cache indexed was pruned from disk. + const reread = await git.readBlob({ fs, dir, oid: targetOid, cache }); + expect(Buffer.from(reread.blob).toString()).toBe(targetContent); + + // Exercise the readTree/readCommit path through the same cache too. + const co = await git.readCommit({ fs, dir, oid: tip, cache }); + const tr = await git.readTree({ fs, dir, oid: co.commit.tree, cache }); + expect(tr.tree.length).toBeGreaterThan(0); + }); + + test("GC that drops a now-unreachable object surfaces NotFound, not stale bytes, under a warm cache", async () => { + const dir = await tempDir(); + await initAgentRepo(dir); + + for (let i = 0; i < 3; i++) { + await commitFile(dir, `state/f${i}.txt`, `keep ${i}`); + } + + // A blob reachable only from a secondary ref. + const orphanContent = "orphan-only-in-old-pack"; + const orphanOid = await git.writeBlob({ + fs, + dir, + blob: Buffer.from(orphanContent), + }); + await git.branch({ fs, dir, ref: "scratch" }); + await commitFileOnRef(dir, "scratch", "state/orphan.txt", orphanContent); + + // Pack everything reachable into pack #1. + await runGC(dir, { retention: "keep-history" }); + expect(countLoose(dir)).toBe(0); + + const cache: object = {}; + const warm = await git.readBlob({ fs, dir, oid: orphanOid, cache }); + expect(Buffer.from(warm.blob).toString()).toBe(orphanContent); + + // Drop the scratch ref so the orphan is unreachable, then GC. + await git.deleteBranch({ fs, dir, ref: "scratch" }); + await commitFile(dir, "state/f9.txt", "keep 9"); + await runGC(dir, { retention: "keep-history" }); + + // The object is genuinely gone — a NotFound throw, never stale bytes. + let threw = false; + try { + const r = await git.readBlob({ fs, dir, oid: orphanOid, cache }); + expect(Buffer.from(r.blob).toString()).not.toBe(orphanContent); + } catch { + threw = true; + } + expect(threw).toBe(true); + }); +}); diff --git a/packages/storage-isogit/src/index.ts b/packages/storage-isogit/src/index.ts index afd86266..653797f1 100644 --- a/packages/storage-isogit/src/index.ts +++ b/packages/storage-isogit/src/index.ts @@ -1,6 +1,6 @@ import type { ContextStore, AuditStore } from "@intx/types/runtime"; import { initAgentRepo } from "./init"; -import { IsogitStore } from "./store"; +import { IsogitStore, type DurableMirrorReads } from "./store"; import type { CommitSigner } from "./signer"; import type { GCPolicy } from "./gc"; @@ -11,6 +11,7 @@ export type { TreeValidatorResult, } from "./pack-receive"; export { IsogitStore }; +export type { DurableMirrorReads }; export { switchBranch, createAndSwitchBranch, @@ -63,7 +64,7 @@ export async function createIsogitStore( dir: string, signer?: CommitSigner, gcPolicy?: GCPolicy, -): Promise { +): Promise { await initAgentRepo(dir); return new IsogitStore(dir, signer, gcPolicy); } diff --git a/packages/storage-isogit/src/pack-receive.ts b/packages/storage-isogit/src/pack-receive.ts index 421414a0..a0a14158 100644 --- a/packages/storage-isogit/src/pack-receive.ts +++ b/packages/storage-isogit/src/pack-receive.ts @@ -138,8 +138,8 @@ async function topLevelNames(dir: string, oid: string): Promise> { * * Derives the set of deploy-managed top-level entries from the union of the * old and new commit trees, removes those entries, then writes the new tree. - * Paths that never appear in any commit tree (e.g. .git, agent.json, state, - * keys) are never touched. + * Paths that never appear in any commit tree (e.g. .git, state, keys) are + * never touched. * * NOTE: The rm-then-write sequence is not atomic. If writeTree fails after rm * succeeds (e.g. disk full), the working tree will be missing the cleared diff --git a/packages/storage-isogit/src/peek-turns.test.ts b/packages/storage-isogit/src/peek-turns.test.ts new file mode 100644 index 00000000..2e49c6bb --- /dev/null +++ b/packages/storage-isogit/src/peek-turns.test.ts @@ -0,0 +1,103 @@ +import { describe, test, expect, afterEach } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { createIsogitStore } from "./index"; +import type { ConversationTurn, TokenUsage } from "@intx/types/runtime"; + +const ZERO_USAGE: TokenUsage = { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + thinking: 0, +}; + +const tempDirs: string[] = []; +async function tempDir(): Promise { + const d = await fs.promises.mkdtemp(path.join(os.tmpdir(), "peek-turns-")); + tempDirs.push(d); + return d; +} +afterEach(async () => { + const dirs = tempDirs.splice(0); + await Promise.all( + dirs.map((d) => fs.promises.rm(d, { recursive: true, force: true })), + ); +}); + +function userTurn(text: string): ConversationTurn { + return { role: "user", content: [{ type: "text", text }], timestamp: 1 }; +} + +describe("DurableMirrorReads.peekTurns", () => { + test("returns [] before any writeTurns", async () => { + const store = await createIsogitStore(await tempDir()); + expect(store.peekTurns()).toEqual([]); + }); + + test("returns the exact array reference handed to writeTurns", async () => { + const store = await createIsogitStore(await tempDir()); + const arr: ConversationTurn[] = [userTurn("a")]; + await store.writeTurns(arr); + expect(store.peekTurns()).toBe(arr); // same reference, not a copy + }); + + test("aliases the caller's array: in-place mutation is visible", async () => { + const store = await createIsogitStore(await tempDir()); + const arr: ConversationTurn[] = [userTurn("a")]; + await store.writeTurns(arr); + arr.push(userTurn("b")); // the reactor's appendTurn does exactly this + // peekTurns now reports 2 turns though only 1 was persisted to disk. + expect(store.peekTurns()).toHaveLength(2); + const onDisk = await store.load(); + expect(onDisk.turns).toHaveLength(1); + }); + + test("peekTurns tracks the last writeTurns array", async () => { + const store = await createIsogitStore(await tempDir()); + const first: ConversationTurn[] = [userTurn("first")]; + await store.writeTurns(first); + const second: ConversationTurn[] = [ + userTurn("second-a"), + userTurn("second-b"), + ]; + await store.writeTurns(second); + expect(store.peekTurns()).toBe(second); + }); +}); + +describe("DurableMirrorReads.loadMetadata", () => { + test("returns fresh empty defaults when metadata.json is absent", async () => { + const store = await createIsogitStore(await tempDir()); + const m1 = await store.loadMetadata(); + expect(m1.pendingOperations).toEqual([]); + expect(m1.tokenUsage).toEqual(ZERO_USAGE); + expect(m1.connectorState).toBeNull(); + // A fresh copy each call: distinct references, and mutating one + // result must not leak into the next. + m1.tokenUsage.input = 999; + const m2 = await store.loadMetadata(); + expect(m2.tokenUsage).toEqual(ZERO_USAGE); + expect(m2.pendingOperations).toEqual([]); + expect(m2.pendingOperations).not.toBe(m1.pendingOperations); + }); + + test("load and loadMetadata agree after writeMetadata", async () => { + const store = await createIsogitStore(await tempDir()); + const usage: TokenUsage = { + input: 10, + output: 20, + cacheRead: 0, + cacheWrite: 0, + thinking: 0, + }; + await store.writeMetadata({ pendingOperations: [], tokenUsage: usage }); + const viaLoad = await store.load(); + const viaMeta = await store.loadMetadata(); + expect(viaMeta.tokenUsage).toEqual(usage); + expect(viaLoad.tokenUsage).toEqual(usage); + expect(viaLoad.pendingOperations).toEqual(viaMeta.pendingOperations); + expect(viaLoad.connectorState).toEqual(viaMeta.connectorState); + }); +}); diff --git a/packages/storage-isogit/src/store.ts b/packages/storage-isogit/src/store.ts index 70c4226c..02dfa284 100644 --- a/packages/storage-isogit/src/store.ts +++ b/packages/storage-isogit/src/store.ts @@ -252,11 +252,40 @@ function parseTurns(text: string): ConversationTurn[] { * `dir`. The caller is responsible for calling `initAgentRepo(dir)` before * constructing. */ -export class IsogitStore implements ContextStore, AuditStore { +/** + * Extra reads the durable WAL mirror needs beyond `ContextStore`, kept + * off the shared interface because they are isogit-specific. + */ +export interface DurableMirrorReads { + /** + * The turns most recently handed to `writeTurns`, by reference -- the + * reactor's live array, not a copy. Lets the WAL mirror slice the new + * turns from memory instead of re-reading and re-parsing `turns.jsonl` + * every boundary. Safe because the local store is single-writer and + * in-process: nothing else writes `turns.jsonl`, so the last-written + * array equals the on-disk state at the mirror boundary. + */ + peekTurns(): ConversationTurn[]; + /** + * Read only `metadata.json` (pending operations, token usage, connector + * state), skipping the O(N) turns parse `load` pays. The mirror gets + * its turns from `peekTurns`. + */ + loadMetadata(): Promise<{ + pendingOperations: PendingOperation[]; + tokenUsage: TokenUsage; + connectorState: ConnectorThreadState | null; + }>; +} + +export class IsogitStore + implements ContextStore, AuditStore, DurableMirrorReads +{ private readonly dir: string; private readonly signer: CommitSigner | undefined; private readonly gcPolicy: GCPolicy | undefined; private pendingConnectorState: ConnectorThreadState | null = null; + private lastTurns: ConversationTurn[] = []; constructor(dir: string, signer?: CommitSigner, gcPolicy?: GCPolicy) { this.dir = dir; @@ -286,7 +315,6 @@ export class IsogitStore implements ContextStore, AuditStore { connectorState: ConnectorThreadState | null; }> { const turnsPath = path.join(this.dir, TURNS_FILE); - const metadataPath = path.join(this.dir, METADATA_FILE); let turns: ConversationTurn[] = []; if (await pathExists(turnsPath)) { @@ -294,20 +322,31 @@ export class IsogitStore implements ContextStore, AuditStore { turns = parseTurns(text); } - let pendingOperations: PendingOperation[] = []; - let tokenUsage: TokenUsage = { ...EMPTY_USAGE }; - let connectorState: ConnectorThreadState | null = null; - - if (await pathExists(metadataPath)) { - const text = await fs.promises.readFile(metadataPath, "utf-8"); - const parsed: unknown = JSON.parse(text); - const data = parseMetadata(parsed); - pendingOperations = data.pendingOperations; - tokenUsage = data.tokenUsage; - connectorState = data.connectorState; - } + const metadata = await this.loadMetadata(); + return { turns, ...metadata }; + } - return { turns, pendingOperations, tokenUsage, connectorState }; + async loadMetadata(): Promise<{ + pendingOperations: PendingOperation[]; + tokenUsage: TokenUsage; + connectorState: ConnectorThreadState | null; + }> { + const metadataPath = path.join(this.dir, METADATA_FILE); + if (!(await pathExists(metadataPath))) { + return { + pendingOperations: [], + tokenUsage: { ...EMPTY_USAGE }, + connectorState: null, + }; + } + const text = await fs.promises.readFile(metadataPath, "utf-8"); + const parsed: unknown = JSON.parse(text); + const data = parseMetadata(parsed); + return { + pendingOperations: data.pendingOperations, + tokenUsage: data.tokenUsage, + connectorState: data.connectorState, + }; } async commit( @@ -471,6 +510,14 @@ export class IsogitStore implements ContextStore, AuditStore { path.join(this.dir, TURNS_FILE), encodeJsonlLines(turns), ); + // Advance the in-memory marker only after the durable write succeeds. + // peekTurns must never surface an array that failed to persist -- a + // write failure leaves it pointing at the last array that did. + this.lastTurns = turns; + } + + peekTurns(): ConversationTurn[] { + return this.lastTurns; } /** diff --git a/packages/tool-packaging/README.md b/packages/tool-packaging/README.md index c002c246..dbaa6d77 100644 --- a/packages/tool-packaging/README.md +++ b/packages/tool-packaging/README.md @@ -85,8 +85,8 @@ and the sidecar's fetch can cause the apply to fail with The substrate's `writeTreePreservingPrefix` primitive makes PUTs and DELETEs against the asset atomic with respect to one another, but it does not couple them to the sidecar's deploy-apply read. The -intentional behavior is: the apply fails loudly through the existing -`deploy.apply.error` frame channel. There is no silent corruption — +intentional behavior is: the apply fails loudly as a rejected +deploy-apply. There is no silent corruption — the integrity check on the sidecar side guarantees that mismatched bytes are rejected — but the operator does see a failed apply if the asset is mutated concurrently with a launch. diff --git a/packages/tool-packaging/src/index.ts b/packages/tool-packaging/src/index.ts index 6b556292..aefd1e39 100644 --- a/packages/tool-packaging/src/index.ts +++ b/packages/tool-packaging/src/index.ts @@ -22,8 +22,7 @@ // applyAtomic({ manifest, loader, … }) — per-deploy-id apply // protocol that stages each deploy into its own never-renamed // directory and maps every loader failure category onto an -// `ApplyAtomicFailure` the caller routes to the -// `deploy.apply.error` frame channel. +// `ApplyAtomicFailure` the caller surfaces as a rejected apply. export { type ClosureResolver, diff --git a/packages/tool-packaging/src/loader.ts b/packages/tool-packaging/src/loader.ts index 6cce817e..ba7f9aae 100644 --- a/packages/tool-packaging/src/loader.ts +++ b/packages/tool-packaging/src/loader.ts @@ -140,8 +140,8 @@ export interface LoaderConfig { /** * Deadline in milliseconds for a single HTTP-registry tarball fetch, * spanning the request and the streamed body read. A stalled registry - * cannot block the fetch -- and the restoring `startSession` awaiting - * it -- past this bound. Defaults to + * cannot block the fetch -- and the deploy's tool materialization + * awaiting it -- past this bound. Defaults to * `DEFAULT_REGISTRY_FETCH_TIMEOUT_MS`. Asset-sourced tarballs read from * the local filesystem and are not subject to it. */ @@ -175,8 +175,8 @@ export const DEFAULT_MAX_REGISTRY_TARBALL_BYTES = 10 * 1024 * 1024; * both the request and the streamed body read. `readResponseWithLimit` * consumes the body through a manual reader loop, so the byte cap bounds * size but nothing bounds time: a registry that accepts the connection - * and then stalls mid-stream would block the fetch -- and the - * `startSession` awaiting it during a sidecar restore -- indefinitely. + * and then stalls mid-stream would block the fetch -- and the deploy's + * tool materialization awaiting it -- indefinitely. * The deadline is generous so a legitimately large tarball on a slow * link still completes within it. Callers that need a different bound * pass `registryFetchTimeoutMs` to `createToolLoader`. @@ -726,7 +726,7 @@ export function createToolLoader(config: LoaderConfig): ToolLoader { entry.tarballUrl ?? defaultTarballUrl(registry.url, entry.name, entry.version); // Bound the whole fetch -- request and streamed body read -- so a - // stalled registry cannot block the awaiting startSession forever. + // stalled registry cannot block the awaiting deploy forever. // npm-registry-fetch honors the signal for the request phase; // readResponseWithLimit honors it for the manual body read. The // timer spans both phases and is cleared only once the read settles. diff --git a/packages/types/src/grant-wire.ts b/packages/types/src/grant-wire.ts index 0074f824..a5e5f389 100644 --- a/packages/types/src/grant-wire.ts +++ b/packages/types/src/grant-wire.ts @@ -3,7 +3,7 @@ // GrantRule.expiresAt is a Date | null at runtime, but JSON round-trips // turn it into a string | null. This validator accepts either form and // coerces strings back to Date instances, making it safe to use when -// deserializing persisted configs (e.g. agent.json on sidecar restart). +// deserializing grants that have round-tripped through JSON. import { type } from "arktype"; diff --git a/packages/types/src/sidecar.test.ts b/packages/types/src/sidecar.test.ts index c33c7e41..a0b6c16b 100644 --- a/packages/types/src/sidecar.test.ts +++ b/packages/types/src/sidecar.test.ts @@ -3,8 +3,7 @@ import { type } from "arktype"; import { AgentDeployFrame, DeployApplyErrorCategory, - DeployApplyErrorFrame, - SidecarFrame, + SourcesUpdateFrame, } from "./sidecar"; describe("DeployApplyErrorCategory", () => { @@ -37,76 +36,6 @@ describe("DeployApplyErrorCategory", () => { }); }); -describe("DeployApplyErrorFrame", () => { - const validFrame = { - type: "deploy.apply.error" as const, - agentAddress: "agt_1", - attemptId: "atp_1", - previousDeployId: "dpl_0", - category: "integrity.mismatch" as const, - message: "fetched bytes for left-pad@1.3.0 did not match pinned integrity", - occurredAt: "2026-06-05T20:00:00.000Z", - }; - - test("accepts a minimal valid frame", () => { - const result = DeployApplyErrorFrame(validFrame); - expect(result instanceof type.errors).toBe(false); - }); - - test("accepts a frame with package context", () => { - const result = DeployApplyErrorFrame({ - ...validFrame, - package: { name: "left-pad", version: "1.3.0" }, - }); - expect(result instanceof type.errors).toBe(false); - }); - - test("rejects a wrong type discriminator", () => { - const result = DeployApplyErrorFrame({ - ...validFrame, - type: "deploy.apply.ok", - }); - expect(result instanceof type.errors).toBe(true); - }); - - test("rejects an unknown category on the frame", () => { - const result = DeployApplyErrorFrame({ - ...validFrame, - category: "network.timeout", - }); - expect(result instanceof type.errors).toBe(true); - }); - - test("rejects a frame missing previousDeployId", () => { - const { previousDeployId: _omitted, ...partial } = validFrame; - const result = DeployApplyErrorFrame(partial); - expect(result instanceof type.errors).toBe(true); - }); - - test("rejects a package field with a missing version", () => { - const result = DeployApplyErrorFrame({ - ...validFrame, - package: { name: "left-pad" }, - }); - expect(result instanceof type.errors).toBe(true); - }); -}); - -describe("SidecarFrame union", () => { - test("accepts a DeployApplyErrorFrame as a member", () => { - const result = SidecarFrame({ - type: "deploy.apply.error", - agentAddress: "agt_1", - attemptId: "atp_1", - previousDeployId: "dpl_0", - category: "manifest.invalid", - message: "schemaVersion was '2', expected '1'", - occurredAt: "2026-06-05T20:00:00.000Z", - }); - expect(result instanceof type.errors).toBe(false); - }); -}); - describe("AgentDeployFrame", () => { const baseConfig = { sessionId: "ses_1", @@ -160,7 +89,7 @@ describe("AgentDeployFrame", () => { stepOrder: ["plan", "act"], steps: { plan: {}, act: {} }, }, - sources: { plan: stepSource, act: stepSource }, + sources: { plan: [stepSource], act: [stepSource] }, }, }); expect(result instanceof type.errors).toBe(false); @@ -191,7 +120,7 @@ describe("AgentDeployFrame", () => { stepOrder: ["plan", "act"], steps: { plan: {}, act: {} }, }, - sources: { plan: stepSource }, + sources: { plan: [stepSource] }, }, }); expect(result instanceof type.errors).toBe(true); @@ -210,9 +139,39 @@ describe("AgentDeployFrame", () => { stepOrder: ["plan"], steps: { plan: {} }, }, - sources: { plan: stepSource }, + sources: { plan: [stepSource] }, }, }); expect(result instanceof type.errors).toBe(true); }); }); + +describe("SourcesUpdateFrame", () => { + const source = { + id: "src_a", + provider: "openai", + baseURL: "https://api.openai.test", + apiKey: "sk-a", + model: "gpt-a", + }; + const base = { + type: "sources.update" as const, + requestId: "req_1", + agentAddress: "agt_1@example.test", + defaultSource: "src_a", + }; + + test("accepts a frame with a non-empty sources list", () => { + const result = SourcesUpdateFrame({ ...base, sources: [source] }); + expect(result instanceof type.errors).toBe(false); + }); + + test("rejects a frame whose sources list is empty", () => { + // The hub never emits an empty rotation -- `pushInstanceSourceUpdate` + // returns early when there is no head source -- so the boundary + // rejects an empty `sources` rather than accepting a rotation the + // agent could not swap to any live source. + const result = SourcesUpdateFrame({ ...base, sources: [] }); + expect(result instanceof type.errors).toBe(true); + }); +}); diff --git a/packages/types/src/sidecar.ts b/packages/types/src/sidecar.ts index e019d568..a194527e 100644 --- a/packages/types/src/sidecar.ts +++ b/packages/types/src/sidecar.ts @@ -9,13 +9,11 @@ import { type } from "arktype"; import { - AbortReason, ConnectorThreadState, HarnessConfig, InferenceEvent, InferenceSource, } from "./runtime"; -import { WireGrantRule } from "./grant-wire"; // --------------------------------------------------------------------------- // Sidecar → Hub @@ -35,9 +33,11 @@ export const RegisterFrame = type({ export type RegisterFrame = typeof RegisterFrame.infer; /** - * Sent on connect when the sidecar has existing agent repositories from a - * previous run. Lists the agent addresses it can serve, triggering the - * challenge/response verification flow. + * Sent on connect when the sidecar has agent repositories or deployments + * from a previous run. Lists the addresses it can serve, triggering the + * challenge/response ownership-verification flow for every one of them -- + * launched agents and workflow deployments alike, so both are proven, not + * routed on trust. */ export const ReconnectFrame = type({ type: "'reconnect'", @@ -141,7 +141,7 @@ export const PingFrame = type({ type: "'ping'" }); export type PingFrame = typeof PingFrame.infer; /** - * Acknowledges a request from the hub (session.abort, grants.update). + * Acknowledges a request from the hub (sources.update). */ export const SessionAckFrame = type({ type: "'session.ack'", @@ -159,17 +159,6 @@ export const SessionErrorFrame = type({ }); export type SessionErrorFrame = typeof SessionErrorFrame.infer; -/** - * Acknowledges that the inference session has started for a provisioned - * agent. Sent in response to a session.start frame after the harness is - * running. - */ -export const SessionStartAckFrame = type({ - type: "'session.start.ack'", - agentAddress: "string", -}); -export type SessionStartAckFrame = typeof SessionStartAckFrame.infer; - /** * Acknowledges that an agent has been fully undeployed: harness stopped, * state pushed (best-effort), and directory deleted. @@ -252,11 +241,10 @@ export const DrainDeliverFrame = type({ export type DrainDeliverFrame = typeof DrainDeliverFrame.infer; /** - * Workflow projection carried on an `agent.deploy` frame for the - * multi-step branch. Presence of the field at the deploy router is the - * discriminator between the trivial-launch path (single-step, - * `config`-driven) and the multi-step path (workflow-process spawn, - * per-step source pins). + * Workflow projection carried on an `agent.deploy` frame. Its presence + * at the deploy router routes the frame to the workflow deploy path -- + * single- or multi-step, both of which spawn the workflow-process child + * -- as opposed to a per-step provision frame. * * `definition` is the wire projection of `WorkflowDefinition` from * `@intx/workflow`. The arktype validator enforces the structural @@ -270,11 +258,16 @@ export type DrainDeliverFrame = typeof DrainDeliverFrame.infer; * of authoring-time primitive shape lives on the workflow definition * surface in `@intx/workflow`, not on the wire. * - * `sources` pins one inference source per step in `definition.stepOrder` - * so the workflow-process child can resolve inference at step invocation - * without a round trip to the hub. Every `stepOrder` entry must have a - * matching `sources` entry; the validator rejects frames that violate - * this invariant at the boundary. + * `sources` pins an ordered, non-empty inference-source list per step in + * `definition.stepOrder` so the workflow-process child can resolve inference + * at step invocation without a round trip to the hub. The list is the step's + * failover chain: element 0 is the active source (its id is the step's + * `defaultSource`), and the reactor fails over forward through the tail on a + * transient inference error. A workflow step pins a single-element list (no + * per-step failover); a single-agent instance pins the instance's full + * ordered source chain. Every `stepOrder` entry must have a matching + * `sources` entry; the validator rejects frames that violate this at the + * boundary. */ export const AgentDeployWorkflow = type({ definition: type({ @@ -285,7 +278,7 @@ export const AgentDeployWorkflow = type({ "state?": "Record", "+": "delete", }), - sources: { "[string]": InferenceSource }, + sources: { "[string]": InferenceSource.array().atLeastLength(1) }, }).narrow((value, ctx) => { for (const stepId of value.definition.stepOrder) { if (!Object.prototype.hasOwnProperty.call(value.sources, stepId)) { @@ -299,13 +292,20 @@ export const AgentDeployWorkflow = type({ export type AgentDeployWorkflow = typeof AgentDeployWorkflow.infer; /** - * Deploy an agent to this sidecar. The sidecar initializes a harness from - * the config. + * Deploy an agent to this sidecar. The sidecar spawns a supervised + * workflow-process child to host the deployment. * - * `workflow` is set only on multi-step deployments. Absent on every - * trivial-shape frame (every frame the system currently emits); the - * deploy router uses field presence to discriminate the two branches - * without consulting `config`. + * The deploy router discriminates two shapes by field presence without + * consulting `config`: + * - `workflow` set: a workflow deployment (single-step head or multi-step) + * that spawns the supervised workflow-process child. + * - `provisionStep` true: a no-spawn per-step provision of a multi-step + * deploy -- the sidecar initializes the step's agent-state repo and + * records the hub key so the follow-up deploy pack applies and verifies, + * but spawns nothing. The deployment-level `workflow` frame (sent once + * after every step is provisioned) spawns the child. + * A frame carrying neither is rejected -- there is no in-process + * fall-through. `workflow` and `provisionStep` are mutually exclusive. */ export const AgentDeployFrame = type({ type: "'agent.deploy'", @@ -314,6 +314,7 @@ export const AgentDeployFrame = type({ config: HarnessConfig, hubPublicKey: "string", "workflow?": AgentDeployWorkflow, + "provisionStep?": "boolean", }); export type AgentDeployFrame = typeof AgentDeployFrame.infer; @@ -329,17 +330,6 @@ export const AgentUndeployFrame = type({ }); export type AgentUndeployFrame = typeof AgentUndeployFrame.infer; -/** - * Start the inference session for a provisioned agent. Sent after the - * deploy pack has been applied so the harness can read deploy-tree tools - * and prompt from disk. - */ -export const SessionStartFrame = type({ - type: "'session.start'", - agentAddress: "string", -}); -export type SessionStartFrame = typeof SessionStartFrame.infer; - /** * Per-address cryptographic challenge. The sidecar must sign * `nonce || utf8(address)` with each agent's private key and respond @@ -361,18 +351,6 @@ export const ChallengeFailedFrame = type({ }); export type ChallengeFailedFrame = typeof ChallengeFailedFrame.infer; -/** - * Kill switch. Aborts a running agent immediately with the given reason. - * Responds with session.ack or session.error. - */ -export const SessionAbortFrame = type({ - type: "'session.abort'", - requestId: "string", - agentAddress: "string", - reason: AbortReason, -}); -export type SessionAbortFrame = typeof SessionAbortFrame.infer; - /** * Keepalive pong sent by the hub in response to a ping frame. * If the sidecar stops receiving pongs, it considers the hub dead. @@ -381,29 +359,19 @@ export const PongFrame = type({ type: "'pong'" }); export type PongFrame = typeof PongFrame.infer; /** - * Push updated grants to a running agent. The sidecar replaces the agent's - * grant snapshot and re-persists the config. Responds with session.ack or - * session.error. - */ -export const GrantsUpdateFrame = type({ - type: "'grants.update'", - requestId: "string", - agentAddress: "string", - grants: WireGrantRule.array(), -}); -export type GrantsUpdateFrame = typeof GrantsUpdateFrame.infer; - -/** - * Push updated inference sources to a running agent. The sidecar hot-swaps - * the active source on the harness (selected by id from `defaultSource`) - * and re-persists the agent config. Responds with session.ack or - * session.error. + * Push an updated inference-source list to a running single-step + * deployment. The sidecar routes it to the deployment's supervisor, which + * delivers it to the warm agent and swaps its sources in place. `sources` + * is non-empty (validated at this boundary, mirroring the deploy frame's + * per-step source arrays). Element 0 is the active source; the producer + * sets `defaultSource` to its id -- that equality is producer-enforced, + * not checked here. Responds with session.ack or session.error. */ export const SourcesUpdateFrame = type({ type: "'sources.update'", requestId: "string", agentAddress: "string", - sources: InferenceSource.array(), + sources: InferenceSource.array().atLeastLength(1), defaultSource: "string", }); export type SourcesUpdateFrame = typeof SourcesUpdateFrame.infer; @@ -651,44 +619,6 @@ export const DeployApplyErrorCategory = type.enumerated( ); export type DeployApplyErrorCategory = typeof DeployApplyErrorCategory.infer; -/** - * Sidecar reports a failed deploy-apply attempt back to the hub. - * - * `previousDeployId` is the atomicity contract for every category - * except `apply.previous-rotation.failed`: it names the deploy the - * instance was running before the rejected attempt, and after this - * frame is emitted the instance continues to run that deploy untouched. - * - * For `apply.previous-rotation.failed` the field has inverted meaning. - * The new deploy was staged and its `active-deploy-id` commit was - * written through the degradation ladder (so the new deploy is live) - * before the persist was found to be non-durable; the field carries - * the **new** deploy id so the hub can record the on-disk truth rather - * than a stale pre-apply id. The atomicity contract is preserved in - * the sense that the field always reflects the deploy id the instance - * is now running — what shifts is whether "now" is pre-apply or - * post-apply, depending on whether the commit landed before the - * failure. Renaming the field to make this explicit would break the - * wire shape; the category's semantics document the override instead. - * - * `package` is set when the failure implicates a specific manifest - * entry; deploy-wide failures (manifest.invalid) omit it. - */ -export const DeployApplyErrorFrame = type({ - type: "'deploy.apply.error'", - agentAddress: "string", - attemptId: "string", - previousDeployId: "string", - category: DeployApplyErrorCategory, - message: "string", - "package?": type({ - name: "string", - version: "string", - }), - occurredAt: "string", -}); -export type DeployApplyErrorFrame = typeof DeployApplyErrorFrame.infer; - /** * Hub requests the sidecar to push its current agent state. The sidecar * responds by sending pack.push frames followed by pack.done using the @@ -716,24 +646,19 @@ export const SidecarFrame = RegisterFrame.or(ReconnectFrame) .or(PingFrame) .or(SessionAckFrame) .or(SessionErrorFrame) - .or(SessionStartAckFrame) .or(AgentUndeployAckFrame) .or(PackPushFrame) .or(PackDoneFrame) .or(PackAckFrame) - .or(PackRejectFrame) - .or(DeployApplyErrorFrame); + .or(PackRejectFrame); export type SidecarFrame = typeof SidecarFrame.infer; /** All frame types the hub sends to the sidecar. */ export const HubFrame = MailInboundFrame.or(AgentDeployFrame) .or(AgentUndeployFrame) - .or(SessionStartFrame) .or(ChallengeFrame) .or(ChallengeFailedFrame) .or(PongFrame) - .or(SessionAbortFrame) - .or(GrantsUpdateFrame) .or(SourcesUpdateFrame) .or(PackPushFrame) .or(PackDoneFrame) diff --git a/packages/workflow-deploy/README.md b/packages/workflow-deploy/README.md index eef96c90..dc12be8b 100644 --- a/packages/workflow-deploy/README.md +++ b/packages/workflow-deploy/README.md @@ -6,26 +6,18 @@ and the workflow deploy orchestrator. This package is the deploy-side counterpart to `@intx/workflow`. It takes a `WorkflowDefinition`, computes the per-step grant declarations the workflow will require, gates them against an -operator-supplied `ApprovalSet`, and routes the deployment along -one of two paths. - -The orchestrator branches on the trivial-vs-multi-step dichotomy: - -- **Trivial workflow** (single step, caller supplies - `trivialBindings`): preserve the caller's existing agent address - and write the underlying agent's deploy tree onto the existing - `agent-state` repo via the legacy path. The on-disk and on-wire - surfaces stay bit-identical to what the pre-collapse - `SessionService.launchSession` produced. +operator-supplied `ApprovalSet`, and routes the deployment by step +count: + +- **Single-step workflow**: the lone step has no distinct address -- + it IS the deployment head. Deploy once at the head + (`ins_@`) through the single-step + hand-off, staging the head's deploy tree and firing the + `agent.deploy` frame in one call. - **Multi-step workflow**: derive per-step agent addresses as `ins_-@`, instantiate one `agent-state` repo per step, and write per-step deploy trees. -The asymmetry is intentional: it is the agent-deploy uniformity -claim's escape hatch. Without it, the existing -`tests/hub-agent/deploy-flow.test.ts` could not pass with zero -source changes after the collapse. - Public surface: - `createWorkflowDeployOrchestrator(opts)` — the orchestrator. diff --git a/packages/workflow-deploy/src/capability-approval.ts b/packages/workflow-deploy/src/capability-approval.ts index ca408118..64c0d26a 100644 --- a/packages/workflow-deploy/src/capability-approval.ts +++ b/packages/workflow-deploy/src/capability-approval.ts @@ -12,11 +12,6 @@ // - A non-empty `unresolvedDirectors` field on the walk result is // itself a deploy-time failure; the gate surfaces it through // `ApprovalDecision` and the orchestrator aborts. -// - The trivial-workflow uniformity bridge: operators populating the -// approval set from the legacy per-agent grant store carry through -// the previously-approved grants so the operator does not see a -// fresh approval prompt for grants that were already approved on the -// legacy agent-deploy path. import type { CapabilityWalkResult } from "./capability-walk"; diff --git a/packages/workflow-deploy/src/capability-walk.test.ts b/packages/workflow-deploy/src/capability-walk.test.ts index 04961593..4bc1dee4 100644 --- a/packages/workflow-deploy/src/capability-walk.test.ts +++ b/packages/workflow-deploy/src/capability-walk.test.ts @@ -41,7 +41,7 @@ function makeTrivialAgent(): AgentDefinition { }); } -function makeTrivialWorkflow( +function makeSingleStepWorkflow( agent: AgentDefinition, ): WorkflowDefinition { return defineWorkflow({ @@ -84,7 +84,7 @@ describe("walkCapabilities", () => { test("trivial workflow grants match the implicit agent-deploy grants", () => { const registry = createDefaultDirectorRegistry(); const agent = makeTrivialAgent(); - const workflow = makeTrivialWorkflow(agent); + const workflow = makeSingleStepWorkflow(agent); const walk = walkCapabilities(workflow, registry); diff --git a/packages/workflow-deploy/src/index.ts b/packages/workflow-deploy/src/index.ts index 0310b3f7..951b31a7 100644 --- a/packages/workflow-deploy/src/index.ts +++ b/packages/workflow-deploy/src/index.ts @@ -8,8 +8,8 @@ // - approval gate: consumes the walk's output plus an operator- // supplied `ApprovalSet` and yields a per-step pending delta. // - orchestrator: validates the workflow, runs the walk + approval -// gate, writes the workflow repo, and branches on the trivial-vs- -// multi-step dichotomy for per-agent launches. +// gate, writes the workflow repo, and branches on the single-step- +// vs-multi-step dichotomy for per-agent launches. export { walkCapabilities, @@ -28,15 +28,18 @@ export { createWorkflowDeployOrchestrator, deriveDeploymentAddress, deriveStepAddress, + resolveStepAddress, deriveStepAgentId, deriveWorkflowRunRepoId, isWorkflowDerivedAddress, - wrapHarnessAsTrivialAgent, + wrapHarnessAsSingleStepWorkflow, CapabilityApprovalDeniedError, MultiStepDeployHandoffMissingError, MultiStepDeploymentArgsMissingError, + SingleStepDeployHandoffMissingError, WorkflowDefinitionInvalidError, type DeployContent, + type DeploySingleStepFn, type DeployWorkflowArgs, type DeployWorkflowResult, type LaunchSessionFn, diff --git a/packages/workflow-deploy/src/orchestrator-fallback-source.test.ts b/packages/workflow-deploy/src/orchestrator-fallback-source.test.ts index aedd9788..b2b540be 100644 --- a/packages/workflow-deploy/src/orchestrator-fallback-source.test.ts +++ b/packages/workflow-deploy/src/orchestrator-fallback-source.test.ts @@ -31,6 +31,7 @@ import { createWorkflowDeployOrchestrator, WorkflowDefinitionInvalidError, type DeployContent, + type DeploySingleStepFn, type LaunchSessionFn, type SendMultiStepDeployFn, type WorkflowRepoWriter, @@ -106,13 +107,29 @@ function createRecordingDeps() { sources.push(params.sources); return { publicKey: "00".repeat(32) }; }; + // A one-step workflow deploys once at the head via this hand-off; it + // records the pinned sources exactly as the multi-step hand-off does so + // the source-pin assertions read the same `sources` array regardless of + // which branch the deploy took. + const singleStep: DeploySingleStepFn = async (params) => { + sources.push(params.sources); + return { publicKey: "00".repeat(32) }; + }; const repoWrites: { workflowRepoId: string }[] = []; const workflowRepo: WorkflowRepoWriter = { async writeWorkflowRepo(params) { repoWrites.push({ workflowRepoId: params.workflowRepoId }); }, }; - return { launches, launch, sources, multiStep, workflowRepo, repoWrites }; + return { + launches, + launch, + sources, + multiStep, + singleStep, + workflowRepo, + repoWrites, + }; } describe("pickStepInferenceSource (agent step)", () => { @@ -126,6 +143,7 @@ describe("pickStepInferenceSource (agent step)", () => { workflowRepo: deps.workflowRepo, launchSession: deps.launch, sendMultiStepDeploy: deps.multiStep, + deploySingleStepAtHead: deps.singleStep, }); // HarnessConfig has only a default-pinned source for openai:default-model, @@ -180,6 +198,7 @@ describe("pickStepInferenceSource (agent step)", () => { workflowRepo: deps.workflowRepo, launchSession: deps.launch, sendMultiStepDeploy: deps.multiStep, + deploySingleStepAtHead: deps.singleStep, }); const config = makeConfig({ @@ -217,17 +236,20 @@ describe("pickStepInferenceSource (agent step)", () => { operatorApprovals: approvals, }); - expect(result.kind).toBe("multi-step"); + expect(result.publicKey).toMatch(/^[0-9a-f]{64}$/); expect(deps.sources).toHaveLength(1); const sources = deps.sources[0]; if (sources === undefined) throw new Error("missing sources"); - expect(sources.only).toEqual({ - id: "src-default", - provider: "openai", - baseURL: "https://api.example/openai", - apiKey: "secret", - model: "default-model", - }); + // The step pins a single-element failover chain around the picked source. + expect(sources.only).toEqual([ + { + id: "src-default", + provider: "openai", + baseURL: "https://api.example/openai", + apiKey: "secret", + model: "default-model", + }, + ]); }); test("uses the agent's preferred source when it matches an approved HarnessConfig source", async () => { @@ -240,6 +262,7 @@ describe("pickStepInferenceSource (agent step)", () => { workflowRepo: deps.workflowRepo, launchSession: deps.launch, sendMultiStepDeploy: deps.multiStep, + deploySingleStepAtHead: deps.singleStep, }); const config = makeConfig({ @@ -281,17 +304,20 @@ describe("pickStepInferenceSource (agent step)", () => { operatorApprovals: approvals, }); - expect(result.kind).toBe("multi-step"); + expect(result.publicKey).toMatch(/^[0-9a-f]{64}$/); expect(deps.sources).toHaveLength(1); const sources = deps.sources[0]; if (sources === undefined) throw new Error("missing sources"); - expect(sources.only).toEqual({ - id: "src-preferred", - provider: "anthropic", - baseURL: "https://api.example/anthropic", - apiKey: "secret-a", - model: "preferred-model", - }); + // The step pins a single-element failover chain around the picked source. + expect(sources.only).toEqual([ + { + id: "src-preferred", + provider: "anthropic", + baseURL: "https://api.example/anthropic", + apiKey: "secret-a", + model: "preferred-model", + }, + ]); }); test("regression: unapproved fallback is also rejected at the approval gate when the walk surfaces it", async () => { @@ -307,6 +333,7 @@ describe("pickStepInferenceSource (agent step)", () => { workflowRepo: deps.workflowRepo, launchSession: deps.launch, sendMultiStepDeploy: deps.multiStep, + deploySingleStepAtHead: deps.singleStep, }); const config = makeConfig({ sources: [ diff --git a/packages/workflow-deploy/src/orchestrator-nonagent-source.test.ts b/packages/workflow-deploy/src/orchestrator-nonagent-source.test.ts index 607c5d6b..5cd584c7 100644 --- a/packages/workflow-deploy/src/orchestrator-nonagent-source.test.ts +++ b/packages/workflow-deploy/src/orchestrator-nonagent-source.test.ts @@ -35,6 +35,7 @@ import { createWorkflowDeployOrchestrator, WorkflowDefinitionInvalidError, type DeployContent, + type DeploySingleStepFn, type LaunchSessionFn, type SendMultiStepDeployFn, type WorkflowRepoWriter, @@ -113,13 +114,29 @@ function createRecordingDeps() { sources.push(params.sources); return { publicKey: "00".repeat(32) }; }; + // A one-step workflow deploys once at the head via this hand-off; it + // records the pinned sources exactly as the multi-step hand-off does so + // the source-pin assertions read the same `sources` array regardless of + // which branch the deploy took. + const singleStep: DeploySingleStepFn = async (params) => { + sources.push(params.sources); + return { publicKey: "00".repeat(32) }; + }; const repoWrites: { workflowRepoId: string }[] = []; const workflowRepo: WorkflowRepoWriter = { async writeWorkflowRepo(params) { repoWrites.push({ workflowRepoId: params.workflowRepoId }); }, }; - return { launches, launch, sources, multiStep, workflowRepo, repoWrites }; + return { + launches, + launch, + sources, + multiStep, + singleStep, + workflowRepo, + repoWrites, + }; } describe("pickStepInferenceSource (non-agent step)", () => { @@ -133,6 +150,7 @@ describe("pickStepInferenceSource (non-agent step)", () => { workflowRepo: deps.workflowRepo, launchSession: deps.launch, sendMultiStepDeploy: deps.multiStep, + deploySingleStepAtHead: deps.singleStep, }); // HarnessConfig carries TWO sources: the agent's preferred and a @@ -209,6 +227,7 @@ describe("pickStepInferenceSource (non-agent step)", () => { workflowRepo: deps.workflowRepo, launchSession: deps.launch, sendMultiStepDeploy: deps.multiStep, + deploySingleStepAtHead: deps.singleStep, }); const config = makeConfig({ @@ -256,6 +275,7 @@ describe("pickStepInferenceSource (non-agent step)", () => { workflowRepo: deps.workflowRepo, launchSession: deps.launch, sendMultiStepDeploy: deps.multiStep, + deploySingleStepAtHead: deps.singleStep, }); const config = makeConfig({ @@ -288,18 +308,21 @@ describe("pickStepInferenceSource (non-agent step)", () => { operatorApprovals: approvals, }); - expect(result.kind).toBe("multi-step"); + expect(result.publicKey).toMatch(/^[0-9a-f]{64}$/); expect(deps.sources).toHaveLength(1); const sources = deps.sources[0]; if (sources === undefined) throw new Error("missing sources"); - // Both steps land on the approved source. + // Both steps land on the approved source, each pinned as a single-element + // failover chain. expect(sources.worker).toBeDefined(); - expect(sources.cooldown).toEqual({ - id: "src-anthropic", - provider: "anthropic", - baseURL: "https://api.example/anthropic", - apiKey: "secret-a", - model: "worker-model", - }); + expect(sources.cooldown).toEqual([ + { + id: "src-anthropic", + provider: "anthropic", + baseURL: "https://api.example/anthropic", + apiKey: "secret-a", + model: "worker-model", + }, + ]); }); }); diff --git a/packages/workflow-deploy/src/orchestrator.test.ts b/packages/workflow-deploy/src/orchestrator.test.ts index fe64d5f4..e98a47a9 100644 --- a/packages/workflow-deploy/src/orchestrator.test.ts +++ b/packages/workflow-deploy/src/orchestrator.test.ts @@ -28,9 +28,11 @@ import { isWorkflowDerivedAddress, MultiStepDeployHandoffMissingError, MultiStepDeploymentArgsMissingError, + SingleStepDeployHandoffMissingError, WorkflowDefinitionInvalidError, - wrapHarnessAsTrivialAgent, + wrapHarnessAsSingleStepWorkflow, type DeployContent, + type DeploySingleStepFn, type LaunchSessionFn, type SendMultiStepDeployFn, type WorkflowRepoWriter, @@ -63,7 +65,7 @@ function makeAgent( }); } -function makeTrivialWorkflow( +function makeSingleStepWorkflow( agent: AgentDefinition, ): WorkflowDefinition { return defineWorkflow({ @@ -204,11 +206,29 @@ function createRecordingMultiStepDeploy(publicKey = "ff".repeat(32)): { return { fn, calls }; } +type RecordedSingleStepDeploy = Parameters[0]; + +function createRecordingSingleStepDeploy(publicKey = "ff".repeat(32)): { + fn: DeploySingleStepFn; + calls: RecordedSingleStepDeploy[]; +} { + const calls: RecordedSingleStepDeploy[] = []; + const fn: DeploySingleStepFn = async (params) => { + calls.push(params); + return { publicKey }; + }; + return { fn, calls }; +} + describe("createWorkflowDeployOrchestrator", () => { - describe("trivial branch", () => { - test("preserves the existing agent address and launches once", async () => { - const agent = makeAgent("legacy-agent"); - const workflow = makeTrivialWorkflow(agent); + describe("deploy provisioning", () => { + test("passes through toolPackagePins to every step launch", async () => { + const workflow = makeMultiStepWorkflow(); + const planAgent = workflow.steps.plan; + const executeAgent = workflow.steps.execute; + if (planAgent?.kind !== "step" || executeAgent?.kind !== "step") { + throw new Error("expected both steps to be step primitives"); + } const directorRegistry = createDefaultDirectorRegistry(); const workflowRepo = createRecordingWorkflowRepoWriter(); const launch = createRecordingLaunch(); @@ -219,72 +239,37 @@ describe("createWorkflowDeployOrchestrator", () => { launchSession: launch.fn, sendMultiStepDeploy: multiStep.fn, }); - - const approvals = approvedGrantsForWorkflow(workflow, [agent]); - - const result = await orchestrator.deployWorkflow({ - workflow, - trivialBindings: { - agentAddress: "ins_legacy-agent@integration.interchange", - agentId: "legacy-agent", - instanceId: "instance-legacy", - }, - config: HARNESS_CONFIG_BASE, - deployContent: DEPLOY_CONTENT_BASE, - operatorApprovals: approvals, - }); - - expect(launch.launches).toHaveLength(1); - const launched = launch.launches[0]; - if (launched === undefined) throw new Error("missing launch"); - expect(launched.agentAddress).toBe( - "ins_legacy-agent@integration.interchange", - ); - expect(launched.agentId).toBe("legacy-agent"); - expect(launched.instanceId).toBe("instance-legacy"); - expect(launched.config).toEqual(HARNESS_CONFIG_BASE); - expect(launched.deployContent).toEqual(DEPLOY_CONTENT_BASE); - // Regression: the trivial branch does NOT invoke the multi-step - // deploy hand-off. - expect(multiStep.calls).toHaveLength(0); - expect(result).toEqual({ kind: "trivial" }); - }); - - test("passes through toolPackagePins to the launch", async () => { - const agent = makeAgent("legacy-agent"); - const workflow = makeTrivialWorkflow(agent); - const directorRegistry = createDefaultDirectorRegistry(); - const workflowRepo = createRecordingWorkflowRepoWriter(); - const launch = createRecordingLaunch(); - const orchestrator = createWorkflowDeployOrchestrator({ - directorRegistry, - workflowRepo, - launchSession: launch.fn, - }); - const approvals = approvedGrantsForWorkflow(workflow, [agent]); + const approvals = approvedGrantsForWorkflow(workflow, [ + planAgent.agent, + executeAgent.agent, + ]); const pins = [{ name: "@vendor/pkg", version: "1.0.0" }] as const; await orchestrator.deployWorkflow({ workflow, - trivialBindings: { - agentAddress: "ins_legacy-agent@integration.interchange", - agentId: "legacy-agent", - instanceId: "instance-legacy", - }, + deploymentId: "dep_pins", + deploymentDomain: "workflow.interchange", config: HARNESS_CONFIG_BASE, deployContent: DEPLOY_CONTENT_BASE, + hubPublicKey: "00".repeat(32), toolPackagePins: pins, operatorApprovals: approvals, }); - expect(launch.launches[0]?.toolPackagePins).toEqual(pins); + expect(launch.launches).toHaveLength(2); + for (const launched of launch.launches) { + expect(launched.toolPackagePins).toEqual(pins); + } }); - test("writes the workflow repo before launching", async () => { - const agent = makeAgent("legacy-agent"); - const workflow = makeTrivialWorkflow(agent); + test("writes the workflow repo before launching any step", async () => { + const workflow = makeMultiStepWorkflow(); + const planAgent = workflow.steps.plan; + const executeAgent = workflow.steps.execute; + if (planAgent?.kind !== "step" || executeAgent?.kind !== "step") { + throw new Error("expected both steps to be step primitives"); + } const directorRegistry = createDefaultDirectorRegistry(); - const workflowRepo = createRecordingWorkflowRepoWriter(); const order: string[] = []; const launch: LaunchSessionFn = async () => { order.push("launch"); @@ -292,39 +277,35 @@ describe("createWorkflowDeployOrchestrator", () => { const recordingRepo: WorkflowRepoWriter = { async writeWorkflowRepo(args) { order.push("repo"); - workflowRepo.writes.push({ - workflowRepoId: args.workflowRepoId, - files: new Map(args.files), - }); + expect(args.files.has("workflow.json")).toBe(true); + expect(args.files.has("capability-declarations.json")).toBe(true); + expect(args.files.has(".gitignore")).toBe(true); }, }; + const multiStep = createRecordingMultiStepDeploy(); const orchestrator = createWorkflowDeployOrchestrator({ directorRegistry, workflowRepo: recordingRepo, launchSession: launch, + sendMultiStepDeploy: multiStep.fn, }); - const approvals = approvedGrantsForWorkflow(workflow, [agent]); + const approvals = approvedGrantsForWorkflow(workflow, [ + planAgent.agent, + executeAgent.agent, + ]); await orchestrator.deployWorkflow({ workflow, - trivialBindings: { - agentAddress: "ins_legacy-agent@integration.interchange", - agentId: "legacy-agent", - instanceId: "instance-legacy", - }, + deploymentId: "dep_order", + deploymentDomain: "workflow.interchange", config: HARNESS_CONFIG_BASE, deployContent: DEPLOY_CONTENT_BASE, + hubPublicKey: "00".repeat(32), operatorApprovals: approvals, }); - expect(order).toEqual(["repo", "launch"]); - expect(workflowRepo.writes).toHaveLength(1); - const write = workflowRepo.writes[0]; - if (write === undefined) throw new Error("missing write"); - expect(write.workflowRepoId).toBe("wf_trivial"); - expect(write.files.has("workflow.json")).toBe(true); - expect(write.files.has("capability-declarations.json")).toBe(true); - expect(write.files.has(".gitignore")).toBe(true); + // The workflow repo lands before any per-step agent-state write. + expect(order).toEqual(["repo", "launch", "launch"]); }); }); @@ -394,10 +375,11 @@ describe("createWorkflowDeployOrchestrator", () => { expect(Object.keys(handoff.sources).sort()).toEqual(["execute", "plan"]); const baseSource = HARNESS_CONFIG_BASE.sources[0]; if (baseSource === undefined) throw new Error("missing base source"); - expect(handoff.sources.plan).toEqual(baseSource); - expect(handoff.sources.execute).toEqual(baseSource); + // Each step pins a single-element failover chain: the multi-step branch + // wraps its one picked source in a list. + expect(handoff.sources.plan).toEqual([baseSource]); + expect(handoff.sources.execute).toEqual([baseSource]); expect(result).toEqual({ - kind: "multi-step", publicKey: "ff".repeat(32), }); }); @@ -516,6 +498,32 @@ describe("createWorkflowDeployOrchestrator", () => { ).rejects.toBeInstanceOf(MultiStepDeployHandoffMissingError); }); + test("throws when deploySingleStepAtHead dep is unwired", async () => { + const agent = makeAgent("only"); + const workflow = makeSingleStepWorkflow(agent); + const directorRegistry = createDefaultDirectorRegistry(); + const workflowRepo = createRecordingWorkflowRepoWriter(); + const launch = createRecordingLaunch(); + const orchestrator = createWorkflowDeployOrchestrator({ + directorRegistry, + workflowRepo, + launchSession: launch.fn, + }); + const approvals = approvedGrantsForWorkflow(workflow, [agent]); + + await expect( + orchestrator.deployWorkflow({ + workflow, + deploymentId: "dep_xyz", + deploymentDomain: "workflow.interchange", + config: HARNESS_CONFIG_BASE, + deployContent: DEPLOY_CONTENT_BASE, + hubPublicKey: "00".repeat(32), + operatorApprovals: approvals, + }), + ).rejects.toBeInstanceOf(SingleStepDeployHandoffMissingError); + }); + test("throws when deploymentId is missing", async () => { const workflow = makeMultiStepWorkflow(); const planAgent = workflow.steps.plan; @@ -578,22 +586,24 @@ describe("createWorkflowDeployOrchestrator", () => { ).rejects.toBeInstanceOf(MultiStepDeploymentArgsMissingError); }); - test("single-step workflow without trivialBindings takes the derived path", async () => { + test("single-step workflow deploys once at the head", async () => { const agent = makeAgent("only"); - const workflow = makeTrivialWorkflow(agent); + const workflow = makeSingleStepWorkflow(agent); const directorRegistry = createDefaultDirectorRegistry(); const workflowRepo = createRecordingWorkflowRepoWriter(); const launch = createRecordingLaunch(); const multiStep = createRecordingMultiStepDeploy(); + const singleStep = createRecordingSingleStepDeploy(); const orchestrator = createWorkflowDeployOrchestrator({ directorRegistry, workflowRepo, launchSession: launch.fn, sendMultiStepDeploy: multiStep.fn, + deploySingleStepAtHead: singleStep.fn, }); const approvals = approvedGrantsForWorkflow(workflow, [agent]); - await orchestrator.deployWorkflow({ + const result = await orchestrator.deployWorkflow({ workflow, deploymentId: "dep_xyz", deploymentDomain: "workflow.interchange", @@ -603,17 +613,23 @@ describe("createWorkflowDeployOrchestrator", () => { operatorApprovals: approvals, }); - expect(launch.launches).toHaveLength(1); - const launched = launch.launches[0]; - if (launched === undefined) throw new Error("missing launch"); + // A one-step workflow deploys ONCE at the head: the lone step + // collapses onto the deployment (head) address, so there is no + // per-step `launchSession` and no separate multi-step frame. + expect(launch.launches).toHaveLength(0); + expect(multiStep.calls).toHaveLength(0); + expect(singleStep.calls).toHaveLength(1); + const call = singleStep.calls[0]; + if (call === undefined) throw new Error("missing single-step deploy"); + expect(call.agentAddress).toBe("ins_dep_xyz@workflow.interchange"); + expect(call.agentId).toBe("ins_dep_xyz"); + expect(call.instanceId).toBe("ins_dep_xyz"); const expectedStepId = workflow.stepOrder[0]; if (expectedStepId === undefined) { throw new Error("missing step id"); } - expect(launched.agentAddress).toBe( - `ins_dep_xyz-${expectedStepId}@workflow.interchange`, - ); - expect(multiStep.calls).toHaveLength(1); + expect(call.sources[expectedStepId]).toBeDefined(); + expect(result.publicKey).toMatch(/^[0-9a-f]{64}$/); }); test("source-pin failure carries workflow.id and names the offending provider+model", async () => { @@ -683,7 +699,7 @@ describe("createWorkflowDeployOrchestrator", () => { describe("approval failures", () => { test("unapproved grant throws CapabilityApprovalDeniedError naming the step", async () => { const agent = makeAgent("legacy-agent"); - const workflow = makeTrivialWorkflow(agent); + const workflow = makeSingleStepWorkflow(agent); const directorRegistry = createDefaultDirectorRegistry(); const workflowRepo = createRecordingWorkflowRepoWriter(); const launch = createRecordingLaunch(); @@ -700,11 +716,8 @@ describe("createWorkflowDeployOrchestrator", () => { try { await orchestrator.deployWorkflow({ workflow, - trivialBindings: { - agentAddress: "ins_legacy-agent@integration.interchange", - agentId: "legacy-agent", - instanceId: "instance-legacy", - }, + deploymentId: "dep_legacy", + deploymentDomain: "workflow.interchange", config: HARNESS_CONFIG_BASE, deployContent: DEPLOY_CONTENT_BASE, operatorApprovals: incompleteApprovals, @@ -726,7 +739,7 @@ describe("createWorkflowDeployOrchestrator", () => { test("zero approved sources throws with the offending step and missing source", async () => { const agent = makeAgent("legacy-agent"); - const workflow = makeTrivialWorkflow(agent); + const workflow = makeSingleStepWorkflow(agent); const directorRegistry = createDefaultDirectorRegistry(); const workflowRepo = createRecordingWorkflowRepoWriter(); const launch = createRecordingLaunch(); @@ -740,11 +753,8 @@ describe("createWorkflowDeployOrchestrator", () => { try { await orchestrator.deployWorkflow({ workflow, - trivialBindings: { - agentAddress: "ins_legacy-agent@integration.interchange", - agentId: "legacy-agent", - instanceId: "instance-legacy", - }, + deploymentId: "dep_legacy", + deploymentDomain: "workflow.interchange", config: HARNESS_CONFIG_BASE, deployContent: DEPLOY_CONTENT_BASE, operatorApprovals: new Set(), @@ -799,11 +809,8 @@ describe("createWorkflowDeployOrchestrator", () => { try { await orchestrator.deployWorkflow({ workflow, - trivialBindings: { - agentAddress: "ins_legacy-agent@integration.interchange", - agentId: "legacy-agent", - instanceId: "instance-legacy", - }, + deploymentId: "dep_legacy", + deploymentDomain: "workflow.interchange", config: HARNESS_CONFIG_BASE, deployContent: DEPLOY_CONTENT_BASE, operatorApprovals: broad, @@ -850,9 +857,9 @@ describe("createWorkflowDeployOrchestrator", () => { }); }); -describe("wrapHarnessAsTrivialAgent", () => { +describe("wrapHarnessAsSingleStepWorkflow", () => { test("derives id from config.agentId", () => { - const agent = wrapHarnessAsTrivialAgent({ + const agent = wrapHarnessAsSingleStepWorkflow({ config: HARNESS_CONFIG_BASE, deployContent: DEPLOY_CONTENT_BASE, }); @@ -863,7 +870,7 @@ describe("wrapHarnessAsTrivialAgent", () => { const customContent: DeployContent = { systemPrompt: "you are the trivial agent", }; - const agent = wrapHarnessAsTrivialAgent({ + const agent = wrapHarnessAsSingleStepWorkflow({ config: HARNESS_CONFIG_BASE, deployContent: customContent, }); @@ -890,7 +897,7 @@ describe("wrapHarnessAsTrivialAgent", () => { }, ], }; - const agent = wrapHarnessAsTrivialAgent({ + const agent = wrapHarnessAsSingleStepWorkflow({ config, deployContent: DEPLOY_CONTENT_BASE, }); @@ -901,7 +908,7 @@ describe("wrapHarnessAsTrivialAgent", () => { }); test("empty toolFactories and capabilities (deploy tree is the source of truth)", () => { - const agent = wrapHarnessAsTrivialAgent({ + const agent = wrapHarnessAsSingleStepWorkflow({ config: HARNESS_CONFIG_BASE, deployContent: DEPLOY_CONTENT_BASE, }); @@ -910,7 +917,7 @@ describe("wrapHarnessAsTrivialAgent", () => { }); test("no director ref (caller carries no director state in the trivial shape)", () => { - const agent = wrapHarnessAsTrivialAgent({ + const agent = wrapHarnessAsSingleStepWorkflow({ config: HARNESS_CONFIG_BASE, deployContent: DEPLOY_CONTENT_BASE, }); diff --git a/packages/workflow-deploy/src/orchestrator.ts b/packages/workflow-deploy/src/orchestrator.ts index 7afac64e..8aa35a36 100644 --- a/packages/workflow-deploy/src/orchestrator.ts +++ b/packages/workflow-deploy/src/orchestrator.ts @@ -1,38 +1,26 @@ // Workflow-deploy orchestrator. // -// ========================================================================= -// THE LOAD-BEARING DICHOTOMY -// ========================================================================= +// A deploy validates the workflow, runs the capability walk, and gates on +// operator approval, then routes by step count. // -// Trivial-workflow path preserves the legacy address shape; multi-step -// uses the derived shape; this asymmetry is the agent-deploy uniformity -// claim's escape hatch. +// A one-step workflow has no distinct step address: the lone step IS the +// deployment head. It deploys once at the head (`deriveDeploymentAddress`) +// through the single-step hand-off -- the tree staging and the +// `agent.deploy` frame collapse onto one head deploy, with no per-step +// provisioning loop. // -// A workflow with exactly one step that arrives with `trivialBindings` -// reuses the deployment's existing `` -// triple unchanged. The deploy tree lands on the same per-agent -// `agent-state` repo the legacy agent-deploy path writes to, and the -// per-step content (system prompt, tool-package pins, asset attachments) -// flows through `launchSession` with bit-identical wire and on-disk -// shape. The legacy `tests/hub-agent/deploy-flow.test.ts` is the -// invariant the trivial branch must round-trip. -// -// A workflow with more than one step (or one without `trivialBindings`) -// derives per-step agent addresses of the form -// `ins_-@`, instantiates one -// agent-state repo per step keyed by the derived address, and writes +// A workflow with more than one step derives per-step agent addresses of +// the form `ins_-@`, instantiates +// one agent-state repo per step keyed by the derived address, and writes // each step's deploy tree onto its own repo. The derivation is a pure // function of `(deploymentId, stepId, deploymentDomain)`, so the // supervisor reconstructs the same addresses at spawn time without any // per-deploy state. // -// Both branches first validate the workflow, run the capability walk, -// and gate on operator approval. The workflow definition envelope plus -// the walk's per-step grant declarations land on a `workflow` repo -// before any agent-state write happens; if the workflow repo write -// fails, no agent-state repo is created. -// -// ========================================================================= +// The workflow definition envelope plus the walk's per-step grant +// declarations land on a `workflow` repo before any agent-state write +// happens; if the workflow repo write fails, no agent-state repo is +// created. import type { AgentDefinition, @@ -115,10 +103,39 @@ export type SendMultiStepDeployFn = (params: { agentId: string; config: HarnessConfig; definition: WorkflowDefinition; - sources: Record; + sources: Record; hubPublicKey: string; }) => Promise; +/** + * Single-step deploy hand-off. A one-step workflow has no distinct steps + * (the lone step IS the head), so it does NOT take the per-step + * provisioning loop: it deploys once at the head, staging the head's + * deploy tree AND firing the deployment `agent.deploy` frame that carries + * the workflow definition and the sole step's source pin. The caller-site + * closure produces the deploy pack, sends the workflow frame (the sidecar + * initializes the head repo on receipt), then delivers the pack to the + * head; it waits on the `agent.deploy.ack` and surfaces the supervisor's + * principal public key back through the result. + * + * This carries the head deploy content and tool pins (which the head-tree + * staging needs) alongside the definition + sources (which the frame + * needs) -- the union of what `LaunchSessionFn` and `SendMultiStepDeployFn` + * carry, because for one step the tree staging and the frame collapse onto + * a single head deploy. + */ +export type DeploySingleStepFn = (params: { + agentAddress: string; + agentId: string; + instanceId: string; + config: HarnessConfig; + deployContent: DeployContent; + definition: WorkflowDefinition; + sources: Record; + hubPublicKey: string; + toolPackagePins?: readonly ToolPackagePin[]; +}) => Promise; + /** * Result returned by `sendMultiStepDeploy`. Surfaces the sidecar * supervisor's principal public key (hex-encoded Ed25519) from the @@ -131,14 +148,13 @@ export interface MultiStepDeployResult { } /** - * Result returned by `deployWorkflow`. The trivial branch returns - * `{ kind: "trivial" }`; the multi-step branch surfaces the supervisor - * public key collected from the sidecar's `agent.deploy.ack` so the - * caller can stash it alongside the deployment record. + * Result returned by `deployWorkflow`. Surfaces the supervisor public key + * collected from the sidecar's `agent.deploy.ack` so the caller can stash + * it alongside the deployment record. */ -export type DeployWorkflowResult = - | { readonly kind: "trivial" } - | { readonly kind: "multi-step"; readonly publicKey: string }; +export type DeployWorkflowResult = { + readonly publicKey: string; +}; /** * Minimal interface for writing the workflow repo. The orchestrator @@ -162,61 +178,51 @@ export interface WorkflowDeployOrchestratorDeps { * orchestrator. */ readonly directorRegistry: DirectorRegistry; - /** - * Writes the workflow repo's deploy tree. The trivial-branch and the - * multi-step branch both call this once. - */ + /** Writes the workflow repo's deploy tree. Every deploy calls this once. */ readonly workflowRepo: WorkflowRepoWriter; /** - * Performs the per-agent deploy + session start. The orchestrator - * calls this once per step (once total in the trivial branch). In - * production this is `SessionService.launchSession`; tests pass a - * tracking stub. + * Performs the per-agent deploy + session start. The multi-step branch + * calls this once per step. In production this is + * `SessionService.launchSession`; tests pass a tracking stub. */ readonly launchSession: LaunchSessionFn; /** * Fires the deployment-level `agent.deploy` frame that carries the * workflow definition and per-step source pins to the sidecar. The * multi-step branch calls this exactly once, after every per-step - * `agent-state` repo has been provisioned via `launchSession`. The - * trivial branch never calls it. + * `agent-state` repo has been provisioned via `launchSession`. * - * Optional so existing callers that only exercise the trivial branch - * (e.g. the legacy agent-deploy passthrough) do not have to wire a - * stub. The multi-step branch fails fast with + * Optional so a caller that only exercises the single-step branch does + * not have to wire a stub. The multi-step branch fails fast with * `MultiStepDeployHandoffMissingError` if the dep is absent. */ readonly sendMultiStepDeploy?: SendMultiStepDeployFn; + /** + * Deploys a single-step workflow once at the head: stages the head's + * deploy tree and fires the deployment `agent.deploy` frame in one + * hand-off (see `DeploySingleStepFn`). The single-step branch calls + * this exactly once and never runs the per-step `launchSession` loop. + * + * Optional for the same reason as `sendMultiStepDeploy`; the single-step + * branch fails fast with `SingleStepDeployHandoffMissingError` if the + * dep is absent. + */ + readonly deploySingleStepAtHead?: DeploySingleStepFn; } export interface DeployWorkflowArgs { /** The workflow definition the orchestrator validates and deploys. */ readonly workflow: WorkflowDefinition; /** - * Pre-existing per-agent address binding. Required for the trivial - * branch (workflow with exactly one step that wraps an existing - * agent-deploy). Absent for the multi-step branch -- the orchestrator - * derives per-step addresses from `deploymentId` and the workflow. - */ - readonly trivialBindings?: { - readonly agentAddress: string; - readonly agentId: string; - readonly instanceId: string; - }; - /** - * Stable identifier the multi-step branch concatenates into derived - * agent addresses. Required when `trivialBindings` is absent. + * Stable identifier the branch concatenates into derived agent + * addresses. Required. */ readonly deploymentId?: string; /** - * Mail-domain for the deployment. Required when `trivialBindings` is - * absent. The multi-step branch derives per-step addresses as - * `ins_-@`. - * - * The plan's `deployWorkflow` shape lists `deploymentId` but elides - * `deploymentDomain`; the derivation needs both, so the API surfaces - * the second parameter explicitly. Trivial deployments do not consume - * it because they reuse the existing `agentAddress`. + * Mail-domain for the deployment. Required. The multi-step branch + * derives per-step addresses as + * `ins_-@`; the single-step + * branch deploys the lone step at `ins_@`. */ readonly deploymentDomain?: string; /** @@ -241,11 +247,9 @@ export interface DeployWorkflowArgs { */ readonly operatorApprovals: ApprovalSet; /** - * Hex-encoded hub Ed25519 public key the multi-step branch threads - * onto the `agent.deploy` frame so the sidecar can verify the - * deploy-tree commit signatures. Required when the multi-step branch - * is reached; ignored by the trivial branch (whose `launchSession` - * delegate already carries the hub key through its own wire surface). + * Hex-encoded hub Ed25519 public key threaded onto the `agent.deploy` + * frame so the sidecar can verify the deploy-tree commit signatures. + * Required for both deploy paths (single-step head and multi-step). */ readonly hubPublicKey?: string; } @@ -277,7 +281,7 @@ export class WorkflowDefinitionInvalidError extends Error { export class MultiStepDeploymentArgsMissingError extends Error { constructor(missing: string) { super( - `multi-step deploy requires ${missing}; supply both deploymentId and deploymentDomain (or pass trivialBindings for a single-step workflow)`, + `deploy requires ${missing}; supply both deploymentId and deploymentDomain`, ); this.name = "MultiStepDeploymentArgsMissingError"; } @@ -285,7 +289,7 @@ export class MultiStepDeploymentArgsMissingError extends Error { /** * Error thrown when the multi-step branch is reached but the - * `sendMultiStepDeploy` dependency was not wired. The trivial branch + * `sendMultiStepDeploy` dependency was not wired. The single-step branch * does not consult this dep, so the dep is optional on the deps record; * callers that may take the multi-step branch must wire it. */ @@ -298,6 +302,21 @@ export class MultiStepDeployHandoffMissingError extends Error { } } +/** + * Error thrown when the single-step branch is reached but the + * `deploySingleStepAtHead` dependency was not wired. Parallel to + * `MultiStepDeployHandoffMissingError`; the multi-step branch does not + * consult this dep, so it is optional on the deps record. + */ +export class SingleStepDeployHandoffMissingError extends Error { + constructor() { + super( + "single-step deploy requires deploySingleStepAtHead dep; wire it on the orchestrator's WorkflowDeployOrchestratorDeps record", + ); + this.name = "SingleStepDeployHandoffMissingError"; + } +} + /** * Error thrown when the capability-approval gate rejects the deploy. * Carries the per-step `pending` delta and the unresolvable director @@ -336,13 +355,19 @@ function formatApprovalDeniedMessage( /** * Build a `WorkflowDeployOrchestrator`. The orchestrator owns the - * trivial-vs-multi-step decision; its deps own everything else. + * step-count routing (single-step head vs multi-step derived); its deps + * own everything else. */ export function createWorkflowDeployOrchestrator( deps: WorkflowDeployOrchestratorDeps, ): WorkflowDeployOrchestrator { - const { directorRegistry, workflowRepo, launchSession, sendMultiStepDeploy } = - deps; + const { + directorRegistry, + workflowRepo, + launchSession, + sendMultiStepDeploy, + deploySingleStepAtHead, + } = deps; return { async deployWorkflow( @@ -365,19 +390,17 @@ export function createWorkflowDeployOrchestrator( workflowRepo, }); - const trivial = isTrivialDeploy(args); - if (trivial !== null) { - await launchSession({ - agentAddress: trivial.bindings.agentAddress, - agentId: trivial.bindings.agentId, - instanceId: trivial.bindings.instanceId, - config: args.config, - deployContent: args.deployContent, - ...(args.toolPackagePins !== undefined - ? { toolPackagePins: args.toolPackagePins } - : {}), + // A one-step workflow has no distinct steps: the lone step IS the + // head. It deploys once at the head (no per-step provisioning loop), + // so it routes through the dedicated single-step hand-off rather + // than `runMultiStepBranch`. The multi-step branch is reached only + // for `stepOrder.length >= 2`. + if (args.workflow.stepOrder.length === 1) { + const result = await runSingleStepAtHead({ + args, + deploySingleStepAtHead, }); - return { kind: "trivial" }; + return { publicKey: result.publicKey }; } const result = await runMultiStepBranch({ @@ -385,26 +408,104 @@ export function createWorkflowDeployOrchestrator( launchSession, sendMultiStepDeploy, }); - return { kind: "multi-step", publicKey: result.publicKey }; + return { publicKey: result.publicKey }; }, }; } /** - * Decide whether the deploy takes the trivial branch. A trivial deploy - * requires a single-step workflow AND a `trivialBindings` triple from - * the caller. Either condition alone is not enough: multi-step - * workflows never reuse a single legacy address, and a single-step - * workflow without `trivialBindings` is the multi-step branch's - * degenerate case (one derived address). The asymmetry is the - * load-bearing escape hatch documented in this module's header. + * Deploy a one-step workflow once at the head. The lone step has no + * distinct per-step address -- it IS the head (`deriveDeploymentAddress`) + * -- so this pins the sole step's inference source, builds the head + * config + deploy content, and hands the whole thing to + * `deploySingleStepAtHead` in a single call. There is no per-step + * `launchSession` loop and no separate deployment frame: the tree staging + * and the `agent.deploy` frame collapse onto one head deploy. The result + * surfaces the sidecar supervisor's principal public key, same as the + * multi-step branch. */ -function isTrivialDeploy( - args: DeployWorkflowArgs, -): { bindings: NonNullable } | null { - if (args.workflow.stepOrder.length !== 1) return null; - if (args.trivialBindings === undefined) return null; - return { bindings: args.trivialBindings }; +async function runSingleStepAtHead(args: { + args: DeployWorkflowArgs; + deploySingleStepAtHead: DeploySingleStepFn | undefined; +}): Promise { + const { args: deploy, deploySingleStepAtHead } = args; + const deploymentId = deploy.deploymentId; + const deploymentDomain = deploy.deploymentDomain; + if (deploymentId === undefined) { + throw new MultiStepDeploymentArgsMissingError("deploymentId"); + } + if (deploymentDomain === undefined) { + throw new MultiStepDeploymentArgsMissingError("deploymentDomain"); + } + if (deploySingleStepAtHead === undefined) { + throw new SingleStepDeployHandoffMissingError(); + } + if (deploy.hubPublicKey === undefined) { + throw new MultiStepDeploymentArgsMissingError("hubPublicKey"); + } + + // The sole step. `validateWorkflowDefinition` already guaranteed + // `stepOrder` is non-empty and every entry has a matching `steps` + // primitive; the index access is re-narrowed here for the compiler. + const stepId = deploy.workflow.stepOrder[0]; + if (stepId === undefined) { + throw new WorkflowDefinitionInvalidError( + deploy.workflow.id, + "single-step deploy requires a non-empty stepOrder", + ); + } + const primitive = deploy.workflow.steps[stepId]; + if (primitive === undefined) { + throw new WorkflowDefinitionInvalidError( + deploy.workflow.id, + `step ${stepId} listed in stepOrder is missing from steps`, + ); + } + const stepAgent = extractAgent(primitive); + const source = pickStepInferenceSource({ + stepAgent, + stepId, + workflowId: deploy.workflow.id, + config: deploy.config, + operatorApprovals: deploy.operatorApprovals, + }); + + // The lone step IS the head: one deploy at the deployment address, no + // per-step derivation. The head's agentId and instanceId are the same + // `ins_` identity. + const headAddress = deriveDeploymentAddress({ + deploymentId, + deploymentDomain, + }); + const headId = deriveDeploymentAgentId({ deploymentId }); + const headConfig: HarnessConfig = { + ...deploy.config, + agentAddress: headAddress, + agentId: headId, + ...(stepAgent !== null ? { systemPrompt: stepAgent.systemPrompt } : {}), + }; + const headDeployContent: DeployContent = + stepAgent !== null + ? { ...deploy.deployContent, systemPrompt: stepAgent.systemPrompt } + : deploy.deployContent; + + return deploySingleStepAtHead({ + agentAddress: headAddress, + agentId: headId, + instanceId: headId, + config: headConfig, + deployContent: headDeployContent, + definition: deploy.workflow, + // A workflow step pins a single source (no per-step failover): wrap the + // one operator-approved source in a one-element list. The per-step + // failover chain is intentionally an instance-only concern; a workflow + // step preserves its prior single-source behavior. + sources: { [stepId]: [source] }, + hubPublicKey: deploy.hubPublicKey, + ...(deploy.toolPackagePins !== undefined + ? { toolPackagePins: deploy.toolPackagePins } + : {}), + }); } async function runMultiStepBranch(args: { @@ -442,7 +543,7 @@ async function runMultiStepBranch(args: { config: HarnessConfig; deployContent: DeployContent; }; - const sources: Record = {}; + const sources: Record = {}; const prepared: PreparedStep[] = []; for (const stepId of deploy.workflow.stepOrder) { const primitive = deploy.workflow.steps[stepId]; @@ -453,13 +554,18 @@ async function runMultiStepBranch(args: { ); } const stepAgent = extractAgent(primitive); - sources[stepId] = pickStepInferenceSource({ - stepAgent, - stepId, - workflowId: deploy.workflow.id, - config: deploy.config, - operatorApprovals: deploy.operatorApprovals, - }); + // A workflow step pins a single source (no per-step failover), wrapped in + // a one-element list. Per-step failover chains are an instance-only + // concern; this preserves prior workflow-step behavior. + sources[stepId] = [ + pickStepInferenceSource({ + stepAgent, + stepId, + workflowId: deploy.workflow.id, + config: deploy.config, + operatorApprovals: deploy.operatorApprovals, + }), + ]; const agentAddress = deriveStepAddress({ deploymentId, stepId, @@ -628,6 +734,37 @@ export function deriveDeploymentAddress(args: { return `ins_${args.deploymentId}@${args.deploymentDomain}`; } +/** + * Resolve where a step's deploy tree lives, given the deployment's step + * count. This is the single owner of the head/step collapse DECISION for + * a consumer that must choose the address without knowing the deploy + * shape: a one-step workflow has no distinct steps, so its lone step IS + * the head (`deriveDeploymentAddress`); a multi-step deployment keeps the + * head distinct from its per-step addresses (`deriveStepAddress`). The + * sidecar child reads its deploy tree from the address this returns, + * keyed only off the deployment mailbox and the host-sourced `stepCount`. + * + * The producers do not route through here -- each handles one shape + * unconditionally: the single-step deploy stages the tree at the head, + * the multi-step deploy at each per-step address. Because `stepCount` is + * the deployed definition's `stepOrder.length`, sourced from the host, + * the consumer's collapse always agrees with whichever producer staged + * the tree; the two processes never derive divergent addresses. + */ +export function resolveStepAddress(args: { + deploymentId: string; + stepId: string; + deploymentDomain: string; + stepCount: number; +}): string { + return args.stepCount === 1 + ? deriveDeploymentAddress({ + deploymentId: args.deploymentId, + deploymentDomain: args.deploymentDomain, + }) + : deriveStepAddress(args); +} + /** * Derive the deployment-level agent id used on the `agent.deploy` * frame's `agentId` field. Pure function of `(deploymentId)`. @@ -768,30 +905,23 @@ function serializeWalk(walk: CapabilityWalkResult): unknown { } /** - * Build a trivial `AgentDefinition` from a `HarnessConfig` and a - * `DeployContent`. The orchestrator's trivial branch hands the resulting - * shape off to consumers that want to inspect an agent definition for a - * legacy agent-deploy that never went through the workflow surface. - * - * The wrap is the load-bearing transformation behind the trivial - * round-trip claim: the legacy deploy-flow exposes `HarnessConfig` plus - * `DeployContent`, and `SessionService.launchSession` collapses onto - * `deployWorkflow` by synthesizing a single-step workflow from those - * two values via this function. The deploy tree itself - * (`deployContent.systemPrompt`, the harness's `tools` and `grants` - * arrays) is the source of truth for runtime behaviour; the wrap - * synthesizes only the surfaces the capability walk needs to gate the - * deploy against the operator-approval set. + * Build an `AgentDefinition` from a `HarnessConfig` and a + * `DeployContent`. `SessionService.deployInstanceAtHead` uses it to wrap + * a single-agent instance's harness as a one-step workflow and deploy it + * at the head. The deploy tree itself (`deployContent.systemPrompt`, the + * harness's `tools` and `grants` arrays) is the source of truth for + * runtime behaviour; the wrap synthesizes only the surfaces the + * capability walk needs to gate the deploy against the operator-approval + * set. * * The walk inspects `agent.toolFactories[i].id` to emit `tool:` * grants. The wrap projects each `HarnessConfig.tools[i].name` onto a * synthesized `AnnotatedToolFactory` whose `id` matches; the factory * function itself is never invoked on the walk path. Skipping this - * projection would let the gate admit every trivial deploy regardless - * of what `HarnessConfig.tools` named, breaking the uniformity claim's - * substance at the approval layer. + * projection would let the gate admit every deploy regardless of what + * `HarnessConfig.tools` named, weakening the approval gate. */ -export function wrapHarnessAsTrivialAgent(args: { +export function wrapHarnessAsSingleStepWorkflow(args: { config: HarnessConfig; deployContent: DeployContent; }): AgentDefinition { @@ -824,8 +954,8 @@ export function wrapHarnessAsTrivialAgent(args: { * `validateNamespacedId` (the constructor `defineTool` runs) is * deliberately skipped: `HarnessConfig.tools[i].name` is the existing * wire shape downstream consumers gate against, and re-validating it - * here would diverge the trivial-wrap's surface from what the harness - * actually loads. The walk and the gate only consult `.id`, so a bare + * here would diverge the single-step wrap's surface from what the + * harness actually loads. The walk and the gate only consult `.id`, so a bare * name still produces a stable grant string. */ function synthesizeWalkToolFactory( @@ -833,7 +963,7 @@ function synthesizeWalkToolFactory( ): AnnotatedToolFactory { const factory = (_env: BaseEnv): never => { throw new Error( - `wrapHarnessAsTrivialAgent synthesized tool factory for ${JSON.stringify(tool.name)} is walk-only; do not instantiate the trivial-wrap agent`, + `wrapHarnessAsSingleStepWorkflow synthesized tool factory for ${JSON.stringify(tool.name)} is walk-only; do not instantiate the single-step wrap agent`, ); }; return Object.assign(factory, { diff --git a/packages/workflow-deploy/src/trivial-wrap-grant-coverage.test.ts b/packages/workflow-deploy/src/single-step-wrap-grant-coverage.test.ts similarity index 75% rename from packages/workflow-deploy/src/trivial-wrap-grant-coverage.test.ts rename to packages/workflow-deploy/src/single-step-wrap-grant-coverage.test.ts index 74e83968..daa3f6ba 100644 --- a/packages/workflow-deploy/src/trivial-wrap-grant-coverage.test.ts +++ b/packages/workflow-deploy/src/single-step-wrap-grant-coverage.test.ts @@ -1,11 +1,11 @@ -// Regression coverage for H-D1: the trivial-wrap path must surface +// Regression coverage for H-D1: the single-step wrap path must surface // the HarnessConfig's tool surface to the capability walk so the // operator-approval gate gates `tool:` grants. Before the fix, -// `wrapHarnessAsTrivialAgent` emitted `toolFactories: []` regardless +// `wrapHarnessAsSingleStepWorkflow` emitted `toolFactories: []` regardless // of `HarnessConfig.tools`, and the walk emitted zero `tool:` grants. -// The gate then trivially admitted every deploy whose only path was -// the trivial branch, defeating the agent-deploy uniformity claim's -// substance at the approval layer. +// The gate then admitted every single-step deploy unconditionally, +// defeating the agent-deploy uniformity claim's substance at the +// approval layer. // // The fix widens the wrap to propagate `HarnessConfig.tools[i].name` // into the synthesized agent's `toolFactories[i].id`, so the existing @@ -22,7 +22,7 @@ import type { HarnessConfig } from "@intx/types/runtime"; import { defineWorkflow } from "@intx/workflow/definition"; import { walkCapabilities } from "./capability-walk"; -import { wrapHarnessAsTrivialAgent } from "./orchestrator"; +import { wrapHarnessAsSingleStepWorkflow } from "./orchestrator"; const HARNESS_CONFIG_WITH_TOOLS: HarnessConfig = { sessionId: "ses-1", @@ -56,20 +56,21 @@ const HARNESS_CONFIG_WITH_TOOLS: HarnessConfig = { defaultSource: "src-1", }; -describe("H-D1: trivial wrap surfaces HarnessConfig.tools to the walk", () => { +describe("H-D1: single-step wrap surfaces HarnessConfig.tools to the walk", () => { test("walk emits a tool: grant per HarnessConfig.tools entry", () => { - const trivialAgent: AgentDefinition = wrapHarnessAsTrivialAgent({ - config: HARNESS_CONFIG_WITH_TOOLS, - deployContent: { systemPrompt: "legacy" }, - }); - expect(trivialAgent.toolFactories.length).toBe(2); - const factoryIds = trivialAgent.toolFactories.map((f) => f.id); + const singleStepAgent: AgentDefinition = + wrapHarnessAsSingleStepWorkflow({ + config: HARNESS_CONFIG_WITH_TOOLS, + deployContent: { systemPrompt: "legacy" }, + }); + expect(singleStepAgent.toolFactories.length).toBe(2); + const factoryIds = singleStepAgent.toolFactories.map((f) => f.id); expect(factoryIds).toContain("sketchy_tool"); expect(factoryIds).toContain("other_tool"); const workflow = defineWorkflow({ - id: "wf_trivial", - agent: trivialAgent, + id: "wf_single_step", + agent: singleStepAgent, trigger: { type: "mail", to: HARNESS_CONFIG_WITH_TOOLS.agentAddress }, }); const walk = walkCapabilities(workflow, createDefaultDirectorRegistry()); @@ -88,15 +89,15 @@ describe("H-D1: trivial wrap surfaces HarnessConfig.tools to the walk", () => { }); test("walk emits no tool: grants when HarnessConfig.tools is empty", () => { - const trivialAgent = wrapHarnessAsTrivialAgent({ + const singleStepAgent = wrapHarnessAsSingleStepWorkflow({ config: { ...HARNESS_CONFIG_WITH_TOOLS, tools: [] }, deployContent: { systemPrompt: "legacy" }, }); - expect(trivialAgent.toolFactories).toEqual([]); + expect(singleStepAgent.toolFactories).toEqual([]); const workflow = defineWorkflow({ - id: "wf_trivial_empty", - agent: trivialAgent, + id: "wf_single_step_empty", + agent: singleStepAgent, trigger: { type: "mail", to: HARNESS_CONFIG_WITH_TOOLS.agentAddress }, }); const walk = walkCapabilities(workflow, createDefaultDirectorRegistry()); diff --git a/packages/workflow-host/README.md b/packages/workflow-host/README.md index cbbba62f..91d211ad 100644 --- a/packages/workflow-host/README.md +++ b/packages/workflow-host/README.md @@ -99,46 +99,30 @@ The constructor argument shape: `deriveStepAddress`, `deriveStepRepoId?`, `ipcKeyPairFactory?` — per-deployment configuration the supervisor needs in its closure state. -- `trivialLaunch: (bindings) => Promise` — host-injected - callback the supervisor invokes on the trivial branch of - `deploy(frame)`. See "Deploy routing" below for the trivial- - branch invariants the callback runs under. ### Deploy routing -`deploy(frame)` is the single ingress for inbound `agent.deploy` -frames. The supervisor decides between two branches and the host -does not re-decide: - -- **Trivial branch (1-step workflows).** The supervisor calls - `bindings.trivialLaunch(frame)` directly. No IPC channel opens. - No workflow-process child spawns. No mail-bus subscription - registers. No workflow-run event is emitted; `signAsPrincipal` - is not invoked. `getCredentialsSnapshot()` continues to return - `null`. The trivial branch is a true passthrough -- the host - callback owns the entire deploy, and the supervisor's other - bindings stay inert. -- **Multi-step branch (`steps.length >= 2`).** The supervisor - routes through the same lifecycle `spawn(opts)` runs: per-step - `agent-state` repo provisioning, key minting, child spawn via - `subprocessSpawner`, mail-bus registration, IPC handshake, and - `credentialsSnapshot` assembly. The multi-step branch's body - lands as `agent.deploy` frames are extended to carry a - `WorkflowDefinition`; the routing seam exists today. - -The `agent.deploy` wire frame currently carries only a -`HarnessConfig` (no workflow definition), so every frame today is -trivial. The supervisor codifies the seam now so the frame-format -extension lands as a pure data-shape change. - -The sidecar's production wiring lives at -`apps/sidecar/src/workflow-host-wiring.ts`: -`createSidecarDeployRouter` constructs a fresh per-deployment -supervisor on every inbound frame whose `trivialLaunch` closes -over `SessionManager.provisionAgent` plus the hub-pairing-key -recording the legacy handler performed inline. The bytes flowing -through the deploy-flow integration test path stay bit-identical -to the pre-supervisor surface. +The sidecar's deploy router is the single ingress for inbound +`agent.deploy` frames; its production wiring lives at +`apps/sidecar/src/workflow-host-wiring.ts` in +`createSidecarDeployRouter`. Every deploy stages through the +workflow-run substrate, and the router decides between two frame +shapes: + +- **Provision-step frame (`provisionStep: true`).** The router + primes the frame's per-step `agent-state` repo and records the + hub key, without constructing a supervisor or spawning a child. + The follow-up full-closure deploy pack then applies into the + primed repo and verifies against the recorded key. +- **Workflow frame (carries a `WorkflowDefinition`).** The router + constructs a fresh per-deployment supervisor and drives its + `spawn(opts)` lifecycle: per-step `agent-state` repo + provisioning, key minting, child spawn via `subprocessSpawner`, + mail-bus registration, IPC handshake, and `credentialsSnapshot` + assembly. + +A frame carrying neither shape is rejected -- there is no +in-process deploy path. ### Lifecycle diff --git a/packages/workflow-host/src/adapters/step-invoker.test.ts b/packages/workflow-host/src/adapters/step-invoker.test.ts index aea86737..f0e179ce 100644 --- a/packages/workflow-host/src/adapters/step-invoker.test.ts +++ b/packages/workflow-host/src/adapters/step-invoker.test.ts @@ -255,6 +255,72 @@ function buildRequest(opts: { }; } +describe("workflow-host StepInvoker adapter - warm source rotation", () => { + test("a rotation during the first warm build reaches the built agent", async () => { + // Finding: a sources rotation that lands during the (async) first warm + // build hits the still-empty cache as a no-op applySources, while the + // in-flight build already captured the prior sources. After the built + // agent is stored, the adapter re-applies the live table so the rotation + // is not lost for the warm agent's life. + const stub = buildStreamingStubAgent([]); + const setSourcesCalls: { + sources: InferenceSource[]; + defaultSource: string; + }[] = []; + stub.agent.setSources = (sources, defaultSource) => { + setSourcesCalls.push({ sources, defaultSource }); + }; + const workflowAuthorize: WorkflowAuthorizeFn = async () => ({ + effect: "allow", + matchingGrants: [], + resolvedBy: null, + }); + const warmCache = createWarmAgentCache(); + const rotated: InferenceSource = { ...STUB_SOURCE, id: "rotated" }; + const sourcesRef = { + current: { "step-1": [STUB_SOURCE] } as Record, + }; + + let releaseBuild!: () => void; + const buildGate = new Promise((resolve) => { + releaseBuild = resolve; + }); + let buildStarted = false; + + const invoker = createWorkflowStepInvoker({ + workflowAuthorize, + buildEnv: async () => stubBuildEnv(), + agentFactory: async () => { + buildStarted = true; + await buildGate; + return stub.agent; + }, + warmCache, + sourcesRef, + }); + + const invokePromise = invoker(buildRequest({ input: { goal: "ping" } })); + while (!buildStarted) { + await new Promise((r) => setTimeout(r, 1)); + } + // A rotation lands DURING the build: the run-loop updates the ref and + // calls applySources -- a no-op here because the cache is still empty. + sourcesRef.current = { "step-1": [rotated] }; + warmCache.applySources([rotated], "rotated"); + expect(setSourcesCalls).toHaveLength(0); + + releaseBuild(); + await invokePromise; + + // The re-apply after store applied the ROTATED table to the built agent. + expect(setSourcesCalls).toEqual([ + { sources: [rotated], defaultSource: "rotated" }, + ]); + + await warmCache.evictAll("test cleanup"); + }); +}); + describe("workflow-host StepInvoker adapter - happy path", () => { test("delivers synthesized message, captures reply, returns output shape", async () => { const stub = buildStubAgent(); diff --git a/packages/workflow-host/src/adapters/step-invoker.ts b/packages/workflow-host/src/adapters/step-invoker.ts index 146a9f44..eb751ee5 100644 --- a/packages/workflow-host/src/adapters/step-invoker.ts +++ b/packages/workflow-host/src/adapters/step-invoker.ts @@ -60,7 +60,7 @@ import { type BaseEnv, } from "@intx/agent"; import { getLogger } from "@intx/log"; -import type { InferenceEvent } from "@intx/types/runtime"; +import type { InferenceEvent, InferenceSource } from "@intx/types/runtime"; import type { AuthorizeContext, StepInvokeRequest, @@ -152,6 +152,16 @@ export interface WorkflowStepInvokerOpts { * agent is never warm-kept. */ warmCache?: WarmAgentCache; + /** + * Live per-step inference-source table the run-loop mutates in place on a + * rotation. Supplied only on the warm path: after building and storing the + * warm agent, the adapter re-applies the current table so a rotation that + * landed during the (async) first build -- which the empty-cache + * `applySources` no-op could not reach, and which the in-flight build had + * already captured the prior sources for -- is not lost for the warm + * agent's life. + */ + sourcesRef?: { current: Record }; /** * Run-boundary hook for the warm path (design §3c durability). When * supplied, the adapter awaits it in the warm path's `finally` -- once @@ -225,8 +235,7 @@ async function invokeColdStep( // `onEvent` sink; absent a sink the agent's `stream()` is never // consumed (stub agents whose `stream()` throws stay untouched). // - // `message.received` is the single intentional exclusion, matching - // the in-process harness's forwarder (`default-harness.ts`): it is an + // `message.received` is the single intentional exclusion: it is an // assembly-internal dequeue signal, and the hub-facing audit chain // expresses per-message work through the `message.run.started` / // `message.run.ended` bracket pair instead. The filter is an @@ -301,6 +310,17 @@ async function invokeWarmStep( if (sink !== null) sink(event); }); warmCache.store(key, agent, eventSinkRef, eventForward); + // Re-apply the live source table to the just-built agent. A rotation + // that arrived during the (async) build hit the still-empty cache as a + // no-op `applySources` while the build had already captured the prior + // sources; now that the entry exists, this applies any such rotation so + // it is not lost for the warm agent's life. No-op when the table is + // unchanged. The wire boundary guarantees element 0 is the default. + const live = opts.sourcesRef?.current[key]; + const head = live?.[0]; + if (live !== undefined && head !== undefined) { + warmCache.applySources(live, head.id); + } } if (opts.onEvent !== undefined) { @@ -426,8 +446,7 @@ async function sendWithAbort( * Forwarding is best-effort observability: a sink that throws is * logged and swallowed so a downstream consumer's failure cannot abort * the step. A failure of the stream iterator itself (the agent's - * teardown surfacing through the iterator) is logged at warn, mirroring - * the in-process harness's forwarder. + * teardown surfacing through the iterator) is logged at warn. */ function subscribeAgentEvents( agent: Agent, diff --git a/packages/workflow-host/src/child/env-bootstrap.ts b/packages/workflow-host/src/child/env-bootstrap.ts index ef25b7da..560bf549 100644 --- a/packages/workflow-host/src/child/env-bootstrap.ts +++ b/packages/workflow-host/src/child/env-bootstrap.ts @@ -18,6 +18,29 @@ import { type } from "arktype"; import { hexDecode } from "@intx/types"; import { IPC_CRYPTO } from "../ipc/index"; +/** + * The required spawn-time env keys, named once so the supervisor-side + * producer (`buildChildSpawnEnv`) and this child-side parser share a + * single required-key contract. `WARM_KEEP` is optional and lives only in + * the shape below. The producer types its output against this list + * (`Record`), so omitting a listed key is a + * compile error. That this list stays in step with the validator shape + * below -- a hand-maintained arktype object -- is covered by the recycle + * env-contract regression test, which drives the real producer through + * this parser. This is the contract whose drift once omitted `STEP_COUNT` + * from the recycle env and broke every recycle. + */ +export const REQUIRED_SPAWN_ENV_KEYS = [ + "IPC_CHANNEL_ID", + "IPC_HMAC_KEY", + "HOST_PUBKEY", + "DEPLOYMENT_ID", + "DEFINITION_HASH", + "MAILBOX_ADDRESS", + "STEP_COUNT", +] as const; +export type RequiredSpawnEnvKey = (typeof REQUIRED_SPAWN_ENV_KEYS)[number]; + /** * Required env keys carried by the supervisor at spawn time. The * validator surface is intentionally narrow: every key documented at @@ -32,6 +55,12 @@ const SpawnTimeEnvShape = type({ DEPLOYMENT_ID: "string > 0", DEFINITION_HASH: "string > 0", MAILBOX_ADDRESS: "string > 0", + // Step count of the deployed `WorkflowDefinition` (`stepOrder.length`), + // stringified by the supervisor. The child's deploy-tree read collapses + // onto the head for a single-step deployment (`resolveStepAddress`), so + // producer and consumer never derive divergent step addresses. Parsed to + // a positive integer below; a non-integer or non-positive value throws. + STEP_COUNT: "string > 0", // Warm-keep signal (design §3b). The supervisor sets this to the // string `"true"` only for the single-step long-lived deployment the // deploy projection marked a warm candidate; any other value (or the @@ -60,6 +89,13 @@ export interface SpawnTimeEnv { definitionHash: string; /** Mail address the deployment registered on the bus. */ mailboxAddress: string; + /** + * Number of steps in the deployed `WorkflowDefinition` + * (`stepOrder.length`). Selects the head/step collapse in the sidecar's + * `resolveStepAddress`: a single-step deployment reads its deploy tree + * at the head, a multi-step deployment at the per-step address. + */ + stepCount: number; /** * Whether this deployment's agent is warm-kept across messages (design * §3b). True only for the single-step long-lived deployment the deploy @@ -113,6 +149,12 @@ export function parseSpawnTimeEnv( `workflow-child IPC_CHANNEL_ID must be ${String(expectedChannelIdHex)} hex chars; got ${String(validated.IPC_CHANNEL_ID.length)}`, ); } + const stepCount = Number(validated.STEP_COUNT); + if (!Number.isInteger(stepCount) || stepCount <= 0) { + throw new Error( + `workflow-child STEP_COUNT must be a positive integer; got ${JSON.stringify(validated.STEP_COUNT)}`, + ); + } return { channelId: validated.IPC_CHANNEL_ID, hmacKey, @@ -120,6 +162,7 @@ export function parseSpawnTimeEnv( deploymentId: validated.DEPLOYMENT_ID, definitionHash: validated.DEFINITION_HASH, mailboxAddress: validated.MAILBOX_ADDRESS, + stepCount, // Strict `=== "true"` so any other value (including the key's // absence) reads false. Warm-keep is opt-in and deterministic; a // typo'd or partial value must not silently enable it. diff --git a/packages/workflow-host/src/child/index.ts b/packages/workflow-host/src/child/index.ts index 7cccf3ef..0d785d26 100644 --- a/packages/workflow-host/src/child/index.ts +++ b/packages/workflow-host/src/child/index.ts @@ -9,6 +9,7 @@ export { type RunWorkflowChildBindings, type RunWorkflowChildOpts, type RunWorkflowChildResult, + type SourcesSnapshotRef, type SubstrateWriteResponseSink, } from "./run-child"; diff --git a/packages/workflow-host/src/child/proxy-repo-store.ts b/packages/workflow-host/src/child/proxy-repo-store.ts index 64c779e6..f2001b0e 100644 --- a/packages/workflow-host/src/child/proxy-repo-store.ts +++ b/packages/workflow-host/src/child/proxy-repo-store.ts @@ -41,6 +41,7 @@ type WriteTreeArgs = Parameters[3]; type WriteTreePreservingPrefixArgs = Parameters< RepoStore["writeTreePreservingPrefix"] >[3]; +type WriteTreeDeltaArgs = Parameters[3]; type SubscribeOpts = Parameters[3]; @@ -166,6 +167,16 @@ export function createProxyWorkflowRunRepoStore( "workflow-child proxy substrate: receivePack is not supported (writes are proxied to the supervisor)", ); }, + writeTreeDelta: ( + _principal: Principal, + _repoId: RepoId, + _ref: string, + _args: WriteTreeDeltaArgs, + ): Promise => { + throw new Error( + "workflow-child proxy substrate: writeTreeDelta is not supported (claim-check writes run supervisor-side)", + ); + }, async writeTreePreservingPrefix( _principal: Principal, repoId: RepoId, diff --git a/packages/workflow-host/src/child/run-child.test.ts b/packages/workflow-host/src/child/run-child.test.ts index d5d57032..8568b79c 100644 --- a/packages/workflow-host/src/child/run-child.test.ts +++ b/packages/workflow-host/src/child/run-child.test.ts @@ -43,12 +43,16 @@ import { type ChildStepInvoker, type RunWorkflowChildBindings, } from "./index"; +import { emitTerminalEvent } from "./run-child"; +import type { RunResult } from "@intx/workflow"; import { createControlChannelSender, generateChannelId, generateHmacKey, receiveControlChannel, receiveEventChannel, + type ControlChannelSender, + type ControlPayload, type FrameReader, type FrameWriter, type NdjsonReader, @@ -159,12 +163,58 @@ function createMemoryFrameStream() { }; } +// Read every file directly under the `runs//events/` prefix into the +// full-path-keyed map the merge callback expects (`appendBatchEvents` +// slices the prefix off each key to recover the seq). +async function readPrefixEntries( + repoDir: string, + prefix: string, +): Promise> { + const entries = new Map(); + const prefixDir = path.join(repoDir, prefix); + let names: string[]; + try { + names = await fs.readdir(prefixDir); + } catch (cause) { + // No prior entries under the prefix yet: the first append starts the + // subtree. + if (cause instanceof Error && "code" in cause && cause.code === "ENOENT") { + return entries; + } + throw cause; + } + for (const name of names) { + const full = path.join(prefixDir, name); + if (!(await fs.stat(full)).isFile()) continue; + entries.set(`${prefix}${name}`, await fs.readFile(full)); + } + return entries; +} + function createStubRepoStore(baseDir: string): RepoStore { const stub: Partial = { getRepoDir(repoId: RepoId): string { return path.join(baseDir, repoId.kind, repoId.id); }, - async writeTreePreservingPrefix(_principal, _repoId, _ref, _args) { + async writeTreePreservingPrefix(_principal, repoId, _ref, args) { + // Persist the run event log the way the real substrate does so the + // `createWorkflowRunRepoStore` adapter's disk read round-trips: run + // the merge callback against the prior entries under the preserved + // prefix, then replace that subtree with the merged result (files + // outside the prefix are untouched). Discarding the write here would + // make a completed run's `result.events` read back empty. + const repoDir = path.join(baseDir, repoId.kind, repoId.id); + const existing = await readPrefixEntries(repoDir, args.preservePrefix); + const merged = await args.merge(existing); + await fs.rm(path.join(repoDir, args.preservePrefix), { + recursive: true, + force: true, + }); + for (const [relPath, content] of Object.entries(merged)) { + const full = path.join(repoDir, relPath); + await fs.mkdir(path.dirname(full), { recursive: true }); + await fs.writeFile(full, content); + } return { commitSha: "deadbeefcafef00d", newlyTerminalRuns: [] }; }, }; @@ -185,6 +235,10 @@ function createStubRepoStore(baseDir: string): RepoStore { async function seedWorkflowDefinition( baseDir: string, repoId: RepoId, + workflow: { steps: Record; stepOrder: string[] } = { + steps: {}, + stepOrder: [], + }, ): Promise { const dir = path.join(baseDir, repoId.kind, repoId.id); await fs.mkdir(dir, { recursive: true }); @@ -193,8 +247,8 @@ async function seedWorkflowDefinition( JSON.stringify({ id: "test-workflow", triggers: [], - steps: {}, - stepOrder: [], + steps: workflow.steps, + stepOrder: workflow.stepOrder, }), ); } @@ -347,6 +401,7 @@ function makeSpawnEnv(opts: { DEPLOYMENT_ID: "deployment-x", DEFINITION_HASH: "definition-hash-abc", MAILBOX_ADDRESS: "deployment-x@example.com", + STEP_COUNT: "1", }; } @@ -714,14 +769,19 @@ describe("runWorkflowChild", () => { data: { runId: "run-1", messageId: "msg-1", receivedAt: 1 }, }); - let sawTerminal = false; + let terminalSeq: number | null = null; for await (const payload of recvIter) { if (payload.type === "terminal.event" && payload.data.runId === "run-1") { - sawTerminal = true; + terminalSeq = payload.data.seq; break; } } - expect(sawTerminal).toBe(true); + // The frame's seq is sourced from the run's committed terminal event, so + // a real (non-zero) seq proves the child test substrate faithfully + // persisted and read back the run's event log rather than round-tripping + // an empty one. + expect(terminalSeq).not.toBeNull(); + expect(terminalSeq).toBeGreaterThan(0); // The run reached terminal AND the completion continuation ran (it // emitted the terminal.event we just observed). On the cold path the // same continuation would have called cleanupRunStorage by now; the @@ -905,6 +965,128 @@ describe("runWorkflowChild", () => { ]); }); + test("sources-updated on a multi-step deployment is rejected", async () => { + const baseDir = await makeTempDir("child-sources-multistep-"); + const supervisorKeyPair = await generateKeyPair(); + const childKeyPair = await generateKeyPair(); + const channelId = generateChannelId(); + const hmacKey = generateHmacKey(); + await seedWorkflowDefinition( + baseDir, + { kind: "workflow", id: "workflow-asset" }, + { + steps: { "step-1": { kind: "step" }, "step-2": { kind: "step" } }, + stepOrder: ["step-1", "step-2"], + }, + ); + + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const eventStream = createMemoryFrameStream(); + + const env = parseSpawnTimeEnv( + makeSpawnEnv({ + channelId, + hmacKeyHex: hexEncode(hmacKey), + hostPubKeyHex: hexEncode(supervisorKeyPair.publicKey), + }), + ); + const bindings = buildBindings({ baseDir, childKeyPair }); + const supervisorSender = createControlChannelSender({ + privateKeySeed: supervisorKeyPair.privateKey, + channelId, + writer: supervisorToChild.writer, + }); + + const runPromise = runWorkflowChild({ + env, + controlReader: supervisorToChild.reader, + controlWriter: childToSupervisor.writer, + eventWriter: eventStream.writer, + bindings, + }); + + await supervisorSender.send({ + type: "sources-updated", + data: { + sources: [ + { + id: "primary", + provider: "anthropic", + baseURL: "https://api.anthropic.com", + apiKey: "sk-x", + model: "claude-test", + }, + ], + defaultSource: "primary", + }, + }); + + await expect(runPromise).rejects.toThrow(/single-step deployment/); + supervisorToChild.close(); + }); + + test("sources-updated with no warm cache is rejected as a routing bug", async () => { + // A single-step definition with WARM_KEEP off leaves the child with no + // warm cache. A sources-updated must never silently no-op there; it is + // a routing bug, so the handler throws. + const baseDir = await makeTempDir("child-sources-nowarm-"); + const supervisorKeyPair = await generateKeyPair(); + const childKeyPair = await generateKeyPair(); + const channelId = generateChannelId(); + const hmacKey = generateHmacKey(); + await seedWorkflowDefinition( + baseDir, + { kind: "workflow", id: "workflow-asset" }, + { steps: { "step-1": { kind: "step" } }, stepOrder: ["step-1"] }, + ); + + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const eventStream = createMemoryFrameStream(); + + const env = parseSpawnTimeEnv( + makeSpawnEnv({ + channelId, + hmacKeyHex: hexEncode(hmacKey), + hostPubKeyHex: hexEncode(supervisorKeyPair.publicKey), + }), + ); + const bindings = buildBindings({ baseDir, childKeyPair }); + const supervisorSender = createControlChannelSender({ + privateKeySeed: supervisorKeyPair.privateKey, + channelId, + writer: supervisorToChild.writer, + }); + + const runPromise = runWorkflowChild({ + env, + controlReader: supervisorToChild.reader, + controlWriter: childToSupervisor.writer, + eventWriter: eventStream.writer, + bindings, + }); + + await supervisorSender.send({ + type: "sources-updated", + data: { + sources: [ + { + id: "primary", + provider: "anthropic", + baseURL: "https://api.anthropic.com", + apiKey: "sk-x", + model: "claude-test", + }, + ], + defaultSource: "primary", + }, + }); + + await expect(runPromise).rejects.toThrow(/no warm cache/); + supervisorToChild.close(); + }); + test("child ready frame is signed by the child's own keypair and bootstraps the supervisor's verification key", async () => { const baseDir = await makeTempDir("child-ready-"); const supervisorKeyPair = await generateKeyPair(); @@ -1107,6 +1289,10 @@ interface WarmAgentSpy { closeCount: number; readonly conversation: string[]; readonly replies: string[]; + readonly sourceRotations: { + sources: InferenceSource[]; + defaultSource: string; + }[]; } /** @@ -1124,6 +1310,7 @@ function buildWarmAgentSpy(): { agent: Agent; spy: WarmAgentSpy } { closeCount: 0, conversation: [], replies: [], + sourceRotations: [], }; let endStream: () => void = () => undefined; const streamEnded = new Promise((resolve) => { @@ -1166,8 +1353,11 @@ function buildWarmAgentSpy(): { agent: Agent; spy: WarmAgentSpy } { setSource(_source: InferenceSource) { throw new Error("stub setSource() not used"); }, - setSources(_sources: InferenceSource[], _defaultSource: string) { - throw new Error("stub setSources() not used"); + setSources(sources: InferenceSource[], defaultSource: string) { + // Live source rotation lands here on the warm agent. Record it so the + // sources-updated round-trip can assert the swap reached the agent + // in place rather than rebuilding it. + spy.sourceRotations.push({ sources, defaultSource }); }, async history() { return []; @@ -1505,6 +1695,176 @@ describe("warm-agent round-trip (Phase 4.4)", () => { expect(spy.closeCount).toBe(1); expect(spy.lspAlive).toBe(false); }); + + test("a sources-updated frame swaps the built warm agent's sources in place", async () => { + const baseDir = await makeTempDir("warm-sources-"); + const supervisorKeyPair = await generateKeyPair(); + const childKeyPair = await generateKeyPair(); + const channelId = generateChannelId(); + const hmacKey = generateHmacKey(); + const stepId = "step-1"; + const deploymentId = "deployment-x"; + const workflowRunRepoId: RepoId = { + kind: "workflow-run", + id: deploymentId, + }; + const workflowDefinitionRepoId: RepoId = { + kind: "workflow-run", + id: "workflow-asset", + }; + + const signingKey: KeyPair = await generateKeyPair(); + const allowAll: AuthorizeFn = () => ({ allowed: true }); + const substrate = createRepoStore({ + dataDir: baseDir, + signingKey, + handlers: { "workflow-run": workflowRunKindHandler }, + authorize: allowAll, + }); + const principalShape = { kind: "workflow-process", deploymentId }; + const principal: Principal = principalShape; + + await substrate.writeTree( + { kind: "hub" }, + workflowRunRepoId, + "refs/heads/main", + { files: { [WORKFLOW_RUN_GITIGNORE_PATH]: "" }, message: "genesis" }, + ); + const runRepoDir = substrate.getRepoDir(workflowRunRepoId); + await seedProcessingEntryInDir(runRepoDir, { + address: "deployment-x@example.com", + messageId: "msg-1", + receivedAt: 1, + text: "alpha body", + }); + await substrate.writeTree( + { kind: "hub" }, + workflowDefinitionRepoId, + "refs/heads/main", + { files: { [WORKFLOW_RUN_GITIGNORE_PATH]: "" }, message: "genesis" }, + ); + await seedOneStepWorkflowDir( + substrate.getRepoDir(workflowDefinitionRepoId), + stepId, + ); + + const { agent, spy } = buildWarmAgentSpy(); + let factoryCalls = 0; + const invokeStep: ChildStepInvoker = async ( + req, + onEvent, + authorize, + warmCache, + ) => + createWorkflowStepInvoker({ + workflowAuthorize: authorize, + buildEnv: async () => stubStepEnv(), + agentFactory: async () => { + factoryCalls += 1; + return agent; + }, + onEvent: (event) => onEvent(event), + ...(warmCache !== undefined ? { warmCache } : {}), + })(req); + + const bindings: RunWorkflowChildBindings = { + substrate, + workflowRunRepoId, + workflowRunRef: "refs/heads/main", + principal, + workflowDefinitionRepoId, + workflowDefinitionRef: "refs/heads/main", + invokeStep, + spawnChild: async () => ({ terminalStatus: "completed" }), + scheduler: { scheduleIn: () => () => undefined }, + evaluateGrants: async () => ({ + effect: "allow" as const, + matchingGrants: [], + resolvedBy: null, + }), + ipcChildKeyPairFactory: () => Promise.resolve(childKeyPair), + initialCredentialsSnapshot: { + steps: [ + { + stepId, + address: "deployment-x@example.com", + grants: [], + contentHash: "deadbeef", + }, + ], + }, + }; + + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const eventStream = createMemoryFrameStream(); + + const env = parseSpawnTimeEnv({ + ...makeSpawnEnv({ + channelId, + hmacKeyHex: hexEncode(hmacKey), + hostPubKeyHex: hexEncode(supervisorKeyPair.publicKey), + }), + WARM_KEEP: "true", + }); + + const supervisorSender = createControlChannelSender({ + privateKeySeed: supervisorKeyPair.privateKey, + channelId, + writer: supervisorToChild.writer, + }); + + const runPromise = runWorkflowChild({ + env, + controlReader: supervisorToChild.reader, + controlWriter: childToSupervisor.writer, + eventWriter: eventStream.writer, + bindings, + }); + + await waitForTriggeredRun(childToSupervisor, (lines) => lines.length > 0); + + // Build the warm agent with one message so the cache holds an entry. + await supervisorSender.send({ + type: "trigger.fire", + data: { runId: "run-1", messageId: "msg-1", receivedAt: 1 }, + }); + await waitForTriggeredRun(childToSupervisor, () => spy.replies.length >= 1); + expect(factoryCalls).toBe(1); + + // Rotate the sources on the live warm agent. + const rotated: InferenceSource[] = [ + { + id: "rotated", + provider: "openai", + baseURL: "https://api.openai.com", + apiKey: "sk-rotated", + model: "gpt-rotated", + }, + ]; + await supervisorSender.send({ + type: "sources-updated", + data: { sources: rotated, defaultSource: "rotated" }, + }); + for (let i = 0; i < 400; i += 1) { + if (spy.sourceRotations.length >= 1) break; + await new Promise((r) => setTimeout(r, 5)); + } + + // The swap reached the built agent in place, carrying the frame's list + // and default, and did NOT rebuild the agent. + expect(spy.sourceRotations).toHaveLength(1); + expect(spy.sourceRotations[0]?.sources).toEqual(rotated); + expect(spy.sourceRotations[0]?.defaultSource).toBe("rotated"); + expect(factoryCalls).toBe(1); + + await supervisorSender.send({ + type: "shutdown", + data: { reason: "undeploy" }, + }); + supervisorToChild.close(); + await runPromise; + }); }); describe("event channel writer wiring", () => { @@ -1548,3 +1908,112 @@ describe("event channel writer wiring", () => { expect(crashes).toHaveLength(0); }); }); + +describe("emitTerminalEvent", () => { + function capturingSender(): { + sender: ControlChannelSender; + sent: ControlPayload[]; + } { + const sent: ControlPayload[] = []; + const sender: ControlChannelSender = { + seq: 0, + send: (payload) => { + sent.push(payload); + return Promise.resolve(); + }, + }; + return { sender, sent }; + } + + test("mirrors a RunCompleted terminal event, sourcing seq and at from it", async () => { + const { sender, sent } = capturingSender(); + const result: RunResult = { + runId: "run-ok", + terminalStatus: "completed", + outputs: {}, + events: [{ kind: "RunCompleted", seq: 5, at: "2026-01-01T00:00:05Z" }], + }; + await emitTerminalEvent(sender, result); + expect(sent).toEqual([ + { + type: "terminal.event", + data: { + runId: "run-ok", + seq: 5, + kind: "RunCompleted", + at: "2026-01-01T00:00:05Z", + }, + }, + ]); + }); + + test("mirrors a RunFailed terminal event, sourcing the error message from it", async () => { + const { sender, sent } = capturingSender(); + const result: RunResult = { + runId: "run-boom", + terminalStatus: "failed", + outputs: {}, + events: [ + { + kind: "RunFailed", + seq: 7, + at: "2026-01-01T00:00:07Z", + error: { message: "step blew up" }, + }, + ], + }; + await emitTerminalEvent(sender, result); + expect(sent).toEqual([ + { + type: "terminal.event", + data: { + runId: "run-boom", + seq: 7, + kind: "RunFailed", + at: "2026-01-01T00:00:07Z", + error: { message: "step blew up" }, + }, + }, + ]); + }); + + test("throws when the committed log carries no terminal event", () => { + const { sender, sent } = capturingSender(); + const result: RunResult = { + runId: "run-nolog", + terminalStatus: "completed", + outputs: {}, + events: [], + }; + // The runtime commits the terminal event last, so its absence is a + // producer bug. Throw rather than emit a seq-0 frame that would desync + // the supervisor from the durable log. + expect(() => emitTerminalEvent(sender, result)).toThrow( + /carries no terminal event/, + ); + expect(sent).toEqual([]); + }); + + test("throws when the terminal event kind disagrees with the terminal status", () => { + const { sender, sent } = capturingSender(); + const result: RunResult = { + runId: "run-mismatch", + terminalStatus: "completed", + outputs: {}, + events: [ + { + kind: "RunFailed", + seq: 9, + at: "2026-01-01T00:00:09Z", + error: { message: "actually failed" }, + }, + ], + }; + // Emitting a RunCompleted frame carrying the RunFailed's seq would claim + // completion while pointing at a failure's audit entry. + expect(() => emitTerminalEvent(sender, result)).toThrow( + /committed terminal event is RunFailed/, + ); + expect(sent).toEqual([]); + }); +}); diff --git a/packages/workflow-host/src/child/run-child.ts b/packages/workflow-host/src/child/run-child.ts index ca3ec430..419772cc 100644 --- a/packages/workflow-host/src/child/run-child.ts +++ b/packages/workflow-host/src/child/run-child.ts @@ -91,6 +91,8 @@ import { type WorkflowHostDrainController, } from "../drain-controller"; +import type { InferenceSource } from "@intx/types/runtime"; + import { createWorkflowRunRepoStore } from "../adapters/repo-store"; import { createWorkflowRunBlobSubstrate } from "../adapters/blob-substrate"; import { @@ -135,6 +137,19 @@ export type CredentialsSnapshotRef = { current: CredentialsSnapshot | null; }; +/** + * Per-step inference-source table the build path reads through a mutable + * reference, keyed by stepId. Each value is the step's ordered failover + * chain (element 0 is the active source). The single-step build resolves + * its sources from `current` at build time, so a rotation that writes + * `current` before the first build is reflected in the built agent. A + * warm agent that is already built does not re-read this ref, so rotating + * its live sources is out of this ref's scope. + */ +export type SourcesSnapshotRef = { + current: Record; +}; + /** * Construct the workflow-level authorize closure backed by a * mutable credentialsSnapshot reference. @@ -228,6 +243,7 @@ export type ChildStepInvoker = ( onEvent: (event: EventPayload) => void, authorize: WorkflowAuthorizeFn, warmCache: WarmAgentCache | undefined, + sourcesRef: SourcesSnapshotRef, ) => Promise; /** @@ -302,6 +318,14 @@ export interface RunWorkflowChildBindings { * value defers to the first `grants-updated` control frame. */ initialCredentialsSnapshot?: CredentialsSnapshot; + /** + * Bootstrap per-step inference-source table (keyed by stepId), parsed + * from the spawn env by the host's substrate factory. Seeds the + * mutable `sourcesRef` the build path reads. Absent value defers to an + * empty table, so a step with no pinned source fails loudly at build + * rather than resolving a default. + */ + initialSources?: Record; /** * Optional override for the child's Ed25519 keypair factory. The * child mints a fresh keypair at startup, holds the private half @@ -424,6 +448,9 @@ export async function runWorkflowChild( const credentialsRef: CredentialsSnapshotRef = { current: opts.bindings.initialCredentialsSnapshot ?? null, }; + const sourcesRef: SourcesSnapshotRef = { + current: opts.bindings.initialSources ?? {}, + }; const directors = opts.bindings.directors ?? createDefaultDirectorRegistry(); const clock = opts.bindings.clock ?? defaultClock; const newId = opts.bindings.newId ?? defaultNewId; @@ -508,6 +535,7 @@ export async function runWorkflowChild( newId, drainController, warmCache, + sourcesRef, onEvent: (event) => { void eventSender.send(event).catch((cause) => { logger.error`event-channel send failed during resume run ${run.runId}: ${String(cause)}`; @@ -590,6 +618,7 @@ export async function runWorkflowChild( drainController, triggeredRunIds, warmCache, + sourcesRef, ...(opts.substrateWriteBridge !== undefined ? { substrateWriteBridge: opts.substrateWriteBridge } : {}), @@ -662,6 +691,7 @@ async function handleControlPayload( drainController: DrainController; triggeredRunIds: string[]; warmCache: WarmAgentCache | undefined; + sourcesRef: SourcesSnapshotRef; substrateWriteBridge?: SubstrateWriteResponseSink; outboundMailBridge?: ChildOutboundMailBridge; }, @@ -695,6 +725,7 @@ async function handleControlPayload( newId: ctx.newId, drainController: ctx.drainController, warmCache: ctx.warmCache, + sourcesRef: ctx.sourcesRef, onEvent: (event) => { void ctx.eventSender.send(event).catch((cause) => { logger.error`event-channel send failed during run ${payload.data.runId}: ${String(cause)}`; @@ -831,7 +862,46 @@ async function handleControlPayload( return true; } case "sources-updated": { - logger.info`workflow-child sources-updated: ${JSON.stringify(payload.data)}`; + // Live inference-source rotation for the warm single-step agent. The + // wire boundary (`SourcesUpdatedData`) already guaranteed the list is + // non-empty, its ids are unique, and its head is the default, so this + // trusts the frame and does not re-validate it. + // + // Only a single-step deployment rotates sources: its sole step's id + // is the sole key in the sources table, so the whole table is + // replaced. A multi-step deployment has no single per-agent source + // identity to swap and is never routed a sources-updated frame; + // assert it so a mis-route fails loudly rather than corrupting the + // table. + if (ctx.definition.stepOrder.length !== 1) { + throw new Error( + `workflow-child sources-updated: only a single-step deployment can rotate sources; got ${String(ctx.definition.stepOrder.length)} steps`, + ); + } + const stepId = ctx.definition.stepOrder[0]; + if (stepId === undefined) { + throw new Error( + "workflow-child sources-updated: single-step deployment has no step id", + ); + } + // A sources-updated only reaches a warm single-step deployment, which + // always builds a warm cache. An absent cache is a routing bug, not a + // silent no-op. + if (ctx.warmCache === undefined) { + throw new Error( + "workflow-child sources-updated: no warm cache; a sources rotation must target a warm single-step deployment", + ); + } + // Swap the built warm agent first (a no-op when none is built yet), + // then update the table the next cold build reads. Applying to the + // agent first means a rotation racing eviction -- a closed-agent + // `setSources` throw -- leaves the table untouched rather than ahead + // of a half-applied swap. + ctx.warmCache.applySources( + payload.data.sources, + payload.data.defaultSource, + ); + ctx.sourcesRef.current = { [stepId]: payload.data.sources }; return false; } case "ready": { @@ -940,6 +1010,7 @@ function buildRuntimeEnv(args: { newId: (prefix: string) => string; drainController: DrainController; warmCache: WarmAgentCache | undefined; + sourcesRef: SourcesSnapshotRef; onEvent: (event: EventPayload) => void; }): WorkflowRuntimeEnv { const signalChannel = createWorkflowHostSignalChannel({ @@ -972,6 +1043,7 @@ function buildRuntimeEnv(args: { args.onEvent, args.authorize, args.warmCache, + args.sourcesRef, ); }; return { @@ -997,29 +1069,34 @@ function buildRuntimeEnv(args: { * loop and any armed drainTimeout accumulator subscribed for the * runId. * - * The mapping is total: every `terminalStatus` the runtime body - * surfaces corresponds to exactly one `kind` in the wire union. The - * `error.message` on `RunFailed` is taken from the last - * `RunFailed`/`StepFailed` event the runtime emitted; when the log - * does not carry one the supervisor's downstream consumers see an - * empty message rather than a thrown error (the wire shape requires - * the field). + * The frame mirrors the run's committed terminal event: every field -- + * `kind`, `seq`, `at`, and (for `RunFailed`) `error.message` -- is + * sourced from that event, which is why the frame's `seq` matches the + * on-disk audit-log entry. `terminalStatus` is only the cross-check: the + * found event's `kind` must agree with it. A missing terminal event, or + * one whose kind disagrees, is a runtime producer bug (the runtime + * commits the terminal event last), and emitting a frame anyway would + * desync the supervisor from the durable log that `discoverInFlightRuns` + * reads on resume -- the supervisor would settle a run the on-disk log + * still shows in-flight. So this throws instead: no frame keeps the + * supervisor and the durable log agreeing that the run is unsettled, and + * the next recycle/restart resumes it. The throw propagates to the + * caller's `complete` continuation, which logs it. * - * Errors flowing out of `upstreamSender.send` are logged but not - * rethrown -- the supervisor's dispatch loop is the authoritative - * settler for the dispatch entry through its cohort abort signal, so a - * lost terminal frame surfaces structurally as a wedged dispatch - * rather than a silent lifecycle failure. + * Errors flowing out of `upstreamSender.send` are a different case -- + * a transport send failure, logged but not rethrown. The supervisor's + * dispatch loop is the authoritative settler through its cohort abort + * signal, so a lost frame surfaces structurally as a wedged dispatch + * rather than a silent lifecycle failure. The invariant throws above run + * before the send so that catch never swallows them. */ -function emitTerminalEvent( +export function emitTerminalEvent( upstreamSender: ControlChannelSender, result: RunResult, ): Promise { - const at = new Date().toISOString(); - // Recover the terminal event blob from the committed event log so - // the wire frame's seq matches the on-disk audit-log entry. The - // runtime body commits the terminal event last; walking from the - // end finds it in one step without rebuilding the state machine. + // Recover the terminal event from the committed event log. The runtime + // body commits the terminal event last; walking from the end finds it in + // one step without rebuilding the state machine. let terminalEvent: (typeof result.events)[number] | null = null; for (let i = result.events.length - 1; i >= 0; i -= 1) { const candidate = result.events[i]; @@ -1033,34 +1110,49 @@ function emitTerminalEvent( break; } } - const seq = terminalEvent?.seq ?? 0; - const eventAt = terminalEvent?.at ?? at; + if (terminalEvent === null) { + throw new Error( + `emitTerminalEvent: run ${result.runId} terminated as ${result.terminalStatus} but its committed event log carries no terminal event (the runtime commits it last; this is a producer bug)`, + ); + } + const expectedKind = + result.terminalStatus === "completed" + ? "RunCompleted" + : result.terminalStatus === "cancelled" + ? "RunCancelled" + : "RunFailed"; + if (terminalEvent.kind !== expectedKind) { + throw new Error( + `emitTerminalEvent: run ${result.runId} terminated as ${result.terminalStatus} but its committed terminal event is ${terminalEvent.kind}`, + ); + } + // The RunFailed-missing-error.message case the supervisor's + // `synthesizeTerminalEvent` guards is unreachable here: `result.events` + // is typed `WorkflowEvent[]`, and `RunFailed.error.message` is a + // non-optional `string`, so a RunFailed reached here always carries one. + // The supervisor needs that guard because it parses untrusted JSON. let payload: Extract["data"]; - if (result.terminalStatus === "completed") { + if (terminalEvent.kind === "RunCompleted") { payload = { runId: result.runId, - seq, + seq: terminalEvent.seq, kind: "RunCompleted", - at: eventAt, + at: terminalEvent.at, }; - } else if (result.terminalStatus === "cancelled") { + } else if (terminalEvent.kind === "RunCancelled") { payload = { runId: result.runId, - seq, + seq: terminalEvent.seq, kind: "RunCancelled", - at: eventAt, + at: terminalEvent.at, }; } else { - const message = - terminalEvent !== null && terminalEvent.kind === "RunFailed" - ? terminalEvent.error.message - : ""; payload = { runId: result.runId, - seq, + seq: terminalEvent.seq, kind: "RunFailed", - at: eventAt, - error: { message }, + at: terminalEvent.at, + error: { message: terminalEvent.error.message }, }; } return upstreamSender diff --git a/packages/workflow-host/src/child/warm-agent-cache.test.ts b/packages/workflow-host/src/child/warm-agent-cache.test.ts new file mode 100644 index 00000000..7746096f --- /dev/null +++ b/packages/workflow-host/src/child/warm-agent-cache.test.ts @@ -0,0 +1,57 @@ +import { describe, test, expect } from "bun:test"; +import type { Agent } from "@intx/agent"; +import type { InferenceSource } from "@intx/types/runtime"; + +import { + createWarmAgentCache, + type WarmEventSinkRef, +} from "./warm-agent-cache"; + +function makeSource(id: string): InferenceSource { + return { + id, + provider: "anthropic", + baseURL: "https://api.anthropic.com", + apiKey: `sk-${id}`, + model: "claude-test", + }; +} + +function stubAgent(): { + agent: Agent; + calls: { sources: InferenceSource[]; defaultSource: string }[]; +} { + const calls: { sources: InferenceSource[]; defaultSource: string }[] = []; + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub: applySources only calls setSources on the entry's agent + const agent = { + setSources(sources: InferenceSource[], defaultSource: string) { + calls.push({ sources, defaultSource }); + }, + } as unknown as Agent; + return { agent, calls }; +} + +describe("warm-agent cache applySources", () => { + test("applies the rotated sources to a built warm agent", () => { + const cache = createWarmAgentCache(); + const { agent, calls } = stubAgent(); + const sinkRef: WarmEventSinkRef = { current: null }; + cache.store("step-1", agent, sinkRef, Promise.resolve()); + + const sources = [makeSource("primary"), makeSource("failover")]; + cache.applySources(sources, "primary"); + + expect(calls).toHaveLength(1); + expect(calls[0]?.sources).toEqual(sources); + expect(calls[0]?.defaultSource).toBe("primary"); + }); + + test("is a no-op when no warm agent is built yet", () => { + const cache = createWarmAgentCache(); + // The pre-first-build window: a rotation arriving before the agent is + // built must not throw; the ref carries it to the next build. + expect(() => + cache.applySources([makeSource("primary")], "primary"), + ).not.toThrow(); + }); +}); diff --git a/packages/workflow-host/src/child/warm-agent-cache.ts b/packages/workflow-host/src/child/warm-agent-cache.ts index bd60ca2f..6e5a7c69 100644 --- a/packages/workflow-host/src/child/warm-agent-cache.ts +++ b/packages/workflow-host/src/child/warm-agent-cache.ts @@ -35,7 +35,7 @@ import { getLogger } from "@intx/log"; import type { Agent } from "@intx/agent"; -import type { InferenceEvent } from "@intx/types/runtime"; +import type { InferenceEvent, InferenceSource } from "@intx/types/runtime"; const logger = getLogger(["workflow-host", "child", "warm-agent-cache"]); @@ -102,6 +102,19 @@ export interface WarmAgentCache { * no-op: the agent may already have been evicted. */ clearEventSink(key: string): void; + /** + * Apply a rotated inference-source list to every retained warm agent in + * place, via `Agent.setSources`. A single-step warm cache holds 0 or 1 + * entry, so this rotates the one built agent -- or is a no-op when none + * is built yet (the pre-first-build window). The swap mutates the + * agent's shared active-source object in place and takes effect on the + * reactor's next inference call; the single-threaded control loop means + * there is no torn read against a concurrent send. `setSources` validates + * the list and throws on an invalid source (or a closed agent, if a + * rotation races eviction), so a bad rotation surfaces rather than being + * swallowed. + */ + applySources(sources: InferenceSource[], defaultSource: string): void; /** * Tear down every cached warm agent: run the wrapped `agent.close()` * (disposing plugins and killing the LSP subprocess) and drain the @@ -160,6 +173,15 @@ export function createWarmAgentCache(): WarmAgentCache { entry.eventSinkRef.current = null; } + function applySources( + sources: InferenceSource[], + defaultSource: string, + ): void { + for (const entry of entries.values()) { + entry.agent.setSources(sources, defaultSource); + } + } + async function evictAll(reason: string): Promise { if (entries.size === 0) return; const toEvict = [...entries.values()]; @@ -195,6 +217,7 @@ export function createWarmAgentCache(): WarmAgentCache { store, setEventSink, clearEventSink, + applySources, evictAll, }; } diff --git a/packages/workflow-host/src/index.ts b/packages/workflow-host/src/index.ts index 3684e30f..b3968312 100644 --- a/packages/workflow-host/src/index.ts +++ b/packages/workflow-host/src/index.ts @@ -21,7 +21,6 @@ export { createWorkflowSupervisor, assembleCredentialsSnapshot, commitCancelRequested, - commitRunEvent, createDrainTimeoutAccumulator, createRecyclePolicy, defaultStepRepoId, @@ -40,11 +39,10 @@ export { type CancelRequestOpts, type CommitCancelRequestedOpts, type CommitCancelRequestedResult, - type CommitRunEventOpts, - type CommitRunEventResult, type CredentialsSnapshot, type CredentialsSnapshotStep, type DeliverSignalOpts, + type DeliverSourcesOpts, type DeriveMailAuditRef, type DeriveStepAddress, type DeriveStepRepoId, @@ -56,7 +54,6 @@ export { type MailAuditRef, type MailBusBindings, type PrincipalSigner, - type RecordRunEvent, type RecycleAttempt, type RecycleContext, type RecycleOpts, @@ -65,17 +62,13 @@ export { type RecyclePolicyBounds, type RecyclePolicyOpts, type SignedPayload, - type SupervisorRunEvent, type SpawnOpts, type SpawnResult, type SubprocessHandle, type SubprocessSpawner, - type SupervisorDeployFrame, type TerminalEventSource, type TerminalRunEvent, type TriggerRecycleOpts, - type TrivialLaunch, - type TrivialLaunchBindings, type DispatchTimingMark, type DispatchStructuralCounters, type DispatchSubstrateLeg, @@ -101,6 +94,7 @@ export { EventPayload, FrameEnvelope, IPC_CRYPTO, + SourcesUpdatedData, MacedEnvelope, OutboundAttachmentPayload, OutboundMessagePayload, @@ -156,6 +150,7 @@ export { type RunWorkflowChildFromProcessEnvOpts, type RunWorkflowChildOpts, type RunWorkflowChildResult, + type SourcesSnapshotRef, type SpawnTimeEnv, type SubstrateFactory, type SubstrateFactoryEnv, diff --git a/packages/workflow-host/src/ipc/control-channel.ts b/packages/workflow-host/src/ipc/control-channel.ts index 92bbafc3..e5ab218c 100644 --- a/packages/workflow-host/src/ipc/control-channel.ts +++ b/packages/workflow-host/src/ipc/control-channel.ts @@ -37,7 +37,7 @@ import { type } from "arktype"; import { hexDecode, hexEncode } from "@intx/types"; -import { InterchangeType } from "@intx/types/runtime"; +import { InferenceSource, InterchangeType } from "@intx/types/runtime"; import { decodeEnvelope, @@ -68,6 +68,44 @@ export const CredentialsSnapshotPayload = type({ steps: CredentialsSnapshotStepPayload.array(), }); +/** + * Wire shape of a `sources-updated` frame's `data`: the full ordered + * inference-source failover chain plus the default source id. Carried + * inline like the grants snapshot -- a single-producer, single-consumer + * supervisor->child push, so a substrate round-trip would only add + * latency. No per-source hash rides along; a source list is flat, with no + * per-item pin for a receiver to cross-check. + * + * The `narrow` pins two frame-structural invariants at this boundary so + * every consumer can trust them without re-checking: source ids are + * unique, and the first element is the default source. The head-is-default + * rule is what keeps the two rotation paths in agreement -- a warm agent's + * `setSources` activates the matched default index, while a cold rebuild + * pins element 0 -- so they pick the same active source only when the + * default is the head. + */ +export const SourcesUpdatedData = type({ + sources: InferenceSource.array().atLeastLength(1), + defaultSource: "string > 0", +}).narrow((data, ctx) => { + const seen = new Set(); + for (const source of data.sources) { + if (seen.has(source.id)) { + return ctx.mustBe( + `a source list with unique ids; "${source.id}" appears more than once`, + ); + } + seen.add(source.id); + } + const head = data.sources[0]; + if (head === undefined || head.id !== data.defaultSource) { + return ctx.mustBe( + "a source list whose first element is the default source", + ); + } + return true; +}); + /** * Wire projection of an attachment on an outbound mail message. The * runtime `MessageAttachment.data` is raw bytes; the NDJSON control @@ -176,9 +214,7 @@ export const ControlPayload = type( }) .or({ type: "'sources-updated'", - data: { - "sourceHashes?": "Record", - }, + data: SourcesUpdatedData, }) .or({ type: "'ready'", diff --git a/packages/workflow-host/src/ipc/index.ts b/packages/workflow-host/src/ipc/index.ts index c030af34..ddc7bcf0 100644 --- a/packages/workflow-host/src/ipc/index.ts +++ b/packages/workflow-host/src/ipc/index.ts @@ -142,6 +142,7 @@ export { ControlPayload, OutboundAttachmentPayload, OutboundMessagePayload, + SourcesUpdatedData, createControlChannelSender, receiveControlChannel, type ControlChannelSender, diff --git a/packages/workflow-host/src/ipc/ipc.test.ts b/packages/workflow-host/src/ipc/ipc.test.ts index 5fda6e35..e11afa8b 100644 --- a/packages/workflow-host/src/ipc/ipc.test.ts +++ b/packages/workflow-host/src/ipc/ipc.test.ts @@ -1442,6 +1442,94 @@ describe("substrate.write.request payload validation", () => { }); }); +describe("sources-updated payload validation", () => { + const source = { + id: "primary", + provider: "anthropic", + baseURL: "https://api.anthropic.com", + apiKey: "sk-x", + model: "claude-3-5", + }; + + test("accepts a well-formed sources-updated frame", () => { + const payload = { + type: "sources-updated", + data: { sources: [source], defaultSource: "primary" }, + }; + const validated = ControlPayload(payload); + expect(validated instanceof type.errors).toBe(false); + }); + + test("rejects an empty sources list", () => { + const payload = { + type: "sources-updated", + data: { sources: [], defaultSource: "primary" }, + }; + const validated = ControlPayload(payload); + expect(validated instanceof type.errors).toBe(true); + }); + + test("rejects a missing sources list", () => { + const payload = { + type: "sources-updated", + data: { defaultSource: "primary" }, + }; + const validated = ControlPayload(payload); + expect(validated instanceof type.errors).toBe(true); + }); + + test("rejects a missing defaultSource", () => { + const payload = { + type: "sources-updated", + data: { sources: [source] }, + }; + const validated = ControlPayload(payload); + expect(validated instanceof type.errors).toBe(true); + }); + + test("rejects an empty-string defaultSource", () => { + const payload = { + type: "sources-updated", + data: { sources: [source], defaultSource: "" }, + }; + const validated = ControlPayload(payload); + expect(validated instanceof type.errors).toBe(true); + }); + + test("rejects a defaultSource that is not the head source", () => { + // The wire boundary owns the head-is-default invariant: the first + // element must be the default source, so the warm-swap and cold-build + // rotation paths agree on the active source. A default that is present + // but not first is still rejected here. + const second = { ...source, id: "secondary" }; + const payload = { + type: "sources-updated", + data: { sources: [source, second], defaultSource: "secondary" }, + }; + const validated = ControlPayload(payload); + expect(validated instanceof type.errors).toBe(true); + }); + + test("rejects duplicate source ids", () => { + const payload = { + type: "sources-updated", + data: { sources: [source, { ...source }], defaultSource: "primary" }, + }; + const validated = ControlPayload(payload); + expect(validated instanceof type.errors).toBe(true); + }); + + test("rejects a source element missing a required field", () => { + const { apiKey: _apiKey, ...sourceWithoutApiKey } = source; + const payload = { + type: "sources-updated", + data: { sources: [sourceWithoutApiKey], defaultSource: "primary" }, + }; + const validated = ControlPayload(payload); + expect(validated instanceof type.errors).toBe(true); + }); +}); + describe("substrate.merge.request payload validation", () => { test("accepts a well-formed substrate.merge.request", () => { const payload = { diff --git a/packages/workflow-host/src/mail-bus/hub-transport-adapter.ts b/packages/workflow-host/src/mail-bus/hub-transport-adapter.ts index f4caf971..26a78007 100644 --- a/packages/workflow-host/src/mail-bus/hub-transport-adapter.ts +++ b/packages/workflow-host/src/mail-bus/hub-transport-adapter.ts @@ -10,9 +10,9 @@ // adapter fans the bytes out to every subscribed handler. The // transport itself is treated as a sink the host already owns -- the // adapter does not register addresses on the transport on the -// supervisor's behalf (production sidecars already register via -// their own `provisionAgent` flow) and does not double-deliver -// messages the transport itself routes. +// supervisor's behalf (production sidecars register the head's address +// through the deploy router before the supervisor spawns) and does not +// double-deliver messages the transport itself routes. import type { OutboundMessage, SendReceipt } from "@intx/types/runtime"; @@ -44,10 +44,9 @@ export interface HubTransportMailBusAdapter extends MailBusBindings { * Wrap an existing `HubTransport` instance as the supervisor-facing * `MailBusBindings` shape. The `transport` argument is held only as * a sink-side reference the adapter does not actively reach into - * today -- production wiring uses the existing `provisionAgent` - * flow to register agent addresses on the transport, and the - * adapter delivers inbound bytes through `routeInbound` directly - * into the per-address subscriber map below. + * today -- the sidecar's deploy router registers the head's address + * on the transport, and the adapter delivers inbound bytes through + * `routeInbound` directly into the per-address subscriber map below. */ export function wrapHubTransportAsMailBus( transport: HubTransport, @@ -55,11 +54,10 @@ export function wrapHubTransportAsMailBus( const subscribers = new Map void>>(); return { registerAddress(address: string) { - // The supervisor's address registration is the seam for the - // multi-step branch where the workflow-process child owns - // its own mailbox; production sidecars register the trivial - // address through `SessionManager.provisionAgent` before any - // workflow-host hook runs, so this method is intentionally + // The supervisor's address registration is the seam where the + // workflow-process child owns its own mailbox; production + // sidecars register the head's address through the deploy router + // before the supervisor spawns, so this method is intentionally // inert. The `transport` reference is retained so a future // routing change that wants the bus to own registration has // the handle in scope. diff --git a/packages/workflow-host/src/supervisor/child-termination.test.ts b/packages/workflow-host/src/supervisor/child-termination.test.ts new file mode 100644 index 00000000..7f9a7c32 --- /dev/null +++ b/packages/workflow-host/src/supervisor/child-termination.test.ts @@ -0,0 +1,144 @@ +import { describe, test, expect } from "bun:test"; + +import { getLogger } from "@intx/log"; + +import { + killChildHandle, + waitDeadline, + defaultSetTimer, +} from "./child-termination"; +import type { SubprocessHandle } from "./types"; + +const logger = getLogger(["workflow-host", "supervisor", "child-termination"]); + +function emptyControlReader(): AsyncIterableIterator { + return (async function* () { + /* no control frames in these unit tests */ + })(); +} + +function emptyEventReader(): AsyncIterableIterator { + return (async function* () { + /* no event frames in these unit tests */ + })(); +} + +/** + * A `SubprocessHandle` whose only live surface is `kill` (recording the + * signals it receives) and `exited`. When `sigtermExits` is false the child + * ignores SIGTERM and settles `exited` only on SIGKILL -- the wedged-child + * shape the escalation branch exists for. The stream fields are inert + * stubs; `killChildHandle` never touches them. + */ +function makeHandle(opts: { sigtermExits: boolean }): { + handle: SubprocessHandle; + killSignals: string[]; +} { + const killSignals: string[] = []; + let resolveExit: ((code: number) => void) | undefined; + const exited = new Promise((resolve) => { + resolveExit = resolve; + }); + const handle: SubprocessHandle = { + pid: 1234, + controlWriter: { write: () => Promise.resolve() }, + controlReader: { read: emptyControlReader }, + eventReader: { read: emptyEventReader }, + kill: (signal?: number | string) => { + const sig = typeof signal === "string" ? signal : String(signal ?? ""); + killSignals.push(sig); + if (opts.sigtermExits || sig === "SIGKILL") { + resolveExit?.(0); + } + }, + exited, + }; + return { handle, killSignals }; +} + +describe("killChildHandle", () => { + test("a child that exits on SIGTERM is not escalated to SIGKILL", async () => { + const { handle, killSignals } = makeHandle({ sigtermExits: true }); + let cleared = 0; + + await killChildHandle(handle, 5_000, { + logger, + // A child that exits before the deadline means the timer callback + // never fires; the deadline handle is still cleared exactly once. + setTimer: () => Symbol("timer"), + clearTimer: () => { + cleared += 1; + }, + }); + + expect(killSignals).toEqual(["SIGTERM"]); + expect(cleared).toBe(1); + }); + + test("a child that ignores SIGTERM is escalated to SIGKILL when the deadline wins", async () => { + const { handle, killSignals } = makeHandle({ sigtermExits: false }); + let cleared = 0; + const scheduled: (() => void)[] = []; + + // waitDeadline calls setTimer synchronously, before killChildHandle's + // first await, so the callback is captured by the time this call returns + // its pending promise. Firing it makes the deadline win the race. + const pending = killChildHandle(handle, 5_000, { + logger, + setTimer: (cb) => { + scheduled.push(cb); + return Symbol("timer"); + }, + clearTimer: () => { + cleared += 1; + }, + }); + expect(scheduled).toHaveLength(1); + scheduled[0]?.(); + await pending; + + expect(killSignals).toEqual(["SIGTERM", "SIGKILL"]); + expect(cleared).toBe(1); + }); + + test("omitting the timer deps falls back to real timers and still escalates", async () => { + const { handle, killSignals } = makeHandle({ sigtermExits: false }); + + // No setTimer/clearTimer keys -- the omit-key path the recycle call site + // takes when its context timers are undefined. A tiny real timeout drives + // the escalation without a fake timer. + await killChildHandle(handle, 5, { logger }); + + expect(killSignals).toEqual(["SIGTERM", "SIGKILL"]); + }); +}); + +describe("waitDeadline", () => { + test("resolves via the injected timer and surfaces the timer handle", async () => { + const scheduled: (() => void)[] = []; + const sentinel = Symbol("timer"); + const { promise, handle } = waitDeadline((cb) => { + scheduled.push(cb); + return sentinel; + }, 1_000); + + expect(handle).toBe(sentinel); + let resolved = false; + void promise.then(() => { + resolved = true; + }); + expect(resolved).toBe(false); + scheduled[0]?.(); + await promise; + expect(resolved).toBe(true); + }); + + test("defaultSetTimer schedules the callback via the real event loop", async () => { + let fired = false; + defaultSetTimer(() => { + fired = true; + }, 1); + await new Promise((r) => setTimeout(r, 15)); + expect(fired).toBe(true); + }); +}); diff --git a/packages/workflow-host/src/supervisor/child-termination.ts b/packages/workflow-host/src/supervisor/child-termination.ts new file mode 100644 index 00000000..13e1f81c --- /dev/null +++ b/packages/workflow-host/src/supervisor/child-termination.ts @@ -0,0 +1,107 @@ +// Child termination plus the resolve-only deadline / injectable-timer +// primitives its escalation drives. Two supervisor paths need to bound a +// wait on a workflow-process child with a timer and then force the child +// down: the recycle path (SIGTERM -> deadline -> SIGKILL between cohorts) +// and the spawn path's ready-handshake timeout (a child that spawns but +// never emits `ready`). Both need the same `killChildHandle` escalation +// and the same resolve-only `waitDeadline` raced against a child event, +// and both need injectable timers so tests can drive the deadline +// deterministically. Factoring them here keeps one implementation instead +// of a copy per path. + +import { getLogger } from "@intx/log"; + +import type { SubprocessHandle } from "./types"; + +/** + * Default kill-timeout between SIGTERM and SIGKILL. Used when a caller + * supplies no override; the recycle path exposes it as a per-deployment + * override and the spawn ready-timeout teardown uses it directly. + */ +export const DEFAULT_KILL_TIMEOUT_MS = 5_000; + +/** + * Default deadline for a child's `ready` handshake -- the window a freshly + * spawned child has to emit `ready` before the supervisor kills it and + * treats the spawn (or recycle respawn) as failed. Used when a caller + * supplies no override. Shared here, alongside the kill default, so both + * the spawn path and the recycle path bound the handshake identically + * without either re-declaring the constant. + */ +export const DEFAULT_READY_TIMEOUT_MS = 30_000; + +/** + * Injected dependencies for `killChildHandle`. `setTimer`/`clearTimer` + * default to the real `setTimeout`/`clearTimeout` when omitted so tests + * can substitute a deterministic timer. `logger` is supplied by the + * caller so the SIGKILL-escalation warning is attributed to the path that + * initiated the kill (recycle vs. spawn) rather than a single shared + * namespace. + */ +export interface KillChildHandleDeps { + setTimer?: (cb: () => void, ms: number) => unknown; + clearTimer?: (handle: unknown) => void; + logger: ReturnType; +} + +/** + * Issue SIGTERM and wait for the child to exit. If the exit does not land + * within `killTimeoutMs`, escalate to SIGKILL and wait again. SIGKILL is + * unignorable, so `exited` is guaranteed to settle -- a child that traps + * or never services SIGTERM cannot wedge this call. The supervisor's + * spawner returns the `exited` promise; this helper does not consult OS + * primitives directly. + */ +export async function killChildHandle( + handle: SubprocessHandle, + killTimeoutMs: number, + deps: KillChildHandleDeps, +): Promise { + const setTimer = deps.setTimer ?? defaultSetTimer; + const clearTimer = deps.clearTimer ?? defaultClearTimer; + + handle.kill("SIGTERM"); + const sigTermDeadline = waitDeadline(setTimer, killTimeoutMs); + const exitedFirst = await Promise.race([ + handle.exited.then(() => "exited" as const), + sigTermDeadline.promise.then(() => "deadline" as const), + ]); + if (exitedFirst === "exited") { + clearTimer(sigTermDeadline.handle); + return; + } + clearTimer(sigTermDeadline.handle); + deps.logger + .warn`child termination: SIGTERM did not land within ${String(killTimeoutMs)}ms; escalating to SIGKILL`; + handle.kill("SIGKILL"); + await handle.exited.catch(() => { + /* swallowed: a non-zero exit on SIGKILL is the expected outcome; + termination treats handle exit as success regardless of code. */ + }); +} + +/** + * A resolve-only deadline: a promise that resolves after `ms` via the + * injected `setTimer`, plus the timer handle so the caller can cancel it + * with the matching `clearTimer` once the race settles. It only ever + * resolves, so it contributes no rejection of its own to a race. + */ +export function waitDeadline( + setTimer: (cb: () => void, ms: number) => unknown, + ms: number, +): { promise: Promise; handle: unknown } { + let h: unknown; + const promise = new Promise((resolve) => { + h = setTimer(() => resolve(), ms); + }); + return { promise, handle: h }; +} + +export function defaultSetTimer(cb: () => void, ms: number): unknown { + return setTimeout(cb, ms); +} + +export function defaultClearTimer(handle: unknown): void { + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- production wiring; the handle is the value `setTimeout` returned, narrowed back at the boundary + clearTimeout(handle as ReturnType); +} diff --git a/packages/workflow-host/src/supervisor/credentials.ts b/packages/workflow-host/src/supervisor/credentials.ts index c249404b..1c6343ef 100644 --- a/packages/workflow-host/src/supervisor/credentials.ts +++ b/packages/workflow-host/src/supervisor/credentials.ts @@ -126,10 +126,11 @@ export type AssembleCredentialsSnapshotOpts = { /** * Default mapping from `(deploymentId, stepId)` to the agent-state - * repo's identity. Multi-step deployments use `- - * ` to keep each step's grants isolated to its own repo; - * trivial deployments collapse to the same convention with the - * sole step's id appended. + * repo id: `-`, isolating each step's grants in + * its own repo. Applied to a one-step `stepOrder` it yields a single + * such repo. The single-step launched-agent deploy overrides this + * default (see `DeriveStepRepoId`) to reuse the legacy agent-state + * repo. */ export function defaultStepRepoId(args: { deploymentId: string; @@ -150,8 +151,8 @@ export function defaultStepRepoId(args: { * through the git object database. * * A missing grants file is treated as "no grants" (empty array) so - * the trivial path with no operator-supplied grants does not crash; - * a malformed file does crash, because the file's presence implies + * a step whose repo carries no grants file does not crash; a + * malformed file does crash, because the file's presence implies * the deploy orchestrator intended a snapshot and a structural * failure is a programming bug at the boundary. */ diff --git a/packages/workflow-host/src/supervisor/index.ts b/packages/workflow-host/src/supervisor/index.ts index 9ef4574b..0fe3b511 100644 --- a/packages/workflow-host/src/supervisor/index.ts +++ b/packages/workflow-host/src/supervisor/index.ts @@ -4,6 +4,7 @@ export { type CancelCommitInfo, type CancelRequestOpts, type DeliverSignalOpts, + type DeliverSourcesOpts, type DrainOpts, type RecycleOpts, type SpawnOpts, @@ -31,13 +32,6 @@ export { type CommitCancelRequestedResult, } from "./cancel-signing"; -export { - commitRunEvent, - type CommitRunEventOpts, - type CommitRunEventResult, - type SupervisorRunEvent, -} from "./run-event-signing"; - export { createDrainTimeoutAccumulator, DEFAULT_DRAIN_TIMEOUT_MS, @@ -46,10 +40,11 @@ export { type DrainTimeoutOpts, } from "./drain-timeout"; +export { DEFAULT_KILL_TIMEOUT_MS } from "./child-termination"; + export { createRecyclePolicy, triggerRecycle, - DEFAULT_KILL_TIMEOUT_MS, DEFAULT_POLICY_INTERVAL_MS, MAX_BUFFERED_MAIL, type ChildWiring, @@ -71,15 +66,11 @@ export type { MailAuditRef, MailBusBindings, PrincipalSigner, - RecordRunEvent, SignedPayload, SubprocessHandle, SubprocessSpawner, - SupervisorDeployFrame, TerminalEventSource, TerminalRunEvent, - TrivialLaunch, - TrivialLaunchBindings, WorkflowSupervisorBindings, WorkflowSupervisorPrincipalKind, } from "./types"; diff --git a/packages/workflow-host/src/supervisor/lifecycle-races.test.ts b/packages/workflow-host/src/supervisor/lifecycle-races.test.ts index a78be829..d08c85b1 100644 --- a/packages/workflow-host/src/supervisor/lifecycle-races.test.ts +++ b/packages/workflow-host/src/supervisor/lifecycle-races.test.ts @@ -343,16 +343,15 @@ async function buildBindings(opts: { subprocessSpawner: opts.spawner, binaryPath: "/fake/bin/workflow-child", substrateEnv: { DATA_DIR: opts.baseDir }, + dynamicSpawnEnv: () => ({}), workflowRunRepoId: { kind: "workflow-run", id: "deployment-x" }, workflowRunRef: "refs/heads/main", deploymentId: "deployment-x", + stepCount: 1, deploymentMailAddress: "deployment-x@example.com", readPrincipal: { kind: "supervisor" }, deriveStepAddress: ({ deploymentId, stepId }) => `${deploymentId}-${stepId}@example.com`, - trivialLaunch: () => { - throw new Error("trivialLaunch must not run in lifecycle-races tests"); - }, ipcKeyPairFactory: () => Promise.resolve(opts.ipcKeypair), inboxPrimitives: createMemoryInboxPrimitives(), }; diff --git a/packages/workflow-host/src/supervisor/outbound-signed-send.test.ts b/packages/workflow-host/src/supervisor/outbound-signed-send.test.ts index d9ddaed3..5e8012e2 100644 --- a/packages/workflow-host/src/supervisor/outbound-signed-send.test.ts +++ b/packages/workflow-host/src/supervisor/outbound-signed-send.test.ts @@ -261,9 +261,11 @@ describe("supervisor-backed outbound signed send (Phase 4.3)", () => { }, binaryPath: "/fake/bin/workflow-child", substrateEnv: {}, + dynamicSpawnEnv: () => ({}), workflowRunRepoId: { kind: "workflow-run", id: "outbound-dep" }, workflowRunRef: "refs/heads/main", deploymentId: "outbound-dep", + stepCount: 1, deploymentMailAddress: AGENT_ADDRESS, readPrincipal: { kind: "supervisor" }, deriveStepAddress: () => AGENT_ADDRESS, @@ -272,9 +274,6 @@ describe("supervisor-backed outbound signed send (Phase 4.3)", () => { // getRepoDir resolves the dir; no grants file is needed because the // outbound path never reads grants. deriveStepRepoId: () => ({ kind: "agent-state", id: "outbound-dep" }), - trivialLaunch: () => { - throw new Error("trivialLaunch not used in this test"); - }, inboxPrimitives: createNoopInboxPrimitives(), ipcKeyPairFactory: () => Promise.resolve(supervisorIpcKeyPair), }); diff --git a/packages/workflow-host/src/supervisor/recycle.test.ts b/packages/workflow-host/src/supervisor/recycle.test.ts index e53db2af..d4d2c1e9 100644 --- a/packages/workflow-host/src/supervisor/recycle.test.ts +++ b/packages/workflow-host/src/supervisor/recycle.test.ts @@ -31,6 +31,7 @@ import { type WorkflowSupervisorBindings, } from "./index"; import { createRecyclePolicy, type RecyclePolicyBounds } from "./recycle"; +import { parseSpawnTimeEnv } from "../child/env-bootstrap"; import { defaultStepRepoId, STEP_GRANTS_PATH } from "./credentials"; import { createControlChannelSender, @@ -243,6 +244,14 @@ async function seedStepGrants( ); } +async function corruptStepGrants( + baseDir: string, + repoId: RepoId, +): Promise { + const dir = path.join(baseDir, repoId.kind, repoId.id); + await fs.writeFile(path.join(dir, STEP_GRANTS_PATH), "{ this is not json"); +} + type FakeChild = { pid: number; channelId: string | undefined; @@ -257,12 +266,18 @@ type FakeChild = { type SpawnTracker = { spawner: SubprocessSpawner; children: FakeChild[]; + spawnEnvs: Record[]; totalSpawns: number; }; -function createSpawnTracker(opts: { sigtermExits?: boolean }): SpawnTracker { +function createSpawnTracker(opts: { + sigtermExits?: boolean; + killThrows?: boolean; +}): SpawnTracker { const children: FakeChild[] = []; + const spawnEnvs: Record[] = []; const spawner: SubprocessSpawner = ({ env }) => { + spawnEnvs.push(env); const supervisorToChild = createMemoryNdjsonStream(); const childToSupervisor = createMemoryNdjsonStream(); const eventChildToSupervisor = createMemoryFrameStream(); @@ -295,6 +310,11 @@ function createSpawnTracker(opts: { sigtermExits?: boolean }): SpawnTracker { childToSupervisor.close(); child.resolveExit?.(0); } + // Settle the exit above BEFORE throwing so a shutdown awaiting + // `exited` does not hang; the throw exercises the kill-step guard. + if (opts.killThrows === true) { + throw new Error("child kill boom"); + } }, exited, }; @@ -303,6 +323,7 @@ function createSpawnTracker(opts: { sigtermExits?: boolean }): SpawnTracker { return { spawner, children, + spawnEnvs, get totalSpawns() { return children.length; }, @@ -468,16 +489,15 @@ async function buildBindings(opts: { subprocessSpawner: opts.spawner, binaryPath: "/fake/bin/workflow-child", substrateEnv: { DATA_DIR: opts.baseDir }, + dynamicSpawnEnv: () => ({}), workflowRunRepoId: { kind: "workflow-run", id: "deployment-x" }, workflowRunRef: "refs/heads/main", deploymentId: "deployment-x", + stepCount: 1, deploymentMailAddress: "deployment-x@example.com", readPrincipal: { kind: "supervisor" }, deriveStepAddress: ({ deploymentId, stepId }) => `${deploymentId}-${stepId}@example.com`, - trivialLaunch: () => { - throw new Error("trivialLaunch must not run in recycle tests"); - }, ipcKeyPairFactory: () => Promise.resolve(opts.ipcKeypair), inboxPrimitives: opts.inboxPrimitives ?? createMemoryInboxPrimitives(), ...(opts.recyclePolicy !== undefined @@ -528,7 +548,221 @@ async function spawnSupervisor(opts: { return { supervisor, spawnResult: result, firstSender }; } +describe("supervisor spawn: failure cleanup", () => { + test("a spawn that times out releases the mail subscription and address", async () => { + // Regression: spawn registers + subscribes the deployment mail address + // before the ready handshake. A handshake that times out (or the child + // exiting mid-handshake) must release that subscription and unregister + // the address; otherwise a live subscription lingers with no dispatch + // loop to service it and a redeploy adds a second one. + const baseDir = await makeTempDir("spawn-timeout-"); + const ipcKeypair = await generateKeyPair(); + const mailBus = createMockMailBus(); + // sigtermExits so the timeout path's kill settles the fake child. + const tracker = createSpawnTracker({ sigtermExits: true }); + await seedStepGrants( + baseDir, + defaultStepRepoId({ deploymentId: "deployment-x", stepId: "step-1" }), + [{ resource: "thing", action: "read" }], + ); + const bindings = await buildBindings({ + baseDir, + spawner: tracker.spawner, + mailBus, + ipcKeypair, + }); + const supervisor = createWorkflowSupervisor({ + ...bindings, + readyTimeoutMs: 20, + }); + + // Spawn but never drive the child's `ready` frame, so the handshake + // times out. + await expect( + supervisor.spawn({ + stepOrder: ["step-1"], + definitionHash: "def-hash-abc", + warmKeep: false, + onInferenceEvent: () => undefined, + }), + ).rejects.toThrow(/did not emit ready/); + + // The address must not remain registered, and the history must carry + // both the register and its matching unregister. + expect(mailBus.registered()).not.toContain("deployment-x@example.com"); + expect(mailBus.registrationHistory()).toContain( + "register:deployment-x@example.com", + ); + expect(mailBus.registrationHistory()).toContain( + "unregister:deployment-x@example.com", + ); + }); + + test("a throwing teardown step during spawn cleanup is absorbed", async () => { + // On a failed spawn the spawn-catch unwinds the cohort through + // shutdownInternal. Its teardown steps are individually guarded and the + // child kill lives in its `finally`, so a throwing step (here the mail + // unsubscribe) is absorbed end to end: the spawn still rejects with the + // real ready-timeout cause rather than the secondary teardown error, + // and the child is still reaped. The spawn-catch guard that preserves + // the cause is now defense-in-depth -- shutdownInternal is total by + // construction -- so the discriminating coverage of the teardown guards + // themselves lives in the "supervisor shutdown: teardown robustness" + // block below, which drives shutdownInternal directly. + const baseDir = await makeTempDir("spawn-teardown-mask-"); + const ipcKeypair = await generateKeyPair(); + const baseMailBus = createMockMailBus(); + // The mail unsubscribe throws during the spawn-failure unwind. + const mailBus: MailBusBindings = { + ...baseMailBus, + subscribeMailForAddress(address, handler) { + baseMailBus.subscribeMailForAddress(address, handler); + return () => { + throw new Error("mail unsubscribe boom during teardown"); + }; + }, + }; + // sigtermExits so the ready-timeout reap settles the fake child. + const tracker = createSpawnTracker({ sigtermExits: true }); + await seedStepGrants( + baseDir, + defaultStepRepoId({ deploymentId: "deployment-x", stepId: "step-1" }), + [{ resource: "thing", action: "read" }], + ); + const bindings = await buildBindings({ + baseDir, + spawner: tracker.spawner, + mailBus, + ipcKeypair, + }); + const supervisor = createWorkflowSupervisor({ + ...bindings, + readyTimeoutMs: 20, + }); + + // Never drive `ready`, so the handshake times out. The spawn cause + // (ready timeout) must surface even though the teardown then throws + // its own error. + await expect( + supervisor.spawn({ + stepOrder: ["step-1"], + definitionHash: "def-hash-abc", + warmKeep: false, + onInferenceEvent: () => undefined, + }), + ).rejects.toThrow(/did not emit ready/); + + // The child was reaped during the unwind despite the throwing + // unsubscribe -- a throwing teardown step leaves nothing orphaned. + const child = tracker.children[0]; + if (child === undefined) { + throw new Error("tracker.children[0] missing"); + } + expect(child.killSignals.length).toBeGreaterThan(0); + }); +}); + +describe("supervisor spawn: dynamic env", () => { + test("a recycle respawn re-pulls dynamicSpawnEnv so a host revision survives", async () => { + // Mechanism behind a source rotation surviving a recycle: the spawn-env + // builder pulls dynamicSpawnEnv() on every spawn AND respawn, so a value + // the host revised between the two reaches the respawned child instead + // of reverting to the frozen substrateEnv value. + const baseDir = await makeTempDir("recycle-dynenv-"); + const ipcKeypair = await generateKeyPair(); + const mailBus = createMockMailBus(); + const tracker = createSpawnTracker({}); + await seedStepGrants( + baseDir, + defaultStepRepoId({ deploymentId: "deployment-x", stepId: "step-1" }), + [{ resource: "thing", action: "read" }], + ); + let hostRevised = "before-rotation"; + const bindings = await buildBindings({ + baseDir, + spawner: tracker.spawner, + mailBus, + ipcKeypair, + }); + const supervisor = createWorkflowSupervisor({ + ...bindings, + dynamicSpawnEnv: () => ({ DYNAMIC_MARKER: hostRevised }), + }); + + const spawnPromise = supervisor.spawn({ + stepOrder: ["step-1"], + definitionHash: "def-hash-abc", + warmKeep: false, + onInferenceEvent: () => undefined, + }); + while (tracker.children.length === 0) { + await new Promise((r) => setTimeout(r, 1)); + } + const firstChild = tracker.children[0]; + if (firstChild === undefined) throw new Error("first child missing"); + await driveReady(firstChild, ipcKeypair); + await spawnPromise; + expect(tracker.spawnEnvs[0]?.DYNAMIC_MARKER).toBe("before-rotation"); + + // The host revises the dynamic value (models a live source rotation), + // then a recycle respawns the child. + hostRevised = "after-rotation"; + const recyclePromise = supervisor.recycle({ reason: "dynenv" }); + while (tracker.children.length < 2) { + await new Promise((r) => setTimeout(r, 1)); + } + const secondChild = tracker.children[1]; + if (secondChild === undefined) throw new Error("second child missing"); + await driveReady(secondChild, ipcKeypair); + await recyclePromise; + + // The respawn env carries the REVISED value -- the rotation survived. + expect(tracker.spawnEnvs[1]?.DYNAMIC_MARKER).toBe("after-rotation"); + }); +}); + describe("supervisor recycle: operator-initiated", () => { + test("the recycle respawn env satisfies the child spawn-time env contract", async () => { + // Regression: the recycle respawn env must carry every required + // spawn-time key. A recycle env that omitted a required key (STEP_COUNT) + // made the respawned child abort in parseSpawnTimeEnv before opening + // IPC, so every recycle tore the deployment down. The fake spawner does + // not run parseSpawnTimeEnv, so this asserts the captured env against it + // directly -- the check the real child binary performs at boot. + const baseDir = await makeTempDir("recycle-env-"); + const ipcKeypair = await generateKeyPair(); + const mailBus = createMockMailBus(); + const tracker = createSpawnTracker({}); + const { supervisor } = await spawnSupervisor({ + baseDir, + tracker, + mailBus, + ipcKeypair, + }); + + const recyclePromise = supervisor.recycle({ reason: "env-contract" }); + while (tracker.children.length < 2) { + await new Promise((r) => setTimeout(r, 1)); + } + const secondChild = tracker.children[1]; + if (secondChild === undefined) { + throw new Error("second child missing after recycle spawn"); + } + await driveReady(secondChild, ipcKeypair); + await recyclePromise; + + expect(tracker.spawnEnvs).toHaveLength(2); + const spawnEnv = tracker.spawnEnvs[0]; + const respawnEnv = tracker.spawnEnvs[1]; + if (spawnEnv === undefined || respawnEnv === undefined) { + throw new Error("expected two captured spawn envs"); + } + // Both the initial spawn and the recycle respawn must satisfy the + // child's spawn-time env contract, with the same step count. + expect(parseSpawnTimeEnv(spawnEnv).stepCount).toBe(1); + expect(parseSpawnTimeEnv(respawnEnv).stepCount).toBe(1); + }); + test("drain mail sends, child is killed, fresh child spawns with a new channelId", async () => { const baseDir = await makeTempDir("recycle-op-"); const ipcKeypair = await generateKeyPair(); @@ -638,6 +872,136 @@ describe("supervisor recycle: failure after the cohort handoff", () => { }); }); +describe("supervisor recycle: respawn handshake bound", () => { + test("a respawn that never emits ready times out, reaps the new child, and reaches a clean stopped state", async () => { + const baseDir = await makeTempDir("recycle-ready-timeout-"); + const ipcKeypair = await generateKeyPair(); + const mailBus = createMockMailBus(); + // sigtermExits:false so the reap escalates SIGTERM -> SIGKILL: a wedged + // respawn is exactly the child that ignores SIGTERM, and the reap must + // still guarantee it dies. + const tracker = createSpawnTracker({ sigtermExits: false }); + await seedStepGrants( + baseDir, + defaultStepRepoId({ deploymentId: "deployment-x", stepId: "step-1" }), + [{ resource: "thing", action: "read" }], + ); + const bindings = await buildBindings({ + baseDir, + spawner: tracker.spawner, + mailBus, + ipcKeypair, + }); + const supervisor = createWorkflowSupervisor({ + ...bindings, + // Collapse the recycle path's ready deadline and kill-escalation + // deadline to the next macrotask so the wedged-respawn path resolves + // without real-time waits. The child never emits ready, so timeout + // always wins the handshake race. The spawn path keeps the default + // real timer, so the first child readies normally. + recyclePolicySetTimer: (cb) => setTimeout(cb, 0), + recyclePolicyClearTimer: () => undefined, + }); + const spawnPromise = supervisor.spawn({ + stepOrder: ["step-1"], + definitionHash: "def-hash-abc", + warmKeep: false, + onInferenceEvent: () => undefined, + }); + while (tracker.children.length === 0) { + await new Promise((r) => setTimeout(r, 1)); + } + const first = tracker.children[0]; + if (first === undefined) throw new Error("first child missing"); + await driveReady(first, ipcKeypair); + await spawnPromise; + + // Recycle spawns a second child but never drives its ready frame, so + // the bounded handshake times out. + let caught: unknown; + try { + await supervisor.recycle({ reason: "ready-timeout-test" }); + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(Error); + expect(caught instanceof Error && caught.message).toMatch( + /did not emit ready/, + ); + + // The wedged respawn was reaped through the full SIGTERM -> SIGKILL + // ladder rather than left running as an orphan. + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + expect(second.killSignals).toEqual(["SIGTERM", "SIGKILL"]); + + // The recycle failure routed through shutdownInternal to `stopped`, so + // a follow-up shutdown is a no-op. + await supervisor.shutdown(); + }); + + test("a respawn whose control channel ends before ready rejects and reaps the new child", async () => { + const baseDir = await makeTempDir("recycle-ready-failed-"); + const ipcKeypair = await generateKeyPair(); + const mailBus = createMockMailBus(); + // sigtermExits:true so the reap's SIGTERM settles the fake child; this + // path proves the child is killed even though the control channel end + // does not itself terminate the process. + const tracker = createSpawnTracker({ sigtermExits: true }); + await seedStepGrants( + baseDir, + defaultStepRepoId({ deploymentId: "deployment-x", stepId: "step-1" }), + [{ resource: "thing", action: "read" }], + ); + const bindings = await buildBindings({ + baseDir, + spawner: tracker.spawner, + mailBus, + ipcKeypair, + }); + // Default (real, 30s) ready timer: the deadline must NOT fire, so the + // control-channel-end failure wins the handshake race deterministically. + const supervisor = createWorkflowSupervisor(bindings); + const spawnPromise = supervisor.spawn({ + stepOrder: ["step-1"], + definitionHash: "def-hash-abc", + warmKeep: false, + onInferenceEvent: () => undefined, + }); + while (tracker.children.length === 0) { + await new Promise((r) => setTimeout(r, 1)); + } + const first = tracker.children[0]; + if (first === undefined) throw new Error("first child missing"); + await driveReady(first, ipcKeypair); + await spawnPromise; + + const recyclePromise = supervisor.recycle({ reason: "ready-failed-test" }); + while (tracker.children.length < 2) { + await new Promise((r) => setTimeout(r, 1)); + } + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + // End the new child's control channel before it emits ready: the + // handshake fails rather than hangs. + second.childToSupervisor.close(); + + let caught: unknown; + try { + await recyclePromise; + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(Error); + expect(caught instanceof Error && caught.message).toMatch( + /control channel ended before child emitted ready/, + ); + expect(second.killSignals).toContain("SIGTERM"); + + await supervisor.shutdown(); + }); +}); + describe("supervisor recycle: deliverSignal phase guard", () => { test("deliverSignal during the recycling window rejects rather than writing to the dying cohort's controlSender", async () => { // M2: during `recycling`, the supervisor's `state.controlSender` @@ -1287,3 +1651,126 @@ describe("supervisor recycle: external drain phase guard", () => { await supervisor.shutdown(); }); }); + +describe("supervisor recycle: respawn credentials-read failure", () => { + test("a respawn whose credentials re-read throws reaps the freshly-spawned child", async () => { + const baseDir = await makeTempDir("recycle-credleak-"); + const ipcKeypair = await generateKeyPair(); + const mailBus = createMockMailBus(); + // sigtermExits so the reap's SIGTERM settles the fake child. + const tracker = createSpawnTracker({ sigtermExits: true }); + const { supervisor } = await spawnSupervisor({ + baseDir, + tracker, + mailBus, + ipcKeypair, + }); + expect(tracker.totalSpawns).toBe(1); + + // Corrupt the step grants so the recycle's pre-handshake + // assembleCredentialsSnapshot re-read throws -- the recycle doubles + // as the grant-refresh path, so a malformed grants file is exactly + // the read that fails here. + await corruptStepGrants( + baseDir, + defaultStepRepoId({ deploymentId: "deployment-x", stepId: "step-1" }), + ); + + let caught: unknown; + try { + await supervisor.recycle({ reason: "credentials-leak-regression" }); + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(Error); + expect(caught instanceof Error && caught.message).toMatch(/not valid JSON/); + + // The respawn spawned the new child before the credentials read + // threw. Its catch must reap that child rather than leak it: the + // supervisor's recycle-failure teardown reaps only the PRIOR cohort + // (state.handle during `recycling`), so a child that failed before + // installation is invisible to it. + expect(tracker.totalSpawns).toBe(2); + const newChild = tracker.children[1]; + if (newChild === undefined) { + throw new Error("tracker.children[1] missing"); + } + expect(newChild.killSignals.length).toBeGreaterThan(0); + }); +}); + +describe("supervisor shutdown: teardown robustness", () => { + test("shutdown still kills the child and reaches stopped when mail unsubscribe throws", async () => { + const baseDir = await makeTempDir("shutdown-unsub-throw-"); + const ipcKeypair = await generateKeyPair(); + const baseMailBus = createMockMailBus(); + // The mail unsubscribe runs BEFORE the child kill in shutdown. Left + // unguarded, a throw here skips the kill (child leak) and leaves the + // supervisor wedged in `stopping`. + const mailBus: MailBusBindings = { + ...baseMailBus, + subscribeMailForAddress(address, handler) { + baseMailBus.subscribeMailForAddress(address, handler); + return () => { + throw new Error("mail unsubscribe boom during shutdown"); + }; + }, + }; + const tracker = createSpawnTracker({ sigtermExits: true }); + const { supervisor } = await spawnSupervisor({ + baseDir, + tracker, + mailBus, + ipcKeypair, + }); + const child = tracker.children[0]; + if (child === undefined) { + throw new Error("tracker.children[0] missing"); + } + + // Shutdown must resolve despite the throwing unsubscribe. + await supervisor.shutdown(); + + // The kill step was reached even though the unsubscribe (which runs + // first) threw. + expect(child.killSignals.length).toBe(1); + + // The supervisor reached `stopped`, not wedged in `stopping`: a second + // shutdown is an idempotent no-op that returns without re-running the + // (throwing) teardown, so no further kill is issued. + await supervisor.shutdown(); + expect(child.killSignals.length).toBe(1); + }); + + test("shutdown reaches stopped when the child kill throws", async () => { + const baseDir = await makeTempDir("shutdown-kill-throw-"); + const ipcKeypair = await generateKeyPair(); + const mailBus = createMockMailBus(); + // The child kill throws (after settling exit). The `finally` must still + // drive the supervisor to `stopped` rather than leaving it in + // `stopping`. + const tracker = createSpawnTracker({ + sigtermExits: true, + killThrows: true, + }); + const { supervisor } = await spawnSupervisor({ + baseDir, + tracker, + mailBus, + ipcKeypair, + }); + const child = tracker.children[0]; + if (child === undefined) { + throw new Error("tracker.children[0] missing"); + } + + // Shutdown must resolve even though kill threw. + await supervisor.shutdown(); + expect(child.killSignals.length).toBe(1); + + // Reached `stopped`: the second shutdown no-ops rather than re-entering + // teardown (which would call the throwing kill again). + await supervisor.shutdown(); + expect(child.killSignals.length).toBe(1); + }); +}); diff --git a/packages/workflow-host/src/supervisor/recycle.ts b/packages/workflow-host/src/supervisor/recycle.ts index 0da6521c..a60a3d0a 100644 --- a/packages/workflow-host/src/supervisor/recycle.ts +++ b/packages/workflow-host/src/supervisor/recycle.ts @@ -83,7 +83,6 @@ import { getLogger } from "@intx/log"; import { generateKeyPair } from "@intx/crypto"; -import { hexEncode } from "@intx/types"; import { createControlChannelSender, @@ -101,6 +100,15 @@ import { type CredentialsSnapshot, } from "./credentials"; import type { SubprocessHandle, WorkflowSupervisorBindings } from "./types"; +import { buildChildSpawnEnv } from "./spawn-env"; +import { + DEFAULT_KILL_TIMEOUT_MS, + DEFAULT_READY_TIMEOUT_MS, + defaultClearTimer, + defaultSetTimer, + killChildHandle, + waitDeadline, +} from "./child-termination"; const logger = getLogger(["workflow-host", "supervisor", "recycle"]); @@ -112,13 +120,6 @@ const logger = getLogger(["workflow-host", "supervisor", "recycle"]); */ export const MAX_BUFFERED_MAIL = 256; -/** - * Default kill-timeout between SIGTERM and SIGKILL during step 2. - * Operator-overridable per deployment via the supervisor's recycle - * bindings; this is the value used when no override is supplied. - */ -export const DEFAULT_KILL_TIMEOUT_MS = 5_000; - /** * Default supervisor-policy check interval. The policy thread wakes * roughly every minute, evaluates the configured bounds against the @@ -231,6 +232,14 @@ export interface RecycleContext { * `DEFAULT_KILL_TIMEOUT_MS`. */ readonly killTimeoutMs?: number; + /** + * Deadline (ms) for the respawned child's `ready` handshake, matching + * the bound the spawn path applies. The supervisor resolves the + * effective value at its edge (`bindings.readyTimeoutMs ?? + * DEFAULT_READY_TIMEOUT_MS`) and passes it through; the `??` fallback + * here only fires for a direct test caller. + */ + readonly readyTimeoutMs?: number; /** * Optional drain deadline (ms) used in step 1. The supervisor's own * drainTimeout accumulator escalates separately; this deadline is @@ -304,7 +313,11 @@ export async function triggerRecycle( // `killTimeoutMs`, the handle's hard kill lands. The injected // spawner's `SubprocessHandle.kill()` is the surface the recycle // path touches; the spawner owns the Node primitives. - await killChildHandle(ctx.current.handle, killTimeoutMs, ctx); + await killChildHandle(ctx.current.handle, killTimeoutMs, { + logger, + ...(ctx.setTimer !== undefined ? { setTimer: ctx.setTimer } : {}), + ...(ctx.clearTimer !== undefined ? { clearTimer: ctx.clearTimer } : {}), + }); // Step 3: respawn. Fresh channelId, fresh HMAC key, fresh Ed25519 // IPC keypair. Per-step credentials are re-read so a grants update @@ -316,16 +329,18 @@ export async function triggerRecycle( const ipcKeypair = await ( ctx.bindings.ipcKeyPairFactory ?? generateKeyPair )(); - const env: Record = { - ...ctx.bindings.substrateEnv, - IPC_CHANNEL_ID: channelId, - IPC_HMAC_KEY: hexEncode(hmacKey), - HOST_PUBKEY: hexEncode(ipcKeypair.publicKey), - DEPLOYMENT_ID: ctx.bindings.deploymentId, - DEFINITION_HASH: ctx.definitionHash, - MAILBOX_ADDRESS: ctx.bindings.deploymentMailAddress, - WARM_KEEP: ctx.warmKeep ? "true" : "false", - }; + const env = buildChildSpawnEnv({ + substrateEnv: ctx.bindings.substrateEnv, + dynamicSpawnEnv: ctx.bindings.dynamicSpawnEnv, + channelId, + hmacKey, + hostPublicKey: ipcKeypair.publicKey, + deploymentId: ctx.bindings.deploymentId, + deploymentMailAddress: ctx.bindings.deploymentMailAddress, + stepCount: ctx.bindings.stepCount, + definitionHash: ctx.definitionHash, + warmKeep: ctx.warmKeep, + }); const handle = ctx.bindings.subprocessSpawner({ binaryPath: ctx.bindings.binaryPath, @@ -346,6 +361,14 @@ export async function triggerRecycle( }); const readyPromise = waitForReady(controlIncoming); + // Attach a benign handler at creation, before the fold below consumes + // the rejection: `readyPromise` is created here but not raced until + // after `assembleCredentialsSnapshot` awaits. A child that exits during + // that window rejects `readyPromise` with no handler yet attached -- an + // unhandled rejection across the await boundary. Mirrors the spawn + // path's identical guard. Attaching `.catch` here and `.then` at the + // race is fine; both observe the same settled value. + void readyPromise.catch(() => undefined); const eventIter = receiveEventChannel({ hmacKey, @@ -355,25 +378,93 @@ export async function triggerRecycle( }); const eventPump = pumpEvents(eventIter, ctx.onInferenceEvent); + // The child handshake below is deadline-bounded and needs these timer + // bindings; the pre-handshake credentials-read reap needs them too, so + // derive them before that read. + const setTimer = ctx.setTimer ?? defaultSetTimer; + const clearTimer = ctx.clearTimer ?? defaultClearTimer; + // Re-read per-step credentials. A grants update that landed during // the previous child's lifetime is picked up here -- the recycle // doubles as the supervisor's grant-refresh path. The deploy tree // is not consulted; this read is against the `agent-state` repos // alone, whose contents are independent of `workflow.json`. - const credentialsSnapshot = await assembleCredentialsSnapshot({ - repoStore: ctx.bindings.repoStore, - principal: ctx.bindings.readPrincipal, - stepOrder: ctx.stepOrder, - deploymentId: ctx.bindings.deploymentId, - deriveStepAddress: ctx.bindings.deriveStepAddress, - ...(ctx.bindings.deriveStepRepoId !== undefined - ? { deriveStepRepoId: ctx.bindings.deriveStepRepoId } - : {}), - }); + // + // This is a substrate read that can reject -- a grants file that + // became malformed is precisely the recycle's grant-refresh path. The + // new child is already spawned and wired but not yet installed on + // `state`, so the supervisor's recycle-failure teardown (which reaps + // the PRIOR cohort) cannot see it; reap it here on failure or it + // leaks. The spawn path routes this same throw through its teardown + // owner. The try wraps only the awaited read: the handle and pumps it + // reaps are all constructed above, so the reap always has live + // handles. + let credentialsSnapshot: CredentialsSnapshot; + try { + credentialsSnapshot = await assembleCredentialsSnapshot({ + repoStore: ctx.bindings.repoStore, + principal: ctx.bindings.readPrincipal, + stepOrder: ctx.stepOrder, + deploymentId: ctx.bindings.deploymentId, + deriveStepAddress: ctx.bindings.deriveStepAddress, + ...(ctx.bindings.deriveStepRepoId !== undefined + ? { deriveStepRepoId: ctx.bindings.deriveStepRepoId } + : {}), + }); + } catch (cause) { + await reapUnreadyChild(handle, eventPump, controlIncoming, { + killTimeoutMs, + setTimer, + clearTimer, + phase: "credentials read failure", + }); + throw cause; + } // Steps 4 + 5: self-discover + resume. These run inside the child - // before it emits `ready`; the supervisor waits. - const readyInfo = await readyPromise; + // before it emits `ready`; the supervisor waits, but bounded -- a child + // that neither readies nor exits must not park the recycle (and thus + // the supervisor) in `recycling` forever. Bound the handshake exactly + // as the spawn path bounds its own: fold ready/failed into values, race + // a resolve-only deadline, clear the timer on every path. + // + // NOTE: `assembleCredentialsSnapshot` above is a substrate read that + // sits OUTSIDE this deadline; a wedged substrate is a supervisor-side + // fault bounded at its own layer, not by overloading this handshake + // timer. The respawn is deadline-bounded on the child handshake, not on + // that read. + const readyTimeoutMs = ctx.readyTimeoutMs ?? DEFAULT_READY_TIMEOUT_MS; + const readyOutcome = readyPromise.then( + (info) => ({ kind: "ready" as const, info }), + (err: unknown) => ({ kind: "failed" as const, err }), + ); + const readyDeadline = waitDeadline(setTimer, readyTimeoutMs); + const readyRace = await Promise.race([ + readyOutcome, + readyDeadline.promise.then(() => ({ kind: "timeout" as const })), + ]); + clearTimer(readyDeadline.handle); + if (readyRace.kind !== "ready") { + // The new child was never installed on `state`, so the supervisor's + // recycle-failure teardown (which reaps the PRIOR cohort) would leak + // it. Reap it here. `killChildHandle` on an already-dead handle is a + // cheap no-op, so the `failed` path (a control-channel end does not + // guarantee the process died, since the event channel is separate) + // is reaped too, not just the timeout. + await reapUnreadyChild(handle, eventPump, controlIncoming, { + killTimeoutMs, + setTimer, + clearTimer, + phase: "handshake failure", + }); + if (readyRace.kind === "timeout") { + throw new Error( + `workflow-host supervisor recycle: child did not emit ready within ${String(readyTimeoutMs)}ms; killed`, + ); + } + throw readyRace.err; + } + const readyInfo = readyRace.info; logger.info`recycle ${opts.origin}: child ready (pid=${String(readyInfo.childPid)}, newChannelId=${channelId})`; const newWiring: ChildWiring = { @@ -404,57 +495,45 @@ export async function triggerRecycle( } /** - * Issue SIGTERM and wait for the child to exit. If the exit does not - * land within `killTimeoutMs`, escalate to SIGKILL and wait again. - * The supervisor's spawner returns the `exited` promise; the recycle - * path does not consult OS primitives directly. + * Reap a respawned child that was spawned and wired but never installed + * on `state`. Such a child is invisible to the supervisor's + * recycle-failure teardown -- that path reaps the PRIOR cohort + * (`state.handle` during `recycling`) -- so a respawn that fails after + * the spawn must reap the new child here or it leaks its OS process and + * both IPC channels. + * + * Kill FIRST, then finalize the pumps: process death drives EOF on both + * channels, which unparks `waitForReady`'s in-flight `iter.next()` + * (letting `controlIncoming.return` complete) and ends `pumpEvents`. + * Awaiting either finalizer before the kill would hang behind the + * still-open channels. `killChildHandle` on an already-dead handle is a + * cheap no-op, so a child that died on its own -- not just one killed on + * timeout -- is reaped safely too. */ -async function killChildHandle( +async function reapUnreadyChild( handle: SubprocessHandle, - killTimeoutMs: number, - ctx: RecycleContext, + eventPump: Promise, + controlIncoming: AsyncGenerator, + deps: { + killTimeoutMs: number; + setTimer: (cb: () => void, ms: number) => unknown; + clearTimer: (handle: unknown) => void; + phase: string; + }, ): Promise { - const setTimer = ctx.setTimer ?? defaultSetTimer; - const clearTimer = ctx.clearTimer ?? defaultClearTimer; - - handle.kill("SIGTERM"); - const sigTermDeadline = waitDeadline(setTimer, killTimeoutMs); - const exitedFirst = await Promise.race([ - handle.exited.then(() => "exited" as const), - sigTermDeadline.promise.then(() => "deadline" as const), - ]); - if (exitedFirst === "exited") { - clearTimer(sigTermDeadline.handle); - return; - } - clearTimer(sigTermDeadline.handle); - logger.warn`recycle: SIGTERM did not land within ${String(killTimeoutMs)}ms; escalating to SIGKILL`; - handle.kill("SIGKILL"); - await handle.exited.catch(() => { - /* swallowed: a non-zero exit on SIGKILL is the expected outcome; - the recycle path treats handle exit as success regardless of - code. */ + await killChildHandle(handle, deps.killTimeoutMs, { + logger, + setTimer: deps.setTimer, + clearTimer: deps.clearTimer, }); -} - -function waitDeadline( - setTimer: (cb: () => void, ms: number) => unknown, - ms: number, -): { promise: Promise; handle: unknown } { - let h: unknown; - const promise = new Promise((resolve) => { - h = setTimer(() => resolve(), ms); + void eventPump.catch((cause: unknown) => { + const message = cause instanceof Error ? cause.message : String(cause); + logger.warn`recycle: reaped child eventPump failed after ${deps.phase}: ${message}`; + }); + void controlIncoming.return(undefined).catch((cause: unknown) => { + const message = cause instanceof Error ? cause.message : String(cause); + logger.warn`recycle: reaped child controlIncoming.return failed after ${deps.phase}: ${message}`; }); - return { promise, handle: h }; -} - -function defaultSetTimer(cb: () => void, ms: number): unknown { - return setTimeout(cb, ms); -} - -function defaultClearTimer(handle: unknown): void { - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- production wiring; the handle is the value `setTimeout` returned, narrowed back at the boundary - clearTimeout(handle as ReturnType); } /** diff --git a/packages/workflow-host/src/supervisor/run-event-signing.test.ts b/packages/workflow-host/src/supervisor/run-event-compaction.test.ts similarity index 99% rename from packages/workflow-host/src/supervisor/run-event-signing.test.ts rename to packages/workflow-host/src/supervisor/run-event-compaction.test.ts index a3f135c4..fa8163fc 100644 --- a/packages/workflow-host/src/supervisor/run-event-signing.test.ts +++ b/packages/workflow-host/src/supervisor/run-event-compaction.test.ts @@ -17,7 +17,7 @@ import type { WorkflowRunSupervisorPrincipal, } from "@intx/hub-sessions"; -import { compactRunEvents } from "./run-event-signing"; +import { compactRunEvents } from "./run-event-compaction"; const REF = "refs/heads/main"; const allowAll: AuthorizeFn = () => ({ allowed: true }); diff --git a/packages/workflow-host/src/supervisor/run-event-compaction.ts b/packages/workflow-host/src/supervisor/run-event-compaction.ts new file mode 100644 index 00000000..a28a22cd --- /dev/null +++ b/packages/workflow-host/src/supervisor/run-event-compaction.ts @@ -0,0 +1,148 @@ +// Run-event compaction path for the per-deployment supervisor. +// +// Once a workflow run terminates, the supervisor folds that run's +// per-event `events/.json` blobs into one combined `events.jsonl` +// and drops the per-event files. This shrinks the workflow-run repo's +// file count -- and every per-commit cost that scales with it -- +// without losing any event. The fold writes under the substrate's +// per-repo lock as the `supervisor` principal, whose `deploymentId` +// the workflow-run kind handler checks against `repoId.id`. + +import type { + RepoId, + RepoStore as SubstrateRepoStore, + WorkflowRunSupervisorPrincipal, +} from "@intx/hub-sessions/substrate"; +import { + WORKFLOW_RUN_EVENTS_FILE, + encodeCombinedEventLog, +} from "@intx/hub-sessions/substrate"; + +import { SUPERVISOR_PRINCIPAL_KIND } from "./cancel-signing"; + +const RUNS_PREFIX = "runs"; +const EVENTS_DIR = "events"; +const EVENT_FILENAME_RE = /^(0|[1-9][0-9]*)\.json$/; +const TERMINAL_EVENT_TYPES = new Set([ + "RunCompleted", + "RunFailed", + "RunCancelled", +]); + +export type CompactRunEventsOpts = { + /** Substrate handle the supervisor writes through. */ + substrate: SubstrateRepoStore; + /** Workflow-run repo for this deployment. */ + repoId: RepoId; + /** Events ref the workflow-run repo writes to. */ + ref: string; + /** Deployment id used to construct the supervisor principal. */ + deploymentId: string; + /** Run to seal. */ + runId: string; +}; + +/** + * Fold a terminated run's per-event `events/.json` blobs into one + * combined `events.jsonl`, dropping the per-event files. This shrinks the + * repo's file count -- and so every per-commit cost that scales with it -- + * without losing any event. + * + * Idempotent and terminal-only: a run already sealed (no `events/` subtree) + * or one whose latest event is not terminal is left untouched, so the call + * is safe to repeat. The live caller fires it once per run, right after the + * run terminates; a bounded recovery sweep that would re-fire it to seal a + * run whose fold a crash interrupted is not yet implemented. + * + * The combined file is the verbatim byte concatenation of the per-event + * blobs in seq order (`encodeCombinedEventLog`), the exact shape the + * workflow-run kind handler's compaction validation requires. It is written + * as a sibling of `events/`, so returning it from the merge while omitting + * the per-event files lets the substrate's prefix clear drop them. + */ +export async function compactRunEvents( + opts: CompactRunEventsOpts, +): Promise<{ compacted: boolean }> { + const fs = await import("node:fs/promises"); + const path = await import("node:path"); + const dir = opts.substrate.getRepoDir(opts.repoId); + const eventsDir = path.join(dir, RUNS_PREFIX, opts.runId, EVENTS_DIR); + + // Cheap pre-check off the working tree to skip an empty commit when there + // is nothing to seal (already combined, or not yet terminal). The merge + // re-reads the prefix under the per-repo lock, so the seal stays + // consistent if another writer raced in between. + let filenames: string[]; + try { + filenames = await fs.readdir(eventsDir); + } catch (cause) { + if (isErrnoNotFound(cause)) return { compacted: false }; + throw cause; + } + const seqs: number[] = []; + for (const name of filenames) { + const match = EVENT_FILENAME_RE.exec(name); + if (match === null || match[1] === undefined) continue; + seqs.push(Number.parseInt(match[1], 10)); + } + if (seqs.length === 0) return { compacted: false }; + const lastRaw = await fs.readFile( + path.join(eventsDir, `${String(Math.max(...seqs))}.json`), + "utf8", + ); + let parsed: unknown; + try { + parsed = JSON.parse(lastRaw); + } catch { + return { compacted: false }; + } + if (typeof parsed !== "object" || parsed === null || !("type" in parsed)) { + return { compacted: false }; + } + const lastType = (parsed as { type?: unknown }).type; + if (typeof lastType !== "string" || !TERMINAL_EVENT_TYPES.has(lastType)) { + return { compacted: false }; + } + + const prefix = `${RUNS_PREFIX}/${opts.runId}/${EVENTS_DIR}/`; + const combinedPath = `${RUNS_PREFIX}/${opts.runId}/${WORKFLOW_RUN_EVENTS_FILE}`; + const principal: WorkflowRunSupervisorPrincipal = { + kind: SUPERVISOR_PRINCIPAL_KIND, + deploymentId: opts.deploymentId, + }; + let sealed = false; + await opts.substrate.writeTreePreservingPrefix( + principal, + opts.repoId, + opts.ref, + { + preservePrefix: prefix, + merge: async (existing) => { + const entries: { seq: number; bytes: Uint8Array }[] = []; + for (const [filepath, bytes] of existing) { + const name = filepath.slice(prefix.length); + const match = EVENT_FILENAME_RE.exec(name); + if (match === null || match[1] === undefined) { + throw new Error( + `supervisor run-event-compaction: unexpected non-event file ${filepath} under run ${opts.runId}; refusing to compact`, + ); + } + entries.push({ seq: Number.parseInt(match[1], 10), bytes }); + } + if (entries.length === 0) return {}; + entries.sort((a, b) => a.seq - b.seq); + sealed = true; + return { + [combinedPath]: encodeCombinedEventLog(entries.map((e) => e.bytes)), + }; + }, + message: `compact run ${opts.runId} events`, + }, + ); + return { compacted: sealed }; +} + +function isErrnoNotFound(cause: unknown): boolean { + if (cause === null || typeof cause !== "object") return false; + return (cause as { code?: unknown }).code === "ENOENT"; +} diff --git a/packages/workflow-host/src/supervisor/run-event-signing.ts b/packages/workflow-host/src/supervisor/run-event-signing.ts deleted file mode 100644 index cbfeb62f..00000000 --- a/packages/workflow-host/src/supervisor/run-event-signing.ts +++ /dev/null @@ -1,364 +0,0 @@ -// Run-event signing path for the per-deployment supervisor. -// -// The supervisor commits the canonical run-lifecycle event chain -// (`RunStarted`, `StepStarted`, `StepCompleted`, `RunCompleted`) to -// the workflow-run repo on behalf of the deployment. Both the trivial -// branch (no IPC, no child) and the multi-step branch's -// supervisor-side bookkeeping route through this module so the on- -// disk shape is identical regardless of process topology. -// -// The substrate-side workflow-run kind handler validates that the -// signing principal is `supervisor` and that the principal's -// `deploymentId` equals `repoId.id`; the surface here mirrors the -// `commitCancelRequested` shape so an audit-log walker sees the same -// envelope conventions across every event kind the supervisor -// authors. - -import type { - RepoId, - RepoStore as SubstrateRepoStore, - WorkflowRunSupervisorPrincipal, -} from "@intx/hub-sessions/substrate"; -import { - WORKFLOW_RUN_EVENTS_FILE, - encodeCombinedEventLog, -} from "@intx/hub-sessions/substrate"; -import { hexEncode } from "@intx/types"; - -import { SUPERVISOR_PRINCIPAL_KIND } from "./cancel-signing"; -import type { PrincipalSigner, SignedPayload } from "./types"; - -const RUNS_PREFIX = "runs"; -const EVENTS_DIR = "events"; -const EVENT_FILENAME_RE = /^(0|[1-9][0-9]*)\.json$/; -const TERMINAL_EVENT_TYPES = new Set([ - "RunCompleted", - "RunFailed", - "RunCancelled", -]); - -/** - * Shape of the run-lifecycle events the supervisor commits inline. - * The discriminator mirrors the workflow state-machine's - * `WorkflowEvent` union without pulling `@intx/workflow` into the - * supervisor module's dependency closure: the supervisor only needs - * to know the on-disk envelope shape, not the full transition - * semantics. The transition function in `@intx/workflow` is the - * authoritative validator for the chain. - */ -export type SupervisorRunEvent = - | { - readonly kind: "RunStarted"; - readonly runId: string; - readonly at: string; - readonly definitionHash: string; - readonly trigger: { readonly type: string; readonly payload: unknown }; - readonly consumedMessageId?: string; - } - | { - readonly kind: "StepStarted"; - readonly runId: string; - readonly at: string; - readonly stepId: string; - readonly attempt: number; - readonly input: { readonly ref: string }; - } - | { - readonly kind: "StepCompleted"; - readonly runId: string; - readonly at: string; - readonly stepId: string; - readonly attempt: number; - readonly output: { readonly ref: string }; - } - | { - readonly kind: "RunCompleted"; - readonly runId: string; - readonly at: string; - }; - -export type CommitRunEventOpts = { - /** Substrate handle the supervisor writes through. */ - substrate: SubstrateRepoStore; - /** Workflow-run repo for this deployment. */ - repoId: RepoId; - /** Events ref the workflow-run repo writes to. */ - ref: string; - /** Deployment id used to construct the supervisor principal. */ - deploymentId: string; - /** Event to commit. */ - event: SupervisorRunEvent; - /** Host-supplied per-principal signing callback. */ - signAsPrincipal: PrincipalSigner; -}; - -export type CommitRunEventResult = { - /** Substrate-assigned commit SHA the append produced. */ - commitSha: string; - /** Per-run sequence number the append landed at. */ - seq: number; - /** Signature the supervisor attached to the event. */ - signature: SignedPayload; -}; - -/** - * Build the canonical bytes the supervisor signs for a run-lifecycle - * event. The on-disk envelope adds a `signature` field after the - * signature is computed; signing the payload without that field keeps - * the verifier's reconstruction trivial (strip `signature`, - * canonicalize, verify). - * - * The substrate-level discriminator field on disk is `type` rather - * than `kind` so the workflow-run kind handler's `EventEnvelope` - * validator matches every supervisor-authored blob without a - * per-event-kind translation. - */ -function buildPayloadBytes(args: { - seq: number; - event: SupervisorRunEvent; -}): Uint8Array { - const canonical = canonicalize({ seq: args.seq, event: args.event }); - return new TextEncoder().encode(JSON.stringify(canonical)); -} - -function canonicalize(args: { - seq: number; - event: SupervisorRunEvent; -}): Record { - const { seq, event } = args; - switch (event.kind) { - case "RunStarted": { - const base: Record = { - type: "RunStarted", - seq, - runId: event.runId, - at: event.at, - definitionHash: event.definitionHash, - trigger: event.trigger, - }; - if (event.consumedMessageId !== undefined) { - base["consumedMessageId"] = event.consumedMessageId; - } - return base; - } - case "StepStarted": - return { - type: "StepStarted", - seq, - runId: event.runId, - at: event.at, - stepId: event.stepId, - attempt: event.attempt, - input: event.input, - }; - case "StepCompleted": - return { - type: "StepCompleted", - seq, - runId: event.runId, - at: event.at, - stepId: event.stepId, - attempt: event.attempt, - output: event.output, - }; - case "RunCompleted": - return { - type: "RunCompleted", - seq, - runId: event.runId, - at: event.at, - }; - } -} - -/** - * Commit a run-lifecycle event signed by the supervisor. The append - * is atomic against concurrent supervisor commits via the substrate's - * `writeTreePreservingPrefix` lock; the merge callback computes the - * next per-run seq by scanning the existing files under the run's - * `events/` subtree. - */ -export async function commitRunEvent( - opts: CommitRunEventOpts, -): Promise { - const prefix = `${RUNS_PREFIX}/${opts.event.runId}/${EVENTS_DIR}/`; - const principal: WorkflowRunSupervisorPrincipal = { - kind: SUPERVISOR_PRINCIPAL_KIND, - deploymentId: opts.deploymentId, - }; - let resolved: { seq: number; signature: SignedPayload } | null = null; - const { commitSha } = await opts.substrate.writeTreePreservingPrefix( - principal, - opts.repoId, - opts.ref, - { - preservePrefix: prefix, - merge: async (existing) => { - let maxSeq = -1; - for (const filepath of existing.keys()) { - const name = filepath.slice(prefix.length); - const match = EVENT_FILENAME_RE.exec(name); - if (match === null) continue; - const seqStr = match[1]; - if (seqStr === undefined) continue; - const seq = Number.parseInt(seqStr, 10); - if (seq > maxSeq) maxSeq = seq; - } - const nextSeq = maxSeq + 1; - const payloadBytes = buildPayloadBytes({ - seq: nextSeq, - event: opts.event, - }); - const signature = await opts.signAsPrincipal( - SUPERVISOR_PRINCIPAL_KIND, - payloadBytes, - ); - resolved = { seq: nextSeq, signature }; - const onDisk: Record = { - ...canonicalize({ seq: nextSeq, event: opts.event }), - signature: serializeSignedPayload(signature), - }; - const files: Record = {}; - for (const [k, v] of existing) files[k] = v; - files[`${prefix}${String(nextSeq)}.json`] = JSON.stringify(onDisk); - return files; - }, - message: `append ${opts.event.kind} for run ${opts.event.runId}`, - }, - ); - if (resolved === null) { - throw new Error( - `supervisor run-event-signing: merge callback did not assign a sequence number for run ${opts.event.runId}`, - ); - } - const final: { seq: number; signature: SignedPayload } = resolved; - return { commitSha, seq: final.seq, signature: final.signature }; -} - -export type CompactRunEventsOpts = { - /** Substrate handle the supervisor writes through. */ - substrate: SubstrateRepoStore; - /** Workflow-run repo for this deployment. */ - repoId: RepoId; - /** Events ref the workflow-run repo writes to. */ - ref: string; - /** Deployment id used to construct the supervisor principal. */ - deploymentId: string; - /** Run to seal. */ - runId: string; -}; - -/** - * Fold a terminated run's per-event `events/.json` blobs into one - * combined `events.jsonl`, dropping the per-event files. This shrinks the - * repo's file count -- and so every per-commit cost that scales with it -- - * without losing any event. - * - * Idempotent and terminal-only: a run already sealed (no `events/` subtree) - * or one whose latest event is not terminal is left untouched, so the call - * is safe to repeat. The live caller fires it once per run, right after the - * run terminates; a bounded recovery sweep that would re-fire it to seal a - * run whose fold a crash interrupted is tracked separately (INTR-229). - * - * The combined file is the verbatim byte concatenation of the per-event - * blobs in seq order (`encodeCombinedEventLog`), the exact shape the - * workflow-run kind handler's compaction validation requires. It is written - * as a sibling of `events/`, so returning it from the merge while omitting - * the per-event files lets the substrate's prefix clear drop them. - */ -export async function compactRunEvents( - opts: CompactRunEventsOpts, -): Promise<{ compacted: boolean }> { - const fs = await import("node:fs/promises"); - const path = await import("node:path"); - const dir = opts.substrate.getRepoDir(opts.repoId); - const eventsDir = path.join(dir, RUNS_PREFIX, opts.runId, EVENTS_DIR); - - // Cheap pre-check off the working tree to skip an empty commit when there - // is nothing to seal (already combined, or not yet terminal). The merge - // re-reads the prefix under the per-repo lock, so the seal stays - // consistent if another writer raced in between. - let filenames: string[]; - try { - filenames = await fs.readdir(eventsDir); - } catch (cause) { - if (isErrnoNotFound(cause)) return { compacted: false }; - throw cause; - } - const seqs: number[] = []; - for (const name of filenames) { - const match = EVENT_FILENAME_RE.exec(name); - if (match === null || match[1] === undefined) continue; - seqs.push(Number.parseInt(match[1], 10)); - } - if (seqs.length === 0) return { compacted: false }; - const lastRaw = await fs.readFile( - path.join(eventsDir, `${String(Math.max(...seqs))}.json`), - "utf8", - ); - let parsed: unknown; - try { - parsed = JSON.parse(lastRaw); - } catch { - return { compacted: false }; - } - if (typeof parsed !== "object" || parsed === null || !("type" in parsed)) { - return { compacted: false }; - } - const lastType = (parsed as { type?: unknown }).type; - if (typeof lastType !== "string" || !TERMINAL_EVENT_TYPES.has(lastType)) { - return { compacted: false }; - } - - const prefix = `${RUNS_PREFIX}/${opts.runId}/${EVENTS_DIR}/`; - const combinedPath = `${RUNS_PREFIX}/${opts.runId}/${WORKFLOW_RUN_EVENTS_FILE}`; - const principal: WorkflowRunSupervisorPrincipal = { - kind: SUPERVISOR_PRINCIPAL_KIND, - deploymentId: opts.deploymentId, - }; - let sealed = false; - await opts.substrate.writeTreePreservingPrefix( - principal, - opts.repoId, - opts.ref, - { - preservePrefix: prefix, - merge: async (existing) => { - const entries: { seq: number; bytes: Uint8Array }[] = []; - for (const [filepath, bytes] of existing) { - const name = filepath.slice(prefix.length); - const match = EVENT_FILENAME_RE.exec(name); - if (match === null || match[1] === undefined) { - throw new Error( - `supervisor run-event-signing: unexpected non-event file ${filepath} under run ${opts.runId}; refusing to compact`, - ); - } - entries.push({ seq: Number.parseInt(match[1], 10), bytes }); - } - if (entries.length === 0) return {}; - entries.sort((a, b) => a.seq - b.seq); - sealed = true; - return { - [combinedPath]: encodeCombinedEventLog(entries.map((e) => e.bytes)), - }; - }, - message: `compact run ${opts.runId} events`, - }, - ); - return { compacted: sealed }; -} - -function isErrnoNotFound(cause: unknown): boolean { - if (cause === null || typeof cause !== "object") return false; - return (cause as { code?: unknown }).code === "ENOENT"; -} - -function serializeSignedPayload(signed: SignedPayload): { - principalKind: string; - sig: string; -} { - return { - principalKind: signed.principalKind, - sig: hexEncode(signed.sig), - }; -} diff --git a/packages/workflow-host/src/supervisor/spawn-env.ts b/packages/workflow-host/src/supervisor/spawn-env.ts new file mode 100644 index 00000000..bb7db397 --- /dev/null +++ b/packages/workflow-host/src/supervisor/spawn-env.ts @@ -0,0 +1,75 @@ +// Supervisor-side producer of the workflow-process child's spawn-time env. +// +// Both the initial spawn and every recycle respawn route the env through +// this single builder, so the required-key contract the child-side parser +// (`parseSpawnTimeEnv`) enforces has exactly one producer. A divergence +// between the two paths -- the recycle env once omitted `STEP_COUNT` and +// broke every recycle -- is no longer expressible: the required keys are +// built as an exactly-typed record, so omitting one is a compile error. + +import { hexEncode } from "@intx/types"; + +import type { RequiredSpawnEnvKey } from "../child/env-bootstrap"; + +export interface ChildSpawnEnvParts { + /** + * The deployment's stable substrate env (DATA_DIR, the adapter manifest, + * and so on), frozen for the deployment's lifetime. Layered UNDER the + * dynamic env fragment (so a host revision wins) and the per-spawn anchors + * (so a required key is never shadowed). + */ + substrateEnv: Record; + /** + * Host-supplied dynamic env fragment, recomputed for every spawn and + * respawn. Its keys layer OVER `substrateEnv` (so a value the host revised + * between spawns wins) and UNDER the required anchors. Returns `{}` when + * the host has no dynamic entries. + */ + dynamicSpawnEnv: () => Record; + /** Supervisor-minted IPC channel id for this spawn. */ + channelId: string; + /** Shared HMAC key for the event channel, minted for this spawn. */ + hmacKey: Uint8Array; + /** Supervisor's Ed25519 public key for this spawn's control channel. */ + hostPublicKey: Uint8Array; + /** Deployment identity the supervisor manages. */ + deploymentId: string; + /** Mail address the deployment registered on the bus. */ + deploymentMailAddress: string; + /** Step count of the deployed workflow (`stepOrder.length`). */ + stepCount: number; + /** Content hash of the deployed workflow definition. */ + definitionHash: string; + /** Whether this deployment's agent is warm-kept across messages. */ + warmKeep: boolean; +} + +/** + * Build the spawn-time env for a workflow-process child. The single + * producer of what `parseSpawnTimeEnv` consumes; the initial spawn and + * every recycle respawn both call it, so neither can drift from the + * required-key contract. + */ +export function buildChildSpawnEnv( + parts: ChildSpawnEnvParts, +): Record { + // Exactly-typed so omitting a required key fails the type-check rather + // than surfacing as a child env-parse abort in production. + const required: Record = { + IPC_CHANNEL_ID: parts.channelId, + IPC_HMAC_KEY: hexEncode(parts.hmacKey), + HOST_PUBKEY: hexEncode(parts.hostPublicKey), + DEPLOYMENT_ID: parts.deploymentId, + DEFINITION_HASH: parts.definitionHash, + MAILBOX_ADDRESS: parts.deploymentMailAddress, + STEP_COUNT: String(parts.stepCount), + }; + return { + ...parts.substrateEnv, + // Host-revised entries win over the frozen substrate env; the required + // anchors below still win over everything. + ...parts.dynamicSpawnEnv(), + ...required, + WARM_KEEP: parts.warmKeep ? "true" : "false", + }; +} diff --git a/packages/workflow-host/src/supervisor/spawn-replay-fifo.test.ts b/packages/workflow-host/src/supervisor/spawn-replay-fifo.test.ts index a26252f0..c1b58fae 100644 --- a/packages/workflow-host/src/supervisor/spawn-replay-fifo.test.ts +++ b/packages/workflow-host/src/supervisor/spawn-replay-fifo.test.ts @@ -545,16 +545,15 @@ async function boot(opts: { prefix: string }): Promise< subprocessSpawner: spawner, binaryPath: "/fake/bin/workflow-child", substrateEnv: { DATA_DIR: baseDir }, + dynamicSpawnEnv: () => ({}), workflowRunRepoId, workflowRunRef: "refs/heads/main", deploymentId: "deployment-x", + stepCount: 1, deploymentMailAddress, readPrincipal: { kind: "supervisor" }, deriveStepAddress: ({ deploymentId, stepId }) => `${deploymentId}-${stepId}@example.com`, - trivialLaunch: () => { - throw new Error("trivialLaunch must not run in this test"); - }, ipcKeyPairFactory: () => Promise.resolve(supervisorIpcKeyPair), inboxPrimitives: gatedInbox.primitives, }; diff --git a/packages/workflow-host/src/supervisor/stale-cohort-routing.test.ts b/packages/workflow-host/src/supervisor/stale-cohort-routing.test.ts index 9a6f3fb5..2632bbca 100644 --- a/packages/workflow-host/src/supervisor/stale-cohort-routing.test.ts +++ b/packages/workflow-host/src/supervisor/stale-cohort-routing.test.ts @@ -442,16 +442,15 @@ describe("H-S2 stale-cohort routing pinch-point", () => { subprocessSpawner: tracker.spawner, binaryPath: "/fake/bin", substrateEnv: { DATA_DIR: baseDir }, + dynamicSpawnEnv: () => ({}), workflowRunRepoId: { kind: "workflow-run", id: "deployment-x" }, workflowRunRef: "refs/heads/main", deploymentId: "deployment-x", + stepCount: 1, deploymentMailAddress: "deployment-x@example.com", readPrincipal: { kind: "supervisor" }, deriveStepAddress: ({ deploymentId, stepId }) => `${deploymentId}-${stepId}@example.com`, - trivialLaunch: () => { - throw new Error("not used"); - }, ipcKeyPairFactory: () => Promise.resolve(ipcKp), inboxPrimitives: wrappedInbox, }; diff --git a/packages/workflow-host/src/supervisor/substrate-write.test.ts b/packages/workflow-host/src/supervisor/substrate-write.test.ts index 3516a75d..887ad3ab 100644 --- a/packages/workflow-host/src/supervisor/substrate-write.test.ts +++ b/packages/workflow-host/src/supervisor/substrate-write.test.ts @@ -517,16 +517,15 @@ async function bootSupervisor(opts: { subprocessSpawner: spawner, binaryPath: "/fake/bin/workflow-child", substrateEnv: { DATA_DIR: baseDir }, + dynamicSpawnEnv: () => ({}), workflowRunRepoId: { kind: "workflow-run", id: "deployment-x" }, workflowRunRef: "refs/heads/main", deploymentId: "deployment-x", + stepCount: 1, deploymentMailAddress: "deployment-x@example.com", readPrincipal: { kind: "supervisor" }, deriveStepAddress: ({ deploymentId, stepId }) => `${deploymentId}-${stepId}@example.com`, - trivialLaunch: () => { - throw new Error("trivialLaunch must not run in this test"); - }, ipcKeyPairFactory: () => Promise.resolve(supervisorIpcKeyPair), inboxPrimitives, ...(opts.terminalWriteWatchdogMs !== undefined diff --git a/packages/workflow-host/src/supervisor/supervisor.test.ts b/packages/workflow-host/src/supervisor/supervisor.test.ts index 1c6ef424..1bad6c8b 100644 --- a/packages/workflow-host/src/supervisor/supervisor.test.ts +++ b/packages/workflow-host/src/supervisor/supervisor.test.ts @@ -7,6 +7,7 @@ import { type } from "arktype"; import { generateKeyPair } from "@intx/crypto"; import { hexDecode, hexEncode } from "@intx/types"; +import type { InferenceSource } from "@intx/types/runtime"; import type { RepoId, RepoStore } from "@intx/hub-sessions"; import { @@ -19,7 +20,6 @@ import { type SubprocessSpawner, type SubprocessHandle, type SignedPayload, - type SupervisorRunEvent, type WorkflowSupervisorBindings, } from "./index"; import { @@ -63,6 +63,26 @@ function parseTriggerFireRunIds(lines: readonly string[]): string[] { return ids; } +function parseSourcesUpdatedFrames( + lines: readonly string[], +): { sources: InferenceSource[]; defaultSource: string }[] { + const out: { sources: InferenceSource[]; defaultSource: string }[] = []; + for (const line of lines) { + if (!line.includes("sources-updated")) continue; + const raw: unknown = JSON.parse(line); + const signed = SignedEnvelope(raw); + if (signed instanceof type.errors) continue; + const payload = ControlPayload(signed.envelope.payload); + if (payload instanceof type.errors) continue; + if (payload.type !== "sources-updated") continue; + out.push({ + sources: payload.data.sources, + defaultSource: payload.data.defaultSource, + }); + } + return out; +} + const CancelRequestedBlob = type({ type: "string", seq: "number", @@ -86,27 +106,6 @@ function readCancelRequestedBlob( return validated; } -const RunEventBlob = type({ - type: "string", - seq: "number", - runId: "string", - at: "string", - signature: { - principalKind: "string", - sig: "string", - }, - "+": "ignore", -}); - -function readRunEventBlob(raw: string): typeof RunEventBlob.infer { - const parsed: unknown = JSON.parse(raw); - const validated = RunEventBlob(parsed); - if (validated instanceof type.errors) { - throw new Error(`unexpected run-event blob shape: ${validated.summary}`); - } - return validated; -} - async function makeTempDir(prefix: string): Promise { const dir = await fs.mkdtemp(path.join(os.tmpdir(), prefix)); return dir; @@ -484,7 +483,6 @@ async function buildBindings(opts: { spawner: SubprocessSpawner; signSpy: (kind: string, payload: Uint8Array) => SignedPayload; mailBus: MailBusBindings; - trivialLaunch?: WorkflowSupervisorBindings["trivialLaunch"]; onWrite?: (args: { principal: { kind: string }; repoId: RepoId; @@ -506,18 +504,15 @@ async function buildBindings(opts: { subprocessSpawner: opts.spawner, binaryPath: "/fake/bin/workflow-child", substrateEnv: { DATA_DIR: opts.baseDir }, + dynamicSpawnEnv: () => ({}), workflowRunRepoId: { kind: "workflow-run", id: "deployment-x" }, workflowRunRef: "refs/heads/main", deploymentId: "deployment-x", + stepCount: 1, deploymentMailAddress: "deployment-x@example.com", readPrincipal: { kind: "supervisor" }, deriveStepAddress: ({ deploymentId, stepId }) => `${deploymentId}-${stepId}@example.com`, - trivialLaunch: - opts.trivialLaunch ?? - (() => { - throw new Error("trivialLaunch not provided to this test binding"); - }), inboxPrimitives: opts.inboxPrimitives ?? createMemoryInboxPrimitives(), }; } @@ -550,7 +545,6 @@ describe("createWorkflowSupervisor", () => { mailBus: createMockMailBus(), }); const supervisor = createWorkflowSupervisor(bindings); - expect(typeof supervisor.deploy).toBe("function"); expect(typeof supervisor.spawn).toBe("function"); expect(typeof supervisor.requestCancel).toBe("function"); expect(typeof supervisor.shutdown).toBe("function"); @@ -739,6 +733,234 @@ describe("createWorkflowSupervisor", () => { expect(mailBus.registered()).not.toContain("deployment-x@example.com"); }); + // Harness for the ready-timeout tests: an injected FakeTimer registry + // (deterministic, per greybeard's ruling against real timers) plus a + // controllable child whose control reader the test can close to model a + // child that exits before signalling ready. `createdTimers` retains every + // timer even after it is cleared, so a test can capture the ready deadline + // and later assert it was cancelled. + async function makeReadyTimeoutHarness(readyTimeoutMs: number) { + type FakeTimer = { cb: () => void; ms: number; cancelled: boolean }; + const timers = new Set(); + const createdTimers: FakeTimer[] = []; + + const baseDir = await makeTempDir("supervisor-ready-timeout-"); + const supervisorIpcKeyPair = await generateKeyPair(); + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const eventChildToSupervisor = createMemoryFrameStream(); + let resolveExit: ((code: number) => void) | undefined; + const exited = new Promise((resolve) => { + resolveExit = resolve; + }); + const killSignals: string[] = []; + + const spawner: SubprocessSpawner = ({ env: _env }) => ({ + pid: 5150, + controlWriter: supervisorToChild.writer, + controlReader: childToSupervisor.reader, + eventReader: eventChildToSupervisor.reader, + kill: (signal) => { + killSignals.push( + typeof signal === "string" ? signal : String(signal ?? ""), + ); + childToSupervisor.close(); + eventChildToSupervisor.close(); + resolveExit?.(0); + }, + exited, + }); + + const mailBus = createMockMailBus(); + const baseBindings = await buildBindings({ + baseDir, + spawner, + signSpy: () => ({ sig: new Uint8Array(64), principalKind: "supervisor" }), + mailBus, + }); + const bindings: WorkflowSupervisorBindings = { + ...baseBindings, + ipcKeyPairFactory: () => Promise.resolve(supervisorIpcKeyPair), + readyTimeoutMs, + setTimer: (cb, ms) => { + const t: FakeTimer = { cb, ms, cancelled: false }; + timers.add(t); + createdTimers.push(t); + return t; + }, + clearTimer: (handle) => { + if (handle === null || typeof handle !== "object") return; + for (const t of timers) { + if (t === handle) { + t.cancelled = true; + timers.delete(t); + return; + } + } + }, + }; + const supervisor = createWorkflowSupervisor(bindings); + + // Resolve once the spawn has armed its ready deadline (which happens + // after the spawner is invoked, so this also confirms the child spawned). + async function waitForReadyDeadline(): Promise { + for (;;) { + const t = createdTimers.find((x) => x.ms === readyTimeoutMs); + if (t !== undefined) return t; + await new Promise((r) => setTimeout(r, 1)); + } + } + + return { supervisor, killSignals, childToSupervisor, waitForReadyDeadline }; + } + + const readyTimeoutSpawnOpts = { + stepOrder: ["step-1"], + definitionHash: "def-hash-abc", + warmKeep: false, + onInferenceEvent: () => { + /* unused in the ready-timeout tests */ + }, + }; + + test("spawn times out, kills the child, rejects, and clears the ready deadline", async () => { + const h = await makeReadyTimeoutHarness(7_777); + // Never send `ready`. Spawn blocks on the handshake until the deadline. + const spawnPromise = h.supervisor.spawn(readyTimeoutSpawnOpts); + const readyDeadline = await h.waitForReadyDeadline(); + readyDeadline.cb(); + + await expect(spawnPromise).rejects.toThrow( + /child did not emit ready within 7777ms; killed/, + ); + expect(h.killSignals).toContain("SIGTERM"); + // The unconditional deadline-timer clear ran on the timeout path. + expect(readyDeadline.cancelled).toBe(true); + }); + + test("spawn clears the ready deadline when the child exits before ready", async () => { + const h = await makeReadyTimeoutHarness(8_888); + const spawnPromise = h.supervisor.spawn(readyTimeoutSpawnOpts); + const readyDeadline = await h.waitForReadyDeadline(); + + // The child exits before signalling ready: closing the control reader + // ends `waitForReady`, rejecting the ready promise. Because the outcomes + // are folded to values, the race resolves to the failed outcome rather + // than rejecting, so the unconditional deadline-timer clear still runs. + // A race that rejected here would skip the clear and leak an armed + // deadline that keeps the event loop alive for up to readyTimeoutMs. + h.childToSupervisor.close(); + + await expect(spawnPromise).rejects.toThrow( + /control channel ended before child emitted ready/, + ); + expect(readyDeadline.cancelled).toBe(true); + }); + + // A spawn that throws AFTER the OS child is running but BEFORE the + // supervisor reaches the ready handshake must not orphan the child or + // leave the mail address registered. `shutdownInternal` owns that + // teardown once the state record enters "starting"; the spawn body + // routes every post-seam throw through it. + async function makePreRegistrationFailureHarness(opts: { + failSubscribe?: boolean; + failDeriveStepAddress?: boolean; + }) { + const baseDir = await makeTempDir("supervisor-spawn-leak-"); + await seedStepGrants( + baseDir, + defaultStepRepoId({ deploymentId: "deployment-x", stepId: "step-1" }), + [{ resource: "thing", action: "read" }], + ); + const supervisorIpcKeyPair = await generateKeyPair(); + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const eventChildToSupervisor = createMemoryFrameStream(); + let resolveExit: ((code: number) => void) | undefined; + const exited = new Promise((resolve) => { + resolveExit = resolve; + }); + const killSignals: string[] = []; + const spawner: SubprocessSpawner = () => ({ + pid: 4321, + controlWriter: supervisorToChild.writer, + controlReader: childToSupervisor.reader, + eventReader: eventChildToSupervisor.reader, + kill: (signal) => { + killSignals.push( + typeof signal === "string" ? signal : String(signal ?? ""), + ); + childToSupervisor.close(); + eventChildToSupervisor.close(); + resolveExit?.(0); + }, + exited, + }); + const mailBus = createMockMailBus(); + const bindingsMailBus: MailBusBindings = { + ...mailBus, + subscribeMailForAddress: + opts.failSubscribe === true + ? () => { + throw new Error("injected subscribe failure"); + } + : mailBus.subscribeMailForAddress, + }; + const baseBindings = await buildBindings({ + baseDir, + spawner, + signSpy: () => ({ sig: new Uint8Array(64), principalKind: "supervisor" }), + mailBus: bindingsMailBus, + }); + const bindings: WorkflowSupervisorBindings = { + ...baseBindings, + ipcKeyPairFactory: () => Promise.resolve(supervisorIpcKeyPair), + ...(opts.failDeriveStepAddress === true + ? { + deriveStepAddress: () => { + throw new Error("injected deriveStepAddress failure"); + }, + } + : {}), + }; + return { + supervisor: createWorkflowSupervisor(bindings), + killSignals, + registered: mailBus.registered, + }; + } + + const preRegistrationSpawnOpts = { + stepOrder: ["step-1"], + definitionHash: "def-hash-abc", + warmKeep: false, + onInferenceEvent: () => { + /* unused in the pre-registration failure tests */ + }, + }; + + test("a spawn whose mail subscription throws kills the child and releases the address", async () => { + const h = await makePreRegistrationFailureHarness({ failSubscribe: true }); + await expect(h.supervisor.spawn(preRegistrationSpawnOpts)).rejects.toThrow( + "injected subscribe failure", + ); + // The address was registered just before subscribe threw; the + // teardown must unregister it so no orphaned registration survives. + expect(h.registered()).toHaveLength(0); + expect(h.killSignals.length).toBeGreaterThan(0); + }); + + test("a spawn whose credentials assembly throws kills the child", async () => { + const h = await makePreRegistrationFailureHarness({ + failDeriveStepAddress: true, + }); + await expect(h.supervisor.spawn(preRegistrationSpawnOpts)).rejects.toThrow( + "injected deriveStepAddress failure", + ); + expect(h.registered()).toHaveLength(0); + expect(h.killSignals.length).toBeGreaterThan(0); + }); + test("drain() forwards the `drain` control frame and arms a drainTimeout accumulator per in-flight run", async () => { const baseDir = await makeTempDir("supervisor-drain-arm-"); await seedStepGrants( @@ -1237,231 +1459,6 @@ describe("createWorkflowSupervisor", () => { expect(onDisk.signature.sig).toMatch(/^01[0-9a-f]+$/); }); - test("deploy routes the trivial branch through the host-injected trivialLaunch callback", async () => { - const baseDir = await makeTempDir("supervisor-deploy-trivial-"); - const trivialCalls: { - agentAddress: string; - agentId: string; - hubPublicKey: string; - config: unknown; - }[] = []; - const signSpyCalls: { kind: string }[] = []; - const bindings = await buildBindings({ - baseDir, - spawner: () => { - throw new Error( - "subprocessSpawner must not be invoked on the trivial branch", - ); - }, - signSpy: (kind) => { - signSpyCalls.push({ kind }); - return { sig: new Uint8Array(64), principalKind: "supervisor" }; - }, - mailBus: createMockMailBus(), - trivialLaunch: async (b) => { - trivialCalls.push({ - agentAddress: b.agentAddress, - agentId: b.agentId, - hubPublicKey: b.hubPublicKey, - config: b.config, - }); - }, - }); - const supervisor = createWorkflowSupervisor(bindings); - const frame = { - agentAddress: "agent-1@example.com", - agentId: "agent-1", - config: { sentinel: "config-bytes" }, - hubPublicKey: "deadbeef", - }; - await supervisor.deploy(frame); - expect(trivialCalls).toHaveLength(1); - expect(trivialCalls[0]).toEqual({ - agentAddress: "agent-1@example.com", - agentId: "agent-1", - hubPublicKey: "deadbeef", - config: { sentinel: "config-bytes" }, - }); - // No signAsPrincipal calls on the trivial branch -- the - // workflow-process-cancel path never engages. - expect(signSpyCalls).toEqual([]); - // credentialsSnapshot is multi-step-only; the trivial deploy - // does not assemble one. - expect(supervisor.getCredentialsSnapshot()).toBeNull(); - }); - - test("deploy does not register a mailbox or open IPC on the trivial branch", async () => { - const baseDir = await makeTempDir("supervisor-deploy-no-mailbus-"); - const mailBus = createMockMailBus(); - const bindings = await buildBindings({ - baseDir, - spawner: () => { - throw new Error("spawner must not be invoked on the trivial branch"); - }, - signSpy: () => ({ - sig: new Uint8Array(64), - principalKind: "supervisor", - }), - mailBus, - trivialLaunch: () => Promise.resolve(), - }); - const supervisor = createWorkflowSupervisor(bindings); - await supervisor.deploy({ - agentAddress: "agent-2@example.com", - agentId: "agent-2", - config: {}, - hubPublicKey: "cafef00d", - }); - // The mail bus is the multi-step branch's seam; the trivial - // branch must not touch it. - expect(mailBus.registered()).not.toContain("deployment-x@example.com"); - }); - - test("deploy hands recordRunEvent into trivialLaunch and commits the canonical four-event chain", async () => { - const baseDir = await makeTempDir("supervisor-deploy-run-events-"); - const signSpyCalls: { kind: string; payload: Uint8Array }[] = []; - const observedWrites: { - principal: { kind: string }; - repoId: RepoId; - ref: string; - files: Record; - }[] = []; - const bindings = await buildBindings({ - baseDir, - spawner: () => { - throw new Error("spawner must not be invoked on the trivial branch"); - }, - signSpy: (kind, payload) => { - signSpyCalls.push({ kind, payload }); - const sig = new Uint8Array(64); - sig[0] = signSpyCalls.length; - return { sig, principalKind: "supervisor" }; - }, - mailBus: createMockMailBus(), - onWrite: (args) => observedWrites.push(args), - statefulWrites: true, - trivialLaunch: async (b) => { - const runId = "run-trivial-1"; - const messageId = "msg-1"; - const stepId = "step-1"; - const chain: readonly SupervisorRunEvent[] = [ - { - kind: "RunStarted", - runId, - at: "2026-01-01T00:00:00.000Z", - definitionHash: "def-hash-trivial", - trigger: { type: "mail", payload: { to: b.agentAddress } }, - consumedMessageId: messageId, - }, - { - kind: "StepStarted", - runId, - at: "2026-01-01T00:00:00.001Z", - stepId, - attempt: 1, - input: { ref: "refs/heads/main" }, - }, - { - kind: "StepCompleted", - runId, - at: "2026-01-01T00:00:00.002Z", - stepId, - attempt: 1, - output: { ref: "refs/heads/main" }, - }, - { - kind: "RunCompleted", - runId, - at: "2026-01-01T00:00:00.003Z", - }, - ]; - for (const event of chain) { - await b.recordRunEvent(event); - } - }, - }); - const supervisor = createWorkflowSupervisor(bindings); - await supervisor.deploy({ - agentAddress: "agent-4@example.com", - agentId: "agent-4", - config: {}, - hubPublicKey: "abc", - }); - - // Every event in the chain flowed through signAsPrincipal with - // kind `"supervisor"` and against payload bytes containing the - // expected discriminator. - expect(signSpyCalls.map((c) => c.kind)).toEqual([ - "supervisor", - "supervisor", - "supervisor", - "supervisor", - ]); - const decodedPayloads = signSpyCalls.map((c) => - new TextDecoder().decode(c.payload), - ); - expect(decodedPayloads[0]).toContain("RunStarted"); - expect(decodedPayloads[0]).toContain("def-hash-trivial"); - expect(decodedPayloads[0]).toContain("msg-1"); - expect(decodedPayloads[1]).toContain("StepStarted"); - expect(decodedPayloads[2]).toContain("StepCompleted"); - expect(decodedPayloads[3]).toContain("RunCompleted"); - - // Every commit went through the supervisor principal against the - // deployment's workflow-run repo. - expect(observedWrites.length).toBe(4); - for (const write of observedWrites) { - expect(write.principal.kind).toBe("supervisor"); - expect(write.repoId).toEqual({ - kind: "workflow-run", - id: "deployment-x", - }); - expect(write.ref).toBe("refs/heads/main"); - } - - // The on-disk envelopes are filed under runs//events/.json - // with monotonically increasing seq from 0. - const expectedTypes = [ - "RunStarted", - "StepStarted", - "StepCompleted", - "RunCompleted", - ]; - for (const [index, write] of observedWrites.entries()) { - const expectedSeq = index; - const expectedPath = `runs/run-trivial-1/events/${String(expectedSeq)}.json`; - const bytes = write.files[expectedPath]; - if (bytes === undefined) { - throw new Error( - `commit ${String(index)} did not contain the expected event blob at ${expectedPath}`, - ); - } - const expectedType = expectedTypes[index]; - if (expectedType === undefined) { - throw new Error( - `unreachable: expectedTypes[${String(index)}] is undefined`, - ); - } - // Every prior commit's blob is also carried through the prefix- - // preserving merge so the substrate's append-only invariant - // holds; assert the count here so a regression that drops a - // prior blob surfaces at the test seam. - expect(Object.keys(write.files)).toHaveLength(expectedSeq + 1); - const blobJson = - typeof bytes === "string" ? bytes : new TextDecoder().decode(bytes); - const blob = readRunEventBlob(blobJson); - expect(blob.type).toBe(expectedType); - expect(blob.seq).toBe(expectedSeq); - expect(blob.runId).toBe("run-trivial-1"); - expect(blob.signature.principalKind).toBe("supervisor"); - expect(blob.signature.sig.length).toBe(128); - } - // The trivial branch does not assemble a credentials snapshot - // even though the event chain landed; observability and - // process topology are independent surfaces. - expect(supervisor.getCredentialsSnapshot()).toBeNull(); - }); - test("drain() threads the per-cohort terminal broadcaster into each accumulator's opts", async () => { const baseDir = await makeTempDir("supervisor-drain-terminal-source-"); await seedStepGrants( @@ -1769,32 +1766,6 @@ describe("createWorkflowSupervisor", () => { await supervisor.shutdown(); }); - test("deploy surfaces a trivialLaunch failure to the caller", async () => { - const baseDir = await makeTempDir("supervisor-deploy-error-"); - const bindings = await buildBindings({ - baseDir, - spawner: () => { - throw new Error("spawner must not be invoked on the trivial branch"); - }, - signSpy: () => ({ - sig: new Uint8Array(64), - principalKind: "supervisor", - }), - mailBus: createMockMailBus(), - trivialLaunch: () => - Promise.reject(new Error("provisionAgent failed in test")), - }); - const supervisor = createWorkflowSupervisor(bindings); - await expect( - supervisor.deploy({ - agentAddress: "agent-3@example.com", - agentId: "agent-3", - config: {}, - hubPublicKey: "abc", - }), - ).rejects.toThrow(/provisionAgent failed in test/); - }); - test("drain() is a no-op when the supervisor is idle (no spawn has run)", async () => { // Pins the defensive contract for an inbound drain.deliver frame // that lands while the supervisor has no in-flight runs to escalate @@ -1881,6 +1852,138 @@ describe("createWorkflowSupervisor", () => { }), ).rejects.toThrow(/deliverSignal called in phase idle/); }); + + test("deliverSources() rejects when the supervisor is idle (no spawn has run)", async () => { + // Same phase-guard contract as deliverSignal: a sources rotation + // landing against a supervisor that is not starting/running throws so + // the sidecar router's rejection surfaces to the hub-link rather than + // writing into a dead child's pipe. + const baseDir = await makeTempDir("supervisor-deliver-sources-idle-"); + const bindings = await buildBindings({ + baseDir, + spawner: () => { + throw new Error("spawner must not be invoked on the idle sources path"); + }, + signSpy: () => ({ + sig: new Uint8Array(64), + principalKind: "supervisor", + }), + mailBus: createMockMailBus(), + }); + const supervisor = createWorkflowSupervisor(bindings); + await expect( + supervisor.deliverSources({ + sources: [ + { + id: "primary", + provider: "anthropic", + baseURL: "https://api.anthropic.com", + apiKey: "sk-x", + model: "claude-test", + }, + ], + defaultSource: "primary", + }), + ).rejects.toThrow(/deliverSources called in phase idle/); + }); + + test("deliverSources() sends a sources-updated frame when running", async () => { + const baseDir = await makeTempDir("supervisor-deliver-sources-running-"); + await seedStepGrants( + baseDir, + defaultStepRepoId({ deploymentId: "deployment-x", stepId: "step-1" }), + [{ resource: "thing", action: "read" }], + ); + + const supervisorIpcKeyPair = await generateKeyPair(); + const childIpcKeyPair = await generateKeyPair(); + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const eventChildToSupervisor = createMemoryFrameStream(); + let resolveExit: ((code: number) => void) | undefined; + const exited = new Promise((resolve) => { + resolveExit = resolve; + }); + + let observedEnv: Record | undefined; + const spawner: SubprocessSpawner = ({ env }) => { + observedEnv = env; + const handle: SubprocessHandle = { + pid: 4321, + controlWriter: supervisorToChild.writer, + controlReader: childToSupervisor.reader, + eventReader: eventChildToSupervisor.reader, + kill: () => { + childToSupervisor.close(); + eventChildToSupervisor.close(); + resolveExit?.(0); + }, + exited, + }; + return handle; + }; + + const baseBindings = await buildBindings({ + baseDir, + spawner, + signSpy: () => ({ sig: new Uint8Array(64), principalKind: "supervisor" }), + mailBus: createMockMailBus(), + }); + const bindings: WorkflowSupervisorBindings = { + ...baseBindings, + ipcKeyPairFactory: () => Promise.resolve(supervisorIpcKeyPair), + }; + const supervisor = createWorkflowSupervisor(bindings); + + const spawnPromise = supervisor.spawn({ + stepOrder: ["step-1"], + definitionHash: "def-hash-abc", + warmKeep: true, + onInferenceEvent: () => undefined, + }); + while (observedEnv === undefined) { + await new Promise((r) => setTimeout(r, 1)); + } + const channelId = observedEnv.IPC_CHANNEL_ID; + if (channelId === undefined) { + throw new Error("IPC_CHANNEL_ID not set in spawn-time env"); + } + const childSender = createControlChannelSender({ + privateKeySeed: childIpcKeyPair.privateKey, + channelId, + writer: { + write(line: string) { + childToSupervisor.inject(line); + }, + }, + }); + await childSender.send({ + type: "ready", + data: { + childPid: 4321, + childPublicKey: Buffer.from(childIpcKeyPair.publicKey).toString("hex"), + }, + }); + await spawnPromise; + + const sources: InferenceSource[] = [ + { + id: "primary", + provider: "anthropic", + baseURL: "https://api.anthropic.com", + apiKey: "sk-primary", + model: "claude-test", + }, + ]; + await supervisor.deliverSources({ sources, defaultSource: "primary" }); + + const frames = parseSourcesUpdatedFrames(supervisorToChild.flushed()); + expect(frames).toHaveLength(1); + expect(frames[0]?.sources).toEqual(sources); + expect(frames[0]?.defaultSource).toBe("primary"); + + await supervisor.shutdown(); + }); }); describe("assembleCredentialsSnapshot", () => { diff --git a/packages/workflow-host/src/supervisor/supervisor.ts b/packages/workflow-host/src/supervisor/supervisor.ts index c67730d6..9cc9c0f9 100644 --- a/packages/workflow-host/src/supervisor/supervisor.ts +++ b/packages/workflow-host/src/supervisor/supervisor.ts @@ -64,7 +64,7 @@ import { } from "@intx/hub-sessions/substrate"; import { base64Decode, base64Encode, hexEncode } from "@intx/types"; import { RepoId } from "@intx/types/sidecar"; -import type { OutboundMessage } from "@intx/types/runtime"; +import type { InferenceSource, OutboundMessage } from "@intx/types/runtime"; import type { CancelOrigin } from "@intx/workflow"; import { @@ -84,7 +84,8 @@ import { type CredentialsSnapshot, } from "./credentials"; import { commitCancelRequested } from "./cancel-signing"; -import { commitRunEvent, compactRunEvents } from "./run-event-signing"; +import { buildChildSpawnEnv } from "./spawn-env"; +import { compactRunEvents } from "./run-event-compaction"; import { createDrainTimeoutAccumulator, DEFAULT_DRAIN_TIMEOUT_MS, @@ -104,9 +105,7 @@ import type { DispatchSubstrateLeg, InboxPrimitives, MailAuditRef, - RecordRunEvent, SubprocessHandle, - SupervisorDeployFrame, TerminalEventSource, TerminalRunEvent, WorkflowSupervisorBindings, @@ -115,6 +114,14 @@ import { createTerminalBroadcaster, type TerminalBroadcaster, } from "./terminal-broadcaster"; +import { + DEFAULT_KILL_TIMEOUT_MS, + DEFAULT_READY_TIMEOUT_MS, + defaultClearTimer, + defaultSetTimer, + killChildHandle, + waitDeadline, +} from "./child-termination"; const logger = getLogger(["workflow-host", "supervisor"]); @@ -140,37 +147,6 @@ export const DEFAULT_TERMINAL_WRITE_WATCHDOG_MS = 30_000; * supervisor's internal state is encapsulated. */ export interface WorkflowSupervisor { - /** - * Single ingress for `agent.deploy` frames. The supervisor owns - * the routing decision between trivial (1-step) and multi-step - * workflows -- the host hands the frame off and never re-decides. - * - * Trivial branch (Option Z, locked): - * - Calls `bindings.trivialLaunch(frame)` directly. - * - Does NOT open an IPC channel. - * - Does NOT spawn a workflow-process child. - * - Does NOT emit any workflow-run event. `signAsPrincipal` - * is not invoked; for a trivial deploy there is no - * workflow-process child to cancel, so cancellation stays - * session-destroy at the host's layer. - * - Does NOT assemble a `credentialsSnapshot`; the trivial - * branch leaves `getCredentialsSnapshot()` returning `null`. - * - * Multi-step branch (`steps.length >= 2`): - * - Provisions per-step `agent-state` repos, mints keys, - * spawns the workflow-process child via `subprocessSpawner`, - * registers the deployment's mail address, waits for the - * child's `ready` frame, and assembles the - * `credentialsSnapshot`. This is the body of `spawn(opts)`, - * which is the multi-step branch's worker today. - * - * The `agent.deploy` wire frame today carries only a - * `HarnessConfig` (no workflow definition); every frame is - * therefore trivial. The supervisor codifies the seam so a frame- - * format extension that carries a `WorkflowDefinition` is a pure - * data-shape change. - */ - deploy(frame: SupervisorDeployFrame): Promise; /** * Spawn the workflow-process child, complete the IPC handshake, * assemble the credentialsSnapshot, register the deployment's mail @@ -227,6 +203,14 @@ export interface WorkflowSupervisor { * for serializing delivery against `spawn` completion. */ deliverSignal(opts: DeliverSignalOpts): Promise; + /** + * Push a rotated inference-source list to the child's warm single-step + * agent. Mirrors `deliverSignal`: the supervisor is the single producer + * of `sources-updated` control frames, and delivery is phase-guarded to + * starting/running so a frame is never written into a recycling child's + * closing pipe. Throws otherwise. + */ + deliverSources(opts: DeliverSourcesOpts): Promise; /** * Current snapshot of the credentials pushed to the child. Surfaced * so the host can audit the per-step contentHash without @@ -307,6 +291,17 @@ export type DeliverSignalOpts = { payload: unknown; }; +export type DeliverSourcesOpts = { + /** + * The rotated ordered inference-source failover chain; element 0 is the + * active source. The wire boundary enforces a non-empty list with unique + * ids whose head is the default. + */ + sources: InferenceSource[]; + /** The default source id; the wire boundary requires it to equal `sources[0].id`. */ + defaultSource: string; +}; + export type RecycleOpts = { reason: string; /** @@ -554,6 +549,14 @@ export function createWorkflowSupervisor( // consumedRetentionMs` for the operator-owned invariant. const consumedRetentionMs = bindings.consumedRetentionMs ?? DEFAULT_CONSUMED_RETENTION_MS; + // Resolve the spawn ready-handshake timeout and its timers once at the + // bindings edge. The timers reuse the same injectable pair the drain + // path resolves (`bindings.setTimer`/`clearTimer`); the ready-timeout + // race and its kill-escalation drive them, and tests substitute a + // deterministic timer through the same bindings. + const readyTimeoutMs = bindings.readyTimeoutMs ?? DEFAULT_READY_TIMEOUT_MS; + const readySetTimer = bindings.setTimer ?? defaultSetTimer; + const readyClearTimer = bindings.clearTimer ?? defaultClearTimer; /** * Resolved on every successful `enqueueInbox`; the dispatch loop * awaits this promise after a null dequeue so it returns to @@ -590,43 +593,6 @@ export function createWorkflowSupervisor( let recyclePolicy: RecyclePolicy | null = null; let recycleInProgress = false; - async function deploy(frame: SupervisorDeployFrame): Promise { - // The `agent.deploy` wire frame currently carries only a - // `HarnessConfig`, which is the trivial-workflow shape (a single - // step derived from the agent's harness). The branching seam - // exists for the multi-step extension; for every frame today - // the supervisor calls trivialLaunch unchanged. - // - // Trivial-branch invariants: - // - No IPC opens, no child spawn, no mail-bus registration. - // - credentialsSnapshot is not assembled; the multi-step - // branch owns it. `getCredentialsSnapshot()` continues to - // return null on the trivial path. - // - Run-lifecycle events are committed inline from the - // supervisor process via `signAsPrincipal`. The supervisor - // hands `recordRunEvent` to the host's `trivialLaunch`; the - // host calls it from its per-message reactor / harness - // lifecycle moments (`message.run.started` / - // `message.run.ended`) so the on-disk event chain matches - // the multi-step branch byte-for-byte. - const recordRunEvent: RecordRunEvent = (event) => - commitRunEvent({ - substrate: bindings.repoStore, - repoId: bindings.workflowRunRepoId, - ref: bindings.workflowRunRef, - deploymentId: bindings.deploymentId, - event, - signAsPrincipal: bindings.signAsPrincipal, - }); - await bindings.trivialLaunch({ - agentAddress: frame.agentAddress, - agentId: frame.agentId, - config: frame.config, - hubPublicKey: frame.hubPublicKey, - recordRunEvent, - }); - } - function onChildCrash(reason: string): void { logger.error`workflow-process control channel crash: {reason}`; void shutdownInternal({ reason }); @@ -1094,7 +1060,7 @@ export function createWorkflowSupervisor( // per-event form, which readers handle. There is no later trigger for // a run whose fold is interrupted here (e.g. by a crash before the // fold commits): the terminal signal fires once. A bounded recovery - // sweep is tracked as INTR-229; until then such a run stays + // sweep is not yet implemented; until then such a run stays // per-event. The fold commit carries no newly-added terminal event, // so it does not re-fire this terminal-write coupling. for (const { runId } of newlyTerminalRuns) { @@ -1350,28 +1316,56 @@ export function createWorkflowSupervisor( const channelId = generateChannelId(); const hmacKey = generateHmacKey(); const ipcKeypair = await (bindings.ipcKeyPairFactory ?? generateKeyPair)(); - const env: Record = { - ...bindings.substrateEnv, - IPC_CHANNEL_ID: channelId, - IPC_HMAC_KEY: hexEncode(hmacKey), - HOST_PUBKEY: hexEncode(ipcKeypair.publicKey), - DEPLOYMENT_ID: bindings.deploymentId, - DEFINITION_HASH: opts.definitionHash, - MAILBOX_ADDRESS: bindings.deploymentMailAddress, - WARM_KEEP: opts.warmKeep ? "true" : "false", - }; + const env = buildChildSpawnEnv({ + substrateEnv: bindings.substrateEnv, + dynamicSpawnEnv: bindings.dynamicSpawnEnv, + channelId, + hmacKey, + hostPublicKey: ipcKeypair.publicKey, + deploymentId: bindings.deploymentId, + deploymentMailAddress: bindings.deploymentMailAddress, + stepCount: bindings.stepCount, + definitionHash: opts.definitionHash, + warmKeep: opts.warmKeep, + }); const handle = bindings.subprocessSpawner({ binaryPath: bindings.binaryPath, env, }); - const wired = await wireChild({ - channelId, - hmacKey, - ipcKeypair, - handle, - onInferenceEvent: opts.onInferenceEvent, + let wired: Awaited>; + try { + wired = await wireChild({ + channelId, + hmacKey, + ipcKeypair, + handle, + onInferenceEvent: opts.onInferenceEvent, + }); + } catch (cause) { + // wireChild threw before any state record owns the handle, so + // shutdownInternal -- which reaches the handle through the + // active-state record -- would early-return on the "idle" phase + // without killing it. Kill the freshly-spawned child directly to + // avoid orphaning the OS process. + await killChildHandle(handle, DEFAULT_KILL_TIMEOUT_MS, { + setTimer: readySetTimer, + clearTimer: readyClearTimer, + logger, + }); + throw cause; + } + + // The ready handshake below folds `wired.readyPromise` into an + // outcome value, handling its rejection. But a startup teardown that + // fires BEFORE the handshake -- a throw during credentials assembly + // or mail registration -- kills the child, and that kill rejects + // `readyPromise` (the control channel ends). Attach a benign handler + // now so the rejection is never unhandled on that path; the + // handshake's own fold still observes the outcome when it runs. + void wired.readyPromise.catch(() => { + /* handled by the ready-handshake fold when the handshake runs */ }); // Cohort abort controller covers terminal-event watcher @@ -1398,199 +1392,269 @@ export function createWorkflowSupervisor( replayDone: null, }; - const credentialsSnapshot = await assembleCredentialsSnapshot({ - repoStore: bindings.repoStore, - principal: bindings.readPrincipal, - stepOrder: opts.stepOrder, - deploymentId: bindings.deploymentId, - deriveStepAddress: bindings.deriveStepAddress, - ...(bindings.deriveStepRepoId !== undefined - ? { deriveStepRepoId: bindings.deriveStepRepoId } - : {}), - }); - state.credentialsSnapshot = credentialsSnapshot; - - // Replay any orphaned `processing/` entries back to `inbox/` - // BEFORE the dispatch loop's first dequeue. A crash mid-dispatch - // in a prior supervisor incarnation can leave an entry in - // `processing/` with no owner; the FIFO contract requires the - // entry move back to `inbox/` so the next dispatch picks it up - // in its original arrival position. The replay runs off the - // spawn critical path (the substrate write may roundtrip through - // the pack-pushing wrap and a slow hub), but `runDispatchLoop` - // takes the promise as an argument and awaits it before its - // first `dequeueToProcessing` so a fresh inbound mail that lands - // during the replay window cannot ship ahead of the orphan once - // the replay completes. - const replayDone = inboxPrimitives - .replayProcessingToInbox( - bindings.repoStore, - inboxWritePrincipal, - bindings.workflowRunRepoId, - bindings.deploymentMailAddress, - ) - .then(() => { - wakeDispatch(); - }) - .catch((cause) => { - // Documented best-effort: a failed replay leaves orphaned - // `processing/` entries parked and the dispatch loop will - // then ship newly-enqueued mail ahead of them, violating - // the FIFO contract described in the comment above. - // Tightening this to a fatal `onChildCrash` was attempted - // but caused spurious crashes in the integration suite - // where the first spawn legitimately has no - // `processing/` directory to replay; resolving that - // requires either a no-op-on-missing variant of - // `replayProcessingToInbox` or a dispatch-loop periodic - // sweep that picks up parked orphans. Left as logged - // best-effort until that lands. - const message = cause instanceof Error ? cause.message : String(cause); - logger.warn`replayProcessingToInbox on spawn failed: ${message}`; + // Everything from here to the successful `return` runs with the state + // record in "starting" (then "running"). A throw at any of these + // steps -- credentials assembly, mail registration, the ready + // handshake, the credentials push, the dispatch-loop start -- routes + // through shutdownInternal, the single owner of starting/running + // teardown: it kills the handle and releases the mail subscription + // and address registration installed below. + try { + const credentialsSnapshot = await assembleCredentialsSnapshot({ + repoStore: bindings.repoStore, + principal: bindings.readPrincipal, + stepOrder: opts.stepOrder, + deploymentId: bindings.deploymentId, + deriveStepAddress: bindings.deriveStepAddress, + ...(bindings.deriveStepRepoId !== undefined + ? { deriveStepRepoId: bindings.deriveStepRepoId } + : {}), }); - // Hold the replay promise on the active-state record so - // `shutdownInternal` awaits its settlement before tearing the - // bindings down. A shutdown that lands while the replay is in - // flight would otherwise leave the substrate write pending past - // the supervisor's exit. - state.replayDone = replayDone; - - bindings.mailBus.registerAddress(bindings.deploymentMailAddress); - const mailUnsubscribe = bindings.mailBus.subscribeMailForAddress( - bindings.deploymentMailAddress, - onMailMessage, - ); - state.mailUnsubscribe = mailUnsubscribe; - - const readyInfo = await wired.readyPromise; - - // Push the assembled credentialsSnapshot to the child before the - // mail buffer drains. Without this, the child's - // `createCredentialsBackedAuthorize` closure observes a null - // snapshot ref on the first authorize call and throws "no - // credentialsSnapshot active"; the run's first step fails before - // the runtime body can commit `StepCompleted`. The send rides the - // same control channel `trigger.fire` uses, so the ordering - // guarantee (`grants-updated` lands before `trigger.fire`) holds - // for buffered and post-ready inbound mail alike. - await wired.wiring.controlSender.send({ - type: "grants-updated", - data: { - snapshot: { - steps: credentialsSnapshot.steps.map((s) => ({ - stepId: s.stepId, - address: s.address, - grants: [...s.grants], - contentHash: s.contentHash, - })), + state.credentialsSnapshot = credentialsSnapshot; + + // Replay any orphaned `processing/` entries back to `inbox/` + // BEFORE the dispatch loop's first dequeue. A crash mid-dispatch + // in a prior supervisor incarnation can leave an entry in + // `processing/` with no owner; the FIFO contract requires the + // entry move back to `inbox/` so the next dispatch picks it up + // in its original arrival position. The replay runs off the + // spawn critical path (the substrate write may roundtrip through + // the pack-pushing wrap and a slow hub), but `runDispatchLoop` + // takes the promise as an argument and awaits it before its + // first `dequeueToProcessing` so a fresh inbound mail that lands + // during the replay window cannot ship ahead of the orphan once + // the replay completes. + const replayDone = inboxPrimitives + .replayProcessingToInbox( + bindings.repoStore, + inboxWritePrincipal, + bindings.workflowRunRepoId, + bindings.deploymentMailAddress, + ) + .then(() => { + wakeDispatch(); + }) + .catch((cause) => { + // Documented best-effort: a failed replay leaves orphaned + // `processing/` entries parked and the dispatch loop will + // then ship newly-enqueued mail ahead of them, violating + // the FIFO contract described in the comment above. + // Tightening this to a fatal `onChildCrash` was attempted + // but caused spurious crashes in the integration suite + // where the first spawn legitimately has no + // `processing/` directory to replay; resolving that + // requires either a no-op-on-missing variant of + // `replayProcessingToInbox` or a dispatch-loop periodic + // sweep that picks up parked orphans. Left as logged + // best-effort until that lands. + const message = + cause instanceof Error ? cause.message : String(cause); + logger.warn`replayProcessingToInbox on spawn failed: ${message}`; + }); + // Hold the replay promise on the active-state record so + // `shutdownInternal` awaits its settlement before tearing the + // bindings down. A shutdown that lands while the replay is in + // flight would otherwise leave the substrate write pending past + // the supervisor's exit. + state.replayDone = replayDone; + + bindings.mailBus.registerAddress(bindings.deploymentMailAddress); + const mailUnsubscribe = bindings.mailBus.subscribeMailForAddress( + bindings.deploymentMailAddress, + onMailMessage, + ); + state.mailUnsubscribe = mailUnsubscribe; + + // Bound the `ready` handshake. `wired.readyPromise` resolves on `ready` + // and rejects when the control channel ends (the child exited); a child + // that neither readies nor exits would block here forever. Fold all three + // outcomes into values so the single `readyClearTimer` below runs on every + // path -- ready, child-exit failure, and timeout -- before we act on the + // result. A `Promise.race` that could reject would skip the clear on the + // child-exit path and leak an armed deadline that keeps the event loop + // alive for up to `readyTimeoutMs`. The deadline is resolve-only, so it + // contributes no rejection of its own. Kill on timeout uses the + // SIGTERM->SIGKILL escalation because a wedged child may ignore SIGTERM; + // SIGKILL guarantees `exited` settles. + const readyOutcome = wired.readyPromise.then( + (info) => ({ kind: "ready" as const, info }), + (err: unknown) => ({ kind: "failed" as const, err }), + ); + const readyDeadline = waitDeadline(readySetTimer, readyTimeoutMs); + const readyRace = await Promise.race([ + readyOutcome, + readyDeadline.promise.then(() => ({ kind: "timeout" as const })), + ]); + readyClearTimer(readyDeadline.handle); + if (readyRace.kind === "timeout") { + await killChildHandle(wired.wiring.handle, DEFAULT_KILL_TIMEOUT_MS, { + setTimer: readySetTimer, + clearTimer: readyClearTimer, + logger, + }); + // The SIGTERM->SIGKILL escalation above is deliberate: a wedged + // child may ignore the plain kill shutdownInternal issues. The + // outer catch then runs shutdownInternal for the "starting"-phase + // teardown (subscription + address release); its kill against the + // already-killed handle is idempotent. + throw new Error( + `workflow-host supervisor: child did not emit ready within ${readyTimeoutMs}ms; killed`, + ); + } + if (readyRace.kind === "failed") { + // The child exited during the handshake; the outer catch releases + // the subscription and registration via shutdownInternal. + throw readyRace.err; + } + const readyInfo = readyRace.info; + + // Push the assembled credentialsSnapshot to the child before the + // mail buffer drains. Without this, the child's + // `createCredentialsBackedAuthorize` closure observes a null + // snapshot ref on the first authorize call and throws "no + // credentialsSnapshot active"; the run's first step fails before + // the runtime body can commit `StepCompleted`. The send rides the + // same control channel `trigger.fire` uses, so the ordering + // guarantee (`grants-updated` lands before `trigger.fire`) holds + // for buffered and post-ready inbound mail alike. + await wired.wiring.controlSender.send({ + type: "grants-updated", + data: { + snapshot: { + steps: credentialsSnapshot.steps.map((s) => ({ + stepId: s.stepId, + address: s.address, + grants: [...s.grants], + contentHash: s.contentHash, + })), + }, }, - }, - }); + }); - // Transition to running. The dispatch loop (started below) - // picks up any pre-ready buffered mail through the FIFO inbox - // queue rather than through an in-memory buffer; arrival order - // is preserved by the envelope's `receivedAt` prefix on the - // inbox filename. - const startingPhaseCohortAbort = state.terminalCohortAbort; - if (startingPhaseCohortAbort === null) { - throw new Error( - "supervisor: terminalCohortAbort missing after spawn handshake", + // Transition to running. The dispatch loop (started below) + // picks up any pre-ready buffered mail through the FIFO inbox + // queue rather than through an in-memory buffer; arrival order + // is preserved by the envelope's `receivedAt` prefix on the + // inbox filename. + const startingPhaseCohortAbort = state.terminalCohortAbort; + if (startingPhaseCohortAbort === null) { + throw new Error( + "supervisor: terminalCohortAbort missing after spawn handshake", + ); + } + const startingPhaseBroadcaster = state.terminalBroadcaster; + const dispatchLoop = runDispatchLoop( + wired.wiring.controlSender, + startingPhaseCohortAbort, + startingPhaseBroadcaster, + replayDone, ); - } - const startingPhaseBroadcaster = state.terminalBroadcaster; - const dispatchLoop = runDispatchLoop( - wired.wiring.controlSender, - startingPhaseCohortAbort, - startingPhaseBroadcaster, - replayDone, - ); - // Surface dispatch-loop failures via the logger; the loop's own - // catch already swallows per-iteration faults, but a structural - // failure (e.g. the cohort abort handler itself throws) lands - // here. - void dispatchLoop.catch((cause) => { - const message = cause instanceof Error ? cause.message : String(cause); - logger.error`dispatch loop terminated with error: ${message}`; - }); - state = { - phase: "running", - handle, - controlSender: wired.wiring.controlSender, - channelId, - eventPump: wired.wiring.eventPump, - onInferenceEvent: opts.onInferenceEvent, - mailUnsubscribe, - credentialsSnapshot, - terminalCohortAbort: startingPhaseCohortAbort, - terminalBroadcaster: startingPhaseBroadcaster, - dispatchLoop, - replayDone, - }; - // Kick the dispatch loop in case mail landed in the inbox - // before the loop's first `await dispatchWake`. A wake against a - // freshly-minted promise is a no-op; the dispatch loop's first - // dequeue happens unconditionally. - wakeDispatch(); + // Surface dispatch-loop failures via the logger; the loop's own + // catch already swallows per-iteration faults, but a structural + // failure (e.g. the cohort abort handler itself throws) lands + // here. + void dispatchLoop.catch((cause) => { + const message = cause instanceof Error ? cause.message : String(cause); + logger.error`dispatch loop terminated with error: ${message}`; + }); + state = { + phase: "running", + handle, + controlSender: wired.wiring.controlSender, + channelId, + eventPump: wired.wiring.eventPump, + onInferenceEvent: opts.onInferenceEvent, + mailUnsubscribe, + credentialsSnapshot, + terminalCohortAbort: startingPhaseCohortAbort, + terminalBroadcaster: startingPhaseBroadcaster, + dispatchLoop, + replayDone, + }; + // Kick the dispatch loop in case mail landed in the inbox + // before the loop's first `await dispatchWake`. A wake against a + // freshly-minted promise is a no-op; the dispatch loop's first + // dequeue happens unconditionally. + wakeDispatch(); - // Cache the spawn context for the recycle path. The recycle path - // reuses the same stepOrder/definitionHash/onInferenceEvent on - // every respawn -- those are the strict-orthogonality anchors - // with redeploy, and the supervisor never mutates them. - const now = bindings.recyclePolicyNow ?? defaultNow; - spawnContext = { - stepOrder: opts.stepOrder, - definitionHash: opts.definitionHash, - warmKeep: opts.warmKeep, - onInferenceEvent: opts.onInferenceEvent, - spawnedAt: now(), - }; + // Cache the spawn context for the recycle path. The recycle path + // reuses the same stepOrder/definitionHash/onInferenceEvent on + // every respawn -- those are the strict-orthogonality anchors + // with redeploy, and the supervisor never mutates them. + const now = bindings.recyclePolicyNow ?? defaultNow; + spawnContext = { + stepOrder: opts.stepOrder, + definitionHash: opts.definitionHash, + warmKeep: opts.warmKeep, + onInferenceEvent: opts.onInferenceEvent, + spawnedAt: now(), + }; - // Start the upstream control pump so the supervisor sees the - // child's `recycle.request` (and any future upstream variant) as - // it arrives. The pump exits when the iterator ends, which - // happens when the child closes its end of the control channel - // -- either on shutdown or on recycle's `kill` step. The pump - // closes over the cohort's broadcaster captured at pump-start - // time so a `terminal.event` frame the iterator dequeues after a - // recycle has minted a new cohort routes to THIS cohort's (now - // disposed) broadcaster, not the successor's. - void pumpUpstreamControl( - wired.controlIncoming, - startingPhaseBroadcaster, - ).catch((cause) => { - const message = cause instanceof Error ? cause.message : String(cause); - logger.error`upstream control pump failed: ${message}`; - }); + // Start the upstream control pump so the supervisor sees the + // child's `recycle.request` (and any future upstream variant) as + // it arrives. The pump exits when the iterator ends, which + // happens when the child closes its end of the control channel + // -- either on shutdown or on recycle's `kill` step. The pump + // closes over the cohort's broadcaster captured at pump-start + // time so a `terminal.event` frame the iterator dequeues after a + // recycle has minted a new cohort routes to THIS cohort's (now + // disposed) broadcaster, not the successor's. + void pumpUpstreamControl( + wired.controlIncoming, + startingPhaseBroadcaster, + ).catch((cause) => { + const message = cause instanceof Error ? cause.message : String(cause); + logger.error`upstream control pump failed: ${message}`; + }); - // Arm the recycle policy. The policy is a no-op when all bounds - // are `undefined`; bounds resolution lives inside `createRecyclePolicy`. - if (bindings.recyclePolicy !== undefined) { - const setTimer = bindings.recyclePolicySetTimer ?? defaultSetTimer; - const clearTimer = bindings.recyclePolicyClearTimer ?? defaultClearTimer; - recyclePolicy = createRecyclePolicy({ - bounds: bindings.recyclePolicy, - now, - spawnedAt: spawnContext.spawnedAt, - ...(bindings.readRssBytes !== undefined - ? { readRssBytes: bindings.readRssBytes } - : {}), - ...(bindings.readGrantsAgeMs !== undefined - ? { readGrantsAgeMs: bindings.readGrantsAgeMs } - : {}), - setTimer, - clearTimer, - trigger: async (reason) => { - await recycle({ reason, origin: "policy" }); - }, + // Arm the recycle policy. The policy is a no-op when all bounds + // are `undefined`; bounds resolution lives inside `createRecyclePolicy`. + if (bindings.recyclePolicy !== undefined) { + const setTimer = bindings.recyclePolicySetTimer ?? defaultSetTimer; + const clearTimer = + bindings.recyclePolicyClearTimer ?? defaultClearTimer; + recyclePolicy = createRecyclePolicy({ + bounds: bindings.recyclePolicy, + now, + spawnedAt: spawnContext.spawnedAt, + ...(bindings.readRssBytes !== undefined + ? { readRssBytes: bindings.readRssBytes } + : {}), + ...(bindings.readGrantsAgeMs !== undefined + ? { readGrantsAgeMs: bindings.readGrantsAgeMs } + : {}), + setTimer, + clearTimer, + trigger: async (reason) => { + await recycle({ reason, origin: "policy" }); + }, + }); + } + + return { + pid: readyInfo.childPid, + channelId, + credentialsSnapshot, + }; + } catch (cause) { + const message = cause instanceof Error ? cause.message : String(cause); + // Defense-in-depth for a distinct invariant: the original spawn + // `cause` must survive the unwind. `shutdownInternal` is designed to + // be total and should not throw, but if it ever regresses this guard + // logs the secondary teardown error rather than letting it replace + // `cause` and hide the real startup failure. Mirrors the + // recycle-failure catch, which preserves its cause the same way. + await shutdownInternal({ + reason: `spawn failed during startup: ${message}`, + }).catch((shutdownCause) => { + const inner = + shutdownCause instanceof Error + ? shutdownCause.message + : String(shutdownCause); + logger.error`shutdown after spawn failure also threw: ${inner}`; }); + throw cause; } - - return { - pid: readyInfo.childPid, - channelId, - credentialsSnapshot, - }; } /** @@ -1911,107 +1975,162 @@ export function createWorkflowSupervisor( if (state.phase === "idle" || state.phase === "stopped") return; const prior = state; state = { phase: "stopping" }; - // Stop every armed drainTimeout accumulator before tearing the - // child down. An accumulator left running would otherwise fire - // its `setTimeout` callback (or its terminal-event watcher's - // settle hook) against a shutdown-mid-flight supervisor; the - // explicit `stop()` makes the lifecycle deterministic. + // shutdownInternal is designed to be TOTAL: when a child is up it must + // always kill it and always reach `stopped`, no matter which teardown + // step throws. Rather than depend on every step being individually + // non-throwing (an approach that has already leaked an escape hatch), + // the whole teardown body runs inside one `try`, and the two + // load-bearing actions -- the child kill and the `phase = "stopped"` + // transition -- live in the `finally`, so a throw anywhere above them + // still runs both. This is the documented shutdown carve-out to the + // fail-loud rule: leaking the child or wedging the supervisor in + // `stopping` is strictly worse than logging and continuing, so the + // steps that can throw surface at `logger.warn` and execution proceeds. + // (`terminalCohortAbort.abort`, `rejectCohortAwaiters`, and + // `wakeDispatch` cannot throw, and the broadcaster's `dispose` is total + // by construction; they sit inside the `try` regardless so the + // invariant survives if that ever changes.) const accumulatorsToDispose = [...drainAccumulators.values()]; - for (const accumulator of accumulatorsToDispose) { - accumulator.stop(); - } - drainAccumulators.clear(); - if ( - prior.phase === "starting" || - prior.phase === "running" || - prior.phase === "recycling" - ) { - prior.terminalCohortAbort.abort(); - // Reject every pending merge round-trip and markConsumed waiter - // so handler closures awaiting them (including fire-and-forget - // `handleSubstrateWriteRequest` instances) cannot outlive the - // dying cohort. Without this, the `await new Promise` inside - // each handler would sit forever on a resolver the dying control - // channel will never invoke. - rejectCohortAwaiters("shutdown"); - // Dispose the cohort broadcaster so any minted iterator settles - // with `done: true` -- the dispatch loop's `waitForRunTerminal` - // and any drainTimeout watcher unblock through the same shutdown - // path the cohort abort drives. - prior.terminalBroadcaster.dispose(); - // Wake the dispatch loop so its `dispatchWake` await settles - // and the loop notices the cohort abort. Without the wake, the - // loop's `Promise.race` would sit on the wake promise until - // some other actor woke it. - wakeDispatch(); - } - // Await every accumulator's `disposed()` so a pending escalation - // commit or terminal-event watcher coroutine cannot outlive the - // supervisor and fire against torn-down bindings. - await Promise.all( - accumulatorsToDispose.map((a) => - a.disposed().catch(() => { - /* swallowed: each accumulator already logs its own failure. */ - }), - ), - ); - if ( - (prior.phase === "running" || prior.phase === "recycling") && - prior.dispatchLoop !== null - ) { - await prior.dispatchLoop.catch(() => { - /* swallowed: dispatch-loop failures are surfaced by the - loop's own logger; the shutdown path only waits for the - loop's last iteration to settle. */ - }); - } - if ( - (prior.phase === "starting" || + try { + // Stop every armed drainTimeout accumulator before tearing the child + // down. An accumulator left running would otherwise fire its + // `setTimeout` callback (or its terminal-event watcher's settle hook) + // against a shutdown-mid-flight supervisor. Guard each `stop` so one + // throwing accumulator does not leave the rest armed. + for (const accumulator of accumulatorsToDispose) { + try { + accumulator.stop(); + } catch (cause) { + const message = + cause instanceof Error ? cause.message : String(cause); + logger.warn`drain accumulator stop threw during shutdown: ${message}`; + } + } + drainAccumulators.clear(); + if ( + prior.phase === "starting" || prior.phase === "running" || - prior.phase === "recycling") && - prior.replayDone !== null - ) { - // Await the spawn-time replayProcessingToInbox before tearing - // the bindings down. The replay's substrate write - // (`processing/` -> `inbox/` rename via a tree commit) must - // settle before the supervisor's exit; without the await the - // substrate I/O outlives the supervisor and a subsequent boot - // can observe a partially-applied replay. - await prior.replayDone.catch(() => { - /* swallowed: the replay's own catch already surfaces the - failure to the supervisor's warn channel; the shutdown - path only waits for the substrate write to settle. */ - }); - } - if (recyclePolicy !== null) { - recyclePolicy.stop(); - recyclePolicy = null; - } - spawnContext = null; - if ( - prior.phase === "starting" || - prior.phase === "running" || - prior.phase === "recycling" - ) { - if (prior.mailUnsubscribe !== null) prior.mailUnsubscribe(); - try { - bindings.mailBus.unregisterAddress(bindings.deploymentMailAddress); - } catch (cause) { - const message = cause instanceof Error ? cause.message : String(cause); - logger.warn`mail bus unregisterAddress threw: ${message}`; + prior.phase === "recycling" + ) { + prior.terminalCohortAbort.abort(); + // Reject every pending merge round-trip and markConsumed waiter + // so handler closures awaiting them (including fire-and-forget + // `handleSubstrateWriteRequest` instances) cannot outlive the + // dying cohort. Without this, the `await new Promise` inside + // each handler would sit forever on a resolver the dying control + // channel will never invoke. + rejectCohortAwaiters("shutdown"); + // Dispose the cohort broadcaster so any minted iterator settles + // with `done: true` -- the dispatch loop's `waitForRunTerminal` + // and any drainTimeout watcher unblock through the same shutdown + // path the cohort abort drives. + prior.terminalBroadcaster.dispose(); + // Wake the dispatch loop so its `dispatchWake` await settles + // and the loop notices the cohort abort. Without the wake, the + // loop's `Promise.race` would sit on the wake promise until + // some other actor woke it. + wakeDispatch(); } - prior.handle.kill(); - await prior.handle.exited.catch(() => { - /* swallowed: the host has already been told the deployment is - coming down; an error surfaced from the spawner is the - process exiting with a non-zero code, which is what the - shutdown path expects. */ - }); - await prior.eventPump.catch(() => { - /* swallowed for the same reason as above. */ - }); + // Await every accumulator's `disposed()` so a pending escalation + // commit or terminal-event watcher coroutine cannot outlive the + // supervisor and fire against torn-down bindings. + await Promise.all( + accumulatorsToDispose.map((a) => + a.disposed().catch(() => { + /* swallowed: each accumulator already logs its own failure. */ + }), + ), + ); + if ( + (prior.phase === "running" || prior.phase === "recycling") && + prior.dispatchLoop !== null + ) { + await prior.dispatchLoop.catch(() => { + /* swallowed: dispatch-loop failures are surfaced by the + loop's own logger; the shutdown path only waits for the + loop's last iteration to settle. */ + }); + } + if ( + (prior.phase === "starting" || + prior.phase === "running" || + prior.phase === "recycling") && + prior.replayDone !== null + ) { + // Await the spawn-time replayProcessingToInbox before tearing + // the bindings down. The replay's substrate write + // (`processing/` -> `inbox/` rename via a tree commit) must + // settle before the supervisor's exit; without the await the + // substrate I/O outlives the supervisor and a subsequent boot + // can observe a partially-applied replay. + await prior.replayDone.catch(() => { + /* swallowed: the replay's own catch already surfaces the + failure to the supervisor's warn channel; the shutdown + path only waits for the substrate write to settle. */ + }); + } + if (recyclePolicy !== null) { + try { + recyclePolicy.stop(); + } catch (cause) { + const message = + cause instanceof Error ? cause.message : String(cause); + logger.warn`recycle policy stop threw during shutdown: ${message}`; + } + recyclePolicy = null; + } + spawnContext = null; + if ( + prior.phase === "starting" || + prior.phase === "running" || + prior.phase === "recycling" + ) { + if (prior.mailUnsubscribe !== null) { + try { + prior.mailUnsubscribe(); + } catch (cause) { + const message = + cause instanceof Error ? cause.message : String(cause); + logger.warn`mail unsubscribe threw during shutdown: ${message}`; + } + } + try { + bindings.mailBus.unregisterAddress(bindings.deploymentMailAddress); + } catch (cause) { + const message = + cause instanceof Error ? cause.message : String(cause); + logger.warn`mail bus unregisterAddress threw: ${message}`; + } + } + } finally { + // Load-bearing: the child kill and the `stopped` transition run + // whatever happened above, so a throwing teardown step can neither + // leak the child nor wedge the supervisor in `stopping`. The kill is + // itself guarded so a throw here cannot re-escape the `finally`. + if ( + prior.phase === "starting" || + prior.phase === "running" || + prior.phase === "recycling" + ) { + try { + prior.handle.kill(); + } catch (cause) { + const message = + cause instanceof Error ? cause.message : String(cause); + logger.warn`child kill threw during shutdown: ${message}`; + } + await prior.handle.exited.catch(() => { + /* swallowed: the host has already been told the deployment is + coming down; an error surfaced from the spawner is the + process exiting with a non-zero code, which is what the + shutdown path expects. */ + }); + await prior.eventPump.catch(() => { + /* swallowed for the same reason as above. */ + }); + } + state = { phase: "stopped" }; } - state = { phase: "stopped" }; logger.info`supervisor shutdown complete (${opts.reason})`; } @@ -2306,6 +2425,9 @@ export function createWorkflowSupervisor( wakeDispatch(); }, onCrash: onChildCrash, + // Edge-resolved once at the supervisor factory; recycle bounds + // the respawn handshake with the same value the spawn path uses. + readyTimeoutMs, ...(bindings.recyclePolicySetTimer !== undefined ? { setTimer: bindings.recyclePolicySetTimer } : {}), @@ -2380,6 +2502,27 @@ export function createWorkflowSupervisor( }); } + async function deliverSources(opts: DeliverSourcesOpts): Promise { + // The supervisor is the single producer of `sources-updated` control + // frames. `recycling` is rejected for the same reason as + // `deliverSignal`: `state.controlSender` still points at the dying + // child, so a frame would either buffer behind the SIGTERM or write + // into a closed pipe and be lost. Rejecting surfaces the race so the + // caller can retry once the recycle completes. + if (state.phase !== "running" && state.phase !== "starting") { + throw new Error( + `supervisor: deliverSources called in phase ${state.phase}; expected starting/running`, + ); + } + await state.controlSender.send({ + type: "sources-updated", + data: { + sources: opts.sources, + defaultSource: opts.defaultSource, + }, + }); + } + function getCredentialsSnapshot(): CredentialsSnapshot | null { if (state.phase === "starting" || state.phase === "running") { return state.credentialsSnapshot; @@ -2388,13 +2531,13 @@ export function createWorkflowSupervisor( } return { - deploy, spawn, requestCancel, shutdown, drain, recycle, deliverSignal, + deliverSources, getCredentialsSnapshot, }; } @@ -2509,15 +2652,6 @@ function defaultNow(): number { return Date.now(); } -function defaultSetTimer(cb: () => void, ms: number): unknown { - return setTimeout(cb, ms); -} - -function defaultClearTimer(handle: unknown): void { - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the handle is the value `setTimeout` returned, narrowed back at the boundary - clearTimeout(handle as ReturnType); -} - /** * Drain the event-channel receive iterator into the host-supplied * sink. The function resolves when the iterator ends (child exit or diff --git a/packages/workflow-host/src/supervisor/terminal-broadcaster.ts b/packages/workflow-host/src/supervisor/terminal-broadcaster.ts index 050ca33b..22ea04cd 100644 --- a/packages/workflow-host/src/supervisor/terminal-broadcaster.ts +++ b/packages/workflow-host/src/supervisor/terminal-broadcaster.ts @@ -18,8 +18,16 @@ // directly so the consumer wakes up even if its own cohort-abort race // has not landed yet). +import { getLogger } from "@intx/log"; + import type { TerminalEventSource, TerminalRunEvent } from "./types"; +const logger = getLogger([ + "workflow-host", + "supervisor", + "terminal-broadcaster", +]); + type Listener = { onEvent: (event: TerminalRunEvent) => void; onDispose: () => void; @@ -203,7 +211,19 @@ export function createTerminalBroadcaster(): TerminalBroadcaster { } listenersByRunId.clear(); for (const listener of snapshot) { - listener.onDispose(); + // Guard each `onDispose` so one throwing listener does not skip + // the rest: dispose must finalise every minted iterator or the + // supervisor's teardown can leak a consumer awaiting a `next()` + // that never settles. A throw here is a listener bug, so surface + // it at error and continue -- this is what lets the shutdown path + // treat `dispose()` as total. + try { + listener.onDispose(); + } catch (cause) { + const message = + cause instanceof Error ? cause.message : String(cause); + logger.error`terminal broadcaster listener onDispose threw: ${message}`; + } } }, get disposed() { diff --git a/packages/workflow-host/src/supervisor/types.ts b/packages/workflow-host/src/supervisor/types.ts index b9cf3b66..f16c93d4 100644 --- a/packages/workflow-host/src/supervisor/types.ts +++ b/packages/workflow-host/src/supervisor/types.ts @@ -19,19 +19,10 @@ import type { RepoStore as SubstrateRepoStore, ReplayProcessingToInboxResult, } from "@intx/hub-sessions/substrate"; -import type { - InferenceSource, - OutboundMessage, - SendReceipt, -} from "@intx/types/runtime"; +import type { OutboundMessage, SendReceipt } from "@intx/types/runtime"; import type { RunCancelled, RunCompleted, RunFailed } from "@intx/workflow"; -import type { WorkflowDefinition } from "@intx/workflow/definition"; import type { FrameReader, NdjsonReader, NdjsonWriter } from "../ipc/index"; -import type { - CommitRunEventResult, - SupervisorRunEvent, -} from "./run-event-signing"; /** * Terminal workflow-run event the supervisor's drain accumulators @@ -193,108 +184,6 @@ export type SubprocessSpawner = (args: { env: Record; }) => SubprocessHandle; -/** - * Frame the supervisor receives at its deploy ingress. The shape is - * a structural projection of the sidecar's `agent.deploy` frame -- - * the supervisor does not depend on `@intx/types` for the wire - * type. `config` is opaque to the supervisor; the host owns its - * interpretation and passes it through to the trivial-launch - * callback (multi-step routing carries it into spawn-time env in - * later commits). - * - * `workflow` is the multi-step projection. Absent on every trivial- - * launch frame; presence is the discriminator the deploy router uses - * to branch into `supervisor.spawn()` instead of `trivialLaunch`. The - * field carries the workflow definition (so the supervisor can - * construct the per-step substrate env without round-tripping the - * hub) and the per-step inference source pins keyed by - * `definition.stepOrder` step ids. - */ -export interface SupervisorDeployFrame { - agentAddress: string; - agentId: string; - config: unknown; - hubPublicKey: string; - workflow?: { - definition: WorkflowDefinition; - sources: Record; - }; -} - -/** - * Callback the supervisor hands to the host so the host's per-message - * reactor / harness lifecycle can drive the canonical run-event chain - * (`RunStarted` -> `StepStarted` -> `StepCompleted` -> `RunCompleted`) - * for the trivial deploy. The supervisor's closure resolves the - * workflow-run repo identity, mints the `signAsPrincipal` signature - * for each event, and commits the on-disk blob; the host calls this - * with the event payload at the appropriate reactor moment. - * - * The on-disk envelope is identical to the one the multi-step branch - * commits via its workflow-process child, which makes a trivial - * deployment's audit trail indistinguishable from a multi-step one - * from a downstream consumer's perspective. The split between the - * two branches is process topology (in-process commit vs IPC-forwarded - * commit), not observability. - */ -export type RecordRunEvent = ( - event: SupervisorRunEvent, -) => Promise; - -/** - * Arguments handed to `trivialLaunch`. Mirrors the deploy frame - * unchanged today (the trivial branch is a true passthrough); kept - * as its own type so future trivial-only context (e.g. a - * supervisor-derived deployment id) can attach without widening - * the deploy frame itself. - * - * `recordRunEvent` is the seam the host wires into its existing - * per-message reactor moments (`message.run.started` / - * `message.run.ended`) to drive the canonical workflow-run event - * chain inline from the supervisor's address space. Hosts that have - * not yet wired the reactor seam supply a `trivialLaunch` body that - * does not invoke the callback; the supervisor commits no events in - * that case but the capability is available without further wiring - * surgery. - */ -export interface TrivialLaunchBindings { - agentAddress: string; - agentId: string; - config: unknown; - hubPublicKey: string; - recordRunEvent: RecordRunEvent; -} - -/** - * Host-injected callback the supervisor invokes on the trivial - * branch. The supervisor's deploy() routes here for every single- - * step deployment; the callback owns the entire trivial deploy. - * - * Invariants preserved by the trivial branch: - * - * - The supervisor does not open an IPC channel. - * - The supervisor does not spawn a workflow-process child. - * - `credentialsSnapshot` is multi-step-only; the trivial branch - * leaves `getCredentialsSnapshot()` returning `null`. - * - * The supervisor DOES emit the canonical workflow-run event chain - * (`RunStarted` / `StepStarted` / `StepCompleted` / `RunCompleted`) - * for the trivial deploy inline from the supervisor process via - * `signAsPrincipal` against the workflow-run repo. The chain fires - * per inbound mail trigger (one run per fire) and is driven by the - * host calling `bindings.recordRunEvent(...)` from its reactor / - * harness lifecycle moments. The trivial-branch observability is - * therefore identical to the multi-step branch's; the two branches - * differ in process topology, not event surface. - * - * The host wires `trivialLaunch` against the legacy single-agent - * provisioning surface so the on-wire bytes and on-disk surfaces - * stay bit-identical to the pre-supervisor path; the `recordRunEvent` - * hook is additive (the host's reactor calls it from existing - * lifecycle brackets without changing the deploy-tree contents). - */ -export type TrivialLaunch = (bindings: TrivialLaunchBindings) => Promise; - /** * Logical pointer to the raw mail bytes the inbox claim-check * envelope stamps. The substrate stores this as the `mailAuditRef` @@ -392,6 +281,17 @@ export interface WorkflowSupervisorBindings { * construction logic owns the shape. */ substrateEnv: Record; + /** + * Per-spawn dynamic substrate-env entries the host recomputes for every + * spawn AND every recycle respawn. Distinct from `substrateEnv`, which is + * frozen for the deployment's lifetime: this callback lets a value the + * host revised between spawns (a live inference-source rotation) reach the + * respawned child. Invoked by the spawn-env builder on each spawn/respawn; + * its keys layer over `substrateEnv` and under the IPC anchors. Like + * `substrateEnv`, the supervisor does not inspect the returned keys -- the + * host owns their shape. A host with no dynamic entries returns `{}`. + */ + dynamicSpawnEnv: () => Record; /** * Workflow-run repo identity for the deployment. The supervisor * commits its own CancelRequested / drain events here. @@ -401,6 +301,16 @@ export interface WorkflowSupervisorBindings { workflowRunRef: string; /** Deployment id baked into the supervisor's principal claims. */ deploymentId: string; + /** + * Number of steps in the deployed `WorkflowDefinition` + * (`stepOrder.length`). The supervisor threads this into the child's + * spawn-time env (`STEP_COUNT`) so the child's deploy-tree read + * (`resolveStepAddress` in the sidecar step tools) collapses onto the + * head for a single-step deployment exactly as the host's producer + * push does -- one source of truth for the head/step collapse across + * the two processes. Fixed for the deployment's lifetime. + */ + stepCount: number; /** * Mail address the deployment registers on the bus. The * supervisor registers this on spawn and unregisters on teardown; @@ -438,13 +348,6 @@ export interface WorkflowSupervisorBindings { privateKey: Uint8Array; publicKey: Uint8Array; }>; - /** - * Host-injected callback the supervisor invokes on the trivial - * branch of `deploy(frame)`. Required for hosts that route deploy - * frames through `supervisor.deploy`; the multi-step branch does - * not consult it. See `TrivialLaunch` for the invariants. - */ - trivialLaunch: TrivialLaunch; /** * Operator-overridable per-deployment `drainTimeout` in * milliseconds. The supervisor's `drain()` path threads this value @@ -468,10 +371,11 @@ export interface WorkflowSupervisorBindings { */ now?: () => number; /** - * Scheduling primitive the supervisor threads into the drainTimeout - * accumulator. Production wires `(cb, ms) => setTimeout(cb, ms)`; - * tests inject a deterministic timer host. Defaults to - * `setTimeout` when omitted. + * General scheduling primitive the supervisor threads into its timed + * waits: the drainTimeout accumulator, the spawn ready-handshake + * timeout, and that timeout's SIGTERM->SIGKILL kill escalation. + * Production wires `(cb, ms) => setTimeout(cb, ms)`; tests inject a + * deterministic timer host. Defaults to `setTimeout` when omitted. */ setTimer?: (cb: () => void, ms: number) => unknown; /** @@ -561,6 +465,17 @@ export interface WorkflowSupervisorBindings { * refused stale enqueue, not silent double-processing). */ consumedRetentionMs?: number; + /** + * Bound on the child's spawn-time `ready` handshake, in milliseconds. + * A spawned child that neither emits `ready` nor exits would block + * `spawn` forever; on expiry the supervisor kills the child (SIGTERM, + * then SIGKILL) and rejects the spawn. The boot edge resolves the + * operator's config and supplies it; absent, `DEFAULT_READY_TIMEOUT_MS` + * (30s) applies. Callers surface the rejection through their existing + * spawn-failure path, so a wedged child fails the deploy (or, on the + * sidecar, is skipped by boot-time restore) instead of hanging it. + */ + readyTimeoutMs?: number; /** * Watchdog timeout (ms) for the supervisor's substrate-write * handler's wait on the dispatch loop's `markConsumed` when a diff --git a/tests/db/deployment-key-lookup.test.ts b/tests/db/deployment-key-lookup.test.ts new file mode 100644 index 00000000..37e4961f --- /dev/null +++ b/tests/db/deployment-key-lookup.test.ts @@ -0,0 +1,130 @@ +import { + afterAll, + beforeAll, + beforeEach, + describe, + expect, + test, +} from "bun:test"; + +import { + createHubSessionLookups, + type AgentRepoStore, +} from "@intx/hub-sessions"; +import { workflowDeployment } from "@intx/db/schema"; +import { + createTestDb, + harnessDbEnvAvailable, + type TestDb, +} from "@intx/test-harness/db-harness"; +import { seedAsset, seedTenants } from "@intx/test-harness/seed"; + +// The reconnect ownership challenge verifies a deployment address against a +// public key resolved by `lookupPublicKey`. These tests pin the workflow- +// deployment side of that lookup: keyed by address, gated on a live +// ("deployed") deployment, fail-closed on a missing/null key, and routed by +// address space so a launched-agent address never resolves against the +// deployment table. +describe.skipIf(!harnessDbEnvAvailable())( + "lookupPublicKey deployment-key routing (real DB)", + () => { + let h: TestDb; + + beforeAll(async () => { + h = await createTestDb(); + }); + + afterAll(async () => { + await h.close(); + }); + + beforeEach(async () => { + await h.reset(); + }); + + // lookupPublicKey never touches the repo store, so a throwing stub keeps + // the AgentRepoStore surface satisfied without a real on-disk store. + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub; lookupPublicKey does not touch the repo store + const stubRepoStore = new Proxy( + {}, + { + get() { + throw new Error("agentRepoStore is not used by lookupPublicKey"); + }, + }, + ) as AgentRepoStore; + + function lookupPublicKey(address: string): Promise { + return createHubSessionLookups({ + db: h.db, + agentRepoStore: stubRepoStore, + }).lookupPublicKey(address); + } + + async function seedDeployment(opts: { + address: string; + publicKey: string | null; + status: "deployed" | "error"; + }): Promise { + await seedTenants(h.db, [{ id: "t1" }]); + await seedAsset(h.db, { + id: "asset1", + tenantId: "t1", + kind: "workflow", + name: "wf", + }); + await h.db.insert(workflowDeployment).values({ + id: "dep1", + tenantId: "t1", + definitionAssetId: "asset1", + address: opts.address, + publicKey: opts.publicKey, + status: opts.status, + }); + } + + test("resolves a deployed deployment's key by address", async () => { + await seedDeployment({ + address: "ins_dep_abc@wf.example", + publicKey: "pk1", + status: "deployed", + }); + expect(await lookupPublicKey("ins_dep_abc@wf.example")).toBe("pk1"); + }); + + test("returns null when the deployment has not yet acked a key", async () => { + await seedDeployment({ + address: "ins_dep_abc@wf.example", + publicKey: null, + status: "deployed", + }); + expect(await lookupPublicKey("ins_dep_abc@wf.example")).toBeNull(); + }); + + test("returns null for a non-deployed (torn-down) deployment", async () => { + await seedDeployment({ + address: "ins_dep_abc@wf.example", + publicKey: "pk1", + status: "error", + }); + expect(await lookupPublicKey("ins_dep_abc@wf.example")).toBeNull(); + }); + + test("returns null for an unknown deployment address", async () => { + expect(await lookupPublicKey("ins_dep_missing@wf.example")).toBeNull(); + }); + + test("routes a launched-agent address to agent_instance, never the deployment table", async () => { + // A workflow_deployment row must not answer for a launched-agent + // (ins_...) address. The two address spaces are disjoint and the lookup + // routes by discriminator, so an absent agent_instance row returns null + // rather than leaking a deployment key. + await seedDeployment({ + address: "ins_dep_abc@wf.example", + publicKey: "pk1", + status: "deployed", + }); + expect(await lookupPublicKey("ins_launched@wf.example")).toBeNull(); + }); + }, +); diff --git a/tests/hub-agent/deploy-flow.test.ts b/tests/hub-agent/deploy-flow.test.ts deleted file mode 100644 index a7238c35..00000000 --- a/tests/hub-agent/deploy-flow.test.ts +++ /dev/null @@ -1,363 +0,0 @@ -// Integration test: full deploy lifecycle through SessionService. -// -// Spins up a real WS server (hub side), spawns a real sidecar subprocess, -// and exercises the complete agent lifecycle orchestrated by SessionService: -// -// launchSession (write → pack → provision → deliver → start) -// → message → sync → endSession -// -// The gap this test fills: nobody else tests that AgentRepoStore.writeDeployTree -// → createDeployPack → sendPack produces a packfile the sidecar actually -// accepts and materializes correctly. -// -// Inference is mocked by a tiny HTTP server that echoes the tools it receives, -// so we can assert the model saw the correct tool definitions without calling -// a real LLM. - -import { describe, test, expect, afterAll, beforeAll } from "bun:test"; -import fs from "node:fs"; -import path from "node:path"; -import { - assembleSignedContent, - assembleMessage, - createDetachedSignatureFromProvider, - type MessageHeaders, -} from "@intx/mime"; -import { generateKeyPair, createEd25519Crypto } from "@intx/crypto"; -import { parseAgentId } from "@intx/hub-sessions"; -import { createAgentRepoStore as createSidecarRepoStore } from "@intx/hub-agent"; -import { base64Encode } from "@intx/types"; -import type { HarnessConfig } from "@intx/types/runtime"; -import git from "isomorphic-git"; - -import { - AGENT_ADDRESS, - AGENT_ID, - SESSION_ID, - SIDECAR_ID, - startDeployFlowEnv, - waitFor, - type DeployFlowEnv, -} from "./lib/deploy-flow-env"; - -let env: DeployFlowEnv; -let setupInferenceRequestIndex: number; -let setupAgentEventsBeforeMessage: number; - -// The downstream tests (send-message side-effect assertions, -// sync-request, endSession) each depend on prior lifecycle stages -// having run. Bun runs tests within a `describe` in source order, so -// the chain works when the file runs end-to-end, but `bun test -t -// ` would skip the prerequisite stages and a filtered run of -// `sync request` would fail with `No sidecar connected for agent` or -// `tree must include at least one state-bearing top-level entry`. -// Running the prerequisite stages here -- launchSession AND the first -// message routing, both of which populate the state the sync test -// then asserts against -- codifies the preconditions once so each -// individual test can run on its own under a `-t` filter. The tests -// below assert against the state these setup steps produced. -beforeAll(async () => { - env = await startDeployFlowEnv(); - - const config: HarnessConfig = { - sessionId: SESSION_ID, - agentId: AGENT_ID, - tenantId: "tenant-1", - principalId: "prin_integration-1", - agentAddress: AGENT_ADDRESS, - systemPrompt: "Fallback prompt (should be overridden by deploy tree)", - tools: [], - grants: [], - sources: [ - { - id: "anthropic:mock-model", - provider: "anthropic", - baseURL: `http://localhost:${env.inference.server.port}`, - apiKey: "sk-mock", - model: "mock-model", - }, - ], - defaultSource: "anthropic:mock-model", - }; - - await env.hub.sessionService.launchSession({ - agentAddress: AGENT_ADDRESS, - agentId: AGENT_ID, - instanceId: AGENT_ID, - config, - deployContent: { - systemPrompt: "You are an integration test agent.", - }, - // The hub is wired with a `workspace-builtins` package-registry - // asset containing a synthetic `@intx/tools-mail@0.1.2` tarball; - // pinning here exercises the resolver → asset-pack fan-out → - // sidecar loader path end-to-end. The model-facing tool surfaces - // in the message test below as - // `@intx/tools-mail/sidecar-bundle:mail_send`. - toolPackagePins: [{ name: "@intx/tools-mail", version: "0.1.2" }], - }); - - // Record the indices before the setup message lands so the - // message-side-effect test can locate the resulting inference - // request and the inference.start event by absolute position - // rather than by delta-from-an-earlier-mutation. - setupInferenceRequestIndex = env.inference.requests.length; - setupAgentEventsBeforeMessage = env.hub.agentEvents.length; - - const messageKeyPair = await generateKeyPair(); - const messageCrypto = createEd25519Crypto(messageKeyPair); - const messageHeaders: MessageHeaders = { - from: "user@integration.interchange", - to: [AGENT_ADDRESS], - cc: undefined, - date: new Date(), - messageId: "", - subject: undefined, - inReplyTo: undefined, - references: undefined, - mimeVersion: "1.0", - interchangeType: "conversation.message", - interchangeCorrelationId: undefined, - interchangeTenantId: undefined, - interchangeAgentId: undefined, - interchangeSessionId: SESSION_ID, - interchangeOfferingId: undefined, - interchangeSchemaVersion: undefined, - traceparent: undefined, - tracestate: undefined, - }; - const messageSignedContent = assembleSignedContent({ - kind: "conversation", - text: "Hello.", - }); - const messageSignature = await createDetachedSignatureFromProvider( - messageSignedContent, - messageCrypto, - ); - const messageRaw = assembleMessage( - messageHeaders, - messageSignedContent, - messageSignature, - ); - env.hub.router.routeMail(AGENT_ADDRESS, base64Encode(messageRaw)); - - // Wait for the FULL message lifecycle to land so the downstream - // sync test sees populated agent state on disk. The inference - // request arriving is not enough -- it only signals the reactor - // dispatched -- so we also wait for the `message.run.ended` - // event, which fires after the reactor's context-store write - // completes. Without this, the sync test races the state file - // appearing on disk and the supervisor pushes an empty pack - // (rejected by the kind handler with `tree must include at - // least one state-bearing top-level entry`). - function setupHasEventType(event: unknown, type: string): boolean { - return ( - typeof event === "object" && - event !== null && - "type" in event && - event.type === type - ); - } - await waitFor( - () => env.inference.requests.length > setupInferenceRequestIndex, - { timeoutMs: 30_000, diagnostics: env.sidecarDiagnostics }, - ); - await waitFor( - () => - env.hub.agentEvents - .slice(setupAgentEventsBeforeMessage) - .some((e) => setupHasEventType(e.event, "message.run.ended")), - { timeoutMs: 30_000, diagnostics: env.sidecarDiagnostics }, - ); -}); - -afterAll(async () => { - await env.teardown(); -}); - -describe("deploy flow integration", () => { - test("sidecar registers with hub", () => { - expect(env.hub.router.getConnectedSidecars()).toContain(SIDECAR_ID); - }); - - test("launchSession writes, packs, provisions, delivers, and starts", async () => { - // The launchSession call lives in `beforeAll` so the downstream - // lifecycle tests can run under a `-t` filter; this test asserts - // the side effects the call produced. - - // The deploy ack should have arrived (provision phase completed). - const publicKey = env.hub.deployAcks.get(AGENT_ADDRESS); - expect(publicKey).toBeDefined(); - if (publicKey === undefined) throw new Error("unreachable"); - expect(publicKey.length).toBeGreaterThan(0); - - // The agent should now be routable (session start completed). - expect(env.hub.router.getRoutableAddresses()).toContain(AGENT_ADDRESS); - - // The deploy tree should have landed on the sidecar's disk. - const agentDir = createSidecarRepoStore({ - dataDir: env.sidecar.dataDir, - }).getAgentDir(AGENT_ADDRESS); - - await waitFor( - async () => { - try { - await fs.promises.access(path.join(agentDir, "deploy", "prompt.md")); - return true; - } catch { - return false; - } - }, - { diagnostics: env.sidecarDiagnostics }, - ); - - const prompt = await fs.promises.readFile( - path.join(agentDir, "deploy", "prompt.md"), - "utf-8", - ); - expect(prompt).toContain("integration test agent"); - }); - - test("send message and inference receives the asset-backed mail tool", async () => { - // The setup message landed in `beforeAll`; this test asserts the - // resulting inference request and event chain. Reading by the - // recorded indices keeps the assertions correct regardless of - // whether prior tests in the suite also bumped the counters. - const req = env.inference.requests[setupInferenceRequestIndex]; - if (req === undefined) throw new Error("unreachable"); - const tools = req.tools ?? []; - const toolNames = tools.map((t) => t.name); - // The synthetic tarball seeded into the workspace-builtins - // package-registry asset publishes a single `mail_send` - // definition under the `@intx/tools-mail/sidecar-bundle` factory - // id; the loader prefixes the definition name with the factory id - // to yield the qualified tool name the model sees. - expect(toolNames).toContain("@intx/tools-mail/sidecar-bundle:mail_send"); - - // reactor.start may or may not have arrived before - // `setupAgentEventsBeforeMessage` was captured (it depends on how - // fast contextStore.load() resolves), so wait until we see an - // inference.start event rather than assuming it is the very first - // new event. - function hasEventType( - event: unknown, - type: string, - ): event is { type: string } { - return ( - typeof event === "object" && - event !== null && - "type" in event && - event.type === type - ); - } - - await waitFor( - () => - env.hub.agentEvents - .slice(setupAgentEventsBeforeMessage) - .some((e) => hasEventType(e.event, "inference.start")), - { diagnostics: env.sidecarDiagnostics }, - ); - - const inferenceStartEvent = env.hub.agentEvents - .slice(setupAgentEventsBeforeMessage) - .find((e) => hasEventType(e.event, "inference.start")); - if (inferenceStartEvent === undefined) throw new Error("unreachable"); - expect(inferenceStartEvent.addr).toBe(AGENT_ADDRESS); - expect(inferenceStartEvent.sid).toBe(SESSION_ID); - }); - - test("sync request triggers state push to hub repo", async () => { - const packCountBefore = env.hub.statePacks.length; - env.hub.router.sendSyncRequest(AGENT_ADDRESS); - - // 30s headroom: the state-pack push travels the - // sidecar -> hub pack-push pipeline and the default - // `waitFor` 10s budget flaked ~10% of runs on busier - // machines. - await waitFor(() => env.hub.statePacks.length > packCountBefore, { - timeoutMs: 30_000, - diagnostics: env.sidecarDiagnostics, - }); - - const last = env.hub.statePacks[env.hub.statePacks.length - 1]; - if (last === undefined) throw new Error("unreachable"); - expect(last.agentAddress).toBe(AGENT_ADDRESS); - expect(last.ref).toMatch(/^refs\//); - expect(last.commitSha).toMatch(/^[0-9a-f]{40}$/); - expect(env.hub.router.getConnectedSidecars()).toContain(SIDECAR_ID); - - // Verify the pack was actually persisted in the hub's git repo. - const hubAgentDir = path.join( - env.hub.hubDataDir, - "agents", - parseAgentId(AGENT_ADDRESS), - ); - const resolvedSha = await git.resolveRef({ - fs, - dir: hubAgentDir, - ref: last.ref, - }); - expect(resolvedSha).toBe(last.commitSha); - - // Verify the commit object is readable (pack was properly indexed). - const { commit } = await git.readCommit({ - fs, - dir: hubAgentDir, - oid: last.commitSha, - }); - expect(commit.tree).toMatch(/^[0-9a-f]{40}$/); - }); - - test("endSession drains an in-flight state read before tearing down the sidecar agent dir", async () => { - const failuresBefore = env.hub.statePackReceiveFailures.filter( - (f) => f.agentAddress === AGENT_ADDRESS, - ).length; - const packCountBefore = env.hub.statePacks.length; - const sidecarLogBefore = env.sidecar.stderr.length; - - // Race the teardown: the sync request makes the sidecar read the agent - // repo (createStatePack) at the same moment endSession deletes that - // repo's directory. The drain serializes the read ahead of the rm, so - // the read runs against a live directory instead of a vanishing one. - env.hub.router.sendSyncRequest(AGENT_ADDRESS); - await env.hub.sessionService.endSession(AGENT_ADDRESS, "test_complete"); - - // Agent should no longer be routable after ack. - expect(env.hub.router.getRoutableAddresses()).not.toContain(AGENT_ADDRESS); - - // The ack is sent after deleteAgentDir completes, so the directory - // is already gone by the time the promise resolves. - const agentDir = createSidecarRepoStore({ - dataDir: env.sidecar.dataDir, - }).getAgentDir(AGENT_ADDRESS); - const dirExists = await fs.promises - .access(agentDir) - .then(() => true) - .catch(() => false); - expect(dirExists).toBe(false); - - // Teardown still pushes agent state to the hub; wait for it to land so - // any sidecar-side read failure has had time to drain into the log. - await waitFor(() => env.hub.statePacks.length > packCountBefore, { - timeoutMs: 30_000, - diagnostics: env.sidecarDiagnostics, - }); - - // Teardown must log no state-push read failure. Both the undeploy push - // and the sync push log "State push failed for
" when their read - // hits a deleted directory; the drain keeps the read ahead of the rm, so - // with the fix this log never appears. This is a one-sided guard: on a - // fast machine the sync read can finish before the rm regardless, so a - // pass does not prove the read raced -- but a build that drops the drain - // and loses the race fails here. The deterministic ordering check is the - // session-manager unit test. The hub-side counter stays flat too, since a - // sidecar-side read throw never reaches the receive boundary. - const newSidecarLog = env.sidecar.stderr.slice(sidecarLogBefore).join(""); - expect(newSidecarLog).not.toContain("State push failed"); - const failuresAfter = env.hub.statePackReceiveFailures.filter( - (f) => f.agentAddress === AGENT_ADDRESS, - ).length; - expect(failuresAfter).toBe(failuresBefore); - }); -}); diff --git a/tests/hub-agent/lib/deploy-flow-env.test.ts b/tests/hub-agent/lib/deploy-flow-env.test.ts index 250e5a66..c62863a6 100644 --- a/tests/hub-agent/lib/deploy-flow-env.test.ts +++ b/tests/hub-agent/lib/deploy-flow-env.test.ts @@ -10,7 +10,6 @@ import { describe, test, expect, afterAll, beforeAll } from "bun:test"; import { WORKFLOW_RUN_TERMINAL_TYPES, - deployWorkflow, injectSignal, readWorkflowRunEvents, simulateProcessingCrash, @@ -20,13 +19,6 @@ import { type DeploymentHandle, type HubEnv, } from "./deploy-flow-env"; -import { deriveTrivialDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; -import { defaultDirectorFactory } from "@intx/agent"; -import { defineWorkflow } from "@intx/workflow/definition"; -import { wrapHarnessAsTrivialAgent } from "@intx/workflow-deploy"; -import type { ApprovalSet } from "@intx/workflow-deploy"; -import type { HarnessConfig } from "@intx/types/runtime"; -import type { SessionService } from "@intx/hub-sessions"; import fs from "node:fs"; @@ -147,88 +139,6 @@ describe("deploy-flow-env helpers smoke tests", () => { ).rejects.toThrow(/timed out/); }); - test("deployWorkflow returns a slug-derived workflowRunRepoId.id for trivial deploys", async () => { - // The supervisor's trivial branch projects the agent address - // through `deriveTrivialDeploymentId` before writing workflow-run - // events; downstream helpers (and the consumer in 13a) read the - // resulting slug off the handle. Verify the helper's reported - // id matches what the supervisor will actually commit to. - const agentAddress = "ins_slug-probe@integration.interchange"; - const agentId = "slug-probe"; - const expectedSlug = deriveTrivialDeploymentId(agentAddress); - expect(expectedSlug).not.toBe(agentAddress); - - const config: HarnessConfig = { - sessionId: "ses-slug-1", - agentId, - tenantId: "tenant-1", - principalId: "prin-1", - agentAddress, - systemPrompt: "slug-probe-prompt", - tools: [], - grants: [], - sources: [ - { - id: "src-slug-1", - provider: "anthropic", - baseURL: "https://api.example/anthropic", - apiKey: "secret-key", - model: "mock-model", - }, - ], - defaultSource: "src-slug-1", - }; - const deployContent = { systemPrompt: "slug-probe-prompt" }; - const trivialAgent = wrapHarnessAsTrivialAgent({ config, deployContent }); - const workflow = defineWorkflow({ - id: `wf_${agentId}`, - agent: trivialAgent, - trigger: { type: "mail", to: agentAddress }, - }); - const approvals = new Set(); - for (const source of config.sources) { - approvals.add(`inference.source:${source.provider}:${source.model}`); - } - approvals.add(`director:${defaultDirectorFactory.id}`); - approvals.add(`mail.address:${agentAddress}`); - const at = agentAddress.lastIndexOf("@"); - if (at < 0 || at >= agentAddress.length - 1) { - throw new Error("unreachable"); - } - approvals.add(`mail.send:${agentAddress.slice(at + 1)}`); - const operatorApprovals: ApprovalSet = approvals; - - // Replace `launchSession` with a no-op so deployWorkflow's - // trivial path completes without engaging sidecar provisioning. - // The slug derivation is the contract under test and happens - // independently of the launch payload. - const originalLaunch = env.hub.sessionService.launchSession.bind( - env.hub.sessionService, - ); - const stubLaunch: SessionService["launchSession"] = async () => { - // The trivial branch's launch is not the contract under test; - // a no-op completes the deploy without engaging the sidecar. - }; - env.hub.sessionService.launchSession = stubLaunch; - try { - const handle = await deployWorkflow(env, workflow, { - config, - deployContent, - operatorApprovals, - trivialBindings: { - agentAddress, - agentId, - instanceId: `inst_${agentId}`, - }, - }); - expect(handle.workflowRunRepoId.kind).toBe("workflow-run"); - expect(handle.workflowRunRepoId.id).toBe(expectedSlug); - expect(handle.workflowRunRepoId.id).not.toBe(agentAddress); - } finally { - env.hub.sessionService.launchSession = originalLaunch; - } - }); - test("simulateProcessingCrash composes enqueueInbox + dequeueToProcessing", async () => { const address = "ins_smoke-test@integration.interchange"; const messageId = ""; diff --git a/tests/hub-agent/lib/deploy-flow-env.ts b/tests/hub-agent/lib/deploy-flow-env.ts index 26aeb549..c1044e5c 100644 --- a/tests/hub-agent/lib/deploy-flow-env.ts +++ b/tests/hub-agent/lib/deploy-flow-env.ts @@ -45,6 +45,7 @@ import { type MessageHeaders, } from "@intx/mime"; import { + bridgeOrchestratorDeployContent, createAgentRepoStore, createSessionService, createSidecarRouter, @@ -75,7 +76,6 @@ import type { HarnessConfig } from "@intx/types/runtime"; import type { ToolPackagePin } from "@intx/types/tool-packages"; import type { ApprovalSet } from "@intx/workflow-deploy"; import type { WorkflowDefinition } from "@intx/workflow"; -import { deriveTrivialDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; export const AGENT_ADDRESS = "ins_test-agent@integration.interchange"; export const AGENT_ID = "ins_test-agent"; @@ -469,6 +469,13 @@ export type HubEnv = { deployAcks: Map; statePacks: { agentAddress: string; ref: string; commitSha: string }[]; statePackReceiveFailures: { agentAddress: string; error: string }[]; + /** + * Every delivered `mail.outbound` frame the sidecar forwarded to the hub + * for persistence, keyed by the signing sender. A frame reaches here only + * after the sidecar signed and delivered the send, so its presence proves + * the sender's identity was registered on the host transport. + */ + outboundMail: { senderAddress: string; recipients: string[] }[]; hubDataDir: string; }; @@ -486,6 +493,7 @@ export async function startHub( const deployAcks = new Map(); const statePacks: HubEnv["statePacks"] = []; const statePackReceiveFailures: HubEnv["statePackReceiveFailures"] = []; + const outboundMail: HubEnv["outboundMail"] = []; const hubDataDir = await fs.promises.mkdtemp( path.join(os.tmpdir(), "hub-data-"), @@ -562,6 +570,14 @@ export async function startHub( } return { accepted: true }; }, + // Capture delivered outbound mail the sidecar forwards for + // persistence. Recording the signing sender is enough for the + // integration assertions; no durable row is minted, so this returns + // an empty result set. + persistMail({ senderAddress, recipients }) { + outboundMail.push({ senderAddress, recipients }); + return Promise.resolve([]); + }, }, }); router.events.on("agent.event", ({ agentAddress, sessionId, event }) => { @@ -722,6 +738,7 @@ export async function startHub( deployAcks, statePacks, statePackReceiveFailures, + outboundMail, hubDataDir, }; } @@ -992,10 +1009,9 @@ const DEFAULT_WORKFLOW_RUN_REF = "refs/heads/main"; /** * Options accepted by `deployWorkflow`. The helper composes the - * workflow-deploy orchestrator against the env's hub substrate and - * routes per-step launches through `env.hub.sessionService.launchSession` - * so the trivial-branch round-trip and the multi-step branch both - * exercise the production code paths. + * workflow-deploy orchestrator against the env's hub substrate and routes + * per-step launches through `env.hub.sessionService.stageWorkflowStep`, the + * stage-only path production's multi-step branch runs. */ export type DeployWorkflowOpts = { /** @@ -1010,22 +1026,14 @@ export type DeployWorkflowOpts = { * branch from the step's agent definition. */ deployContent: { systemPrompt: string }; - /** Pre-existing per-agent address binding for the trivial branch. */ - trivialBindings?: { - agentAddress: string; - agentId: string; - instanceId: string; - }; /** - * Stable identifier the multi-step branch concatenates into derived - * agent addresses. Required when `trivialBindings` is absent. + * Stable identifier the branch concatenates into derived agent + * addresses. Required. */ - deploymentId?: string; + deploymentId: string; /** - * Mail-domain for the deployment. Required when `trivialBindings` is - * absent. Defaults to the integration test's canonical domain so - * callers that exercise the multi-step branch with the default - * fixture wiring do not have to thread the domain through. + * Mail-domain for the deployment. Defaults to the integration test's + * canonical domain so callers do not have to thread the domain through. */ deploymentDomain?: string; /** Tool-package pins to ship with every step's deploy. */ @@ -1043,8 +1051,7 @@ export type DeployWorkflowOpts = { workflowRunRef?: string; /** * Optional override for the deployment's mail address. Defaults to - * the trivial bindings' `agentAddress` (trivial branch) or - * `ins_@` (multi-step branch). + * `ins_@`. */ deploymentMailAddress?: string; }; @@ -1063,11 +1070,9 @@ export type DeployWorkflowHandle = { /** * Build a workflow-deploy orchestrator wired against the env's hub - * substrate and run it against the supplied workflow. Trivial - * single-step workflows route through `env.hub.sessionService.launchSession` - * (which itself invokes the orchestrator's trivial branch); multi-step - * workflows derive per-step addresses and route each launch through - * the same `launchSession` callback. + * substrate and run it against the supplied workflow. The orchestrator + * derives per-step addresses and routes each launch through the + * `launchSession` callback. * * Registers the resulting handle on `env.deployments` so the other * Phase I helpers (event reads, signal injection, drain initiation, @@ -1080,70 +1085,34 @@ export async function deployWorkflow( ): Promise { const workflowRunRef = opts.workflowRunRef ?? DEFAULT_WORKFLOW_RUN_REF; - let deploymentId: string; - let mailAddress: string; - if (opts.trivialBindings !== undefined) { - if (workflow.stepOrder.length !== 1) { - throw new Error( - `deployWorkflow: trivialBindings supplied for a ${String(workflow.stepOrder.length)}-step workflow; trivial deploy requires exactly one step`, - ); - } - deploymentId = opts.trivialBindings.agentAddress; - mailAddress = - opts.deploymentMailAddress ?? opts.trivialBindings.agentAddress; - } else { - const explicit = opts.deploymentId; - if (explicit === undefined) { - throw new Error( - "deployWorkflow: deploymentId is required when trivialBindings is absent", - ); - } - deploymentId = explicit; - const deploymentDomain = opts.deploymentDomain ?? DEFAULT_DEPLOYMENT_DOMAIN; - mailAddress = - opts.deploymentMailAddress ?? `ins_${deploymentId}@${deploymentDomain}`; - } + const deploymentId = opts.deploymentId; + const deploymentDomain = opts.deploymentDomain ?? DEFAULT_DEPLOYMENT_DOMAIN; + const mailAddress = + opts.deploymentMailAddress ?? `ins_${deploymentId}@${deploymentDomain}`; - // The supervisor's trivial branch projects the agent address into - // a substrate-safe slug via `deriveTrivialDeploymentId` before - // writing workflow-run events (see - // `apps/sidecar/src/workflow-host-wiring.ts`). The fixture must - // report the same slug so downstream helpers query the repo the - // supervisor actually committed to. Multi-step deployments supply - // their own substrate-safe `deploymentId` and pass through - // unchanged. - const workflowRunRepoSlug = - opts.trivialBindings !== undefined - ? deriveTrivialDeploymentId(opts.trivialBindings.agentAddress) - : deploymentId; + // The deployment supplies its own substrate-safe `deploymentId`, which + // is the workflow-run repo slug the supervisor commits under. const workflowRunRepoId: RepoId = { kind: "workflow-run", - id: workflowRunRepoSlug, + id: deploymentId, }; - // Route every per-step launch through the session service. The - // session service's `launchSession` itself routes through the - // orchestrator's trivial branch, so this preserves the bit-identical - // trivial round-trip the existing deploy-flow test asserts. - // - // `launchSession`'s `deployContent` parameter widens - // `toolPackageManifest` to `unknown` in the orchestrator's surface - // shape; the session-service's `bridgeOrchestratorDeployContent` - // narrows it back at the inner boundary, so the cast here only - // crosses the structural-shape gap between the orchestrator's - // `OrchestratorDeployContent` and the session-service's - // `DeployContent`. + // Route every per-step launch through the session service, mirroring + // the production multi-step branch, which drives the orchestrator's + // per-step launch callback against `stageWorkflowStep` (stage-only, no + // warm harness). The orchestrator's `DeployContent` widens + // `toolPackageManifest` to `unknown`; `bridgeOrchestratorDeployContent` + // narrows and validates it back to the hub-sessions shape -- the same + // bridge production uses. const launchSession: LaunchSessionFn = async (orchestratorParams) => { - const deployContent = orchestratorParams.deployContent; - await env.hub.sessionService.launchSession({ + await env.hub.sessionService.stageWorkflowStep({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, config: orchestratorParams.config, - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the session-service's launchSession invokes the orchestrator internally and re-narrows `toolPackageManifest` via arktype; this fixture forwards the orchestrator-shaped deploy content as-is - deployContent: deployContent as Parameters< - SessionService["launchSession"] - >[0]["deployContent"], + deployContent: bridgeOrchestratorDeployContent( + orchestratorParams.deployContent, + ), ...(orchestratorParams.toolPackagePins !== undefined ? { toolPackagePins: orchestratorParams.toolPackagePins } : {}), @@ -1178,18 +1147,11 @@ export async function deployWorkflow( await orchestrator.deployWorkflow({ workflow, + deploymentId, + deploymentDomain, config: opts.config, deployContent: opts.deployContent, operatorApprovals: opts.operatorApprovals, - ...(opts.trivialBindings !== undefined - ? { trivialBindings: opts.trivialBindings } - : {}), - ...(opts.trivialBindings === undefined - ? { - deploymentId, - deploymentDomain: opts.deploymentDomain ?? DEFAULT_DEPLOYMENT_DOMAIN, - } - : {}), ...(opts.toolPackagePins !== undefined ? { toolPackagePins: opts.toolPackagePins } : {}), diff --git a/tests/hub-api/asset-routes.test.ts b/tests/hub-api/asset-routes.test.ts index 3f459b0c..88a6ed4e 100644 --- a/tests/hub-api/asset-routes.test.ts +++ b/tests/hub-api/asset-routes.test.ts @@ -142,11 +142,11 @@ function createMockSidecarRouter(): SidecarRouter { routeMail: () => notImpl("routeMail"), sendAgentDeploy: () => notImpl("sendAgentDeploy"), sendAgentUndeploy: () => notImpl("sendAgentUndeploy"), - sendSessionStart: () => notImpl("sendSessionStart"), - sendSessionAbort: () => notImpl("sendSessionAbort"), - sendGrantsUpdate: () => notImpl("sendGrantsUpdate"), sendSourcesUpdate: () => notImpl("sendSourcesUpdate"), sendPack: () => notImpl("sendPack"), + sendProvisionStep: () => notImpl("sendProvisionStep"), + bindStepRoute: () => notImpl("bindStepRoute"), + unbindStepRoute: () => notImpl("unbindStepRoute"), sendSyncRequest: () => notImpl("sendSyncRequest"), sendSignalDeliver: () => notImpl("sendSignalDeliver"), sendDrain: () => notImpl("sendDrain"), @@ -161,10 +161,14 @@ function createMockSidecarRouter(): SidecarRouter { function createMockSessionService(): SessionService { return { - launchSession: notImplemented("sessionService.launchSession"), + stageWorkflowStep: notImplemented("sessionService.stageWorkflowStep"), + deployInstanceAtHead: notImplemented("sessionService.deployInstanceAtHead"), deployWorkflowDefinition: notImplemented( "sessionService.deployWorkflowDefinition", ), + deploySingleStepAtHead: notImplemented( + "sessionService.deploySingleStepAtHead", + ), sendUserMessage: notImplemented("sessionService.sendUserMessage"), endSession: notImplemented("sessionService.endSession"), }; diff --git a/tests/workflow-deploy/README.md b/tests/workflow-deploy/README.md new file mode 100644 index 00000000..a6686f86 --- /dev/null +++ b/tests/workflow-deploy/README.md @@ -0,0 +1,41 @@ +# Workflow-run latency benches + +Two off-by-default measurement benches profile the sustained per-message +latency of the workflow-run substrate. They are `*.bench.ts`, not +`*.test.ts`, so `make test` never runs them; `make build` type-checks +them via this directory's tsconfig. Both drive one warm agent +back-to-back with inference mocked -- message N+1 fires only after reply +N (no pipelining) -- and discard the first (cold) message so agent +build, tool materialization, and LSP spawn cost are excluded. The +steady-state per-message round-trip is what they measure. + +## Running + +Each bench takes `--messages N` (default in-file) and `--out ` and +prints its table to stdout. Send output to a throwaway `tmp/` dir +(gitignored): + +``` +bun run tests/workflow-deploy/latency-gate.bench.ts --messages 200 --out tmp/latency/gate +bun run tests/workflow-deploy/latency-d2-attribution.bench.ts --messages 200 --out tmp/latency/d2 +``` + +A bench flushes its result JSON and prints its table before teardown, +and teardown can hang; once the table has printed (the JSON is already +written), interrupt the process. + +## What each measures + +- **`latency-gate.bench.ts`** -- the dispatch->reply gate. Runs a + baseline (in-process agent) and the unified path (subprocess child + + IPC-proxied commits); writes `raw-baseline.csv`, `raw-unified.csv`, + and `results.json`. Read the `unified`/`baseline` percentiles and + `trend.unified.slopeMsPerMessage` (ms added per sustained message). +- **`latency-d2-attribution.bench.ts`** -- splits the unified path's + per-message substrate cost across the individual git-commit legs + (dequeue, run-event, agent-state WAL, ...); writes `d2-results.json`. + Read `total.slopeMsPerMessage` and `perLeg[].slopeMsPerMessage`. + +A healthy substrate holds the slopes flat (near zero) as `--messages` +grows; a rising slope is the per-message growth these benches exist to +catch. diff --git a/tests/workflow-deploy/child-workflow-roundtrip.test.ts b/tests/workflow-deploy/child-workflow-roundtrip.test.ts index 5d4133c8..d11f3c28 100644 --- a/tests/workflow-deploy/child-workflow-roundtrip.test.ts +++ b/tests/workflow-deploy/child-workflow-roundtrip.test.ts @@ -37,7 +37,7 @@ import { type SendMultiStepDeployFn, type WorkflowRepoWriter, } from "@intx/workflow-deploy"; -import { deriveTrivialDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; +import { deriveDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; import type { RepoId, WorkflowRunHubPrincipal } from "@intx/hub-sessions"; import { DEFAULT_ASSET_REF } from "@intx/hub-sessions"; @@ -194,7 +194,7 @@ describe("parent -> child workflow round-trip", () => { }); const launchSession: LaunchSessionFn = async (orchestratorParams) => { - await env.hub.sessionService.launchSession({ + await env.hub.sessionService.stageWorkflowStep({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, @@ -245,6 +245,10 @@ describe("parent -> child workflow round-trip", () => { workflowRepo, launchSession, sendMultiStepDeploy, + // A single-step (child-spawning) workflow deploys once at the head + // through the deploy core's single-step hand-off. + deploySingleStepAtHead: (params) => + env.hub.sessionService.deploySingleStepAtHead(params), }); const childResult = await orchestrator.deployWorkflow({ @@ -262,7 +266,7 @@ describe("parent -> child workflow round-trip", () => { deploymentDomain: DEPLOYMENT_DOMAIN, hubPublicKey: "00".repeat(32), }); - expect(childResult.kind).toBe("multi-step"); + expect(childResult.publicKey).toBeTruthy(); const parentResult = await orchestrator.deployWorkflow({ workflow: parentWorkflowDefinition, @@ -279,11 +283,11 @@ describe("parent -> child workflow round-trip", () => { deploymentDomain: DEPLOYMENT_DOMAIN, hubPublicKey: "00".repeat(32), }); - expect(parentResult.kind).toBe("multi-step"); + expect(parentResult.publicKey).toBeTruthy(); const parentWorkflowRunRepoId: RepoId = { kind: "workflow-run", - id: deriveTrivialDeploymentId(parentMailAddress), + id: deriveDeploymentId(parentMailAddress), }; env.registerDeployment({ deploymentId: PARENT_DEPLOYMENT_ID, @@ -569,7 +573,7 @@ describe("parent -> child workflow round-trip", () => { }); const launchSession: LaunchSessionFn = async (orchestratorParams) => { - await env.hub.sessionService.launchSession({ + await env.hub.sessionService.stageWorkflowStep({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, @@ -620,6 +624,10 @@ describe("parent -> child workflow round-trip", () => { workflowRepo, launchSession, sendMultiStepDeploy, + // A single-step (child-spawning) workflow deploys once at the head + // through the deploy core's single-step hand-off. + deploySingleStepAtHead: (params) => + env.hub.sessionService.deploySingleStepAtHead(params), }); // Deploy grandchild and child as workflow assets so the parent's @@ -641,7 +649,7 @@ describe("parent -> child workflow round-trip", () => { deploymentDomain: DEPLOYMENT_DOMAIN, hubPublicKey: "00".repeat(32), }); - expect(grandchildResult.kind).toBe("multi-step"); + expect(grandchildResult.publicKey).toBeTruthy(); const childResult = await orchestrator.deployWorkflow({ workflow: childWorkflowDefinition, @@ -658,7 +666,7 @@ describe("parent -> child workflow round-trip", () => { deploymentDomain: DEPLOYMENT_DOMAIN, hubPublicKey: "00".repeat(32), }); - expect(childResult.kind).toBe("multi-step"); + expect(childResult.publicKey).toBeTruthy(); const parentResult = await orchestrator.deployWorkflow({ workflow: parentWorkflowDefinition, @@ -675,11 +683,11 @@ describe("parent -> child workflow round-trip", () => { deploymentDomain: DEPLOYMENT_DOMAIN, hubPublicKey: "00".repeat(32), }); - expect(parentResult.kind).toBe("multi-step"); + expect(parentResult.publicKey).toBeTruthy(); const parentWorkflowRunRepoId: RepoId = { kind: "workflow-run", - id: deriveTrivialDeploymentId(parentMailAddress), + id: deriveDeploymentId(parentMailAddress), }; env.registerDeployment({ deploymentId: NESTED_PARENT_DEPLOYMENT_ID, @@ -924,7 +932,7 @@ describe("parent -> child workflow round-trip", () => { }); const launchSession: LaunchSessionFn = async (orchestratorParams) => { - await env.hub.sessionService.launchSession({ + await env.hub.sessionService.stageWorkflowStep({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, @@ -975,6 +983,10 @@ describe("parent -> child workflow round-trip", () => { workflowRepo, launchSession, sendMultiStepDeploy, + // A single-step (child-spawning) workflow deploys once at the head + // through the deploy core's single-step hand-off. + deploySingleStepAtHead: (params) => + env.hub.sessionService.deploySingleStepAtHead(params), }); for (let i = 0; i < SIBLINGS_CHILD_COUNT; i += 1) { @@ -999,7 +1011,7 @@ describe("parent -> child workflow round-trip", () => { deploymentDomain: DEPLOYMENT_DOMAIN, hubPublicKey: "00".repeat(32), }); - expect(r.kind).toBe("multi-step"); + expect(r.publicKey).toBeTruthy(); } const parentResult = await orchestrator.deployWorkflow({ @@ -1017,11 +1029,11 @@ describe("parent -> child workflow round-trip", () => { deploymentDomain: DEPLOYMENT_DOMAIN, hubPublicKey: "00".repeat(32), }); - expect(parentResult.kind).toBe("multi-step"); + expect(parentResult.publicKey).toBeTruthy(); const parentWorkflowRunRepoId: RepoId = { kind: "workflow-run", - id: deriveTrivialDeploymentId(parentMailAddress), + id: deriveDeploymentId(parentMailAddress), }; env.registerDeployment({ deploymentId: SIBLINGS_PARENT_DEPLOYMENT_ID, diff --git a/tests/workflow-deploy/conversation-state-wal.test.ts b/tests/workflow-deploy/conversation-state-wal.test.ts index 38372349..9169cd02 100644 --- a/tests/workflow-deploy/conversation-state-wal.test.ts +++ b/tests/workflow-deploy/conversation-state-wal.test.ts @@ -26,6 +26,7 @@ import { generateKeyPair } from "@intx/crypto"; import { createSSHSignature } from "@intx/crypto"; import type { KeyPair, + ConnectorThreadState, ConversationTurn, PendingOperation, TokenUsage, @@ -467,4 +468,311 @@ describe("durable conversation store WAL + checkpoint (Phase D1)", () => { reconstructDurableConversation(h.agentStateDir, AGENT_KEY), ).rejects.toThrow(/seq gap/); }); + + test("a turn appended during the WAL write is not skipped by the next mirror", async () => { + // Regression guard: mirrorToSubstrate slices its new-turn delta from the + // reactor's live array BEFORE the appendWalEntry await, then advances the + // mirrored turn count AFTER it. peekTurns returns that array by reference, + // so a turn the reactor appends DURING the await must be counted as the + // count actually persisted -- not the post-await live length -- or the + // next mirror slices past it and drops it from the WAL permanently. + const liveTurns: ConversationTurn[] = [userTurn("a")]; + let injected = false; + + // Wrap the substrate so the first WAL append (boundary 0) appends a turn + // to the reactor's live array mid-write, reproducing the concurrent + // append in the between-slice-and-count window. + const writeTreePreservingPrefix: RepoStore["writeTreePreservingPrefix"] = ( + principal, + repoId, + ref, + args, + ) => { + if (!injected && args.preservePrefix.includes("/wal/")) { + injected = true; + liveTurns.push(userTurn("b")); + } + return h.substrate.writeTreePreservingPrefix( + principal, + repoId, + ref, + args, + ); + }; + const wrappedSubstrate = new Proxy(h.substrate, { + get(target, prop, receiver): unknown { + if (prop === "writeTreePreservingPrefix") { + return writeTreePreservingPrefix; + } + return Reflect.get(target, prop, receiver); + }, + }); + + const store = await createDurableConversationStore({ + localStoreDir: localDir, + signer: h.signer, + substrate: wrappedSubstrate, + workflowRunRepoId: h.workflowRunRepoId, + workflowRunRef: WORKFLOW_RUN_REF, + principal: PRINCIPAL, + agentKey: AGENT_KEY, + }); + + // Boundary 0: the local store holds [a]; during its WAL append the + // wrapper appends b to the reactor's live array. + await store.storage.writeTurns(liveTurns); + await store.storage.writeMetadata({ + pendingOperations: [], + tokenUsage: tokenUsage(0), + }); + await store.storage.commit({ message: "turn-a" }); + await store.mirrorToSubstrate(); + expect(injected).toBe(true); + + // Boundary 1: the reactor has since persisted [a, b] locally. The mirror + // must pick up b. The pre-fix code counted b as already mirrored at + // boundary 0 (reading the live array length after the await) and sliced + // past it here, so boundary 1's WAL entry was an empty delta and b was + // lost from the durable log. + await store.storage.writeTurns(liveTurns); + await store.storage.writeMetadata({ + pendingOperations: [], + tokenUsage: tokenUsage(1), + }); + await store.storage.commit({ message: "turn-b" }); + await store.mirrorToSubstrate(); + + const entry = WalEntryShape(readWalEntry(h.agentStateDir, 1)); + if (entry instanceof type.errors) { + throw new Error(`WAL entry 1 failed validation: ${entry.summary}`); + } + expect(entry.turns.length).toBe(1); + + // Reconstruction yields both turns; under the bug it would yield only [a]. + const reconstructed = await reconstructDurableConversation( + h.agentStateDir, + AGENT_KEY, + ); + if (reconstructed === null) throw new Error("expected a reconstruction"); + expect(reconstructed.turns).toEqual([userTurn("a"), userTurn("b")]); + }); + + test("overlapping mirror runs serialize onto distinct boundaries and lose no turns", async () => { + // The dormant race this guards against: the awaited onRunBoundary mirror + // and the fire-and-forget onStateChanged mirror share the mirrored-count + // state with no serialization. Two overlapping runs both read the same + // boundary seq AND the same mirroredTurnCount, so they collide on one + // boundary and double-advance the turn count -- and a following mirror + // then slices past a genuinely new turn and drops it from the log. + // + // The barrier makes the interleave deterministic: the first WAL append is + // held until a would-be-concurrent second mirror has had time to reach + // its own append. Without serialization the second append runs on the + // stale counts; with it the second mirror does not start until the first + // advances them. + let releaseFirstAppend!: () => void; + const firstAppendHeld = new Promise((resolve) => { + releaseFirstAppend = resolve; + }); + let heldOnce = false; + const writeTreePreservingPrefix: RepoStore["writeTreePreservingPrefix"] = + async (principal, repoId, ref, args) => { + if (!heldOnce && args.preservePrefix.includes("/wal/")) { + heldOnce = true; + await firstAppendHeld; + } + return h.substrate.writeTreePreservingPrefix( + principal, + repoId, + ref, + args, + ); + }; + const wrappedSubstrate = new Proxy(h.substrate, { + get(target, prop, receiver): unknown { + if (prop === "writeTreePreservingPrefix") { + return writeTreePreservingPrefix; + } + return Reflect.get(target, prop, receiver); + }, + }); + + const store = await createDurableConversationStore({ + localStoreDir: localDir, + signer: h.signer, + substrate: wrappedSubstrate, + workflowRunRepoId: h.workflowRunRepoId, + workflowRunRef: WORKFLOW_RUN_REF, + principal: PRINCIPAL, + agentKey: AGENT_KEY, + }); + + // The local store holds [a, b]; fire two overlapping mirrors of it. + const ab = [userTurn("a"), userTurn("b")]; + await store.storage.writeTurns(ab); + await store.storage.writeMetadata({ + pendingOperations: [], + tokenUsage: tokenUsage(0), + }); + await store.storage.commit({ message: "ab" }); + + const both = Promise.all([ + store.mirrorToSubstrate(), + store.mirrorToSubstrate(), + ]); + // Let a would-be-concurrent second mirror reach its append before the + // first is released. Under serialization the second has not started, so + // only the first append is held here. + await new Promise((resolve) => setTimeout(resolve, 40)); + releaseFirstAppend(); + await both; + + // A following mirror carries a genuinely new turn. If the overlapping + // pair over-counted the mirrored turn count, this mirror slices past `c` + // and drops it from the durable log. + const abc = [...ab, userTurn("c")]; + await store.storage.writeTurns(abc); + await store.storage.writeMetadata({ + pendingOperations: [], + tokenUsage: tokenUsage(1), + }); + await store.storage.commit({ message: "abc" }); + await store.mirrorToSubstrate(); + + const reconstructed = await reconstructDurableConversation( + h.agentStateDir, + AGENT_KEY, + ); + if (reconstructed === null) throw new Error("expected a reconstruction"); + expect(reconstructed.turns).toEqual(abc); + }); + + test("a mirror re-entered by restore's connector-state change cannot clobber a concurrent boundary", async () => { + // Restore-reentrancy guard. restoreFromSubstrate calls + // connectorRouter.restore(), which fires onStateChanged synchronously when + // the restored connector state differs from the router's initial null -- + // and onStateChanged enqueues a (turnless) mirror. That reentrant mirror + // shares the mirrored-count state with restore and with the next real + // mirror. Unless restore runs on the same serialization tail (and + // establishes the counts before the connector restore), the reentrant + // append and a following real mirror can land on the SAME boundary seq, + // and the turnless one can overwrite the real turn. + // + // Seed two durable boundaries, the second carrying a non-null connector + // state -- exactly the future condition (a non-null connector state) under + // which this otherwise-dormant path activates. + const seed = await makeStore(h, localDir); + await seed.storage.writeTurns([userTurn("a")]); + await seed.storage.writeMetadata({ + pendingOperations: [], + tokenUsage: tokenUsage(0), + }); + await seed.storage.commit({ message: "a" }); + await seed.mirrorToSubstrate(); + + const connectorState: ConnectorThreadState = { + threadRoot: "", + lastMessageId: "", + replyTo: "user@example.com", + cc: [], + }; + seed.storage.setConnectorState(connectorState); + await seed.storage.writeTurns([userTurn("a"), userTurn("b")]); + await seed.storage.writeMetadata({ + pendingOperations: [], + tokenUsage: tokenUsage(1), + }); + await seed.storage.commit({ message: "b" }); + await seed.mirrorToSubstrate(); + + // A fresh store restores those two boundaries; a barrier holds the first + // WAL append (the reentrant turnless mirror) until a real `c` mirror is + // enqueued behind it. Every boundary seq the substrate writes is recorded + // (parsed from the append's commit message) so a same-seq clobber fails + // loud even if a future change masks the turn-loss symptom. + const seqsWritten: number[] = []; + let releaseFirstAppend!: () => void; + const firstAppendHeld = new Promise((resolve) => { + releaseFirstAppend = resolve; + }); + let heldOnce = false; + const writeTreePreservingPrefix: RepoStore["writeTreePreservingPrefix"] = + async (principal, repoId, ref, args) => { + const matched = /WAL boundary (\d+)/.exec(args.message); + if (matched !== null) seqsWritten.push(Number(matched[1])); + if (!heldOnce && args.preservePrefix.includes("/wal/")) { + heldOnce = true; + await firstAppendHeld; + } + return h.substrate.writeTreePreservingPrefix( + principal, + repoId, + ref, + args, + ); + }; + const wrappedSubstrate = new Proxy(h.substrate, { + get(target, prop, receiver): unknown { + if (prop === "writeTreePreservingPrefix") { + return writeTreePreservingPrefix; + } + return Reflect.get(target, prop, receiver); + }, + }); + + const freshLocalDir = path.join(h.baseDir, "reentry-local"); + const store = await createDurableConversationStore({ + localStoreDir: freshLocalDir, + signer: h.signer, + substrate: wrappedSubstrate, + workflowRunRepoId: h.workflowRunRepoId, + workflowRunRef: WORKFLOW_RUN_REF, + principal: PRINCIPAL, + agentKey: AGENT_KEY, + }); + + const restored = await store.restoreFromSubstrate(); + expect(restored).toBe(true); + // Wait until the reentrant mirror's append is parked, so it is + // unambiguously the first (held) WAL append. + for (let i = 0; i < 200 && !heldOnce; i += 1) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } + expect(heldOnce).toBe(true); + + // A real mirror carrying a genuinely new turn `c`, enqueued behind the + // parked reentrant mirror. + await store.storage.writeTurns([ + userTurn("a"), + userTurn("b"), + userTurn("c"), + ]); + await store.storage.writeMetadata({ + pendingOperations: [], + tokenUsage: tokenUsage(2), + }); + await store.storage.commit({ message: "c" }); + const cMirror = store.mirrorToSubstrate(); + + // Let a would-be-concurrent `c` mirror reach and pass its own append + // before the reentrant one is released. + await new Promise((resolve) => setTimeout(resolve, 40)); + releaseFirstAppend(); + await cMirror; + // The reentrant mirror is fire-and-forget; let it settle after release. + await new Promise((resolve) => setTimeout(resolve, 40)); + + const reconstructed = await reconstructDurableConversation( + h.agentStateDir, + AGENT_KEY, + ); + if (reconstructed === null) throw new Error("expected a reconstruction"); + expect(reconstructed.turns).toEqual([ + userTurn("a"), + userTurn("b"), + userTurn("c"), + ]); + // No two WAL appends targeted the same boundary seq. + expect(new Set(seqsWritten).size).toBe(seqsWritten.length); + }); }); diff --git a/tests/workflow-deploy/cross-process-custom-adapter.test.ts b/tests/workflow-deploy/cross-process-custom-adapter.test.ts index 76a9ab56..b8cd1e7d 100644 --- a/tests/workflow-deploy/cross-process-custom-adapter.test.ts +++ b/tests/workflow-deploy/cross-process-custom-adapter.test.ts @@ -35,11 +35,12 @@ import { createWorkflowDeployOrchestrator, deriveDeploymentAddress, type ApprovalSet, + type DeploySingleStepFn, type LaunchSessionFn, type SendMultiStepDeployFn, type WorkflowRepoWriter, } from "@intx/workflow-deploy"; -import { deriveTrivialDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; +import { deriveDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; import type { RepoId, WorkflowRunHubPrincipal } from "@intx/hub-sessions"; import { DEFAULT_ASSET_REF } from "@intx/hub-sessions"; @@ -157,7 +158,7 @@ async function deployCustomProviderWorkflow( ]); const launchSession: LaunchSessionFn = async (orchestratorParams) => { - await env.hub.sessionService.launchSession({ + await env.hub.sessionService.stageWorkflowStep({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, @@ -183,6 +184,9 @@ async function deployCustomProviderWorkflow( sources: params.sources, }); + const deploySingleStepAtHead: DeploySingleStepFn = (params) => + env.hub.sessionService.deploySingleStepAtHead(params); + const workflowRepo: WorkflowRepoWriter = { async writeWorkflowRepo(args) { const repoId: RepoId = { kind: "workflow", id: args.workflowRepoId }; @@ -208,6 +212,7 @@ async function deployCustomProviderWorkflow( workflowRepo, launchSession, sendMultiStepDeploy, + deploySingleStepAtHead, }); const result = await orchestrator.deployWorkflow({ @@ -219,11 +224,11 @@ async function deployCustomProviderWorkflow( deploymentDomain: DEPLOYMENT_DOMAIN, hubPublicKey: "00".repeat(32), }); - expect(result.kind).toBe("multi-step"); + expect(result.publicKey).toBeTruthy(); const workflowRunRepoId: RepoId = { kind: "workflow-run", - id: deriveTrivialDeploymentId(deploymentMailAddress), + id: deriveDeploymentId(deploymentMailAddress), }; env.registerDeployment({ deploymentId, @@ -289,9 +294,12 @@ describe("cross-process custom inference adapter (INTR-233)", () => { test("a provider absent from the manifest is rejected at the source gate", async () => { // The firewall: the operator registry holds only the built-ins plus the // manifest's "custom-x". A provider id that no manifest entry and no - // built-in supplies is rejected by `canBuildSource` against that same - // registry, before the agent ever launches -- so a provider string (which - // deploy/tenant config controls) cannot conjure an adapter. + // built-in supplies is rejected by the sidecar deploy router's + // source-admission gate (which reuses `canBuildSource` against that same + // registry) before the workflow-process child is spawned -- so a + // provider string (which deploy/tenant config controls) cannot conjure + // an adapter, and the deploy is rejected synchronously at deploy time + // rather than failing the first run. await expect( deployCustomProviderWorkflow( "cross-process-custom-negative", diff --git a/tests/workflow-deploy/drain-roundtrip.test.ts b/tests/workflow-deploy/drain-roundtrip.test.ts index 451a5759..935d8482 100644 --- a/tests/workflow-deploy/drain-roundtrip.test.ts +++ b/tests/workflow-deploy/drain-roundtrip.test.ts @@ -32,10 +32,9 @@ // at this test. // // The orchestrator's multi-step branch is composed in-test (matching -// the multi-step signal round-trip) because the pre-landed -// `deploy-flow-env` fixture wires only the trivial `launchSession` -// callback against `env.hub.sessionService.launchSession`; the -// multi-step `sendMultiStepDeploy` hand-off is supplied here against +// the multi-step signal round-trip): the per-step launch callback drives +// `env.hub.sessionService.stageWorkflowStep` (the stage-only path, no warm +// harness) and the `sendMultiStepDeploy` hand-off is supplied against // `env.hub.router.sendAgentDeploy` so the sidecar's deploy router // takes the workflow-process spawn path. @@ -57,7 +56,7 @@ import { type SendMultiStepDeployFn, type WorkflowRepoWriter, } from "@intx/workflow-deploy"; -import { deriveTrivialDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; +import { deriveDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; import type { RepoId, WorkflowRunHubPrincipal } from "@intx/hub-sessions"; import { DEFAULT_ASSET_REF } from "@intx/hub-sessions"; @@ -168,7 +167,7 @@ describe("drain round-trip", () => { const launchSession: LaunchSessionFn = async (orchestratorParams) => { const deployContent = orchestratorParams.deployContent; - await env.hub.sessionService.launchSession({ + await env.hub.sessionService.stageWorkflowStep({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, @@ -240,11 +239,11 @@ describe("drain round-trip", () => { { cause }, ); } - expect(result.kind).toBe("multi-step"); + expect(result.publicKey).toBeTruthy(); const workflowRunRepoId: RepoId = { kind: "workflow-run", - id: deriveTrivialDeploymentId(deploymentMailAddress), + id: deriveDeploymentId(deploymentMailAddress), }; env.registerDeployment({ deploymentId: DEPLOYMENT_ID, @@ -412,7 +411,7 @@ describe("drain round-trip", () => { ]); const launchSession: LaunchSessionFn = async (orchestratorParams) => { const deployContent = orchestratorParams.deployContent; - await env.hub.sessionService.launchSession({ + await env.hub.sessionService.stageWorkflowStep({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, @@ -470,11 +469,11 @@ describe("drain round-trip", () => { deploymentDomain: DEPLOYMENT_DOMAIN, hubPublicKey: "00".repeat(32), }); - expect(result.kind).toBe("multi-step"); + expect(result.publicKey).toBeTruthy(); const workflowRunRepoId: RepoId = { kind: "workflow-run", - id: deriveTrivialDeploymentId(deploymentMailAddress), + id: deriveDeploymentId(deploymentMailAddress), }; env.registerDeployment({ deploymentId: waitDeploymentId, diff --git a/tests/workflow-deploy/fifo-mail-load.test.ts b/tests/workflow-deploy/fifo-mail-load.test.ts index 1b559c96..14a87605 100644 --- a/tests/workflow-deploy/fifo-mail-load.test.ts +++ b/tests/workflow-deploy/fifo-mail-load.test.ts @@ -31,11 +31,12 @@ import { createWorkflowDeployOrchestrator, deriveDeploymentAddress, type ApprovalSet, + type DeploySingleStepFn, type LaunchSessionFn, type SendMultiStepDeployFn, type WorkflowRepoWriter, } from "@intx/workflow-deploy"; -import { deriveTrivialDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; +import { deriveDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; import type { RepoId, WorkflowRunHubPrincipal } from "@intx/hub-sessions"; import { DEFAULT_ASSET_REF } from "@intx/hub-sessions"; @@ -133,13 +134,10 @@ describe("FIFO mail-trigger serialization under load", () => { test(`${String(LOAD_MAIL_COUNT)} mails dispatch in arrival order under load`, async () => { // Coverage-gap follow-up to the 3-mail case in - // fifo-mail.test.ts. The orchestrator's `isTrivialDeploy` requires - // `stepOrder.length === 1` AND `trivialBindings` present; - // absent the trivial-bindings (this fixture does not pass - // them), a single-step workflow still routes through the - // supervisor's FIFO inbox dispatch loop. The load test - // therefore uses a single-step workflow to keep per-mail - // commit pressure tractable in CI: every mail run still + // fifo-mail.test.ts. A single-step workflow still routes through + // the supervisor's FIFO inbox dispatch loop, so the load test + // uses one to keep per-mail commit pressure tractable in CI: + // every mail run still // exercises inbox -> processing -> trigger.fire -> wait for // terminal -> markConsumed, but trims one StepStarted + // StepCompleted commit pair off the per-run pack-push @@ -198,7 +196,7 @@ describe("FIFO mail-trigger serialization under load", () => { const launchSession: LaunchSessionFn = async (orchestratorParams) => { const deployContent = orchestratorParams.deployContent; - await env.hub.sessionService.launchSession({ + await env.hub.sessionService.stageWorkflowStep({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, @@ -224,6 +222,9 @@ describe("FIFO mail-trigger serialization under load", () => { sources: params.sources, }); + const deploySingleStepAtHead: DeploySingleStepFn = (params) => + env.hub.sessionService.deploySingleStepAtHead(params); + const workflowRepo: WorkflowRepoWriter = { async writeWorkflowRepo(args) { const repoId: RepoId = { kind: "workflow", id: args.workflowRepoId }; @@ -249,6 +250,7 @@ describe("FIFO mail-trigger serialization under load", () => { workflowRepo, launchSession, sendMultiStepDeploy, + deploySingleStepAtHead, }); let result: Awaited>; @@ -270,11 +272,11 @@ describe("FIFO mail-trigger serialization under load", () => { { cause }, ); } - expect(result.kind).toBe("multi-step"); + expect(result.publicKey).toBeTruthy(); const workflowRunRepoId: RepoId = { kind: "workflow-run", - id: deriveTrivialDeploymentId(deploymentMailAddress), + id: deriveDeploymentId(deploymentMailAddress), }; env.registerDeployment({ deploymentId: DEPLOYMENT_ID_LOAD, diff --git a/tests/workflow-deploy/fifo-mail.test.ts b/tests/workflow-deploy/fifo-mail.test.ts index e60abbae..44310714 100644 --- a/tests/workflow-deploy/fifo-mail.test.ts +++ b/tests/workflow-deploy/fifo-mail.test.ts @@ -42,12 +42,11 @@ // end) or a pre-spawn seeding hook on `deployWorkflow`. Adding either // is a separate landing. // -// The orchestrator's multi-step branch is composed in-test because -// the pre-landed `deploy-flow-env` fixture wires only the trivial -// `launchSession` callback against `env.hub.sessionService.launchSession`; -// the multi-step `sendMultiStepDeploy` hand-off is supplied here -// against `env.hub.router.sendAgentDeploy` so the sidecar's deploy -// router takes the workflow-process spawn path. This mirrors the +// The orchestrator's multi-step branch is composed in-test: the per-step +// launch callback drives `env.hub.sessionService.stageWorkflowStep` (the +// stage-only path, no warm harness) and the `sendMultiStepDeploy` hand-off +// is supplied against `env.hub.router.sendAgentDeploy` so the sidecar's +// deploy router takes the workflow-process spawn path. This mirrors the // multistep-signal and drain-roundtrip tests' shape so a regression // in any of the seven hops surfaces uniformly across the three. // @@ -67,7 +66,7 @@ import { type SendMultiStepDeployFn, type WorkflowRepoWriter, } from "@intx/workflow-deploy"; -import { deriveTrivialDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; +import { deriveDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; import type { RepoId, WorkflowRunHubPrincipal } from "@intx/hub-sessions"; import { DEFAULT_ASSET_REF } from "@intx/hub-sessions"; @@ -184,7 +183,7 @@ describe("FIFO mail-trigger serialization", () => { const launchSession: LaunchSessionFn = async (orchestratorParams) => { const deployContent = orchestratorParams.deployContent; - await env.hub.sessionService.launchSession({ + await env.hub.sessionService.stageWorkflowStep({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, @@ -256,11 +255,11 @@ describe("FIFO mail-trigger serialization", () => { { cause }, ); } - expect(result.kind).toBe("multi-step"); + expect(result.publicKey).toBeTruthy(); const workflowRunRepoId: RepoId = { kind: "workflow-run", - id: deriveTrivialDeploymentId(deploymentMailAddress), + id: deriveDeploymentId(deploymentMailAddress), }; env.registerDeployment({ deploymentId: DEPLOYMENT_ID, diff --git a/tests/workflow-deploy/instance-failover-real-agent.test.ts b/tests/workflow-deploy/instance-failover-real-agent.test.ts new file mode 100644 index 00000000..1d348453 --- /dev/null +++ b/tests/workflow-deploy/instance-failover-real-agent.test.ts @@ -0,0 +1,310 @@ +// Inference-source failover real-agent round-trip integration test. +// +// The proof that the per-step inference-source failover chain is HONORED at +// runtime by a REAL spawned workflow-process child -- not merely threaded +// through the wire. A single-agent instance is deployed with a two-element +// source chain whose head is a dead provider (HTTP 500) and whose tail is the +// healthy mock inference server. The child's reactor exhausts its mechanical +// retries against the head, fails over forward to the tail, and returns the +// tail's live reply. This closes the gap the unit tests leave open: they prove +// the child builds `env.sources` from the whole chain, but only a real child +// against a real provider pair proves the reactor actually advances across it. +// +// A dead head maps to `retryable` (HTTP 5xx), so the harness retries it a few +// times before the reactor fails over -- exercising the full +// harness-retry-then-reactor-failover path. +// +// The discriminating proof is that the step reply EQUALS the healthy tail's +// output. Run completion alone proves nothing: a total inference failure with +// no failover target ALSO completes -- the agent synthesizes a graceful +// provider-error reply and the run lands `RunCompleted`. The second test here +// pins that contract as a negative control, so the first test's reliance on +// reply-equality (rather than run completion) is grounded rather than assumed. + +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; + +import type { HarnessConfig } from "@intx/types/runtime"; +import { wrapHarnessAsSingleStepWorkflow } from "@intx/workflow-deploy"; +import { defineWorkflow, type WorkflowDefinition } from "@intx/workflow"; +import { deriveDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; +import type { RepoId } from "@intx/hub-sessions"; + +import { + SESSION_ID, + SIDECAR_ID, + fireMailTrigger, + readWorkflowRunEvents, + startDeployFlowEnv, + waitForFirstRunId, + waitForWorkflowRunComplete, + type DeployFlowEnv, +} from "../hub-agent/lib/deploy-flow-env"; + +const DEPLOYMENT_DOMAIN = "integration.interchange"; +// A real instance identity, distinct from the reroute test's `b`-filled id so +// the two integration tests never collide on a substrate slug. +const INSTANCE_ID = `ins_${"c".repeat(32)}`; +const AGENT_ID = "agent-instance-failover"; +const WORKFLOW_RUN_REF = "refs/heads/main"; + +// The mock inference server's reply for an empty tool set. +const EXPECTED_REPLY = "I see these tools:"; + +let env: DeployFlowEnv; + +// The dead head source. Always returns HTTP 500 (a `retryable` category), so +// the child's reactor fails over off it. `startDeployFlowEnv().teardown()` does +// not own this server, so it is started and stopped symmetrically here. +let headRequests = 0; +let deadHead: ReturnType; + +beforeAll(async () => { + env = await startDeployFlowEnv(); + deadHead = Bun.serve({ + port: 0, + fetch() { + headRequests += 1; + return new Response("upstream boom", { status: 500 }); + }, + }); +}); + +afterAll(async () => { + await env.teardown(); + await deadHead.stop(true); +}); + +describe("instance inference-source failover real-agent round-trip", () => { + test("sidecar registers with hub", () => { + expect(env.hub.router.getConnectedSidecars()).toContain(SIDECAR_ID); + }); + + test("the child fails over from a dead head source to the healthy tail", async () => { + const agentAddress = `${INSTANCE_ID}@${DEPLOYMENT_DOMAIN}`; + + // A two-element failover chain: element 0 is the dead head (default), the + // tail is the healthy mock inference server. The head must be element 0 and + // equal `defaultSource` -- the reactor resolves its initial source by id and + // fails over forward, so the deploy asserts that invariant. + const config: HarnessConfig = { + sessionId: SESSION_ID, + agentId: AGENT_ID, + tenantId: "tenant-1", + principalId: "prin_integration-failover-1", + agentAddress, + systemPrompt: "You are the failover single-agent instance.", + tools: [], + grants: [], + sources: [ + { + id: "anthropic:dead-head", + provider: "anthropic", + baseURL: `http://localhost:${deadHead.port}`, + apiKey: "sk-dead", + model: "mock-model", + }, + { + id: "anthropic:mock-model", + provider: "anthropic", + baseURL: `http://localhost:${env.inference.server.port}`, + apiKey: "sk-mock", + model: "mock-model", + }, + ], + defaultSource: "anthropic:dead-head", + }; + const deployContent = { systemPrompt: config.systemPrompt }; + + // Reconstruct the definition `deployInstanceAtHead` builds internally, for + // the run-observation handle. Same deterministic inputs -> same wrap. + const workflow: WorkflowDefinition = defineWorkflow({ + id: `wf_${AGENT_ID}`, + agent: wrapHarnessAsSingleStepWorkflow({ config, deployContent }), + trigger: { type: "mail", to: agentAddress }, + }); + + const { publicKey } = await env.hub.sessionService.deployInstanceAtHead({ + agentAddress, + agentId: AGENT_ID, + instanceId: INSTANCE_ID, + config, + deployContent, + }); + expect(publicKey).toMatch(/^[0-9a-f]{64}$/); + + const workflowRunRepoId: RepoId = { + kind: "workflow-run", + id: deriveDeploymentId(agentAddress), + }; + env.registerDeployment({ + deploymentId: INSTANCE_ID, + workflowDefinition: workflow, + workflowRunRepoId, + workflowRunRef: WORKFLOW_RUN_REF, + mailAddress: agentAddress, + }); + + expect(env.hub.router.getRoutableAddresses()).toContain(agentAddress); + + await fireMailTrigger(env, agentAddress, { + messageId: "", + }); + + const runId = await waitForFirstRunId(env, workflowRunRepoId, { + diagnostics: env.sidecarDiagnostics, + timeoutMs: 20_000, + }); + + const terminal = await waitForWorkflowRunComplete(env, INSTANCE_ID, runId, { + timeoutMs: 20_000, + diagnostics: env.sidecarDiagnostics, + }); + // A sanity check, not the proof: the run reaches a normal terminal. This + // does NOT discriminate working from broken failover on its own -- the + // negative-control test below shows a total inference failure also lands + // `RunCompleted`. The reply-equality assertion is what proves failover. + expect(terminal.type).toBe("RunCompleted"); + + const events = await readWorkflowRunEvents(env, INSTANCE_ID, runId); + const stepCompleted = events.find((e) => e.type === "StepCompleted"); + if (stepCompleted === undefined) { + throw new Error("missing StepCompleted for the wrapped single step"); + } + + // The proof of failover: the reply EQUALS the healthy tail's output. The + // dead head returns nothing but 500s, so it cannot produce this string; a + // broken failover would instead surface the agent's synthesized + // provider-error reply (the negative control below). + const reply = readStepReply(stepCompleted.body); + expect(reply).toBe(EXPECTED_REPLY); + + // The head was tried first (the default pins to element 0 every cycle), then + // the healthy tail actually served the reply. Both counts are `>= 1` rather + // than exact: the precise retry count is the harness retry policy's contract, + // not this cross-process test's. + expect(headRequests).toBeGreaterThanOrEqual(1); + expect(env.inference.requests.length).toBeGreaterThanOrEqual(1); + }); + + test("a chain with no healthy source completes with a synthesized error reply", async () => { + // Negative control grounding the test above: with the dead head as the SOLE + // source there is nothing to fail over to, yet the run still lands + // `RunCompleted` -- the agent synthesizes a graceful provider-error reply + // rather than failing the run. This is why the failover test cannot lean on + // the terminal type and instead proves failover through reply-equality. + const soleDeadInstanceId = `ins_${"d".repeat(32)}`; + const soleDeadAgentId = "agent-instance-failover-negctl"; + const agentAddress = `${soleDeadInstanceId}@${DEPLOYMENT_DOMAIN}`; + + const config: HarnessConfig = { + sessionId: SESSION_ID, + agentId: soleDeadAgentId, + tenantId: "tenant-1", + principalId: "prin_integration-failover-negctl-1", + agentAddress, + systemPrompt: "You are the failover negative-control instance.", + tools: [], + grants: [], + sources: [ + { + id: "anthropic:dead-head", + provider: "anthropic", + baseURL: `http://localhost:${deadHead.port}`, + apiKey: "sk-dead", + model: "mock-model", + }, + ], + defaultSource: "anthropic:dead-head", + }; + const deployContent = { systemPrompt: config.systemPrompt }; + + const workflow: WorkflowDefinition = defineWorkflow({ + id: `wf_${soleDeadAgentId}`, + agent: wrapHarnessAsSingleStepWorkflow({ config, deployContent }), + trigger: { type: "mail", to: agentAddress }, + }); + + await env.hub.sessionService.deployInstanceAtHead({ + agentAddress, + agentId: soleDeadAgentId, + instanceId: soleDeadInstanceId, + config, + deployContent, + }); + + const workflowRunRepoId: RepoId = { + kind: "workflow-run", + id: deriveDeploymentId(agentAddress), + }; + env.registerDeployment({ + deploymentId: soleDeadInstanceId, + workflowDefinition: workflow, + workflowRunRepoId, + workflowRunRef: WORKFLOW_RUN_REF, + mailAddress: agentAddress, + }); + + await fireMailTrigger(env, agentAddress, { + messageId: "", + }); + + const runId = await waitForFirstRunId(env, workflowRunRepoId, { + diagnostics: env.sidecarDiagnostics, + timeoutMs: 20_000, + }); + + const terminal = await waitForWorkflowRunComplete( + env, + soleDeadInstanceId, + runId, + { timeoutMs: 20_000, diagnostics: env.sidecarDiagnostics }, + ); + // Total inference failure still completes the run. + expect(terminal.type).toBe("RunCompleted"); + + const events = await readWorkflowRunEvents(env, soleDeadInstanceId, runId); + const stepCompleted = events.find((e) => e.type === "StepCompleted"); + if (stepCompleted === undefined) { + throw new Error("missing StepCompleted for the sole-dead-source step"); + } + + // The reply is the synthesized error, NOT the healthy tail's output -- + // exactly the outcome the failover test's reply-equality assertion rules out. + const reply = readStepReply(stepCompleted.body); + expect(reply).not.toBe(EXPECTED_REPLY); + }); +}); + +/** + * Extract the agent's reply string from a `StepCompleted` event body. A small + * `{ reply, turn }` output inlines as `inline:`. + */ +function readStepReply(body: Record): string { + const output = body["output"]; + if (typeof output !== "object" || output === null || !("ref" in output)) { + throw new Error( + `StepCompleted output is not a { ref } record: ${JSON.stringify(output)}`, + ); + } + const ref: unknown = output.ref; + if (typeof ref !== "string") { + throw new Error(`StepCompleted output ref is not a string: ${String(ref)}`); + } + const INLINE_PREFIX = "inline:"; + if (!ref.startsWith(INLINE_PREFIX)) { + throw new Error(`expected an inline output ref, got ${ref}`); + } + const parsed: unknown = JSON.parse(ref.slice(INLINE_PREFIX.length)); + if (typeof parsed !== "object" || parsed === null || !("reply" in parsed)) { + throw new Error( + `step output does not carry a reply field: ${JSON.stringify(parsed)}`, + ); + } + const reply: unknown = parsed.reply; + if (typeof reply !== "string") { + throw new Error( + `step output reply is not a string: ${JSON.stringify(parsed)}`, + ); + } + return reply; +} diff --git a/tests/workflow-deploy/instance-reroute-real-agent.test.ts b/tests/workflow-deploy/instance-reroute-real-agent.test.ts new file mode 100644 index 00000000..c21bd023 --- /dev/null +++ b/tests/workflow-deploy/instance-reroute-real-agent.test.ts @@ -0,0 +1,192 @@ +// Instance-reroute real-agent round-trip integration test. +// +// The proof that a single-agent INSTANCE deploy, routed through +// `SessionService.deployInstanceAtHead` (which wraps the harness as a +// one-step workflow via `wrapHarnessAsSingleStepWorkflow` and deploys it at the +// head), runs a REAL agent in the spawned workflow-process child -- against +// the real hub + real sidecar subprocess + mock inference fixture. This +// closes the gap that unit tests (mock spawner) leave open: that the +// walk-only single-step wrap agent instantiates and runs in a real child. +// +// The child never invokes the wrap's walk-only tool factories: it +// materializes tools from the deploy tree and calls the real `createAgent` +// reading only id/systemPrompt/capabilities. A tool-less instance gets an +// empty materialization slot and never hits the bare-createAgent fallback, +// so the wrapped agent runs to a real inference reply. +// +// The mock inference server returns `I see these tools: `; with an +// empty tool set the reply is the stable prefix `I see these tools:`. + +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; + +import type { HarnessConfig } from "@intx/types/runtime"; +import { wrapHarnessAsSingleStepWorkflow } from "@intx/workflow-deploy"; +import { defineWorkflow, type WorkflowDefinition } from "@intx/workflow"; +import { deriveDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; +import type { RepoId } from "@intx/hub-sessions"; + +import { + SESSION_ID, + SIDECAR_ID, + fireMailTrigger, + readWorkflowRunEvents, + startDeployFlowEnv, + waitForFirstRunId, + waitForWorkflowRunComplete, + type DeployFlowEnv, +} from "../hub-agent/lib/deploy-flow-env"; + +const DEPLOYMENT_DOMAIN = "integration.interchange"; +// A real instance identity: `ins_` + 32 hex, NOT a `dep_`-prefixed deployment +// id. The head address IS the instance address; the reroute keeps that +// identity rather than deriving a synthetic deployment agent id. +const INSTANCE_ID = `ins_${"b".repeat(32)}`; +const AGENT_ID = "agent-instance-reroute"; +const WORKFLOW_RUN_REF = "refs/heads/main"; + +// The mock inference server's reply for an empty tool set. +const EXPECTED_REPLY = "I see these tools:"; + +let env: DeployFlowEnv; + +beforeAll(async () => { + env = await startDeployFlowEnv(); +}); + +afterAll(async () => { + await env.teardown(); +}); + +describe("instance-reroute real-agent round-trip", () => { + test("sidecar registers with hub", () => { + expect(env.hub.router.getConnectedSidecars()).toContain(SIDECAR_ID); + }); + + test("deployInstanceAtHead runs the wrapped single agent in a real child", async () => { + const agentAddress = `${INSTANCE_ID}@${DEPLOYMENT_DOMAIN}`; + + const config: HarnessConfig = { + sessionId: SESSION_ID, + agentId: AGENT_ID, + tenantId: "tenant-1", + principalId: "prin_integration-reroute-1", + agentAddress, + systemPrompt: "You are the rerouted single-agent instance.", + tools: [], + grants: [], + sources: [ + { + id: "anthropic:mock-model", + provider: "anthropic", + baseURL: `http://localhost:${env.inference.server.port}`, + apiKey: "sk-mock", + model: "mock-model", + }, + ], + defaultSource: "anthropic:mock-model", + }; + const deployContent = { systemPrompt: config.systemPrompt }; + + // Reconstruct the definition `deployInstanceAtHead` builds internally, for + // the run-observation handle. Same deterministic inputs -> same wrap. + const workflow: WorkflowDefinition = defineWorkflow({ + id: `wf_${AGENT_ID}`, + agent: wrapHarnessAsSingleStepWorkflow({ config, deployContent }), + trigger: { type: "mail", to: agentAddress }, + }); + + // Deploy the instance through the reroute entry point under test. + const { publicKey } = await env.hub.sessionService.deployInstanceAtHead({ + agentAddress, + agentId: AGENT_ID, + instanceId: INSTANCE_ID, + config, + deployContent, + }); + expect(publicKey).toMatch(/^[0-9a-f]{64}$/); + + const workflowRunRepoId: RepoId = { + kind: "workflow-run", + id: deriveDeploymentId(agentAddress), + }; + env.registerDeployment({ + deploymentId: INSTANCE_ID, + workflowDefinition: workflow, + workflowRunRepoId, + workflowRunRef: WORKFLOW_RUN_REF, + mailAddress: agentAddress, + }); + + // The instance's head address is routable from the hub after the deploy. + expect(env.hub.router.getRoutableAddresses()).toContain(agentAddress); + + const { messageId } = await fireMailTrigger(env, agentAddress, { + messageId: "", + }); + + const runId = await waitForFirstRunId(env, workflowRunRepoId, { + diagnostics: env.sidecarDiagnostics, + timeoutMs: 20_000, + }); + + const terminal = await waitForWorkflowRunComplete(env, INSTANCE_ID, runId, { + timeoutMs: 20_000, + diagnostics: env.sidecarDiagnostics, + }); + expect(terminal.type).toBe("RunCompleted"); + + const events = await readWorkflowRunEvents(env, INSTANCE_ID, runId); + const runStartedBody = events.find((e) => e.type === "RunStarted")?.body; + if (runStartedBody === undefined) throw new Error("missing RunStarted"); + expect(runStartedBody["consumedMessageId"]).toBe(messageId); + + const stepCompleted = events.find((e) => e.type === "StepCompleted"); + if (stepCompleted === undefined) { + throw new Error("missing StepCompleted for the wrapped single step"); + } + + // The proof: the step output is the REAL agent reply from `agent.send` + // (the mock provider's deterministic output) -- the wrapped single-step agent + // instantiated and ran in the child rather than throwing on its walk-only + // tool factories. + const reply = readStepReply(stepCompleted.body); + expect(reply).toBe(EXPECTED_REPLY); + + // The agent's inference call actually reached the mock provider. + expect(env.inference.requests.length).toBeGreaterThan(0); + }); +}); + +/** + * Extract the agent's reply string from a `StepCompleted` event body. A small + * `{ reply, turn }` output inlines as `inline:`. + */ +function readStepReply(body: Record): string { + const output = body["output"]; + if (typeof output !== "object" || output === null || !("ref" in output)) { + throw new Error( + `StepCompleted output is not a { ref } record: ${JSON.stringify(output)}`, + ); + } + const ref: unknown = output.ref; + if (typeof ref !== "string") { + throw new Error(`StepCompleted output ref is not a string: ${String(ref)}`); + } + const INLINE_PREFIX = "inline:"; + if (!ref.startsWith(INLINE_PREFIX)) { + throw new Error(`expected an inline output ref, got ${ref}`); + } + const parsed: unknown = JSON.parse(ref.slice(INLINE_PREFIX.length)); + if (typeof parsed !== "object" || parsed === null || !("reply" in parsed)) { + throw new Error( + `step output does not carry a reply field: ${JSON.stringify(parsed)}`, + ); + } + const reply: unknown = parsed.reply; + if (typeof reply !== "string") { + throw new Error( + `step output reply is not a string: ${JSON.stringify(parsed)}`, + ); + } + return reply; +} diff --git a/tests/workflow-deploy/latency-d2-attribution.bench.ts b/tests/workflow-deploy/latency-d2-attribution.bench.ts index 63991d09..4e8c4dae 100644 --- a/tests/workflow-deploy/latency-d2-attribution.bench.ts +++ b/tests/workflow-deploy/latency-d2-attribution.bench.ts @@ -64,7 +64,7 @@ import { type SendMultiStepDeployFn, type WorkflowRepoWriter, } from "@intx/workflow-deploy"; -import { deriveTrivialDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; +import { deriveDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; import type { RepoId, WorkflowRunHubPrincipal } from "@intx/hub-sessions"; import { DEFAULT_ASSET_REF } from "@intx/hub-sessions"; @@ -403,7 +403,7 @@ async function runUnifiedD2(opts: { ]); const launchSession: LaunchSessionFn = async (orchestratorParams) => { - await env.hub.sessionService.launchSession({ + await env.hub.sessionService.stageWorkflowStep({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, @@ -462,15 +462,13 @@ async function runUnifiedD2(opts: { deploymentDomain: DEPLOYMENT_DOMAIN, hubPublicKey: "00".repeat(32), }); - if (result.kind !== "multi-step") { - throw new Error( - `d2 bench: expected multi-step deploy, got ${result.kind}`, - ); + if (!result.publicKey) { + throw new Error("d2 bench: deploy did not return a public key"); } const workflowRunRepoId: RepoId = { kind: "workflow-run", - id: deriveTrivialDeploymentId(deploymentMailAddress), + id: deriveDeploymentId(deploymentMailAddress), }; env.registerDeployment({ deploymentId: DEPLOYMENT_ID, diff --git a/tests/workflow-deploy/latency-gate.bench.ts b/tests/workflow-deploy/latency-gate.bench.ts index 468e734a..00d22a9a 100644 --- a/tests/workflow-deploy/latency-gate.bench.ts +++ b/tests/workflow-deploy/latency-gate.bench.ts @@ -59,7 +59,8 @@ import { defineAgent, createDefaultDirectorRegistry } from "@intx/agent"; import { noopAuditStore, permissiveAuthorize } from "@intx/agent/testing"; import { createAgent, type Agent } from "@intx/agent"; import type { BaseEnv } from "@intx/agent"; -import { createIsogitStore } from "@intx/storage-isogit"; +import { generateKeyPair, createSSHSignature } from "@intx/crypto"; +import { createIsogitStore, type CommitSigner } from "@intx/storage-isogit"; import type { HarnessConfig, InferenceSource } from "@intx/types/runtime"; import { defineWorkflow, step, type WorkflowDefinition } from "@intx/workflow"; import { @@ -70,7 +71,7 @@ import { type SendMultiStepDeployFn, type WorkflowRepoWriter, } from "@intx/workflow-deploy"; -import { deriveTrivialDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; +import { deriveDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; import type { RepoId, WorkflowRunHubPrincipal } from "@intx/hub-sessions"; import { DEFAULT_ASSET_REF } from "@intx/hub-sessions"; @@ -251,7 +252,13 @@ async function runBaseline(opts: { // A real isogit ContextStore (the same store kind the unified step // agent uses for its conversation) so the baseline pays the same // per-turn context-commit cost the in-process runtime pays today. - const storage = await createIsogitStore(storeDir); + // Sign each commit's sshsig exactly as `agent-repo.ts` does for the + // unified path; without a signer the baseline would skip the + // per-commit signature and undercount its own floor. + const signingKey = await generateKeyPair(); + const signer: CommitSigner = (payload) => + createSSHSignature(payload, signingKey.privateKey, signingKey.publicKey); + const storage = await createIsogitStore(storeDir, signer); const def = defineAgent({ id: "latency-baseline-agent", @@ -359,7 +366,7 @@ async function runUnified(opts: { ]); const launchSession: LaunchSessionFn = async (orchestratorParams) => { - await env.hub.sessionService.launchSession({ + await env.hub.sessionService.stageWorkflowStep({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, @@ -418,15 +425,13 @@ async function runUnified(opts: { deploymentDomain: DEPLOYMENT_DOMAIN, hubPublicKey: "00".repeat(32), }); - if (result.kind !== "multi-step") { - throw new Error( - `unified bench: expected multi-step deploy, got ${result.kind}`, - ); + if (!result.publicKey) { + throw new Error("unified bench: deploy did not return a public key"); } const workflowRunRepoId: RepoId = { kind: "workflow-run", - id: deriveTrivialDeploymentId(deploymentMailAddress), + id: deriveDeploymentId(deploymentMailAddress), }; env.registerDeployment({ deploymentId: DEPLOYMENT_ID, diff --git a/tests/workflow-deploy/launch-session-bridge.ts b/tests/workflow-deploy/launch-session-bridge.ts index 55bb2277..68149566 100644 --- a/tests/workflow-deploy/launch-session-bridge.ts +++ b/tests/workflow-deploy/launch-session-bridge.ts @@ -1,23 +1,23 @@ -import type { DeployContent as LaunchDeployContent } from "@intx/hub-sessions"; +import { + bridgeOrchestratorDeployContent, + type DeployContent as LaunchDeployContent, +} from "@intx/hub-sessions"; import type { DeployContent as OrchestratorDeployContent } from "@intx/workflow-deploy"; /** * Bridge the orchestrator's `DeployContent` to hub-sessions' * `DeployContent` at the `launchSession` boundary. * - * `@intx/workflow-deploy` widens `toolPackageManifest` to `unknown` - * so the package does not need a runtime dep on `@intx/hub-sessions` - * (see the `DeployContent` docblock in - * `packages/workflow-deploy/src/orchestrator.ts`). That docblock - * declares the orchestrator type a structural mirror of hub-sessions' - * `DeployContent`, so the narrowing is sound -- but unprovable to - * the type checker. This helper centralizes the one allowed - * `eslint-disable` so each `LaunchSessionFn` callback in the test - * fixtures does not need its own. + * `@intx/workflow-deploy` widens `toolPackageManifest` to `unknown` so + * the package does not need a runtime dep on `@intx/hub-sessions` (see + * the `DeployContent` docblock in + * `packages/workflow-deploy/src/orchestrator.ts`). This delegates to the + * same `bridgeOrchestratorDeployContent` the production multi-step + * callback uses, which validates and narrows `toolPackageManifest` back + * to the canonical shape rather than casting `unknown`. */ export function toLaunchDeployContent( deployContent: OrchestratorDeployContent, ): LaunchDeployContent { - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- see file docblock; the orchestrator's DeployContent docblock declares this structural mirror - return deployContent as LaunchDeployContent; + return bridgeOrchestratorDeployContent(deployContent); } diff --git a/tests/workflow-deploy/mail-edge-cases.test.ts b/tests/workflow-deploy/mail-edge-cases.test.ts index 30e5573c..677510d6 100644 --- a/tests/workflow-deploy/mail-edge-cases.test.ts +++ b/tests/workflow-deploy/mail-edge-cases.test.ts @@ -60,11 +60,12 @@ import { createWorkflowDeployOrchestrator, deriveDeploymentAddress, type ApprovalSet, + type DeploySingleStepFn, type LaunchSessionFn, type SendMultiStepDeployFn, type WorkflowRepoWriter, } from "@intx/workflow-deploy"; -import { deriveTrivialDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; +import { deriveDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; import type { RepoId, WorkflowRunHubPrincipal } from "@intx/hub-sessions"; import { DEFAULT_ASSET_REF } from "@intx/hub-sessions"; @@ -404,7 +405,7 @@ async function deployEdgeWorkflow( const launchSession: LaunchSessionFn = async (orchestratorParams) => { const deployContent = orchestratorParams.deployContent; - await env.hub.sessionService.launchSession({ + await env.hub.sessionService.stageWorkflowStep({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, @@ -430,6 +431,9 @@ async function deployEdgeWorkflow( sources: params.sources, }); + const deploySingleStepAtHead: DeploySingleStepFn = (params) => + env.hub.sessionService.deploySingleStepAtHead(params); + const workflowRepo: WorkflowRepoWriter = { async writeWorkflowRepo(args) { const repoId: RepoId = { kind: "workflow", id: args.workflowRepoId }; @@ -455,6 +459,7 @@ async function deployEdgeWorkflow( workflowRepo, launchSession, sendMultiStepDeploy, + deploySingleStepAtHead, }); const result = await orchestrator.deployWorkflow({ @@ -466,13 +471,11 @@ async function deployEdgeWorkflow( deploymentDomain: DEPLOYMENT_DOMAIN, hubPublicKey: "00".repeat(32), }); - if (result.kind !== "multi-step") { - throw new Error(`expected multi-step deploy; got ${result.kind}`); - } + expect(result.publicKey).toBeTruthy(); const workflowRunRepoId: RepoId = { kind: "workflow-run", - id: deriveTrivialDeploymentId(deploymentMailAddress), + id: deriveDeploymentId(deploymentMailAddress), }; env.registerDeployment({ deploymentId, diff --git a/tests/workflow-deploy/multistep-signal.test.ts b/tests/workflow-deploy/multistep-signal.test.ts index 04aff63d..2f46d322 100644 --- a/tests/workflow-deploy/multistep-signal.test.ts +++ b/tests/workflow-deploy/multistep-signal.test.ts @@ -6,12 +6,12 @@ // `SignalAwaited`, injects a signal via the host-side signal channel, // and asserts the runtime resumes through `step2` to `RunCompleted`. // -// The orchestrator's multi-step branch is composed in-test because the -// pre-landed `deploy-flow-env` fixture wires only the trivial -// `launchSession` callback against `env.hub.sessionService.launchSession`; -// the multi-step `sendMultiStepDeploy` hand-off is supplied here against -// `env.hub.router.sendAgentDeploy` so the sidecar's deploy router takes -// the workflow-process spawn path. The deployment handle is registered +// The orchestrator's multi-step branch is composed in-test: the per-step +// launch callback drives `env.hub.sessionService.stageWorkflowStep` (the +// stage-only path, no warm harness) and the `sendMultiStepDeploy` hand-off +// is supplied against `env.hub.router.sendAgentDeploy` so the sidecar's +// deploy router takes the workflow-process spawn path. The deployment +// handle is registered // on the env via `registerDeployment` so the fixture's `injectSignal`, // `readWorkflowRunEvents`, and `waitForWorkflowRunComplete` helpers can // resolve it by id. @@ -45,7 +45,7 @@ import { type SendMultiStepDeployFn, type WorkflowRepoWriter, } from "@intx/workflow-deploy"; -import { deriveTrivialDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; +import { deriveDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; import type { RepoId, WorkflowRunHubPrincipal } from "@intx/hub-sessions"; import { DEFAULT_ASSET_REF } from "@intx/hub-sessions"; @@ -160,7 +160,7 @@ describe("multi-step workflow round-trip with signal-await", () => { // `agent-state` repo is provisioned end-to-end. const launchSession: LaunchSessionFn = async (orchestratorParams) => { const deployContent = orchestratorParams.deployContent; - await env.hub.sessionService.launchSession({ + await env.hub.sessionService.stageWorkflowStep({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, @@ -241,15 +241,15 @@ describe("multi-step workflow round-trip with signal-await", () => { { cause }, ); } - expect(result.kind).toBe("multi-step"); + expect(result.publicKey).toBeTruthy(); // The sidecar's deploy router slugs the deployment mail address - // into the workflow-run repo id via `deriveTrivialDeploymentId`; + // into the workflow-run repo id via `deriveDeploymentId`; // the helper queries `runs//events/.json` against the // same id. const workflowRunRepoId: RepoId = { kind: "workflow-run", - id: deriveTrivialDeploymentId(deploymentMailAddress), + id: deriveDeploymentId(deploymentMailAddress), }; env.registerDeployment({ deploymentId: DEPLOYMENT_ID, diff --git a/tests/workflow-deploy/multistep-signed-send.test.ts b/tests/workflow-deploy/multistep-signed-send.test.ts new file mode 100644 index 00000000..0804c52f --- /dev/null +++ b/tests/workflow-deploy/multistep-signed-send.test.ts @@ -0,0 +1,302 @@ +// Multi-step signed outbound send regression test. +// +// A genuine multi-step (2+ step) workflow deployment must register a +// signing identity for its deployment mail address on the host transport, +// exactly as a single-step deployment does. Every step of a multi-step +// deployment signs its outbound mail as the ONE deployment-wide address +// (`ins_@`), so if that address is not registered a +// step's `env.transport.send` rejects with "not registered", the step +// fails, and the run fails. +// +// This deploys a two-step workflow whose sending step calls the +// transport-backed `mail_send` tool (the mock inference drives the tool +// call on the first request that exposes it). The tool routes through the +// real outbound chain -- supervisor-backed transport -> outbound bridge -> +// `outbound.message` IPC -> supervisor `sendOutbound` -> host transport +// SIGNED send -- so the send reaches the host transport as the deployment +// address. The sidecar forwards the delivered `mail.outbound` frame to the +// hub for persistence, where the fixture captures its signing sender. A +// captured frame whose sender is the deployment address is a load-bearing +// proof that the address held a registered signing identity; a registration +// gap would reject the send inside the step and forward no frame. +// +// The sender is a step of a multi-step deployment (not a single-step head), +// so this covers the deployment-scoped registration the single-step path +// already had and the multi-step path lacked. + +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; + +import { defineAgent, createDefaultDirectorRegistry } from "@intx/agent"; +import type { HarnessConfig } from "@intx/types/runtime"; +import type { ToolPackagePin } from "@intx/types/tool-packages"; +import { WireGrantRule } from "@intx/types/grant-wire"; +import { defineWorkflow, step, type WorkflowDefinition } from "@intx/workflow"; +import { + createWorkflowDeployOrchestrator, + deriveDeploymentAddress, + type ApprovalSet, + type LaunchSessionFn, + type SendMultiStepDeployFn, + type WorkflowRepoWriter, +} from "@intx/workflow-deploy"; +import { deriveDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; +import type { RepoId, WorkflowRunHubPrincipal } from "@intx/hub-sessions"; +import { DEFAULT_ASSET_REF } from "@intx/hub-sessions"; + +import { + SESSION_ID, + SIDECAR_ID, + fireMailTrigger, + startDeployFlowEnv, + waitFor, + waitForFirstRunId, + waitForWorkflowRunComplete, + type DeployFlowEnv, +} from "../hub-agent/lib/deploy-flow-env"; +import { toLaunchDeployContent } from "./launch-session-bridge"; + +const DEPLOYMENT_DOMAIN = "integration.interchange"; +const DEPLOYMENT_ID = "multistep-signed-send-1"; +const WORKFLOW_RUN_REF = "refs/heads/main"; +// The first step in `stepOrder`; the mock drives its inference to call the +// mail tool, so it is the step that performs the signed send. +const SENDER_STEP_ID = "send"; + +const TOOL_NAME = "@intx/tools-mail/sidecar-bundle:mail_send"; +const GRANTED_RESOURCE = `tool:${TOOL_NAME}`; +const SENTINEL_FILENAME = "multistep-signed-send-receipt.txt"; + +const TOOL_PINS: readonly ToolPackagePin[] = [ + { name: "@intx/tools-mail", version: "0.1.2" }, +]; + +const GRANTED_RULE: WireGrantRule = { + id: "grant-tool-invoke", + resource: GRANTED_RESOURCE, + action: "invoke", + effect: "allow", + origin: "creator", + conditions: null, + expiresAt: null, + roleId: null, + principalId: null, +}; + +let env: DeployFlowEnv; +let deploymentMailAddress: string; + +beforeAll(async () => { + deploymentMailAddress = deriveDeploymentAddress({ + deploymentId: DEPLOYMENT_ID, + deploymentDomain: DEPLOYMENT_DOMAIN, + }); + // The transport-backed `mail_send` bundle sends through the real outbound + // chain and sentinels on receipt; `inferenceToolCall` drives the model to + // call it. The send targets the deployment's own address -- a local, + // registered recipient once the deployment identity is on the transport -- + // so the signed send delivers without a remote leg. + env = await startDeployFlowEnv({ + transportBackedMailTool: true, + inferenceToolCall: { + toolName: TOOL_NAME, + input: { to: deploymentMailAddress, body: SENTINEL_FILENAME }, + }, + }); +}); + +afterAll(async () => { + await env.teardown(); +}); + +describe("multi-step signed outbound send", () => { + test("sidecar registers with hub", () => { + expect(env.hub.router.getConnectedSidecars()).toContain(SIDECAR_ID); + }); + + test("a step of a multi-step deployment signs and sends outbound mail", async () => { + const sendAgent = defineAgent({ + id: "agent-multistep-send", + systemPrompt: "You are the sending step agent.", + tools: [], + capabilities: [], + inference: { + sources: [{ provider: "anthropic", model: "mock-model" }], + }, + }); + const tailAgent = defineAgent({ + id: "agent-multistep-tail", + systemPrompt: "You are the trailing step agent.", + tools: [], + capabilities: [], + inference: { + sources: [{ provider: "anthropic", model: "mock-model" }], + }, + }); + + const workflow: WorkflowDefinition = defineWorkflow({ + id: `wf_${DEPLOYMENT_ID}`, + trigger: { type: "mail", to: deploymentMailAddress }, + steps: { + [SENDER_STEP_ID]: step({ agent: sendAgent }), + tail: step({ agent: tailAgent, after: [SENDER_STEP_ID] }), + }, + }); + + const config: HarnessConfig = { + sessionId: SESSION_ID, + agentId: `ins_${DEPLOYMENT_ID}`, + tenantId: "tenant-1", + principalId: "prin_integration-1", + agentAddress: deploymentMailAddress, + systemPrompt: "Fallback prompt (overridden per step by orchestrator)", + tools: [], + grants: [GRANTED_RULE], + sources: [ + { + id: "anthropic:mock-model", + provider: "anthropic", + baseURL: `http://localhost:${env.inference.server.port}`, + apiKey: "sk-mock", + model: "mock-model", + }, + ], + defaultSource: "anthropic:mock-model", + }; + + const operatorApprovals: ApprovalSet = new Set([ + "inference.source:anthropic:mock-model", + "director:@intx/agent/default", + `mail.address:${deploymentMailAddress}`, + `mail.send:${DEPLOYMENT_DOMAIN}`, + ]); + + const launchSession: LaunchSessionFn = async (orchestratorParams) => { + await env.hub.sessionService.stageWorkflowStep({ + agentAddress: orchestratorParams.agentAddress, + agentId: orchestratorParams.agentId, + instanceId: orchestratorParams.instanceId, + config: orchestratorParams.config, + deployContent: toLaunchDeployContent(orchestratorParams.deployContent), + ...(orchestratorParams.toolPackagePins !== undefined + ? { toolPackagePins: orchestratorParams.toolPackagePins } + : {}), + }); + }; + + const sendMultiStepDeploy: SendMultiStepDeployFn = async (params) => + env.hub.router.sendAgentDeploy(params.agentAddress, params.config, { + definition: { + id: params.definition.id, + triggers: [...params.definition.triggers], + stepOrder: [...params.definition.stepOrder], + steps: params.definition.steps as Record, + ...(params.definition.state !== undefined + ? { state: params.definition.state } + : {}), + }, + sources: params.sources, + }); + + const workflowRepo: WorkflowRepoWriter = { + async writeWorkflowRepo(args) { + const repoId: RepoId = { kind: "workflow", id: args.workflowRepoId }; + const principal: WorkflowRunHubPrincipal = { kind: "hub" }; + const files: Record = {}; + for (const [k, v] of args.files) { + files[k] = v; + } + await env.hub.agentRepoStore.repoStore.writeTree( + principal, + repoId, + DEFAULT_ASSET_REF, + { + files, + message: `multistep-signed-send test: write workflow repo ${args.workflowRepoId}`, + }, + ); + }, + }; + + const orchestrator = createWorkflowDeployOrchestrator({ + directorRegistry: createDefaultDirectorRegistry(), + workflowRepo, + launchSession, + sendMultiStepDeploy, + }); + + let result: Awaited>; + try { + result = await orchestrator.deployWorkflow({ + workflow, + config, + deployContent: { systemPrompt: config.systemPrompt }, + operatorApprovals, + deploymentId: DEPLOYMENT_ID, + deploymentDomain: DEPLOYMENT_DOMAIN, + hubPublicKey: "00".repeat(32), + toolPackagePins: TOOL_PINS, + }); + } catch (cause) { + const message = cause instanceof Error ? cause.message : String(cause); + const diag = env.sidecarDiagnostics(); + throw new Error( + `deployWorkflow failed: ${message}\n${diag.length > 0 ? diag : ""}`, + { cause }, + ); + } + expect(result.publicKey).toBeTruthy(); + + const workflowRunRepoId: RepoId = { + kind: "workflow-run", + id: deriveDeploymentId(deploymentMailAddress), + }; + env.registerDeployment({ + deploymentId: DEPLOYMENT_ID, + workflowDefinition: workflow, + workflowRunRepoId, + workflowRunRef: WORKFLOW_RUN_REF, + mailAddress: deploymentMailAddress, + }); + + expect(env.hub.router.getRoutableAddresses()).toContain( + deploymentMailAddress, + ); + + await fireMailTrigger(env, deploymentMailAddress, { + messageId: "", + }); + + const runId = await waitForFirstRunId(env, workflowRunRepoId, { + diagnostics: env.sidecarDiagnostics, + timeoutMs: 20_000, + }); + + const terminal = await waitForWorkflowRunComplete( + env, + DEPLOYMENT_ID, + runId, + { timeoutMs: 20_000, diagnostics: env.sidecarDiagnostics }, + ); + expect(terminal.type).toBe("RunCompleted"); + + // The proof of the fix: the sidecar signed and delivered a `mail.outbound` + // frame whose SIGNING SENDER is the deployment mail address. A multi-step + // step signs its outbound sends as that one deployment address, so the + // frame reaches the hub only when the address holds a registered signing + // identity on the host transport. Without the registration the send throws + // "not registered" inside the step and no frame is ever forwarded, leaving + // `outboundMail` empty. + await waitFor( + () => + env.hub.outboundMail.some( + (m) => m.senderAddress === deploymentMailAddress, + ), + { timeoutMs: 20_000, diagnostics: env.sidecarDiagnostics }, + ); + const signedSend = env.hub.outboundMail.find( + (m) => m.senderAddress === deploymentMailAddress, + ); + expect(signedSend).toBeDefined(); + expect(signedSend?.recipients).toContain(deploymentMailAddress); + }); +}); diff --git a/tests/workflow-deploy/single-step-conversation-durability.test.ts b/tests/workflow-deploy/single-step-conversation-durability.test.ts index 06f233a0..69e6d9cf 100644 --- a/tests/workflow-deploy/single-step-conversation-durability.test.ts +++ b/tests/workflow-deploy/single-step-conversation-durability.test.ts @@ -622,6 +622,7 @@ describe("single-step conversation durability across respawn (Phase 4.5)", () => DEPLOYMENT_ID, DEFINITION_HASH: "definition-hash", MAILBOX_ADDRESS: MAILBOX, + STEP_COUNT: "1", WARM_KEEP: "true", }); diff --git a/tests/workflow-deploy/single-step-full-lifecycle.test.ts b/tests/workflow-deploy/single-step-full-lifecycle.test.ts index 8d57df6d..94f30254 100644 --- a/tests/workflow-deploy/single-step-full-lifecycle.test.ts +++ b/tests/workflow-deploy/single-step-full-lifecycle.test.ts @@ -78,11 +78,12 @@ import { deriveDeploymentAddress, isWorkflowDerivedAddress, type ApprovalSet, + type DeploySingleStepFn, type LaunchSessionFn, type SendMultiStepDeployFn, type WorkflowRepoWriter, } from "@intx/workflow-deploy"; -import { deriveTrivialDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; +import { deriveDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; import { createDurableConversationRegistry, reconstructDurableConversation, @@ -237,7 +238,7 @@ describe("single-step full lifecycle on the unified child path (Phase 4.6)", () ]); const launchSession: LaunchSessionFn = async (orchestratorParams) => { - await env.hub.sessionService.launchSession({ + await env.hub.sessionService.stageWorkflowStep({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, @@ -263,6 +264,14 @@ describe("single-step full lifecycle on the unified child path (Phase 4.6)", () sources: params.sources, }); + // A single-step workflow routes through the deploy core's single-step + // hand-off, which deploys once at the head: the service stages the + // head's deploy tree and fires the workflow frame in one call. The + // per-step `launchSession`/`sendMultiStepDeploy` callbacks above are + // reached only by the multi-step branch (>= 2 steps). + const deploySingleStepAtHead: DeploySingleStepFn = (params) => + env.hub.sessionService.deploySingleStepAtHead(params); + const workflowRepo: WorkflowRepoWriter = { async writeWorkflowRepo(args) { const repoId: RepoId = { kind: "workflow", id: args.workflowRepoId }; @@ -288,6 +297,7 @@ describe("single-step full lifecycle on the unified child path (Phase 4.6)", () workflowRepo, launchSession, sendMultiStepDeploy, + deploySingleStepAtHead, }); let result: Awaited>; @@ -310,7 +320,7 @@ describe("single-step full lifecycle on the unified child path (Phase 4.6)", () { cause }, ); } - expect(result.kind).toBe("multi-step"); + expect(result.publicKey).toBeTruthy(); // 4.1 identity preserved: the deploy-ack fired for the legacy // `ins_` address and persisted a public key. @@ -324,7 +334,7 @@ describe("single-step full lifecycle on the unified child path (Phase 4.6)", () const workflowRunRepoId: RepoId = { kind: "workflow-run", - id: deriveTrivialDeploymentId(deploymentMailAddress), + id: deriveDeploymentId(deploymentMailAddress), }; env.registerDeployment({ deploymentId: DEPLOYMENT_ID, diff --git a/tests/workflow-deploy/single-step-grants-bridge.test.ts b/tests/workflow-deploy/single-step-grants-bridge.test.ts index d6630c5b..21d9978a 100644 --- a/tests/workflow-deploy/single-step-grants-bridge.test.ts +++ b/tests/workflow-deploy/single-step-grants-bridge.test.ts @@ -53,11 +53,12 @@ import { deriveDeploymentAddress, isWorkflowDerivedAddress, type ApprovalSet, + type DeploySingleStepFn, type LaunchSessionFn, type SendMultiStepDeployFn, type WorkflowRepoWriter, } from "@intx/workflow-deploy"; -import { deriveTrivialDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; +import { deriveDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; import { generateKeyPair } from "@intx/crypto"; import { assembleCredentialsSnapshot, @@ -199,7 +200,7 @@ describe("single-step launched-agent grants bridge via spawned child", () => { ]); const launchSession: LaunchSessionFn = async (orchestratorParams) => { - await env.hub.sessionService.launchSession({ + await env.hub.sessionService.stageWorkflowStep({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, @@ -225,6 +226,9 @@ describe("single-step launched-agent grants bridge via spawned child", () => { sources: params.sources, }); + const deploySingleStepAtHead: DeploySingleStepFn = (params) => + env.hub.sessionService.deploySingleStepAtHead(params); + const workflowRepo: WorkflowRepoWriter = { async writeWorkflowRepo(args) { const repoId: RepoId = { kind: "workflow", id: args.workflowRepoId }; @@ -250,6 +254,7 @@ describe("single-step launched-agent grants bridge via spawned child", () => { workflowRepo, launchSession, sendMultiStepDeploy, + deploySingleStepAtHead, }); let result: Awaited>; @@ -272,7 +277,7 @@ describe("single-step launched-agent grants bridge via spawned child", () => { { cause }, ); } - expect(result.kind).toBe("multi-step"); + expect(result.publicKey).toBeTruthy(); // (a) identity: the deploy-ack fired for the legacy `ins_` // address and persisted a public key. The ack listener keys on the @@ -290,7 +295,7 @@ describe("single-step launched-agent grants bridge via spawned child", () => { const workflowRunRepoId: RepoId = { kind: "workflow-run", - id: deriveTrivialDeploymentId(deploymentMailAddress), + id: deriveDeploymentId(deploymentMailAddress), }; env.registerDeployment({ deploymentId: DEPLOYMENT_ID, @@ -335,7 +340,7 @@ describe("single-step launched-agent grants bridge via spawned child", () => { repoStore: readbackRepoStore, principal: { kind: "hub" }, stepOrder: [STEP_ID], - deploymentId: deriveTrivialDeploymentId(deploymentMailAddress), + deploymentId: deriveDeploymentId(deploymentMailAddress), deriveStepAddress: () => deploymentMailAddress, deriveStepRepoId: () => legacyAgentStateRepoId, }); diff --git a/tests/workflow-deploy/single-step-message-input.test.ts b/tests/workflow-deploy/single-step-message-input.test.ts index 52e51ace..957ff826 100644 --- a/tests/workflow-deploy/single-step-message-input.test.ts +++ b/tests/workflow-deploy/single-step-message-input.test.ts @@ -29,11 +29,12 @@ import { createWorkflowDeployOrchestrator, deriveDeploymentAddress, type ApprovalSet, + type DeploySingleStepFn, type LaunchSessionFn, type SendMultiStepDeployFn, type WorkflowRepoWriter, } from "@intx/workflow-deploy"; -import { deriveTrivialDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; +import { deriveDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; import type { RepoId, WorkflowRunHubPrincipal } from "@intx/hub-sessions"; import { DEFAULT_ASSET_REF } from "@intx/hub-sessions"; @@ -126,7 +127,7 @@ describe("single-step message-input round-trip", () => { const launchSession: LaunchSessionFn = async (orchestratorParams) => { const deployContent = orchestratorParams.deployContent; - await env.hub.sessionService.launchSession({ + await env.hub.sessionService.stageWorkflowStep({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, @@ -152,6 +153,9 @@ describe("single-step message-input round-trip", () => { sources: params.sources, }); + const deploySingleStepAtHead: DeploySingleStepFn = (params) => + env.hub.sessionService.deploySingleStepAtHead(params); + const workflowRepo: WorkflowRepoWriter = { async writeWorkflowRepo(args) { const repoId: RepoId = { kind: "workflow", id: args.workflowRepoId }; @@ -177,6 +181,7 @@ describe("single-step message-input round-trip", () => { workflowRepo, launchSession, sendMultiStepDeploy, + deploySingleStepAtHead, }); let result: Awaited>; @@ -198,11 +203,11 @@ describe("single-step message-input round-trip", () => { { cause }, ); } - expect(result.kind).toBe("multi-step"); + expect(result.publicKey).toBeTruthy(); const workflowRunRepoId: RepoId = { kind: "workflow-run", - id: deriveTrivialDeploymentId(deploymentMailAddress), + id: deriveDeploymentId(deploymentMailAddress), }; env.registerDeployment({ deploymentId: DEPLOYMENT_ID, diff --git a/tests/workflow-deploy/single-step-posix-tool.test.ts b/tests/workflow-deploy/single-step-posix-tool.test.ts index 01754d07..d2eea3c9 100644 --- a/tests/workflow-deploy/single-step-posix-tool.test.ts +++ b/tests/workflow-deploy/single-step-posix-tool.test.ts @@ -31,13 +31,13 @@ import { defineWorkflow, step, type WorkflowDefinition } from "@intx/workflow"; import { createWorkflowDeployOrchestrator, deriveDeploymentAddress, - deriveStepAddress, type ApprovalSet, + type DeploySingleStepFn, type LaunchSessionFn, type SendMultiStepDeployFn, type WorkflowRepoWriter, } from "@intx/workflow-deploy"; -import { deriveTrivialDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; +import { deriveDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; import { sanitizeAddress } from "@intx/hub-agent"; import type { RepoId, WorkflowRunHubPrincipal } from "@intx/hub-sessions"; import { DEFAULT_ASSET_REF } from "@intx/hub-sessions"; @@ -158,7 +158,7 @@ describe("single-step posix-tool in-child execution", () => { ]); const launchSession: LaunchSessionFn = async (orchestratorParams) => { - await env.hub.sessionService.launchSession({ + await env.hub.sessionService.stageWorkflowStep({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, @@ -184,6 +184,9 @@ describe("single-step posix-tool in-child execution", () => { sources: params.sources, }); + const deploySingleStepAtHead: DeploySingleStepFn = (params) => + env.hub.sessionService.deploySingleStepAtHead(params); + const workflowRepo: WorkflowRepoWriter = { async writeWorkflowRepo(args) { const repoId: RepoId = { kind: "workflow", id: args.workflowRepoId }; @@ -209,6 +212,7 @@ describe("single-step posix-tool in-child execution", () => { workflowRepo, launchSession, sendMultiStepDeploy, + deploySingleStepAtHead, }); let result: Awaited>; @@ -231,11 +235,11 @@ describe("single-step posix-tool in-child execution", () => { { cause }, ); } - expect(result.kind).toBe("multi-step"); + expect(result.publicKey).toBeTruthy(); const workflowRunRepoId: RepoId = { kind: "workflow-run", - id: deriveTrivialDeploymentId(deploymentMailAddress), + id: deriveDeploymentId(deploymentMailAddress), }; env.registerDeployment({ deploymentId: DEPLOYMENT_ID, @@ -312,20 +316,21 @@ describe("single-step posix-tool in-child execution", () => { expect(fs.readFileSync(sentinelPath, "utf-8")).toBe(SENTINEL_CONTENT); // The step's deploy tree (carrying the resolved tool-package - // manifest) landed at the step's legacy agent dir on the child -- - // the on-disk source the child read for materialization. - const stepAddress = deriveStepAddress({ + // manifest) landed at the HEAD's legacy agent dir on the child -- the + // on-disk source the child read for materialization. A single-step + // workflow collapses its lone step onto the head, so the deploy tree + // is staged at the deployment (head) address, not a per-step address. + const headAddress = deriveDeploymentAddress({ deploymentId: DEPLOYMENT_ID, - stepId: STEP_ID, deploymentDomain: DEPLOYMENT_DOMAIN, }); - const stepDeployDir = path.join( + const headDeployDir = path.join( env.sidecar.dataDir, - sanitizeAddress(stepAddress), + sanitizeAddress(headAddress), "deploy", ); expect( - fs.existsSync(path.join(stepDeployDir, "tool-packages-manifest.json")), + fs.existsSync(path.join(headDeployDir, "tool-packages-manifest.json")), ).toBe(true); }); }); diff --git a/tests/workflow-deploy/single-step-real-agent.test.ts b/tests/workflow-deploy/single-step-real-agent.test.ts index 4a4a5894..db0f9075 100644 --- a/tests/workflow-deploy/single-step-real-agent.test.ts +++ b/tests/workflow-deploy/single-step-real-agent.test.ts @@ -33,11 +33,12 @@ import { createWorkflowDeployOrchestrator, deriveDeploymentAddress, type ApprovalSet, + type DeploySingleStepFn, type LaunchSessionFn, type SendMultiStepDeployFn, type WorkflowRepoWriter, } from "@intx/workflow-deploy"; -import { deriveTrivialDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; +import { deriveDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; import { reconstructDurableConversation } from "@intx/sidecar-app/src/conversation-state"; import type { RepoId, WorkflowRunHubPrincipal } from "@intx/hub-sessions"; import { DEFAULT_ASSET_REF } from "@intx/hub-sessions"; @@ -134,7 +135,7 @@ describe("single-step real-agent round-trip", () => { const launchSession: LaunchSessionFn = async (orchestratorParams) => { const deployContent = orchestratorParams.deployContent; - await env.hub.sessionService.launchSession({ + await env.hub.sessionService.stageWorkflowStep({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, @@ -160,6 +161,9 @@ describe("single-step real-agent round-trip", () => { sources: params.sources, }); + const deploySingleStepAtHead: DeploySingleStepFn = (params) => + env.hub.sessionService.deploySingleStepAtHead(params); + const workflowRepo: WorkflowRepoWriter = { async writeWorkflowRepo(args) { const repoId: RepoId = { kind: "workflow", id: args.workflowRepoId }; @@ -185,6 +189,7 @@ describe("single-step real-agent round-trip", () => { workflowRepo, launchSession, sendMultiStepDeploy, + deploySingleStepAtHead, }); let result: Awaited>; @@ -206,11 +211,11 @@ describe("single-step real-agent round-trip", () => { { cause }, ); } - expect(result.kind).toBe("multi-step"); + expect(result.publicKey).toBeTruthy(); const workflowRunRepoId: RepoId = { kind: "workflow-run", - id: deriveTrivialDeploymentId(deploymentMailAddress), + id: deriveDeploymentId(deploymentMailAddress), }; env.registerDeployment({ deploymentId: DEPLOYMENT_ID, diff --git a/tests/workflow-deploy/trivial-roundtrip.test.ts b/tests/workflow-deploy/trivial-roundtrip.test.ts deleted file mode 100644 index 1f3c2f7d..00000000 --- a/tests/workflow-deploy/trivial-roundtrip.test.ts +++ /dev/null @@ -1,297 +0,0 @@ -// Trivial-workflow round-trip integration test. -// -// The runtime-side uniformity gate. Deploys a single-step workflow via -// the workflow-deploy orchestrator's trivial branch against the real -// hub + real sidecar subprocess + mock inference fixture, fires a mail -// trigger at the deployment's address, and asserts the canonical -// workflow-run event chain (`RunStarted` -> `StepStarted` -> -// `StepCompleted` -> `RunCompleted`) materializes in the workflow-run -// repo with the expected `consumedMessageId` correlation against the -// mail's `Message-Id`. The test additionally asserts the legacy -// deploy-flow surface (inference receives tools, the `agent.event` -// capture sees `inference.start`) still holds. -// -// The pre-landed `deploy-flow-env` fixture supplies every helper; this -// file does not modify the fixture. - -import fs from "node:fs"; - -import { afterAll, beforeAll, describe, expect, test } from "bun:test"; -import git from "isomorphic-git"; - -import { defineAgent } from "@intx/agent"; -import type { HarnessConfig } from "@intx/types/runtime"; -import { defineWorkflow } from "@intx/workflow"; - -import { - AGENT_ADDRESS, - AGENT_ID, - SESSION_ID, - SIDECAR_ID, - deployWorkflow, - fireMailTrigger, - readWorkflowRunEvents, - startDeployFlowEnv, - waitFor, - WORKFLOW_RUN_TERMINAL_TYPES, - type DeployFlowEnv, - type WorkflowRunEvent, -} from "../hub-agent/lib/deploy-flow-env"; - -let env: DeployFlowEnv; - -beforeAll(async () => { - env = await startDeployFlowEnv(); -}); - -afterAll(async () => { - await env.teardown(); -}); - -describe("trivial workflow round-trip", () => { - test("sidecar registers with hub", () => { - expect(env.hub.router.getConnectedSidecars()).toContain(SIDECAR_ID); - }); - - test("canonical event chain materializes end-to-end", async () => { - const config: HarnessConfig = { - sessionId: SESSION_ID, - agentId: AGENT_ID, - tenantId: "tenant-1", - principalId: "prin_integration-1", - agentAddress: AGENT_ADDRESS, - systemPrompt: "Fallback prompt (overridden by deploy tree)", - tools: [], - grants: [], - sources: [ - { - id: "anthropic:mock-model", - provider: "anthropic", - baseURL: `http://localhost:${env.inference.server.port}`, - apiKey: "sk-mock", - model: "mock-model", - }, - ], - defaultSource: "anthropic:mock-model", - }; - - // Mirror the legacy deploy-flow fixture's agent shape so the - // trivial-vs-multi-step dichotomy fires the trivial branch: a - // single step, single inference source, no tool factories, no - // capabilities. The workflow-deploy orchestrator's `isTrivialDeploy` - // routes on `stepOrder.length === 1 && trivialBindings !== - // undefined`; supplying both forces the trivial branch. - const agent = defineAgent({ - id: AGENT_ID, - systemPrompt: "You are an integration test agent.", - tools: [], - capabilities: [], - inference: { - sources: [{ provider: "anthropic", model: "mock-model" }], - }, - }); - - const workflow = defineWorkflow({ - id: `wf_${AGENT_ID}`, - agent, - trigger: { type: "mail", to: AGENT_ADDRESS }, - }); - - // Approval shape matches what `buildTrivialApprovalSet` computes - // for a `wrapHarnessAsTrivialAgent` deploy: per-source inference - // grants, the default-director grant, and the mail-address + - // mail-send grants derived from the deployment's mailbox. - const mailDomain = AGENT_ADDRESS.slice(AGENT_ADDRESS.lastIndexOf("@") + 1); - const operatorApprovals = new Set([ - `inference.source:anthropic:mock-model`, - `director:@intx/agent/default`, - `mail.address:${AGENT_ADDRESS}`, - `mail.send:${mailDomain}`, - ]); - - const deployHandle = await deployWorkflow(env, workflow, { - config, - deployContent: { systemPrompt: "You are an integration test agent." }, - trivialBindings: { - agentAddress: AGENT_ADDRESS, - agentId: AGENT_ID, - instanceId: AGENT_ID, - }, - operatorApprovals, - toolPackagePins: [{ name: "@intx/tools-mail", version: "0.1.2" }], - }); - - // The deploy ack arrives once `executeLaunchPhases` finishes - // provisioning the sidecar. This is the trivial branch's "ready" - // signal: the supervisor's trivial-launch path round-trips through - // the same wire surface the legacy agent-deploy uses. - const publicKey = env.hub.deployAcks.get(AGENT_ADDRESS); - expect(publicKey).toBeDefined(); - if (publicKey === undefined) throw new Error("unreachable"); - expect(publicKey.length).toBeGreaterThan(0); - - expect(env.hub.router.getRoutableAddresses()).toContain(AGENT_ADDRESS); - - const { messageId } = await fireMailTrigger(env, AGENT_ADDRESS, { - messageId: "", - }); - - // Legacy assertion #1: inference saw the deploy-tree's tool - // surfaces. The trivial round-trip must preserve the wire-level - // surface the legacy deploy-flow test asserts. - await waitFor(() => env.inference.requests.length > 0, { - diagnostics: env.sidecarDiagnostics, - }); - const inferenceReq = env.inference.requests[0]; - if (inferenceReq === undefined) throw new Error("unreachable"); - const toolNames = (inferenceReq.tools ?? []).map((t) => t.name); - expect(toolNames).toContain("@intx/tools-mail/sidecar-bundle:mail_send"); - - // Legacy assertion #2: the `agent.event` capture saw an - // `inference.start` event for this session. - function hasEventType( - event: unknown, - type: string, - ): event is { type: string } { - return ( - typeof event === "object" && - event !== null && - "type" in event && - event.type === type - ); - } - await waitFor( - () => - env.hub.agentEvents.some((e) => - hasEventType(e.event, "inference.start"), - ), - { diagnostics: env.sidecarDiagnostics }, - ); - const inferenceStartEvent = env.hub.agentEvents.find((e) => - hasEventType(e.event, "inference.start"), - ); - if (inferenceStartEvent === undefined) throw new Error("unreachable"); - expect(inferenceStartEvent.addr).toBe(AGENT_ADDRESS); - expect(inferenceStartEvent.sid).toBe(SESSION_ID); - - // The runtime-side uniformity gate. The deployment's workflow-run - // repo must hold the canonical event chain for the run this mail - // fired. The supervisor mints the runId internally (sha256 of the - // raw RFC 2822 message bytes in the current wiring), so the test - // does not know it up front; the helper below polls `runs/` in - // the workflow-run repo for the first run that reaches a terminal - // event. - const terminal = await waitForTerminalEventForAnyRun( - env, - deployHandle.deploymentId, - { diagnostics: env.sidecarDiagnostics }, - ); - - expect(terminal.event.type).toBe("RunCompleted"); - - const events = await readWorkflowRunEvents( - env, - deployHandle.deploymentId, - terminal.runId, - ); - - const types = events.map((e) => e.type); - const runStartedIdx = types.indexOf("RunStarted"); - const stepStartedIdx = types.indexOf("StepStarted"); - const stepCompletedIdx = types.indexOf("StepCompleted"); - const runCompletedIdx = types.indexOf("RunCompleted"); - - expect(runStartedIdx).toBeGreaterThanOrEqual(0); - expect(stepStartedIdx).toBeGreaterThan(runStartedIdx); - expect(stepCompletedIdx).toBeGreaterThan(stepStartedIdx); - expect(runCompletedIdx).toBeGreaterThan(stepCompletedIdx); - - const runStartedBody = events[runStartedIdx]?.body; - if (runStartedBody === undefined) throw new Error("unreachable"); - expect(runStartedBody["consumedMessageId"]).toBe(messageId); - - const stepStartedBody = events[stepStartedIdx]?.body; - if (stepStartedBody === undefined) throw new Error("unreachable"); - expect(stepStartedBody["attempt"]).toBe(1); - expect(typeof stepStartedBody["stepId"]).toBe("string"); - }); -}); - -/** - * Poll the deployment's workflow-run repo for the first run that - * reaches a terminal event. The trivial-roundtrip test does not know - * the runId the runtime mints (the supervisor derives it from the - * inbound mail bytes), so the helper walks `runs/` in the repo tree - * and returns the first run whose log has reached terminal status. - */ -async function waitForTerminalEventForAnyRun( - env: DeployFlowEnv, - deploymentId: string, - opts: { timeoutMs?: number; diagnostics?: () => string } = {}, -): Promise<{ runId: string; event: WorkflowRunEvent }> { - const { timeoutMs = 10_000, diagnostics } = opts; - const start = Date.now(); - const handle = env.deployments.get(deploymentId); - if (handle === undefined) { - throw new Error( - `waitForTerminalEventForAnyRun: no deployment registered for ${deploymentId}`, - ); - } - let repoDir: string; - try { - repoDir = env.hub.agentRepoStore.repoStore.getRepoDir( - handle.workflowRunRepoId, - ); - } catch (cause) { - const message = cause instanceof Error ? cause.message : String(cause); - throw new Error( - `waitForTerminalEventForAnyRun: workflow-run repo identity for deployment ${deploymentId} is structurally invalid at the substrate boundary (${message}); the trivial-deploy path uses the agent address as the deploymentId, but the workflow-run repo id constraint rejects characters present in mail addresses. This is the runtime-side uniformity gate failure: the trivial branch cannot write workflow-run events because it cannot construct a valid workflow-run repo id.`, - ); - } - for (;;) { - let runIds: string[] = []; - try { - const oid = await git.resolveRef({ - fs, - dir: repoDir, - ref: handle.workflowRunRef, - }); - try { - const runsTree = await git.readTree({ - fs, - dir: repoDir, - oid, - filepath: "runs", - }); - runIds = runsTree.tree - .filter((entry) => entry.type === "tree") - .map((entry) => entry.path); - } catch { - runIds = []; - } - } catch { - runIds = []; - } - for (const runId of runIds) { - const events = await readWorkflowRunEvents(env, deploymentId, runId); - const terminal = events.find((e) => - WORKFLOW_RUN_TERMINAL_TYPES.has(e.type), - ); - if (terminal !== undefined) { - return { runId, event: terminal }; - } - } - if (Date.now() - start > timeoutMs) { - const diag = diagnostics?.(); - const ctx = diag ? `\n${diag}` : ""; - const inspected = - runIds.length === 0 - ? "" - : runIds.join(","); - throw new Error( - `waitForTerminalEventForAnyRun timed out after ${String(timeoutMs)}ms for ${deploymentId}; runs observed: ${inspected}; expected the canonical event chain (RunStarted -> StepStarted -> StepCompleted -> RunCompleted) under runs//events/.${ctx}`, - ); - } - await new Promise((r) => setTimeout(r, 50)); - } -} diff --git a/tests/workflow-deploy/unresolvable-director.test.ts b/tests/workflow-deploy/unresolvable-director.test.ts index 385bb2cf..a664f23e 100644 --- a/tests/workflow-deploy/unresolvable-director.test.ts +++ b/tests/workflow-deploy/unresolvable-director.test.ts @@ -130,11 +130,7 @@ describe("unresolvable-director deploy rejection", () => { await deployWorkflow(env, workflow, { config, deployContent: { systemPrompt: "You are an integration test agent." }, - trivialBindings: { - agentAddress: AGENT_ADDRESS, - agentId: AGENT_ID, - instanceId: AGENT_ID, - }, + deploymentId: AGENT_ID, operatorApprovals, }); } catch (err) {