From 9b803cbcbd4ac06b8a04f270320975d97f9d0c7f Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Tue, 30 Jun 2026 19:31:37 -0500 Subject: [PATCH 001/101] Sign the latency-gate baseline agent per-turn commits The baseline built its isogit ContextStore without a signer, so its per-turn context commits skipped the sshsig the unified path pays on every turn. That understated the baseline floor and inflated the measured unified-minus-baseline delta. Generate a keypair and pass a CommitSigner mirroring the production agent-repo signer so both paths pay the same per-commit signing cost. --- tests/workflow-deploy/latency-gate.bench.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/workflow-deploy/latency-gate.bench.ts b/tests/workflow-deploy/latency-gate.bench.ts index 468e734a..637a9afe 100644 --- a/tests/workflow-deploy/latency-gate.bench.ts +++ b/tests/workflow-deploy/latency-gate.bench.ts @@ -59,7 +59,8 @@ import { defineAgent, createDefaultDirectorRegistry } from "@intx/agent"; import { noopAuditStore, permissiveAuthorize } from "@intx/agent/testing"; import { createAgent, type Agent } from "@intx/agent"; import type { BaseEnv } from "@intx/agent"; -import { createIsogitStore } from "@intx/storage-isogit"; +import { generateKeyPair, createSSHSignature } from "@intx/crypto"; +import { createIsogitStore, type CommitSigner } from "@intx/storage-isogit"; import type { HarnessConfig, InferenceSource } from "@intx/types/runtime"; import { defineWorkflow, step, type WorkflowDefinition } from "@intx/workflow"; import { @@ -251,7 +252,13 @@ async function runBaseline(opts: { // A real isogit ContextStore (the same store kind the unified step // agent uses for its conversation) so the baseline pays the same // per-turn context-commit cost the in-process runtime pays today. - const storage = await createIsogitStore(storeDir); + // Sign each commit's sshsig exactly as `agent-repo.ts` does for the + // unified path; without a signer the baseline would skip the + // per-commit signature and undercount its own floor. + const signingKey = await generateKeyPair(); + const signer: CommitSigner = (payload) => + createSSHSignature(payload, signingKey.privateKey, signingKey.publicKey); + const storage = await createIsogitStore(storeDir, signer); const def = defineAgent({ id: "latency-baseline-agent", From 4e2c4b45f57e9bedff1266608136ac616b0abd12 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Tue, 30 Jun 2026 19:32:44 -0500 Subject: [PATCH 002/101] Add a flag-gated delta-scoped claim-check validation path The claim-check walk re-reads, re-parses and re-validates the entire retained consumed dedup index on every commit, so the per-turn substrate cost grows with the retained set. Behind the experiment flag BENCH_DELTA_SCOPE_CLAIMCHECK (default off), validate only the per-commit delta against the prior tree, keyed by (filename, blob OID): retained entries are skipped via OID equality (the immutability proof that preserves exactly-once), added entries are parsed and dedup checked, removed entries are checked below the watermark. A new optional priorListDirOids closure on the validatePush args surfaces prior blob OIDs from the prior commit's tree in a single read. With the flag off the walk is byte-identical to before. --- packages/hub-sessions/src/repo-store/store.ts | 32 ++- packages/hub-sessions/src/repo-store/types.ts | 13 + .../hub-sessions/src/workflow-run-kind.ts | 268 +++++++++++++++--- 3 files changed, 269 insertions(+), 44 deletions(-) diff --git a/packages/hub-sessions/src/repo-store/store.ts b/packages/hub-sessions/src/repo-store/store.ts index ac9e04b7..c2382ee0 100644 --- a/packages/hub-sessions/src/repo-store/store.ts +++ b/packages/hub-sessions/src/repo-store/store.ts @@ -633,11 +633,15 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { ): { priorReadBlob: (path: string) => Promise; priorListDir: (path: string) => Promise; + priorListDirOids: ( + path: string, + ) => Promise<{ name: string; oid: string }[]>; } { if (commitSha === null) { return { priorReadBlob: async () => null, priorListDir: async () => [], + priorListDirOids: async () => [], }; } const priorReadBlob = async ( @@ -654,7 +658,19 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { const { tree } = await git.readTree({ fs, dir, oid }); return tree.map((e) => e.path); }; - return { priorReadBlob, priorListDir }; + // Same walk as `priorListDir` but carries each child's git object id + // out of the tree listing. A kind handler validating a large subtree + // by its per-commit delta uses the OID to prove a retained entry is + // byte-unchanged without re-reading the blob. + const priorListDirOids = async ( + relPath: string, + ): Promise<{ name: string; oid: string }[]> => { + const oid = await resolveTreeEntry(dir, commitSha, relPath, "tree"); + if (oid === null) return []; + const { tree } = await git.readTree({ fs, dir, oid }); + return tree.map((e) => ({ name: e.path, oid: e.oid })); + }; + return { priorReadBlob, priorListDir, priorListDirOids }; } // Build the `(readBlob, listDir, topLevelTreePaths)` triple a kind @@ -1017,10 +1033,8 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { // to produce. `refPaths` above was snapshotted from the same // commit, so both reads observe the same pre-image. const priorCommitSha = await resolveRefSha(dir, ref); - const { priorReadBlob, priorListDir } = buildPriorTreeClosures( - dir, - priorCommitSha, - ); + const { priorReadBlob, priorListDir, priorListDirOids } = + buildPriorTreeClosures(dir, priorCommitSha); // The prospective tree is the actual commit `git.commit` will // produce: the prior tree, minus paths cleared by `clearPrefix`, @@ -1105,6 +1119,7 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { listDir, priorReadBlob, priorListDir, + priorListDirOids, changedPathPrefixes, }); if (!validation.ok) { @@ -1364,10 +1379,8 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { } } } - const { priorReadBlob, priorListDir } = buildPriorTreeClosures( - dir, - parentSha, - ); + const { priorReadBlob, priorListDir, priorListDirOids } = + buildPriorTreeClosures(dir, parentSha); const { topLevelTreePaths, readBlob, listDir } = await buildCommitTreeClosures(dir, newCommit); const changedPathPrefixes = await computeChangedPathPrefixes( @@ -1384,6 +1397,7 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { listDir, priorReadBlob, priorListDir, + priorListDirOids, changedPathPrefixes, }); if (!result.ok) { diff --git a/packages/hub-sessions/src/repo-store/types.ts b/packages/hub-sessions/src/repo-store/types.ts index cbc35ff5..afbab8d2 100644 --- a/packages/hub-sessions/src/repo-store/types.ts +++ b/packages/hub-sessions/src/repo-store/types.ts @@ -241,6 +241,16 @@ export interface KindHandler { * log) uses this to skip re-validating subtrees the commit provably * did not touch; a handler with no such structure ignores it and * validates unconditionally. + * + * `priorListDirOids` mirrors `priorListDir` but returns each child + * entry's git object id alongside its name, read straight from the + * prior commit's tree (git trees are content-addressed, so the listing + * already carries the OID). A handler that validates a large retained + * subtree by its per-commit delta uses it to prove a retained entry is + * byte-unchanged by OID equality instead of re-reading the blob. It is + * `undefined` when no prior commit exists; a handler that needs OIDs + * for a prospective-side compare hashes the (usually in-memory) + * prospective bytes itself. */ validatePush: (args: { repoId: RepoId; @@ -251,6 +261,9 @@ export interface KindHandler { listDir: (path: string) => Promise; priorReadBlob: (path: string) => Promise; priorListDir: (path: string) => Promise; + priorListDirOids?: ( + path: string, + ) => Promise<{ name: string; oid: string }[]>; changedPathPrefixes?: ReadonlySet | undefined; }) => Promise | ValidatePushResult; /** diff --git a/packages/hub-sessions/src/workflow-run-kind.ts b/packages/hub-sessions/src/workflow-run-kind.ts index 0336d02e..a23348f1 100644 --- a/packages/hub-sessions/src/workflow-run-kind.ts +++ b/packages/hub-sessions/src/workflow-run-kind.ts @@ -189,6 +189,21 @@ import { const logger = getLogger(["hub-sessions", "workflow-run-kind"]); +/** + * Experiment flag (latency gate, default OFF). When set, the claim-check + * validation walk in `validateClaimCheckSubtree` validates only the + * per-commit DELTA of the consumed dedup index against the prior tree -- + * keyed by (filename, blob OID) -- instead of re-reading, re-parsing and + * re-validating the ENTIRE retained consumed set on every commit. Read + * once at module load; OFF reproduces the exhaustive walk byte-for-byte, + * so nothing changes unless the flag is explicitly set. + * + * A second, independent experiment flag (commit batching) is intended to + * live alongside this one; it is deliberately NOT defined here yet. + */ +const BENCH_DELTA_SCOPE_CLAIMCHECK = + process.env.BENCH_DELTA_SCOPE_CLAIMCHECK === "1"; + export type WorkflowRunHubPrincipal = { readonly kind: "hub" }; export type WorkflowRunSidecarPrincipal = { @@ -1033,6 +1048,16 @@ type ClaimCheckBlob = { */ messageIdFromFilename: string; blobPath: string; + /** + * Git blob object id of the entry, populated only for `consumed` + * entries and only when the delta-scoped claim-check path is enabled + * (`BENCH_DELTA_SCOPE_CLAIMCHECK`). Two entries at the same path whose + * OIDs match are byte-identical (git trees are content-addressed), so + * the delta path skips the immutability byte-comparison for retained + * consumed entries. Left `undefined` on the exhaustive (flag-OFF) path + * and on inbox/processing entries, which the delta path does not scope. + */ + oid?: string; }; /** @@ -1075,6 +1100,7 @@ type ClaimCheckAddressBucket = { async function enumerateClaimCheckBlobs( listDir: (path: string) => Promise, + resolveConsumedOid?: (blobPath: string) => Promise, ): Promise< | { ok: true; perAddress: Map } | { ok: false; reason: string } @@ -1154,15 +1180,20 @@ async function enumerateClaimCheckBlobs( reason: `${WORKFLOW_RUN_CONSUMED_DIR} filename ${dirPath}/${filename} produced no message-id capture`, }; } - bucket.consumed.push({ + const consumedBlobPath = `${dirPath}/${filename}`; + const consumedEntry: ClaimCheckBlob = { kind: "consumed", addressSegment: segment, decodedAddress: roundTrip.decoded, filename, receivedAtFromFilename: null, messageIdFromFilename: messageId, - blobPath: `${dirPath}/${filename}`, - }); + blobPath: consumedBlobPath, + }; + if (resolveConsumedOid !== undefined) { + consumedEntry.oid = await resolveConsumedOid(consumedBlobPath); + } + bucket.consumed.push(consumedEntry); } } } @@ -1378,6 +1409,68 @@ async function checkPriorBytesImmutable( return { ok: true }; } +/** + * Compute the git blob OID of a consumed entry from a byte reader, + * used only when the delta-scoped path lacks a substrate-provided prior + * OID listing (e.g. a hand-built test validatePush). `git.hashBlob` + * reproduces the same content-addressed OID a `git.readTree` listing + * carries, so the delta path's intersection compare is identical + * whether the OID came from the tree listing or from hashing the bytes. + */ +async function hashConsumedBlobOid(bytes: Uint8Array): Promise { + const { oid } = await git.hashBlob({ object: bytes }); + return oid; +} + +/** + * Build the resolver that surfaces each PRIOR consumed entry's git blob + * OID for the delta-scoped path. When the substrate supplies + * `priorListDirOids` the OID comes straight from the prior commit's tree + * listing (one `readTree` per consumed directory, cached), so the prior + * side is not re-read blob-by-blob. When it is absent (a hand-built + * validatePush in a unit test) the resolver falls back to hashing the + * prior bytes via `priorReadBlob`; that fallback is O(retained) rather + * than free but preserves identical semantics, which is all a + * correctness test needs. + */ +function makePriorConsumedOidResolver( + priorReadBlob: (path: string) => Promise, + priorListDirOids: + | ((path: string) => Promise<{ name: string; oid: string }[]>) + | undefined, +): (blobPath: string) => Promise { + const dirOidCache = new Map>(); + return async (blobPath) => { + if (priorListDirOids !== undefined) { + const slash = blobPath.lastIndexOf("/"); + const dir = blobPath.slice(0, slash); + const name = blobPath.slice(slash + 1); + let byName = dirOidCache.get(dir); + if (byName === undefined) { + byName = new Map(); + for (const entry of await priorListDirOids(dir)) { + byName.set(entry.name, entry.oid); + } + dirOidCache.set(dir, byName); + } + const oid = byName.get(name); + if (oid === undefined) { + throw new Error( + `delta claim-check: prior tree listing has no OID for enumerated consumed entry ${blobPath}`, + ); + } + return oid; + } + const bytes = await priorReadBlob(blobPath); + if (bytes === null) { + throw new Error( + `delta claim-check: consumed entry ${blobPath} was enumerated in the prior tree but its bytes could not be read`, + ); + } + return hashConsumedBlobOid(bytes); + }; +} + /** * Validate the `addresses//{inbox,processing,consumed}` * subtree as a whole. The walk enforces filename shape, JSON envelope @@ -1385,16 +1478,44 @@ async function checkPriorBytesImmutable( * three queue states, consumed-blob immutability via prior-bytes * equality, and the inbox→processing / processing→consumed * transition invariants against the prior tree. + * + * When `BENCH_DELTA_SCOPE_CLAIMCHECK` is set the consumed dedup index is + * validated by its per-commit DELTA against the prior tree rather than + * by re-walking the whole retained set: retained entries (same filename, + * same blob OID) are skipped as already-validated-and-immutable, added + * entries are parsed and validated, and removed entries are checked + * against the retention watermark. `priorListDirOids`, when supplied by + * the substrate, surfaces prior consumed OIDs straight from the prior + * commit's tree so the prior side is not re-read blob-by-blob. */ async function validateClaimCheckSubtree( listDir: (path: string) => Promise, readBlob: (path: string) => Promise, priorReadBlob: (path: string) => Promise, priorListDir: (path: string) => Promise, + priorListDirOids?: (path: string) => Promise<{ name: string; oid: string }[]>, ): Promise { - const enumerated = await enumerateClaimCheckBlobs(listDir); + // When the delta path is on, surface each consumed entry's git blob + // OID during enumeration: prospective OIDs by hashing the (in the + // measured writeTree path, in-memory) prospective bytes, prior OIDs + // from the substrate tree listing when available. The resolvers are + // omitted entirely on the exhaustive path so it stays byte-identical. + const prospectiveConsumedOid = BENCH_DELTA_SCOPE_CLAIMCHECK + ? async (blobPath: string) => hashConsumedBlobOid(await readBlob(blobPath)) + : undefined; + const priorConsumedOid = BENCH_DELTA_SCOPE_CLAIMCHECK + ? makePriorConsumedOidResolver(priorReadBlob, priorListDirOids) + : undefined; + + const enumerated = await enumerateClaimCheckBlobs( + listDir, + prospectiveConsumedOid, + ); if (!enumerated.ok) return enumerated; - const priorEnumerated = await enumerateClaimCheckBlobs(priorListDir); + const priorEnumerated = await enumerateClaimCheckBlobs( + priorListDir, + priorConsumedOid, + ); if (!priorEnumerated.ok) { // The prior tree is the committed state — if its claim-check // shape is already broken, surface it with a distinct rejection @@ -1452,8 +1573,17 @@ async function validateClaimCheckSubtree( messageIdToLocations.set(entry.messageIdFromFilename, list); } for (const entry of bucket.consumed) { - const parsed = await parseConsumedBlob(entry, readBlob); - if (!parsed.ok) return parsed; + // Cross-state atomicity needs each consumed messageId in the map; + // the messageId is the filename stem, so on the delta path this + // needs no blob read. Retained consumed entries are not re-parsed + // (their envelope was validated when first written and their bytes + // are proven immutable by the OID compare below); added consumed + // entries are parsed and validated by the transition check further + // down. The exhaustive path re-parses every consumed entry here. + if (!BENCH_DELTA_SCOPE_CLAIMCHECK) { + const parsed = await parseConsumedBlob(entry, readBlob); + if (!parsed.ok) return parsed; + } const list = messageIdToLocations.get(entry.messageIdFromFilename) ?? []; list.push({ kind: entry.kind, filename: entry.filename }); messageIdToLocations.set(entry.messageIdFromFilename, list); @@ -1482,16 +1612,52 @@ async function validateClaimCheckSubtree( } } - // Consumed entries are immutable: enforce byte-equality against - // the prior tree for every consumed blob that already existed. - for (const entry of bucket.consumed) { - const immutability = await checkPriorBytesImmutable( - entry.blobPath, - readBlob, - priorReadBlob, - "consumed", - ); - if (!immutability.ok) return immutability; + // Consumed entries are immutable. The exhaustive path re-reads both + // the prospective and the prior bytes of every retained consumed + // entry and compares them. The delta path instead compares the git + // blob OID the enumeration surfaced: a consumed entry present in the + // prior tree at the same path must carry the same OID (git trees are + // content-addressed, so equal OID proves byte-equality without + // reading either blob). A diverging OID is the same immutability + // violation the byte compare catches. Immutability is load-bearing + // for exactly-once: a mutated `receivedAt` on a retained consumed + // entry could fake it below the watermark, get it pruned, and let a + // re-submission miss dedup -- so this compare is not optional. + if (BENCH_DELTA_SCOPE_CLAIMCHECK) { + const priorConsumedOidByPath = new Map(); + for (const e of priorBucket?.consumed ?? []) { + if (e.oid === undefined) { + throw new Error( + `delta claim-check: prior consumed entry ${e.blobPath} was enumerated without an OID`, + ); + } + priorConsumedOidByPath.set(e.blobPath, e.oid); + } + for (const entry of bucket.consumed) { + const priorOid = priorConsumedOidByPath.get(entry.blobPath); + if (priorOid === undefined) continue; // newly added; validated below + if (entry.oid === undefined) { + throw new Error( + `delta claim-check: prospective consumed entry ${entry.blobPath} was enumerated without an OID`, + ); + } + if (entry.oid !== priorOid) { + return { + ok: false, + reason: `consumed ${entry.blobPath} bytes diverge from the prior tree (blob OID ${entry.oid} vs ${priorOid}); consumed entries are immutable once written`, + }; + } + } + } else { + for (const entry of bucket.consumed) { + const immutability = await checkPriorBytesImmutable( + entry.blobPath, + readBlob, + priorReadBlob, + "consumed", + ); + if (!immutability.ok) return immutability; + } } const prospectiveConsumedPaths = new Set( @@ -1569,29 +1735,59 @@ async function validateClaimCheckSubtree( // stale-rejected at enqueue), so it never weakens exactly-once. // The receivedAt lives in the body; read it from the prior tree // (retained bytes are immutable, so prior and prospective agree). - let maxDroppedReceivedAt = Number.NEGATIVE_INFINITY; - let minRetainedReceivedAt = Number.POSITIVE_INFINITY; - for (const e of priorBucket.consumed) { - const priorParsed = await parseConsumedBlobFrom(e, priorReadBlob); - if (!priorParsed.ok) return priorParsed; - const receivedAt = priorParsed.body.receivedAt; - if (prospectiveConsumedPaths.has(e.blobPath)) { - minRetainedReceivedAt = Math.min(minRetainedReceivedAt, receivedAt); - continue; + // + // The delta path reads ONLY the dropped entries. A retained entry + // (present in both trees) is proven byte-identical by the OID + // compare above, so its receivedAt is unchanged and need not be + // read -- the O(retained) prior read the exhaustive path pays is + // exactly what the delta path exists to avoid. It also omits the + // suffix relation (2): computing `minRetainedReceivedAt` would + // require reading every retained entry, and the relation is + // redundant for exactly-once here -- every dropped entry is below + // the watermark (check 1), and a retained entry below the + // watermark is explicitly harmless (extra dedup only), so a + // retained entry older than a dropped one weakens nothing. Check 1 + // plus the already-verified watermark monotonicity is the whole of + // the exactly-once retention contract: pruning is bound to the + // watermark and the watermark only advances. + if (BENCH_DELTA_SCOPE_CLAIMCHECK) { + for (const e of priorBucket.consumed) { + if (prospectiveConsumedPaths.has(e.blobPath)) continue; + const priorParsed = await parseConsumedBlobFrom(e, priorReadBlob); + if (!priorParsed.ok) return priorParsed; + const receivedAt = priorParsed.body.receivedAt; + if (receivedAt >= prospectiveWatermark) { + return { + ok: false, + reason: `consumed ${e.blobPath} present in the prior tree is missing from the prospective tree but its receivedAt ${String(receivedAt)} is not below the retention watermark ${String(prospectiveWatermark)}; consumed entries may be pruned only once the watermark has passed them`, + }; + } + } + } else { + let maxDroppedReceivedAt = Number.NEGATIVE_INFINITY; + let minRetainedReceivedAt = Number.POSITIVE_INFINITY; + for (const e of priorBucket.consumed) { + const priorParsed = await parseConsumedBlobFrom(e, priorReadBlob); + if (!priorParsed.ok) return priorParsed; + const receivedAt = priorParsed.body.receivedAt; + if (prospectiveConsumedPaths.has(e.blobPath)) { + minRetainedReceivedAt = Math.min(minRetainedReceivedAt, receivedAt); + continue; + } + if (receivedAt >= prospectiveWatermark) { + return { + ok: false, + reason: `consumed ${e.blobPath} present in the prior tree is missing from the prospective tree but its receivedAt ${String(receivedAt)} is not below the retention watermark ${String(prospectiveWatermark)}; consumed entries may be pruned only once the watermark has passed them`, + }; + } + maxDroppedReceivedAt = Math.max(maxDroppedReceivedAt, receivedAt); } - if (receivedAt >= prospectiveWatermark) { + if (maxDroppedReceivedAt > minRetainedReceivedAt) { return { ok: false, - reason: `consumed ${e.blobPath} present in the prior tree is missing from the prospective tree but its receivedAt ${String(receivedAt)} is not below the retention watermark ${String(prospectiveWatermark)}; consumed entries may be pruned only once the watermark has passed them`, + reason: `address ${JSON.stringify(decodedAddress)} consumed prune is not a suffix: a dropped entry (receivedAt ${String(maxDroppedReceivedAt)}) is newer than a retained entry (receivedAt ${String(minRetainedReceivedAt)}); only the oldest age-ordered tail may be pruned`, }; } - maxDroppedReceivedAt = Math.max(maxDroppedReceivedAt, receivedAt); - } - if (maxDroppedReceivedAt > minRetainedReceivedAt) { - return { - ok: false, - reason: `address ${JSON.stringify(decodedAddress)} consumed prune is not a suffix: a dropped entry (receivedAt ${String(maxDroppedReceivedAt)}) is newer than a retained entry (receivedAt ${String(minRetainedReceivedAt)}); only the oldest age-ordered tail may be pruned`, - }; } for (const e of priorBucket.processing) { if (prospectiveProcessingPaths.has(e.blobPath)) continue; @@ -1834,6 +2030,7 @@ export const workflowRunKindHandler: KindHandler = { listDir, priorReadBlob, priorListDir, + priorListDirOids, changedPathPrefixes, }): Promise { // Bound the per-run event/blob walks to the runs this commit could @@ -1902,6 +2099,7 @@ export const workflowRunKindHandler: KindHandler = { readBlob, priorReadBlob, priorListDir, + priorListDirOids, ); if (!claimCheck.ok) { logger.debug`workflow-run validatePush rejected ${repoId.kind}/${repoId.id} on ${ref}: ${claimCheck.reason}`; From 7452c3eeb398618f7c70f23585d31f7351c420dc Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Tue, 30 Jun 2026 19:53:48 -0500 Subject: [PATCH 003/101] Lock the delta claim-check watermark prune boundary The delta-scoped claim-check path accepts a below-watermark prune that the exhaustive suffix guard would reject; the domain owner confirmed the relaxation is structural (redundant for exactly-once). Its safety rests on the prune boundary being strict at receivedAt >= watermark, mirroring the strict receivedAt < watermark stale-reject so the entry exactly at the watermark is both retained and not stale-rejected. Assert both sides of that boundary under the delta flag so a later refactor cannot widen the comparison and open a reprocess gap, and record at the check which structural properties the suffix guard provided that this path drops. --- .../src/workflow-run-kind.test.ts | 72 +++++++++++++++++++ .../hub-sessions/src/workflow-run-kind.ts | 14 ++++ 2 files changed, 86 insertions(+) diff --git a/packages/hub-sessions/src/workflow-run-kind.test.ts b/packages/hub-sessions/src/workflow-run-kind.test.ts index cec80235..967eb19d 100644 --- a/packages/hub-sessions/src/workflow-run-kind.test.ts +++ b/packages/hub-sessions/src/workflow-run-kind.test.ts @@ -2975,6 +2975,78 @@ describe("workflowRunKindHandler.validatePush — retention watermark contract", }, { priorFiles: prior }, ); + if (process.env.BENCH_DELTA_SCOPE_CLAIMCHECK === "1") { + // The delta-scoped path does NOT enforce the suffix relation: + // computing the min receivedAt over all retained entries would + // require reading every retained consumed blob, the O(retained) + // work the delta walk exists to avoid. WHY DROPPING IT IS SAFE + // HERE: both the dropped (receivedAt 200) and the retained + // (receivedAt 100) entries are strictly below the stored watermark + // (201), so `claim_check_stale_enqueue` refuses every resubmit in + // that region at the enqueue boundary regardless of the consumed/ + // shape -- the retained entry only adds dedup, so a non-suffix + // prune below the watermark cannot open a reprocess. The delta path + // therefore accepts this prune; the exhaustive path below rejects + // it. The production markConsumed writer always prunes the oldest + // tail, so it never produces a non-suffix tree on either path. + expect(r.ok).toBe(true); + + // Boundary lock. The delta removed-check rejects a dropped entry at + // receivedAt >= watermark (strict, mirroring the strict + // `receivedAt < watermark` stale-reject so the entry AT the + // watermark is both retained and not stale-rejected -- no gap). + // Assert BOTH sides of the boundary so a later refactor cannot + // silently widen `>=` to `>` and drop the entry sitting exactly at + // the watermark, which would let a resubmit at that receivedAt miss + // dedup. + // (a) strictly above the watermark -> rejected. + const droppedAbove = await validate( + { + [WORKFLOW_RUN_GITIGNORE_PATH]: "", + [watermarkPathFor(ADDRESS_SEG)]: watermarkBody(200), + }, + { + priorFiles: { + [watermarkPathFor(ADDRESS_SEG)]: watermarkBody(200), + [consumedPathFor(ADDRESS_SEG, "msg-above")]: consumedBody( + "msg-above", + 300, + "run-above", + 350, + ), + }, + }, + ); + expect(droppedAbove.ok).toBe(false); + if (droppedAbove.ok) throw new Error("unreachable"); + expect(droppedAbove.reason).toMatch(/not below the retention watermark/); + + // (b) exactly equal to the watermark -> rejected (the off-by-one + // that widening `>=` to `>` would open). + const droppedAtBoundary = await validate( + { + [WORKFLOW_RUN_GITIGNORE_PATH]: "", + [watermarkPathFor(ADDRESS_SEG)]: watermarkBody(200), + }, + { + priorFiles: { + [watermarkPathFor(ADDRESS_SEG)]: watermarkBody(200), + [consumedPathFor(ADDRESS_SEG, "msg-at")]: consumedBody( + "msg-at", + 200, + "run-at", + 250, + ), + }, + }, + ); + expect(droppedAtBoundary.ok).toBe(false); + if (droppedAtBoundary.ok) throw new Error("unreachable"); + expect(droppedAtBoundary.reason).toMatch( + /not below the retention watermark/, + ); + return; + } expect(r.ok).toBe(false); if (r.ok) throw new Error("unreachable"); expect(r.reason).toMatch(/prune is not a suffix/); diff --git a/packages/hub-sessions/src/workflow-run-kind.ts b/packages/hub-sessions/src/workflow-run-kind.ts index a23348f1..0f9ed64d 100644 --- a/packages/hub-sessions/src/workflow-run-kind.ts +++ b/packages/hub-sessions/src/workflow-run-kind.ts @@ -1751,6 +1751,20 @@ async function validateClaimCheckSubtree( // the exactly-once retention contract: pruning is bound to the // watermark and the watermark only advances. if (BENCH_DELTA_SCOPE_CLAIMCHECK) { + // Re-review before any productionization: relative to the + // exhaustive path this delta check drops two properties the suffix + // guard also provided. (1) It no longer proves consumed/ is an + // age-ordered contiguous window -- a below-watermark prune may now + // leave a "hole" (a dropped entry older than a retained one), + // which is harmless for exactly-once (every dropped entry is below + // the watermark, so a resubmit is stale-rejected) but means + // consumed/ is no longer a provable suffix. (2) It reduces + // defense-in-depth against a buggy claim-check writer: the suffix + // relation would have caught a writer that selectively dropped a + // recent entry, whereas here only the below-watermark bound + // catches it. The below-watermark bound plus watermark + // monotonicity is the whole of the exactly-once contract; these + // two dropped properties are structural hardening, not correctness. for (const e of priorBucket.consumed) { if (prospectiveConsumedPaths.has(e.blobPath)) continue; const priorParsed = await parseConsumedBlobFrom(e, priorReadBlob); From 1188a7ae57daba9502b83af5b8a8efaeac593ddc Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Tue, 30 Jun 2026 21:33:09 -0500 Subject: [PATCH 004/101] Propagate the delta-scope claim-check flag to the spawned sidecar The latency benches spawn a sidecar subprocess through a curated env allowlist that never inherited the parent process env, so setting BENCH_DELTA_SCOPE_CLAIMCHECK on the bench command never reached the process that runs validatePush. The supervisor owning every workflow-run write lives in that sidecar, and the child proxies its writes to it, so the flag gates a module const the benched process was always reading as unset -- the delta path stayed dormant and the per-leg slopes measured the exhaustive walk regardless of the flag. Forward the flag through the spawn env so parent and child agree, and emit a one-line startup marker from the sidecar reflecting the resolved module const, so a run's effective state is observable directly instead of inferred from timing. Unset stays unset, preserving default-OFF. --- apps/sidecar/src/index.ts | 25 +++++++++++++++++-- packages/hub-sessions/src/index.ts | 1 + .../hub-sessions/src/workflow-run-kind.ts | 12 +++++++++ tests/hub-agent/lib/deploy-flow-env.ts | 12 +++++++++ 4 files changed, 48 insertions(+), 2 deletions(-) diff --git a/apps/sidecar/src/index.ts b/apps/sidecar/src/index.ts index 22762c19..934ce1fb 100644 --- a/apps/sidecar/src/index.ts +++ b/apps/sidecar/src/index.ts @@ -1,6 +1,6 @@ import { appendFileSync } from "node:fs"; import path from "node:path"; -import { setup } from "@intx/log"; +import { getLogger, setup } from "@intx/log"; import { createInMemoryTransport } from "@intx/mail-memory"; import { createEd25519Crypto, @@ -10,7 +10,10 @@ import { } from "@intx/crypto"; import { createSidecarOrchestrator, type HubLink } from "@intx/hub-agent"; import { hexEncode } from "@intx/types"; -import { createAgentRepoStore } from "@intx/hub-sessions"; +import { + claimCheckDeltaScopeEnabled, + createAgentRepoStore, +} from "@intx/hub-sessions"; import { createTarballCache } from "@intx/tool-packaging"; import { loadAdapterRegistry } from "@intx/inference/providers"; @@ -47,6 +50,24 @@ import { loadOrMintSidecarKeypair } from "./signing-keypair"; await setup(); +const logger = getLogger(["sidecar"]); + +// One-line startup marker naming the effective delta-scope claim-check +// state for THIS process. The sidecar's supervisor owns every +// workflow-run write, so this is the process whose module const decides +// which `validateClaimCheckSubtree` path a benchmark actually exercises. +// Emitting the imported const (not a fresh env read) makes a run's live +// state observable straight from the captured sidecar output, so a +// latency bench never has to be inferred back from its per-leg slopes. +// Each branch's message is a static template so the marker prints +// verbatim (`claim-check-delta-scope=ON`) under any log renderer, rather +// than as a quoted interpolated value. +if (claimCheckDeltaScopeEnabled) { + logger.info`claim-check-delta-scope=ON`; +} else { + logger.info`claim-check-delta-scope=OFF`; +} + function requireEnv(name: string): string { const value = process.env[name]; if (value === undefined) { diff --git a/packages/hub-sessions/src/index.ts b/packages/hub-sessions/src/index.ts index 6b48c71e..302f903e 100644 --- a/packages/hub-sessions/src/index.ts +++ b/packages/hub-sessions/src/index.ts @@ -76,6 +76,7 @@ export { export { workflowRunKindHandler, workflowRunAuthorize, + claimCheckDeltaScopeEnabled, enqueueInbox, dequeueToProcessing, readProcessingEntry, diff --git a/packages/hub-sessions/src/workflow-run-kind.ts b/packages/hub-sessions/src/workflow-run-kind.ts index 0f9ed64d..f596b4e8 100644 --- a/packages/hub-sessions/src/workflow-run-kind.ts +++ b/packages/hub-sessions/src/workflow-run-kind.ts @@ -204,6 +204,18 @@ const logger = getLogger(["hub-sessions", "workflow-run-kind"]); const BENCH_DELTA_SCOPE_CLAIMCHECK = process.env.BENCH_DELTA_SCOPE_CLAIMCHECK === "1"; +/** + * The resolved delta-scope claim-check state for the process that loaded + * this module. Exported so the process actually running `validatePush` + * (the sidecar's supervisor, which owns every workflow-run write) can + * emit an unambiguous startup marker reflecting the ACTUAL module const, + * rather than a fresh `process.env` read that could diverge from what the + * validation walk observes. This is the observable that proves whether + * the delta path is live for a given process, so a benchmark run can be + * read directly rather than inferred from its timing slopes. + */ +export const claimCheckDeltaScopeEnabled = BENCH_DELTA_SCOPE_CLAIMCHECK; + export type WorkflowRunHubPrincipal = { readonly kind: "hub" }; export type WorkflowRunSidecarPrincipal = { diff --git a/tests/hub-agent/lib/deploy-flow-env.ts b/tests/hub-agent/lib/deploy-flow-env.ts index 26aeb549..edcb09d0 100644 --- a/tests/hub-agent/lib/deploy-flow-env.ts +++ b/tests/hub-agent/lib/deploy-flow-env.ts @@ -765,6 +765,18 @@ export async function startSidecarSubprocess(opts: { SIDECAR_ID, SIDECAR_TOKEN: TOKEN, SIDECAR_DATA_DIR: dataDir, + // The spawned sidecar runs the supervisor that owns every + // workflow-run write, so its process is the one whose + // `workflow-run-kind` module const decides which claim-check + // validation path `validatePush` takes. The curated env above does + // not inherit the parent's `process.env`, so a bench that sets + // `BENCH_DELTA_SCOPE_CLAIMCHECK` on its own command would otherwise + // leave the child on the default (exhaustive) path -- the flag would + // never reach the process it gates. Forward it here so parent and + // child agree. Unset in the parent stays `undefined`, which + // `Bun.spawn` drops, so the default-OFF behaviour is unchanged for + // every normal test run. + BENCH_DELTA_SCOPE_CLAIMCHECK: process.env["BENCH_DELTA_SCOPE_CLAIMCHECK"], ...(opts.extraEnv ?? {}), }; From 5ea0f827647e345b92f2532311cf92f76bd36b41 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Wed, 1 Jul 2026 00:27:28 -0500 Subject: [PATCH 005/101] Thread a per-repo cache through the repo-store git calls The repo-store passed no isomorphic-git cache to its git.* calls, so every git.add / remove / updateIndex / commit / listFiles re-read and re-parsed the whole on-disk index from scratch. On a workflow-run repo the per-message write sequence flips between the events and workflow-run refs, forcing a full resetIndexToRef each leg, so the repeated index re-parse grew the per-message cost with accumulated history. Give each repo directory one long-lived cache object and thread it through every object, index, and commit call the store makes. The store is the single writer under withRepoLock, and the on-disk repo stays authoritative on every axis the cache touches: the index is persisted on each mutation and stat-guarded on read, cached objects are OID-keyed and content-addressed, packs are enumerated from disk per read, and refs are never cached. So the cache is a pure accelerator that can be dropped at any instant without changing a result. Two bounds keep a warm store from retaining parsed packfiles without limit: a dir's cache is rebuilt after a fixed number of calls, and the store keeps a bounded, least-recently-used set of dir caches. The cache is also dropped after a received pack, which writes objects and advances a ref past the cache, so the next read rebuilds from the mutated repo. The ref-reading calls (resolveRef, listBranches, listTags, currentBranch) take no cache parameter and read only refs, so they are left untouched. --- packages/hub-sessions/src/repo-store/store.ts | 236 +++++++++++++++--- 1 file changed, 205 insertions(+), 31 deletions(-) diff --git a/packages/hub-sessions/src/repo-store/store.ts b/packages/hub-sessions/src/repo-store/store.ts index c2382ee0..5fc151f2 100644 --- a/packages/hub-sessions/src/repo-store/store.ts +++ b/packages/hub-sessions/src/repo-store/store.ts @@ -126,6 +126,62 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { const { dataDir, signingKey, handlers, authorize, signingCallback, gc } = config; + // Per-repo-directory isomorphic-git memoization cache. Every git.* + // call the store makes against a `dir` threads that dir's cache, so + // isomorphic-git reuses the parsed on-disk index across the repo's + // serialized write sequence instead of re-reading and re-parsing it + // on every git.add / remove / updateIndex / commit / listFiles, and + // reuses parsed packfile indexes across object reads. The store is + // the single writer under withRepoLock, so a long-lived per-repo + // cache never races a concurrent mutator. + // + // The cache stays a pure accelerator, never a second source of truth, + // because the on-disk repo remains authoritative on every axis: + // isomorphic-git persists the index to disk on each dirty mutation and + // re-reads it on a stat mismatch; the object cache is OID-keyed + // (content-addressed, so it is never stale for a committed object); + // object reads enumerate pack files from disk on every call (so a GC + // repack's now-pruned packs are simply never consulted again); and + // refs are not cached at all — resolveRef and friends take no cache — + // so the cache can never serve a stale ref tip. Those same properties + // make it safe to drop a dir's cache at any instant: the next call + // just re-reads from disk. + // + // Two bounds keep a long-lived store from retaining parsed packfiles + // (and their pack bytes) without limit: a dir's cache is rebuilt once + // it has served GIT_CACHE_MAX_OPS calls, and the store holds at most + // GIT_CACHE_MAX_REPOS dir caches, evicting the least-recently-used. + // `invalidateGitCache` additionally drops a dir's cache after an + // out-of-band mutation that bypasses it — a received pack writes + // objects and advances a ref without threading this cache — so the + // next read rebuilds from the mutated repo. + const GIT_CACHE_MAX_OPS = 8192; + const GIT_CACHE_MAX_REPOS = 256; + type RepoGitCache = { cache: object; ops: number }; + const gitCaches = new Map(); + function cacheFor(dir: string): object { + let entry = gitCaches.get(dir); + if (entry === undefined) { + entry = { cache: {}, ops: 0 }; + } else { + // Re-insert so this dir ranks most-recently-used for the LRU + // eviction below; rebuild once it has spent its op budget. + gitCaches.delete(dir); + if (entry.ops >= GIT_CACHE_MAX_OPS) entry = { cache: {}, ops: 0 }; + } + entry.ops += 1; + gitCaches.set(dir, entry); + while (gitCaches.size > GIT_CACHE_MAX_REPOS) { + const lru = gitCaches.keys().next().value; + if (lru === undefined) break; + gitCaches.delete(lru); + } + return entry.cache; + } + function invalidateGitCache(dir: string): void { + gitCaches.delete(dir); + } + // Per-(repoId, ref) seq cache. Value is the seq of the ref's // current tip; the next commit on the ref gets `cached + 1`. The // cache is populated lazily — on each ref update we either bump @@ -202,7 +258,7 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { // tip's seq is `count - 1`). async function countCommits(dir: string, ref: string): Promise { try { - const entries = await git.log({ fs, dir, ref }); + const entries = await git.log({ fs, dir, cache: cacheFor(dir), ref }); return entries.length; } catch (err) { if (hasCode(err) && err.code === "NotFoundError") return 0; @@ -220,7 +276,7 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { ): Promise { let entries: Awaited>; try { - entries = await git.log({ fs, dir, ref }); + entries = await git.log({ fs, dir, cache: cacheFor(dir), ref }); } catch (err) { if (hasCode(err) && err.code === "NotFoundError") return []; throw err; @@ -452,11 +508,16 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { // walk becomes O(prefix) rather than O(repo). A prefix with no // tracked entries (a first write) yields an empty matrix and clears // nothing. - const matrix = await git.statusMatrix({ fs, dir, filepaths: [prefix] }); + const matrix = await git.statusMatrix({ + fs, + dir, + cache: cacheFor(dir), + filepaths: [prefix], + }); for (const row of matrix) { const filepath = row[0]; if (filepath.startsWith(prefix)) { - await git.remove({ fs, dir, filepath }); + await git.remove({ fs, dir, cache: cacheFor(dir), filepath }); } } } @@ -469,7 +530,7 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { const fullPath = path.join(dir, relPath); await fs.promises.mkdir(path.dirname(fullPath), { recursive: true }); await fs.promises.writeFile(fullPath, contents); - await git.add({ fs, dir, filepath: relPath }); + await git.add({ fs, dir, cache: cacheFor(dir), filepath: relPath }); } // Read the set of paths present in the tree at the given ref so a @@ -487,7 +548,12 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { } throw err; } - const entries = await git.listFiles({ fs, dir, ref: sha }); + const entries = await git.listFiles({ + fs, + dir, + cache: cacheFor(dir), + ref: sha, + }); return new Set(entries); } @@ -518,7 +584,7 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { continue; } try { - await git.remove({ fs, dir, filepath: relPath }); + await git.remove({ fs, dir, cache: cacheFor(dir), filepath: relPath }); } catch (err) { if (!hasCode(err) || err.code !== "NotFoundError") throw err; } @@ -550,6 +616,7 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { const result = await git.readBlob({ fs, dir, + cache: cacheFor(dir), oid: refSha, filepath: relPath, }); @@ -561,7 +628,7 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { const fullPath = path.join(dir, relPath); await fs.promises.mkdir(path.dirname(fullPath), { recursive: true }); await fs.promises.writeFile(fullPath, blob); - await git.add({ fs, dir, filepath: relPath }); + await git.add({ fs, dir, cache: cacheFor(dir), filepath: relPath }); } } } @@ -575,6 +642,7 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { await git.checkout({ fs, dir, + cache: cacheFor(dir), ref: "HEAD", force: true, filepaths: [args.clearedPrefix], @@ -597,7 +665,12 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { relPath: string, expectedType: "blob" | "tree", ): Promise { - const { commit } = await git.readCommit({ fs, dir, oid: commitSha }); + const { commit } = await git.readCommit({ + fs, + dir, + cache: cacheFor(dir), + oid: commitSha, + }); if (relPath === "") { return expectedType === "tree" ? commit.tree : null; } @@ -607,7 +680,12 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { const segment = segments[i]; if (segment === undefined) throw new Error("unreachable"); const isLast = i === segments.length - 1; - const { tree } = await git.readTree({ fs, dir, oid: currentOid }); + const { tree } = await git.readTree({ + fs, + dir, + cache: cacheFor(dir), + oid: currentOid, + }); const entry = tree.find((e) => e.path === segment); if (entry === undefined) return null; if (isLast) { @@ -649,13 +727,23 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { ): Promise => { const oid = await resolveTreeEntry(dir, commitSha, relPath, "blob"); if (oid === null) return null; - const { blob } = await git.readBlob({ fs, dir, oid }); + const { blob } = await git.readBlob({ + fs, + dir, + cache: cacheFor(dir), + oid, + }); return blob; }; const priorListDir = async (relPath: string): Promise => { const oid = await resolveTreeEntry(dir, commitSha, relPath, "tree"); if (oid === null) return []; - const { tree } = await git.readTree({ fs, dir, oid }); + const { tree } = await git.readTree({ + fs, + dir, + cache: cacheFor(dir), + oid, + }); return tree.map((e) => e.path); }; // Same walk as `priorListDir` but carries each child's git object id @@ -667,7 +755,12 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { ): Promise<{ name: string; oid: string }[]> => { const oid = await resolveTreeEntry(dir, commitSha, relPath, "tree"); if (oid === null) return []; - const { tree } = await git.readTree({ fs, dir, oid }); + const { tree } = await git.readTree({ + fs, + dir, + cache: cacheFor(dir), + oid, + }); return tree.map((e) => ({ name: e.path, oid: e.oid })); }; return { priorReadBlob, priorListDir, priorListDirOids }; @@ -686,10 +779,16 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { readBlob: (path: string) => Promise; listDir: (path: string) => Promise; }> { - const { commit } = await git.readCommit({ fs, dir, oid: commitSha }); + const { commit } = await git.readCommit({ + fs, + dir, + cache: cacheFor(dir), + oid: commitSha, + }); const { tree: rootTree } = await git.readTree({ fs, dir, + cache: cacheFor(dir), oid: commit.tree, }); const topLevelTreePaths = rootTree.map((e) => e.path); @@ -700,7 +799,12 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { `readBlob: path ${relPath} not found in commit ${commitSha} tree`, ); } - const { blob } = await git.readBlob({ fs, dir, oid }); + const { blob } = await git.readBlob({ + fs, + dir, + cache: cacheFor(dir), + oid, + }); return blob; }; const listDir = async (relPath: string): Promise => { @@ -715,7 +819,12 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { // `path_violation` reason instead of a raw substrate throw. A real // read fault inside `git.readTree` below still bubbles. if (oid === null) return []; - const { tree } = await git.readTree({ fs, dir, oid }); + const { tree } = await git.readTree({ + fs, + dir, + cache: cacheFor(dir), + oid, + }); return tree.map((e) => e.path); }; return { topLevelTreePaths, readBlob, listDir }; @@ -732,10 +841,17 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { ): Promise | null> { const oid = relPath === "" - ? (await git.readCommit({ fs, dir, oid: commitSha })).commit.tree + ? ( + await git.readCommit({ + fs, + dir, + cache: cacheFor(dir), + oid: commitSha, + }) + ).commit.tree : await resolveTreeEntry(dir, commitSha, relPath, "tree"); if (oid === null) return null; - const { tree } = await git.readTree({ fs, dir, oid }); + const { tree } = await git.readTree({ fs, dir, cache: cacheFor(dir), oid }); const out = new Map(); for (const e of tree) out.set(e.path, e.oid); return out; @@ -803,7 +919,7 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { if (out.has(oid)) continue; let parsed: Awaited>; try { - parsed = await git.readCommit({ fs, dir, oid }); + parsed = await git.readCommit({ fs, dir, cache: cacheFor(dir), oid }); } catch (err) { // A previously-received single-commit pack may leave the // tip's `parent` field pointing at a SHA the receiver has @@ -865,7 +981,12 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { if (existingCommits.has(current)) break; let parsed: Awaited>; try { - parsed = await git.readCommit({ fs, dir, oid: current }); + parsed = await git.readCommit({ + fs, + dir, + cache: cacheFor(dir), + oid: current, + }); } catch (err) { if (hasCode(err) && err.code === "NotFoundError") break; throw err; @@ -912,11 +1033,13 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { throw err; } } - const indexFiles = new Set(await git.listFiles({ fs, dir })); + const indexFiles = new Set( + await git.listFiles({ fs, dir, cache: cacheFor(dir) }), + ); if (refSha === null) { for (const filepath of indexFiles) { try { - await git.remove({ fs, dir, filepath }); + await git.remove({ fs, dir, cache: cacheFor(dir), filepath }); } catch (err) { if (!hasCode(err) || err.code !== "NotFoundError") throw err; } @@ -927,13 +1050,20 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { for (const filepath of indexFiles) { if (refBlobs.has(filepath)) continue; try { - await git.remove({ fs, dir, filepath }); + await git.remove({ fs, dir, cache: cacheFor(dir), filepath }); } catch (err) { if (!hasCode(err) || err.code !== "NotFoundError") throw err; } } for (const [filepath, oid] of refBlobs) { - await git.updateIndex({ fs, dir, filepath, oid, add: true }); + await git.updateIndex({ + fs, + dir, + cache: cacheFor(dir), + filepath, + oid, + add: true, + }); } } @@ -947,9 +1077,19 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { commitSha: string, ): Promise> { const out = new Map(); - const { commit } = await git.readCommit({ fs, dir, oid: commitSha }); + const { commit } = await git.readCommit({ + fs, + dir, + cache: cacheFor(dir), + oid: commitSha, + }); const recurse = async (treeOid: string, prefix: string): Promise => { - const { tree } = await git.readTree({ fs, dir, oid: treeOid }); + const { tree } = await git.readTree({ + fs, + dir, + cache: cacheFor(dir), + oid: treeOid, + }); for (const entry of tree) { const childPath = prefix === "" ? entry.path : `${prefix}/${entry.path}`; @@ -1161,6 +1301,7 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { const commitSha = await git.commit({ fs, dir, + cache: cacheFor(dir), message: content.message, author: AUTHOR, parent: [parentSha], @@ -1221,21 +1362,36 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { if (!repoExists) return out; const commitSha = await resolveRefSha(dir, ref); if (commitSha === null) return out; - const { commit } = await git.readCommit({ fs, dir, oid: commitSha }); + const { commit } = await git.readCommit({ + fs, + dir, + cache: cacheFor(dir), + oid: commitSha, + }); let currentOid = commit.tree; const segments = prefix .replace(/\/$/, "") .split("/") .filter((s) => s !== ""); for (const segment of segments) { - const { tree } = await git.readTree({ fs, dir, oid: currentOid }); + const { tree } = await git.readTree({ + fs, + dir, + cache: cacheFor(dir), + oid: currentOid, + }); const entry = tree.find((e) => e.path === segment); if (entry === undefined || entry.type !== "tree") { return out; } currentOid = entry.oid; } - const { tree } = await git.readTree({ fs, dir, oid: currentOid }); + const { tree } = await git.readTree({ + fs, + dir, + cache: cacheFor(dir), + oid: currentOid, + }); // N+1 isomorphic-git round-trips: one tree read plus one // readBlob per blob entry. Acceptable at the current scale // (single-digit tarballs per registry); when a registry grows @@ -1247,6 +1403,7 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { const { blob } = await git.readBlob({ fs, dir, + cache: cacheFor(dir), oid: commitSha, filepath: `${prefix}${entry.path}`, }); @@ -1340,6 +1497,7 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { const { commit: parsed } = await git.readCommit({ fs, dir, + cache: cacheFor(dir), oid: newCommit, }); const parents = parsed.parent; @@ -1368,7 +1526,12 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { let parentSha: string | null = null; if (declaredParent !== null) { try { - await git.readCommit({ fs, dir, oid: declaredParent }); + await git.readCommit({ + fs, + dir, + cache: cacheFor(dir), + oid: declaredParent, + }); parentSha = declaredParent; } catch (err) { if (!hasCode(err) || err.code !== "NotFoundError") throw err; @@ -1415,6 +1578,11 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { // index cache must be cleared so the next writeTreeUnderLock // resets the index against the chosen target ref. indexRefCache.delete(indexCacheKey(repoId)); + // receivePackObjects wrote new objects and advanced the ref + // straight to disk without threading the memoization cache, so + // drop the dir's cache; the next read rebuilds against the packed + // objects and the new tip. + invalidateGitCache(dir); for (const sha of newCommitsFromPack) existingCommits.add(sha); await handler.onRefUpdated({ repoId, ref, oldSha, newSha: commitSha }); await emitRefUpdate(repoId, ref, oldSha, commitSha); @@ -1447,6 +1615,7 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { const result = await git.packObjects({ fs, dir, + cache: cacheFor(dir), oids, write: false, }); @@ -1489,7 +1658,12 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { for (const o of perCommit) seen.add(o); let parsed: Awaited>; try { - parsed = await git.readCommit({ fs, dir, oid: current }); + parsed = await git.readCommit({ + fs, + dir, + cache: cacheFor(dir), + oid: current, + }); } catch (err) { if (hasCode(err) && err.code === "NotFoundError") break; throw err; From 73ce4174ddab44efd7ee0ed2678540898c9844dd Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Wed, 1 Jul 2026 01:12:56 -0500 Subject: [PATCH 006/101] Assemble repo-store commit trees without the on-disk index MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit writeTreeUnderLock built each commit by resetting the single shared on-disk index to the target ref, staging every file, and committing the index-derived tree. Because one index is shared across a repo's refs, a workflow-run repo that alternates the events and workflow-run refs every message reset the whole index each leg, and each staged blob re-serialized the entire index, so per-message latency grew with accumulated history. Assemble the commit tree directly instead. Pin the ref tip under the lock, splice the write's blobs and its clearPrefix deletion onto that commit's root tree while reusing every untouched subtree by object id, and commit the assembled tree oid via git.commit({ tree }). The index is off the write path entirely, so the per-commit cost tracks the size of the change rather than the repo. validatePush now sources its prospective-tree closures from the assembled tree oid — the exact tree the commit will carry — while the prior-tree closures still read the parent commit. The assembly writes only unreferenced blob and tree objects until the commit lands, so a rejected push needs no rollback: it leaves the ref, the index, and the working tree untouched. The working tree is still materialized for the paths a write touches (the workflow-run claim-check scan reads those files from disk), but only after validation passes. This removes the index-reset, path-snapshot, prefix-clear, file-staging, and working-tree-rollback helpers, along with the ref-to-index cache they served. --- packages/hub-sessions/src/repo-store/store.ts | 689 +++++++----------- 1 file changed, 272 insertions(+), 417 deletions(-) diff --git a/packages/hub-sessions/src/repo-store/store.ts b/packages/hub-sessions/src/repo-store/store.ts index 5fc151f2..25ad542e 100644 --- a/packages/hub-sessions/src/repo-store/store.ts +++ b/packages/hub-sessions/src/repo-store/store.ts @@ -1,6 +1,6 @@ import fs from "node:fs"; import path from "node:path"; -import git from "isomorphic-git"; +import git, { type TreeEntry } from "isomorphic-git"; import { createSSHSignature } from "@intx/crypto"; import { initRepo as storageInitRepo, @@ -195,14 +195,8 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { // subscribers do not interfere with each other. const subscribers = new Map>(); - // Per-repoId cache of "the ref whose tree currently matches the - // on-disk index". When the next writeTreeUnderLock call targets - // the same `(repoId, ref)` pair as the last successful commit, the - // index already mirrors that ref's tip and the resetIndexToRef - // pass is a no-op — skip it. The cache is invalidated on any - // commit, ref update, or pack receive that advances or replaces - // the ref. - const indexRefCache = new Map(); + // Repo-scoped cache key shared by the per-repoId caches below + // (existing-commit set, and any future per-repo bookkeeping). function indexCacheKey(repoId: RepoId): string { return `${repoId.kind}/${repoId.id}`; } @@ -499,183 +493,21 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { } } - async function clearIndexPrefix(dir: string, prefix: string): Promise { - // `filepaths: [prefix]` scopes statusMatrix to the cleared subtree - // instead of walking every tracked file in the repo and discarding - // the non-prefix rows. The narrowed call returns status rows only - // for files under `prefix` -- exactly the set this loop then - // removes -- so the removal set is identical while the per-commit - // walk becomes O(prefix) rather than O(repo). A prefix with no - // tracked entries (a first write) yields an empty matrix and clears - // nothing. - const matrix = await git.statusMatrix({ - fs, - dir, - cache: cacheFor(dir), - filepaths: [prefix], - }); - for (const row of matrix) { - const filepath = row[0]; - if (filepath.startsWith(prefix)) { - await git.remove({ fs, dir, cache: cacheFor(dir), filepath }); - } - } - } - - async function writeFileEntry( + // Walk from a tree object's root to the tree-or-blob entry at + // `relPath`. Returns `null` when any path segment is missing, or when + // the final entry does not match `expectedType`. `relPath === ""` + // resolves to the root tree itself, and only when a tree is expected. + async function resolveTreeOid( dir: string, - relPath: string, - contents: string | Uint8Array, - ): Promise { - const fullPath = path.join(dir, relPath); - await fs.promises.mkdir(path.dirname(fullPath), { recursive: true }); - await fs.promises.writeFile(fullPath, contents); - await git.add({ fs, dir, cache: cacheFor(dir), filepath: relPath }); - } - - // Read the set of paths present in the tree at the given ref so a - // post-validation rollback can distinguish genuinely-new staged paths - // (which the rollback must unlink) from paths that existed at the - // ref's tip and were merely overwritten by the staging pass (which - // the rollback must restore, not delete). - async function refPathSet(dir: string, ref: string): Promise> { - let sha: string; - try { - sha = await git.resolveRef({ fs, dir, ref }); - } catch (err) { - if (hasCode(err) && err.code === "NotFoundError") { - return new Set(); - } - throw err; - } - const entries = await git.listFiles({ - fs, - dir, - cache: cacheFor(dir), - ref: sha, - }); - return new Set(entries); - } - - // Undo the staged writes from a failed writeTreeUnderLock pass. For - // each newly-staged file: if the push target ref held that path, - // restore it from the ref's tip so the working tree and index match - // the ref bit-for-bit; otherwise drop the file from disk and the - // index because it had no prior presence to restore. Then re-checkout - // any prefix that the clearPrefix step removed wholesale. The - // combination leaves the working tree and index bit-identical to the - // ref for every path the failed pass touched, so a subsequent - // legitimate push does not pick up half-applied state and does not - // lose ref content that the rejected push happened to overwrite - // outside `clearedPrefix`. - async function resetWorkingTreeToRef( - dir: string, - ref: string, - args: { - stagedFiles: string[]; - clearedPrefix: string | undefined; - refPaths: ReadonlySet; - }, - ): Promise { - const toRestore: string[] = []; - for (const relPath of args.stagedFiles) { - if (args.refPaths.has(relPath)) { - toRestore.push(relPath); - continue; - } - try { - await git.remove({ fs, dir, cache: cacheFor(dir), filepath: relPath }); - } catch (err) { - if (!hasCode(err) || err.code !== "NotFoundError") throw err; - } - try { - await fs.promises.unlink(path.join(dir, relPath)); - } catch (err) { - if (!hasCode(err) || err.code !== "ENOENT") throw err; - } - } - if (toRestore.length > 0) { - // Read each ref-existing path's blob directly and rewrite the - // working-tree file plus stage it. `git.checkout` with the - // `filepaths` narrowing does not consistently re-materialize a - // working-tree file that the rollback flow has already touched - // (isomorphic-git treats the narrowed checkout as a subtree-walk - // hint and leaves the working tree untouched for paths already - // present in the index), so the rollback restores each path - // explicitly from the ref's tree. - let refSha: string | undefined; - try { - refSha = await git.resolveRef({ fs, dir, ref }); - } catch (err) { - if (!hasCode(err) || err.code !== "NotFoundError") throw err; - } - if (refSha !== undefined) { - for (const relPath of toRestore) { - let blob: Uint8Array; - try { - const result = await git.readBlob({ - fs, - dir, - cache: cacheFor(dir), - oid: refSha, - filepath: relPath, - }); - blob = result.blob; - } catch (err) { - if (hasCode(err) && err.code === "NotFoundError") continue; - throw err; - } - const fullPath = path.join(dir, relPath); - await fs.promises.mkdir(path.dirname(fullPath), { recursive: true }); - await fs.promises.writeFile(fullPath, blob); - await git.add({ fs, dir, cache: cacheFor(dir), filepath: relPath }); - } - } - } - if (args.clearedPrefix !== undefined) { - // Re-checkout the prefix from HEAD so the files the clearPrefix - // step removed reappear in the working tree and the index. - // `force: true` overwrites any leftover writes from the failed - // pass; the filepaths array narrows the checkout to just the - // prefix so we do not perturb unrelated paths. - try { - await git.checkout({ - fs, - dir, - cache: cacheFor(dir), - ref: "HEAD", - force: true, - filepaths: [args.clearedPrefix], - }); - } catch (err) { - if (!hasCode(err) || err.code !== "NotFoundError") throw err; - } - } - } - - // Walk from a commit's root tree to the tree-or-blob entry at - // `relPath`. Returns `null` when any path segment is missing, or - // when the final entry does not match `expectedType`. Used to back - // both `priorReadBlob` and `priorListDir` so a handler can inspect - // the parent commit's tree from inside validatePush without each - // call duplicating the tree-walk. - async function resolveTreeEntry( - dir: string, - commitSha: string, + rootTreeOid: string, relPath: string, expectedType: "blob" | "tree", ): Promise { - const { commit } = await git.readCommit({ - fs, - dir, - cache: cacheFor(dir), - oid: commitSha, - }); if (relPath === "") { - return expectedType === "tree" ? commit.tree : null; + return expectedType === "tree" ? rootTreeOid : null; } const segments = relPath.split("/").filter((s) => s !== ""); - let currentOid = commit.tree; + let currentOid = rootTreeOid; for (let i = 0; i < segments.length; i++) { const segment = segments[i]; if (segment === undefined) throw new Error("unreachable"); @@ -698,6 +530,25 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { return currentOid; } + // Walk from a commit's root tree to the tree-or-blob entry at + // `relPath`. Used to back both `priorReadBlob` and `priorListDir` so a + // handler can inspect the parent commit's tree from inside validatePush + // without each call duplicating the tree-walk. + async function resolveTreeEntry( + dir: string, + commitSha: string, + relPath: string, + expectedType: "blob" | "tree", + ): Promise { + const { commit } = await git.readCommit({ + fs, + dir, + cache: cacheFor(dir), + oid: commitSha, + }); + return resolveTreeOid(dir, commit.tree, relPath, expectedType); + } + // Build the `(priorReadBlob, priorListDir)` pair fed into a kind // handler's validatePush. When `commitSha` is `null`, both closures // surface the "ref had no prior commit" state — readBlob returns @@ -1006,102 +857,175 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { return chain.reverse(); } - // Reset the index so it bit-matches the tree at `ref`'s tip - // without re-materialising the working tree. The substrate uses - // the index — not the working tree — when it builds the next - // commit's tree via `git.commit({ref})`, so re-pointing index - // entries at the ref-tree's oids is correct and substantially - // cheaper than walking disk for every path. When the ref does not - // yet exist the index is cleared so the next staging pass starts - // from an empty repo state. + // Assemble a new root tree by splicing `puts` (repo-root-relative + // path -> blob oid) and an optional `clearPrefix` deletion onto the + // parent's root tree, reusing every unchanged subtree by oid. The + // result is committed directly via `git.commit({ tree })`, so the + // on-disk index is never touched. // - // isomorphic-git's index is a single repo-global structure shared - // across refs, and `git.add` mutates it in place; without this - // reset, every commit's tree is the union of `ref`'s tree and - // whatever was last staged for a different ref. The reset reads - // `ref` (the push target) rather than HEAD because HEAD may point - // at a different branch the repo was initialised on, so seeding - // from HEAD would mis-construct trees that target a named ref. - async function resetIndexToRef(dir: string, ref: string): Promise { - let refSha: string | null; - try { - refSha = await git.resolveRef({ fs, dir, ref }); - } catch (err) { - if (hasCode(err) && err.code === "NotFoundError") { - refSha = null; - } else { - throw err; + // Recursion is scoped to the touched subtrees: a level is read and + // rewritten only when a put lands under it or `clearPrefix` deletes + // within it. Every sibling the write does not touch is carried + // forward by its existing oid without a re-hash or a walk, so the + // per-commit cost tracks the size of the change rather than the repo. + // Deletions are confined to `clearPrefix`; outside it the splice is + // additive, so reusing a sibling subtree can never drop a deletion. + // Returns the new tree oid, or `null` when the subtree ends up empty + // (the caller writes an empty root tree when the whole repo empties). + async function assembleTree( + dir: string, + baseTreeOid: string | null, + prefix: string, + puts: ReadonlyMap, + clearPrefix: string | undefined, + ): Promise { + // At the exact node `clearPrefix` names, the base subtree is dropped + // wholesale — only merge-output puts under the prefix survive. + const cleared = clearPrefix !== undefined && clearPrefix === prefix; + const baseEntries = new Map< + string, + { mode: string; oid: string; type: TreeEntry["type"] } + >(); + if (baseTreeOid !== null && !cleared) { + const { tree } = await git.readTree({ + fs, + dir, + cache: cacheFor(dir), + oid: baseTreeOid, + }); + for (const e of tree) { + baseEntries.set(e.path, { mode: e.mode, oid: e.oid, type: e.type }); } } - const indexFiles = new Set( - await git.listFiles({ fs, dir, cache: cacheFor(dir) }), - ); - if (refSha === null) { - for (const filepath of indexFiles) { - try { - await git.remove({ fs, dir, cache: cacheFor(dir), filepath }); - } catch (err) { - if (!hasCode(err) || err.code !== "NotFoundError") throw err; + + // Names to rebuild at this level: a direct blob put, a subtree some + // put descends into, or the subtree `clearPrefix` descends into (so + // its deletion is applied even when no put touches that subtree). + const blobPutsHere = new Set(); + const subtreeNames = new Set(); + for (const full of puts.keys()) { + if (prefix !== "" && !full.startsWith(prefix)) continue; + const rest = full.slice(prefix.length); + if (rest.length === 0) continue; + const slash = rest.indexOf("/"); + if (slash === -1) blobPutsHere.add(rest); + else subtreeNames.add(rest.slice(0, slash)); + } + if ( + clearPrefix !== undefined && + clearPrefix !== prefix && + clearPrefix.startsWith(prefix) + ) { + const rest = clearPrefix.slice(prefix.length); + const slash = rest.indexOf("/"); + const seg = slash === -1 ? rest : rest.slice(0, slash); + if (seg.length > 0) subtreeNames.add(seg); + } + + const names = new Set([ + ...baseEntries.keys(), + ...blobPutsHere, + ...subtreeNames, + ]); + const entries: TreeEntry[] = []; + for (const name of names) { + const full = prefix + name; + const putOid = puts.get(full); + if (putOid !== undefined) { + // A direct blob put overrides whatever the base held here. + entries.push({ + mode: "100644", + path: name, + oid: putOid, + type: "blob", + }); + continue; + } + if (subtreeNames.has(name)) { + const base = baseEntries.get(name); + const baseChildOid = + base !== undefined && base.type === "tree" ? base.oid : null; + const childOid = await assembleTree( + dir, + baseChildOid, + `${full}/`, + puts, + clearPrefix, + ); + if (childOid !== null) { + entries.push({ + mode: "040000", + path: name, + oid: childOid, + type: "tree", + }); } + continue; } - return; + const base = baseEntries.get(name); + if (base === undefined) continue; + entries.push({ + mode: base.mode, + path: name, + oid: base.oid, + type: base.type, + }); } - const refBlobs = await walkRefBlobs(dir, refSha); - for (const filepath of indexFiles) { - if (refBlobs.has(filepath)) continue; - try { - await git.remove({ fs, dir, cache: cacheFor(dir), filepath }); - } catch (err) { - if (!hasCode(err) || err.code !== "NotFoundError") throw err; + if (entries.length === 0) return null; + return await git.writeTree({ fs, dir, tree: entries }); + } + + // Prospective-side validatePush closures sourced from the assembled + // tree oid, so a handler sees exactly the tree the commit will carry. + // The blobs and trees the assembly wrote are unreferenced objects + // until the commit lands, so reading them here advances nothing. + function buildTreeReadClosures( + dir: string, + rootTreeOid: string, + ): { + topLevelTreePaths: () => Promise; + readBlob: (relPath: string) => Promise; + listDir: (relPath: string) => Promise; + } { + const topLevelTreePaths = async (): Promise => { + const { tree } = await git.readTree({ + fs, + dir, + cache: cacheFor(dir), + oid: rootTreeOid, + }); + return tree.map((e) => e.path); + }; + const readBlob = async (relPath: string): Promise => { + const oid = await resolveTreeOid(dir, rootTreeOid, relPath, "blob"); + if (oid === null) { + throw new Error( + `readBlob: path ${relPath} not present in prospective tree`, + ); } - } - for (const [filepath, oid] of refBlobs) { - await git.updateIndex({ + const { blob } = await git.readBlob({ fs, dir, cache: cacheFor(dir), - filepath, oid, - add: true, }); - } - } - - // Walk every blob entry reachable from a commit's tree, returning - // a (filepath -> blob oid) map. Avoids the O(n^2) cost of pairing - // `git.listFiles` with per-file `resolveTreeEntry` calls — both of - // which walk the tree top-to-bottom — by doing a single recursive - // walk and emitting blob entries as we encounter them. - async function walkRefBlobs( - dir: string, - commitSha: string, - ): Promise> { - const out = new Map(); - const { commit } = await git.readCommit({ - fs, - dir, - cache: cacheFor(dir), - oid: commitSha, - }); - const recurse = async (treeOid: string, prefix: string): Promise => { + return blob; + }; + const listDir = async (relPath: string): Promise => { + const oid = + relPath === "" + ? rootTreeOid + : await resolveTreeOid(dir, rootTreeOid, relPath, "tree"); + if (oid === null) return []; const { tree } = await git.readTree({ fs, dir, cache: cacheFor(dir), - oid: treeOid, + oid, }); - for (const entry of tree) { - const childPath = - prefix === "" ? entry.path : `${prefix}/${entry.path}`; - if (entry.type === "blob") { - out.set(childPath, entry.oid); - } else if (entry.type === "tree") { - await recurse(entry.oid, childPath); - } - } + return tree.map((e) => e.path); }; - await recurse(commit.tree, ""); - return out; + return { topLevelTreePaths, readBlob, listDir }; } // Unlocked body of writeTree. The caller is responsible for @@ -1120,188 +1044,125 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { const handler = handlerFor(repoId); - // Reset the working tree and index to the target ref's tip so a - // commit to ref A does not pick up index state left behind by a - // prior commit to ref B. isomorphic-git's index is a single - // repo-global structure shared across refs, and `git.add` / - // `writeFileEntry` mutate it in place; without this reset, every - // commit's tree is the union of `ref`'s tree and whatever was last - // staged for a different ref. The reset reads `ref` (the push - // target) rather than HEAD: writes target a named ref, and HEAD - // may point at a different branch the repo was initialized on, so - // checking out HEAD would seed the new commit from the wrong tip. - // - // Skip the reset when the index cache says the on-disk index - // already mirrors `ref` — i.e. the previous commit on this repo - // also targeted this ref. The cache is invalidated on every - // successful commit so a write to a different ref always pays the - // reset cost on the next return. - const indexKey = indexCacheKey(repoId); - if (indexRefCache.get(indexKey) !== ref) { - await resetIndexToRef(dir, ref); - indexRefCache.set(indexKey, ref); - } - - // Snapshot the target ref's path set before the clearPrefix + - // writeFileEntry pass mutates the index. A post-validation rollback - // uses this set to decide whether each staged path needs to be - // restored from the ref's tip (it existed there and was - // overwritten) or unlinked (it had no prior presence). Captured - // here, not inside resetWorkingTreeToHead, because the rollback - // runs after the index has been mutated. The snapshot reads `ref` - // (the push target) rather than HEAD: writes target a named ref - // and HEAD may point at a different branch the repo was - // initialized on, so reading HEAD would mis-classify every staged - // path as new and fall back to the unlink path. - const refPaths = await refPathSet(dir, ref); - + // Assemble the commit's tree directly and commit that tree oid, + // never staging into the on-disk index. The index is a single + // repo-global structure shared across refs; routing writes through + // it forced a full index rebuild on every events<->workflow-run ref + // flip and re-serialized the whole index once per staged blob. + // Splicing the tree from the parent's root tree instead reuses every + // untouched subtree by oid, so the per-commit cost tracks the change + // rather than the accumulated history. if (content.clearPrefix !== undefined) { validateClearPrefix(content.clearPrefix); - await clearIndexPrefix(dir, content.clearPrefix); - await fs.promises.rm(path.join(dir, content.clearPrefix), { - recursive: true, - force: true, + } + const clearPrefix = content.clearPrefix; + + // Pin the parent under the lock: the new commit's parent is the + // ref's tip, and the splice runs against THAT commit's root tree, + // read under the same lock, so the pre-image is race-free. A ref + // that does not yet exist has no base tree (the splice starts from + // empty) and parents on HEAD, matching the prior index-reset path + // which seeded an empty index for a missing ref. + const parentCommitSha = await resolveRefSha(dir, ref); + let baseRootTreeOid: string | null = null; + if (parentCommitSha !== null) { + const { commit } = await git.readCommit({ + fs, + dir, + cache: cacheFor(dir), + oid: parentCommitSha, }); + baseRootTreeOid = commit.tree; } + // Write the merge-output blobs, then splice them (and the + // clearPrefix deletion) onto the parent root tree. writeBlob and + // writeTree emit unreferenced objects; nothing the ref can reach + // moves until the commit below lands. + const puts = new Map(); for (const [relPath, contents] of Object.entries(content.files)) { - await writeFileEntry(dir, relPath, contents); - } - - // The prior tree is whatever the ref points at the moment - // validatePush runs — i.e. the parent of the commit we are about - // to produce. `refPaths` above was snapshotted from the same - // commit, so both reads observe the same pre-image. - const priorCommitSha = await resolveRefSha(dir, ref); - const { priorReadBlob, priorListDir, priorListDirOids } = - buildPriorTreeClosures(dir, priorCommitSha); - - // The prospective tree is the actual commit `git.commit` will - // produce: the prior tree, minus paths cleared by `clearPrefix`, - // unioned with paths newly staged via `writeFileEntry`. The - // closures handed to validatePush must reflect that union — not - // just `content.files` — so a kind handler that walks the prior - // tree (workflow-run's append-only event check) can see every - // path the commit will carry. Building the closures from - // `content.files` alone would advertise only the writer's - // explicitly-staged paths, hiding prior-tree paths the writer - // did not touch and tripping prior-vs-prospective comparisons - // that the substrate does not actually delete. - const clearedPrefix = content.clearPrefix; - const survivingPriorPaths: string[] = []; - for (const p of refPaths) { - if (clearedPrefix !== undefined && p.startsWith(clearedPrefix)) continue; - if (Object.prototype.hasOwnProperty.call(content.files, p)) continue; - survivingPriorPaths.push(p); + const bytes = + typeof contents === "string" + ? new TextEncoder().encode(contents) + : contents; + const oid = await git.writeBlob({ fs, dir, blob: bytes }); + puts.set(relPath, oid); } - const prospectivePaths = new Set([ - ...Object.keys(content.files), - ...survivingPriorPaths, - ]); - const survivingPriorPathSet = new Set(survivingPriorPaths); - const topLevelTreePaths = Array.from( - new Set( - Array.from(prospectivePaths, (p) => { - const slash = p.indexOf("/"); - return slash === -1 ? p : p.substring(0, slash); - }), - ), + const assembled = await assembleTree( + dir, + baseRootTreeOid, + "", + puts, + clearPrefix, ); - const readBlob = async (relPath: string): Promise => { - const entry = content.files[relPath]; - if (entry !== undefined) { - if (typeof entry === "string") { - return new TextEncoder().encode(entry); - } - return entry; - } - if (survivingPriorPathSet.has(relPath)) { - const blob = await priorReadBlob(relPath); - if (blob === null) { - throw new Error( - `readBlob: path ${relPath} listed in prior tree but unreadable from prior commit ${String(priorCommitSha)}`, - ); - } - return blob; - } - throw new Error( - `readBlob: path ${relPath} not present in prospective tree`, - ); - }; - const listDir = async (dirPath: string): Promise => { - const prefix = dirPath === "" ? "" : `${dirPath}/`; - const names = new Set(); - for (const p of prospectivePaths) { - if (prefix !== "" && !p.startsWith(prefix)) continue; - const rest = p.slice(prefix.length); - if (rest.length === 0) continue; - const slash = rest.indexOf("/"); - names.add(slash === -1 ? rest : rest.substring(0, slash)); - } - return Array.from(names); - }; - // A writeTree carrying a `clearPrefix` mutates only paths under that - // prefix: the clearPrefix step removes the prefix's prior entries and - // `writeFileEntry` restages the writer's files, while every other - // prior path is carried forward verbatim in `survivingPriorPaths` - // above. So the change set the handler may scope to is exactly - // `{clearPrefix}`. Without a clearPrefix the write is purely additive - // over an arbitrary set of paths the substrate does not summarise, so - // leave the change set unbounded (validate-all). + const newRootTreeOid = + assembled ?? (await git.writeTree({ fs, dir, tree: [] })); + + // validatePush sees the full prospective tree via closures over the + // assembled tree oid; the prior-side closures read the parent commit + // (`parentCommitSha`) exactly as before. The change set the handler + // may scope to is `{clearPrefix}` when one is present — writes under + // it are a full replace and every other path is carried forward — and + // unbounded (validate-all) otherwise. + const { priorReadBlob, priorListDir, priorListDirOids } = + buildPriorTreeClosures(dir, parentCommitSha); + const prospective = buildTreeReadClosures(dir, newRootTreeOid); const changedPathPrefixes = - clearedPrefix === undefined ? undefined : new Set([clearedPrefix]); + clearPrefix === undefined ? undefined : new Set([clearPrefix]); const validation = await handler.validatePush({ repoId, ref, principal, - topLevelTreePaths, - readBlob, - listDir, + topLevelTreePaths: await prospective.topLevelTreePaths(), + readBlob: prospective.readBlob, + listDir: prospective.listDir, priorReadBlob, priorListDir, priorListDirOids, changedPathPrefixes, }); if (!validation.ok) { - // Roll the working tree and index back to HEAD before - // surfacing the validation refusal. Otherwise the staged - // writeFileEntry / clearPrefix steps above leave files and - // index entries on disk that the next legitimate push would - // either include silently (statusMatrix re-reports them) or - // trip a second validation refusal against. The reset is - // best-effort but logged loudly: a failure here means the - // repo is in a half-applied state the operator needs to fix - // by hand, which is strictly better than the silent-leak - // alternative. - try { - await resetWorkingTreeToRef(dir, ref, { - stagedFiles: Object.keys(content.files), - clearedPrefix: content.clearPrefix, - refPaths, - }); - } catch (resetErr) { - logger.warn`repo-store rollback after validation refusal failed for ${repoId.kind}/${repoId.id}; the repo is in a half-applied state and may need manual cleanup — ${resetErr instanceof Error ? resetErr.message : String(resetErr)}`; - } + // Nothing was staged and no ref advanced: the assembly wrote only + // unreferenced blob and tree objects, which the next GC reclaims. + // There is no half-applied index or working-tree state to roll + // back, so the refusal just surfaces. throw new Error(`path_violation: ${validation.reason}`); } - // Resolve the previous ref value for the post-update hook (precise: - // null only when the ref truly doesn't exist) and the parent SHA for - // the new commit (best-effort: any error falls back to HEAD, so a - // first write on a never-touched ref produces a commit parented on - // the repo's initial commit instead of failing). - const oldSha = await resolveRefSha(dir, ref); - let parentSha: string; - try { - parentSha = await git.resolveRef({ fs, dir, ref }); - } catch { - parentSha = await git.resolveRef({ fs, dir, ref: "HEAD" }); + // Materialize the working tree for the paths this write touched: + // clear the replaced prefix, then write each put file. The store's + // own reads resolve through the object store, but some consumers + // (the workflow-run claim-check processing scan) read these files + // straight from disk, so the working tree must mirror the committed + // change. Only merge-output paths are written and only the cleared + // prefix is removed, so this is O(change), not O(repo). It runs only + // after validation passes, so a rejected push leaves the working + // tree untouched — the failure atomicity the old index rollback + // provided, now without any rollback. + if (clearPrefix !== undefined) { + await fs.promises.rm(path.join(dir, clearPrefix), { + recursive: true, + force: true, + }); } + for (const [relPath, contents] of Object.entries(content.files)) { + const fullPath = path.join(dir, relPath); + await fs.promises.mkdir(path.dirname(fullPath), { recursive: true }); + await fs.promises.writeFile(fullPath, contents); + } + + // The parent is the pinned tip, or HEAD for a never-written ref so a + // first write parents on the repo's initial commit. `oldSha` is + // precise: null only when the ref truly does not exist. + const oldSha = parentCommitSha; + const parentSha = + parentCommitSha ?? (await git.resolveRef({ fs, dir, ref: "HEAD" })); const commitSha = await git.commit({ fs, dir, cache: cacheFor(dir), + tree: newRootTreeOid, message: content.message, author: AUTHOR, parent: [parentSha], @@ -1572,12 +1433,6 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { }, ); - // receivePack does not touch the index — it goes straight from - // pack bytes to ref pointer — so the index now points at - // whatever was last committed (possibly a different ref). The - // index cache must be cleared so the next writeTreeUnderLock - // resets the index against the chosen target ref. - indexRefCache.delete(indexCacheKey(repoId)); // receivePackObjects wrote new objects and advanced the ref // straight to disk without threading the memoization cache, so // drop the dir's cache; the next read rebuilds against the packed From 79430eafab45f7589b31995226c9383877ed899e Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Wed, 1 Jul 2026 09:31:49 -0500 Subject: [PATCH 007/101] Narrow claim-check writes to a targeted tree delta MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every claim-check leg — enqueue, dequeue, markConsumed, replay — cleared and rebuilt the whole addresses// subtree from a full merge output that re-supplied every consumed/ dedup entry's bytes, and read the whole subtree (readAddressSubtree) to run its dedup checks. So each leg's cost grew with the retained consumed set, which grows one entry per message. Add a writeTreeDelta primitive: computeDelta runs under the per-repo lock against the pinned parent tip and returns the exact files to put and the paths to delete; the substrate assembles the new tree by splicing those onto the parent root tree and carrying every untouched entry — sibling subtrees and sibling blobs alike — forward by its object id. assembleTree now takes a delete-set (exact files or subtree prefixes) rather than a single clear-prefix; writeTree and writeTreePreservingPrefix map their clear-prefix onto it and keep byte-identical behavior. Rewrite the four claim-check ops on top of it. Each names only the entries it moves — enqueue puts one inbox entry; dequeue moves inbox to processing; markConsumed moves processing to consumed and advances the watermark; replay moves processing back to inbox — so the untouched consumed/ index is carried by oid instead of re-hashed and re-materialized. The dedup checks read filenames and blob oids from the parent tree (consumed/ is keyed by messageId, an exact lookup; inbox/ and processing/ are the bounded queues), reading blob bytes only for the one entry a leg actually moves. markConsumed still reads each consumed entry's bytes to find the below-watermark tail to prune — its filenames carry only the messageId, so the receivedAt lives in the bytes — which is the one leg that still scans the consumed index. A new OID-equivalence test pins the delta path byte-identical to a full-replace writeTree across the move shapes, the same guard the index-free assembly carries. --- apps/sidecar/src/workflow-run-pack-client.ts | 25 + .../hub-api/src/git-http/receive-pack.test.ts | 3 + packages/hub-api/src/routes/workflows.test.ts | 1 + .../src/hub-session-orchestrator.test.ts | 1 + packages/hub-sessions/src/repo-store/store.ts | 226 +++++-- packages/hub-sessions/src/repo-store/types.ts | 38 ++ .../src/repo-store/write-tree-delta.test.ts | 200 ++++++ .../hub-sessions/src/session-service.test.ts | 2 + .../hub-sessions/src/workflow-run-kind.ts | 611 +++++++++--------- .../src/workflow-run-reader.test.ts | 1 + .../src/child/proxy-repo-store.ts | 11 + 11 files changed, 742 insertions(+), 377 deletions(-) create mode 100644 packages/hub-sessions/src/repo-store/write-tree-delta.test.ts diff --git a/apps/sidecar/src/workflow-run-pack-client.ts b/apps/sidecar/src/workflow-run-pack-client.ts index 7417d498..51bc37ec 100644 --- a/apps/sidecar/src/workflow-run-pack-client.ts +++ b/apps/sidecar/src/workflow-run-pack-client.ts @@ -502,6 +502,31 @@ export function createWorkflowRunPackPushingRepoStore( schedulePush(agentAddress, repoId, ref); return result; }, + async writeTreeDelta(principal, repoId, ref, args) { + if (repoId.kind === "workflow-run") { + const latched = takeLatchedError(repoId, ref); + if (latched !== null) { + throw latched; + } + } + const result = await underlying.writeTreeDelta( + principal, + repoId, + ref, + args, + ); + if (repoId.kind !== "workflow-run") { + return result; + } + const agentAddress = registry.resolve(repoId.id); + if (agentAddress === null) { + throw new Error( + `workflow-run pack push: no agent address registered for deployment ${repoId.id}; the deploy router must record the mapping before the supervisor commits run events`, + ); + } + schedulePush(agentAddress, repoId, ref); + return result; + }, }; return wrapped; } diff --git a/packages/hub-api/src/git-http/receive-pack.test.ts b/packages/hub-api/src/git-http/receive-pack.test.ts index 01a97920..d05586a1 100644 --- a/packages/hub-api/src/git-http/receive-pack.test.ts +++ b/packages/hub-api/src/git-http/receive-pack.test.ts @@ -97,6 +97,9 @@ function createStubStore(stub: ReceivePackStub): { writeTreePreservingPrefix: async () => { throw new Error("writeTreePreservingPrefix: not used in this test"); }, + writeTreeDelta: async () => { + throw new Error("writeTreeDelta: not used in this test"); + }, createPack: async () => { throw new Error("createPack: not used in this test"); }, diff --git a/packages/hub-api/src/routes/workflows.test.ts b/packages/hub-api/src/routes/workflows.test.ts index 31bde7cb..606e03d7 100644 --- a/packages/hub-api/src/routes/workflows.test.ts +++ b/packages/hub-api/src/routes/workflows.test.ts @@ -267,6 +267,7 @@ function createStubRepoStore(repoDirById?: Map): RepoStore { initRepo: unused, writeTree: unused, writeTreePreservingPrefix: unused, + writeTreeDelta: unused, receivePack: unused, createPack: unused, resolveRef: unused, diff --git a/packages/hub-sessions/src/hub-session-orchestrator.test.ts b/packages/hub-sessions/src/hub-session-orchestrator.test.ts index e838bb68..1c8d3055 100644 --- a/packages/hub-sessions/src/hub-session-orchestrator.test.ts +++ b/packages/hub-sessions/src/hub-session-orchestrator.test.ts @@ -251,6 +251,7 @@ function unusedRepoStore(): RepoStore { initRepo: unused, writeTree: unused, writeTreePreservingPrefix: unused, + writeTreeDelta: unused, receivePack: unused, createPack: unused, resolveRef: unused, diff --git a/packages/hub-sessions/src/repo-store/store.ts b/packages/hub-sessions/src/repo-store/store.ts index 25ad542e..8664db05 100644 --- a/packages/hub-sessions/src/repo-store/store.ts +++ b/packages/hub-sessions/src/repo-store/store.ts @@ -26,6 +26,7 @@ import type { RepoStoreSubscribeEvent, TreeContent, WriteResult, + WriteTreeDeltaArgs, WriteTreePreservingPrefixArgs, } from "./types"; import { SAFE_REPO_ID } from "./types"; @@ -404,6 +405,21 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { } } + // Validate a delta write path. A put is always a file (no trailing + // slash); a delete is either an exact file or a subtree prefix + // (trailing slash allowed). Both reject empties, absolute paths, and + // any `..` traversal segment. + function validateDeltaPath(p: string, isDelete: boolean): void { + const malformed = + p.length === 0 || + p.startsWith("/") || + p.split("/").includes("..") || + (!isDelete && p.endsWith("/")); + if (malformed) { + throw new Error(`delta_path_invalid: ${JSON.stringify(p)}`); + } + } + function storageOptsFor( repoId: RepoId, opts: InitRepoOpts | undefined, @@ -857,31 +873,42 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { return chain.reverse(); } - // Assemble a new root tree by splicing `puts` (repo-root-relative - // path -> blob oid) and an optional `clearPrefix` deletion onto the - // parent's root tree, reusing every unchanged subtree by oid. The - // result is committed directly via `git.commit({ tree })`, so the - // on-disk index is never touched. + // Assemble a new root tree by splicing `puts` (repo-root-relative path + // -> blob oid) and `deletes` onto the parent's root tree, reusing every + // unchanged entry by oid. The result is committed directly via + // `git.commit({ tree })`, so the on-disk index is never touched. + // + // `deletes` is a set of repo-root-relative paths. Removal is by NAME, + // not by base-entry type: a trailing-slash entry names a subtree + // prefix and clears it (the clear-prefix shape); a no-slash entry + // clears whatever sits at that exact path -- a blob, or an entire + // subtree if the name happens to be a directory in the base. A + // no-slash delete is therefore NOT git-rm single-file semantics, and + // a trailing-slash delete whose leading segment is a base blob drops + // that blob rather than no-op'ing. Every caller emits only exact + // paths of entries that exist as blobs, or an intended clear-prefix, + // so neither name-vs-type case arises; a caller that builds deletes + // more freely must not rely on per-type git-rm behavior here. A `put` + // at a path overrides a delete of the same path. // // Recursion is scoped to the touched subtrees: a level is read and - // rewritten only when a put lands under it or `clearPrefix` deletes - // within it. Every sibling the write does not touch is carried - // forward by its existing oid without a re-hash or a walk, so the - // per-commit cost tracks the size of the change rather than the repo. - // Deletions are confined to `clearPrefix`; outside it the splice is - // additive, so reusing a sibling subtree can never drop a deletion. - // Returns the new tree oid, or `null` when the subtree ends up empty - // (the caller writes an empty root tree when the whole repo empties). + // rewritten only when a put lands under it or a delete removes within + // it. Every entry the write does not touch — sibling subtrees AND + // sibling blobs inside a subtree being touched — is carried forward by + // its existing oid without a re-hash or a walk, so the per-commit cost + // tracks the size of the change rather than the repo. Returns the new + // tree oid, or `null` when the subtree ends up empty (the caller writes + // an empty root tree when the whole repo empties). async function assembleTree( dir: string, baseTreeOid: string | null, prefix: string, puts: ReadonlyMap, - clearPrefix: string | undefined, + deletes: ReadonlySet, ): Promise { - // At the exact node `clearPrefix` names, the base subtree is dropped - // wholesale — only merge-output puts under the prefix survive. - const cleared = clearPrefix !== undefined && clearPrefix === prefix; + // A subtree-delete naming exactly this node drops the base wholesale; + // only puts under it survive. + const cleared = prefix !== "" && deletes.has(prefix); const baseEntries = new Map< string, { mode: string; oid: string; type: TreeEntry["type"] } @@ -898,11 +925,11 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { } } - // Names to rebuild at this level: a direct blob put, a subtree some - // put descends into, or the subtree `clearPrefix` descends into (so - // its deletion is applied even when no put touches that subtree). + // Classify what changes at this level: direct blob puts, exact-file + // deletes here, and subtrees a put or a delete descends into. const blobPutsHere = new Set(); const subtreeNames = new Set(); + const fileDeletesHere = new Set(); for (const full of puts.keys()) { if (prefix !== "" && !full.startsWith(prefix)) continue; const rest = full.slice(prefix.length); @@ -911,15 +938,13 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { if (slash === -1) blobPutsHere.add(rest); else subtreeNames.add(rest.slice(0, slash)); } - if ( - clearPrefix !== undefined && - clearPrefix !== prefix && - clearPrefix.startsWith(prefix) - ) { - const rest = clearPrefix.slice(prefix.length); + for (const del of deletes) { + if (prefix !== "" && !del.startsWith(prefix)) continue; + const rest = del.slice(prefix.length); + if (rest.length === 0) continue; // del === prefix, handled by `cleared` const slash = rest.indexOf("/"); - const seg = slash === -1 ? rest : rest.slice(0, slash); - if (seg.length > 0) subtreeNames.add(seg); + if (slash === -1) fileDeletesHere.add(rest); + else subtreeNames.add(rest.slice(0, slash)); } const names = new Set([ @@ -932,7 +957,8 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { const full = prefix + name; const putOid = puts.get(full); if (putOid !== undefined) { - // A direct blob put overrides whatever the base held here. + // A put overrides whatever the base held and any delete of the + // same path. entries.push({ mode: "100644", path: name, @@ -941,6 +967,7 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { }); continue; } + if (fileDeletesHere.has(name)) continue; // file removed if (subtreeNames.has(name)) { const base = baseEntries.get(name); const baseChildOid = @@ -950,7 +977,7 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { baseChildOid, `${full}/`, puts, - clearPrefix, + deletes, ); if (childOid !== null) { entries.push({ @@ -1037,7 +1064,12 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { principal: Principal, repoId: RepoId, ref: string, - content: TreeContent, + w: { + files: Record; + deletes: ReadonlySet; + changedPathPrefixes: ReadonlySet | undefined; + message: string; + }, ): Promise { const dir = repoDir(repoId); await storageInitRepo(dir, storageOptsFor(repoId, undefined)); @@ -1049,13 +1081,10 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { // repo-global structure shared across refs; routing writes through // it forced a full index rebuild on every events<->workflow-run ref // flip and re-serialized the whole index once per staged blob. - // Splicing the tree from the parent's root tree instead reuses every - // untouched subtree by oid, so the per-commit cost tracks the change - // rather than the accumulated history. - if (content.clearPrefix !== undefined) { - validateClearPrefix(content.clearPrefix); - } - const clearPrefix = content.clearPrefix; + // Splicing the tree from the parent's root tree — applying `w.files` + // as puts and `w.deletes` as removals, reusing every untouched entry + // by oid — keeps the per-commit cost tracking the change rather than + // the accumulated history. // Pin the parent under the lock: the new commit's parent is the // ref's tip, and the splice runs against THAT commit's root tree, @@ -1075,12 +1104,11 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { baseRootTreeOid = commit.tree; } - // Write the merge-output blobs, then splice them (and the - // clearPrefix deletion) onto the parent root tree. writeBlob and - // writeTree emit unreferenced objects; nothing the ref can reach - // moves until the commit below lands. + // Write the put blobs, then splice them and `w.deletes` onto the + // parent root tree. writeBlob and writeTree emit unreferenced + // objects; nothing the ref can reach moves until the commit lands. const puts = new Map(); - for (const [relPath, contents] of Object.entries(content.files)) { + for (const [relPath, contents] of Object.entries(w.files)) { const bytes = typeof contents === "string" ? new TextEncoder().encode(contents) @@ -1093,22 +1121,20 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { baseRootTreeOid, "", puts, - clearPrefix, + w.deletes, ); const newRootTreeOid = assembled ?? (await git.writeTree({ fs, dir, tree: [] })); // validatePush sees the full prospective tree via closures over the // assembled tree oid; the prior-side closures read the parent commit - // (`parentCommitSha`) exactly as before. The change set the handler - // may scope to is `{clearPrefix}` when one is present — writes under - // it are a full replace and every other path is carried forward — and - // unbounded (validate-all) otherwise. + // (`parentCommitSha`) exactly as before. `w.changedPathPrefixes` is + // the handler's scoping hint — the prefixes this write may have + // touched — and stays undefined for an unbounded (validate-all) write. const { priorReadBlob, priorListDir, priorListDirOids } = buildPriorTreeClosures(dir, parentCommitSha); const prospective = buildTreeReadClosures(dir, newRootTreeOid); - const changedPathPrefixes = - clearPrefix === undefined ? undefined : new Set([clearPrefix]); + const changedPathPrefixes = w.changedPathPrefixes; const validation = await handler.validatePush({ repoId, ref, @@ -1130,22 +1156,23 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { } // Materialize the working tree for the paths this write touched: - // clear the replaced prefix, then write each put file. The store's - // own reads resolve through the object store, but some consumers - // (the workflow-run claim-check processing scan) read these files - // straight from disk, so the working tree must mirror the committed - // change. Only merge-output paths are written and only the cleared - // prefix is removed, so this is O(change), not O(repo). It runs only - // after validation passes, so a rejected push leaves the working - // tree untouched — the failure atomicity the old index rollback - // provided, now without any rollback. - if (clearPrefix !== undefined) { - await fs.promises.rm(path.join(dir, clearPrefix), { + // remove each deleted path, then write each put file. The store's own + // reads resolve through the object store, but some consumers (the + // workflow-run claim-check processing scan) read these files straight + // from disk, so the working tree must mirror the committed change. + // Only the deleted paths and the put paths are touched, so this is + // O(change), not O(repo). It runs only after validation passes, so a + // rejected push leaves the working tree untouched — the failure + // atomicity the old index rollback provided, now without any + // rollback. `rm` with `force` no-ops a missing path and `recursive` + // covers both a file delete and a subtree-prefix delete. + for (const del of w.deletes) { + await fs.promises.rm(path.join(dir, del), { recursive: true, force: true, }); } - for (const [relPath, contents] of Object.entries(content.files)) { + for (const [relPath, contents] of Object.entries(w.files)) { const fullPath = path.join(dir, relPath); await fs.promises.mkdir(path.dirname(fullPath), { recursive: true }); await fs.promises.writeFile(fullPath, contents); @@ -1163,7 +1190,7 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { dir, cache: cacheFor(dir), tree: newRootTreeOid, - message: content.message, + message: w.message, author: AUTHOR, parent: [parentSha], ref, @@ -1187,6 +1214,33 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { return { commitSha, newlyTerminalRuns: validation.newlyTerminalRuns ?? [] }; } + // Normalize a `TreeContent` (files + optional clearPrefix) into the + // puts/deletes/scope shape writeTreeUnderLock consumes. A clearPrefix + // becomes a single subtree-delete and the handler's change scope; its + // absence is a purely-additive write validated in full. + function normalizeTreeContent(content: TreeContent): { + files: Record; + deletes: ReadonlySet; + changedPathPrefixes: ReadonlySet | undefined; + message: string; + } { + if (content.clearPrefix !== undefined) { + validateClearPrefix(content.clearPrefix); + return { + files: content.files, + deletes: new Set([content.clearPrefix]), + changedPathPrefixes: new Set([content.clearPrefix]), + message: content.message, + }; + } + return { + files: content.files, + deletes: new Set(), + changedPathPrefixes: undefined, + message: content.message, + }; + } + async function writeTree( principal: Principal, repoId: RepoId, @@ -1195,12 +1249,12 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { ): Promise { gateAccess(principal, repoId, ref, "writeTree"); - // The lock spans the entire substrate body of writeTree: index - // mutation, validatePush, commit, and the onRefUpdated hook. Holding + // The lock spans the entire substrate body of writeTree: tree + // assembly, validatePush, commit, and the onRefUpdated hook. Holding // the lock through onRefUpdated keeps post-update consumers // serialized against the same ref's next writer. return withRepoLock(repoId, () => - writeTreeUnderLock(principal, repoId, ref, content), + writeTreeUnderLock(principal, repoId, ref, normalizeTreeContent(content)), ); } @@ -1291,7 +1345,44 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { const files = await args.merge(existing); return writeTreeUnderLock(principal, repoId, ref, { files, - clearPrefix: args.preservePrefix, + deletes: new Set([args.preservePrefix]), + changedPathPrefixes: new Set([args.preservePrefix]), + message: args.message, + }); + }); + } + + // Commit a targeted delta: `computeDelta` runs under the per-repo lock + // against the pinned parent tip and returns the exact files to put and + // paths to delete; everything else is carried forward by oid. Unlike + // writeTreePreservingPrefix — which clears and rebuilds a whole prefix + // from a full merge output — a delta touches only the entries it names, + // so a caller that mutates one file in a large directory (a claim-check + // move that adds one entry and deletes another) does not re-hash or + // re-materialize the untouched siblings. `changedPathPrefixes` is the + // handler's scoping hint for the touched region; the caller supplies it + // because the delta has no single clear-prefix to derive it from. + async function writeTreeDelta( + principal: Principal, + repoId: RepoId, + ref: string, + args: WriteTreeDeltaArgs, + ): Promise { + gateAccess(principal, repoId, ref, "writeTree"); + return withRepoLock(repoId, async () => { + const dir = repoDir(repoId); + await storageInitRepo(dir, storageOptsFor(repoId, undefined)); + // Pin the parent tip under the lock and hand it to computeDelta so + // its dedup reads and the tree assembly below observe the same + // pre-image — no lost-update window between the read and the write. + const parentCommitSha = await resolveRefSha(dir, ref); + const delta = await args.computeDelta(parentCommitSha); + for (const p of Object.keys(delta.puts)) validateDeltaPath(p, false); + for (const d of delta.deletes) validateDeltaPath(d, true); + return writeTreeUnderLock(principal, repoId, ref, { + files: delta.puts, + deletes: new Set(delta.deletes), + changedPathPrefixes: args.changedPathPrefixes, message: args.message, }); }); @@ -1707,6 +1798,7 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { initRepo, writeTree, writeTreePreservingPrefix, + writeTreeDelta, receivePack, createPack, resolveRef, diff --git a/packages/hub-sessions/src/repo-store/types.ts b/packages/hub-sessions/src/repo-store/types.ts index afbab8d2..11e93a99 100644 --- a/packages/hub-sessions/src/repo-store/types.ts +++ b/packages/hub-sessions/src/repo-store/types.ts @@ -184,6 +184,28 @@ export type WriteTreePreservingPrefixArgs = { message: string; }; +/** + * Per-call options for `RepoStore.writeTreeDelta`. `computeDelta` runs + * under the per-repo lock against the pinned parent tip (`parentCommitSha`, + * null when the ref does not yet exist) and returns the exact files to + * write and paths to delete; the substrate carries every other entry + * forward by its object id. A delete ending in `/` removes a whole + * subtree; any other delete removes a single file. + * + * `changedPathPrefixes` is the validation scoping hint for the touched + * region (e.g. `addresses//`), supplied by the caller because a + * delta has no single clear-prefix to derive it from; `undefined` means + * validate the whole tree. + */ +export type WriteTreeDeltaArgs = { + computeDelta: (parentCommitSha: string | null) => Promise<{ + puts: Record; + deletes: readonly string[]; + }>; + changedPathPrefixes: ReadonlySet | undefined; + message: string; +}; + export interface KindHandler { kind: RepoKind; /** @@ -317,6 +339,22 @@ export interface RepoStore { ref: string, args: WriteTreePreservingPrefixArgs, ): Promise; + /** + * Delta variant for mutating a few named entries in a large subtree + * without re-materializing the untouched siblings. `computeDelta` runs + * under the per-repo lock against the pinned parent tip and returns the + * files to put and the paths to delete; everything else is carried + * forward by object id. Use this over `writeTreePreservingPrefix` when + * the untouched remainder of the prefix is large (e.g. a claim-check + * move that adds one entry and deletes another while the consumed dedup + * index carries forward unchanged). + */ + writeTreeDelta( + principal: Principal, + repoId: RepoId, + ref: string, + args: WriteTreeDeltaArgs, + ): Promise; /** * Receive a packfile and advance `ref` to `commitSha`. * diff --git a/packages/hub-sessions/src/repo-store/write-tree-delta.test.ts b/packages/hub-sessions/src/repo-store/write-tree-delta.test.ts new file mode 100644 index 00000000..c450abbd --- /dev/null +++ b/packages/hub-sessions/src/repo-store/write-tree-delta.test.ts @@ -0,0 +1,200 @@ +// OID-equivalence guard for the delta write path (writeTreeDelta) added +// alongside the claim-check narrowing. A delta ({puts, deletes} carried +// against the parent tree, reusing untouched entries by oid) MUST commit +// the byte-identical tree that a full-replace writeTree of the same +// logical final state produces. SHA-1 tree hashing is deterministic +// given identical content and structure, so equal commit-tree oids prove +// the delta path never diverges from canonical git for the claim-check +// move shapes: enqueue-add, dequeue-move, markConsumed-move, +// empty->first-entry, and prune-to-empty. + +import { test, expect, afterAll, beforeAll } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import git from "isomorphic-git"; +import { generateKeyPair } from "@intx/crypto"; +import type { KeyPair } from "@intx/types/runtime"; +import { createRepoStore } from "./store"; +import type { AuthorizeFn, Principal, RepoId } from "./types"; + +const tempDirs: string[] = []; +let signingKey: KeyPair; + +beforeAll(async () => { + signingKey = await generateKeyPair(); +}); + +afterAll(async () => { + for (const d of tempDirs.splice(0)) { + await fs.promises + .rm(d, { recursive: true, force: true }) + .catch(() => undefined); + } +}); + +const allowAll: AuthorizeFn = () => ({ allowed: true }); +const principal: Principal = { kind: "test" }; +const REF = "refs/heads/events"; +const SCOPE = "addresses/a/"; + +// Permissive store: the delta path is what is under test, so the handler +// accepts every prospective tree. +function makeStore(dataDir: string) { + return createRepoStore({ + dataDir, + signingKey, + handlers: { + "agent-state": { + kind: "agent-state", + directoryPrefix: "repos-under-test", + validatePush: () => ({ ok: true }), + onRefUpdated: () => undefined, + }, + }, + authorize: allowAll, + }); +} + +async function freshStore(label: string) { + const dataDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), label)); + tempDirs.push(dataDir); + return makeStore(dataDir); +} + +const repoId: RepoId = { kind: "agent-state", id: "subject" }; + +async function commitTreeOid( + store: ReturnType, +): Promise { + const dir = store.getRepoDir(repoId); + const sha = await git.resolveRef({ fs, dir, ref: REF }); + const { commit } = await git.readCommit({ fs, dir, oid: sha }); + return commit.tree; +} + +// Apply a delta to a flat path->content base, mirroring assembleTree's +// delete semantics (trailing-slash prefix delete vs exact file delete). +function applyDelta( + base: Record, + puts: Record, + deletes: readonly string[], +): Record { + const exactDeletes = new Set(deletes.filter((d) => !d.endsWith("/"))); + const prefixDeletes = deletes.filter((d) => d.endsWith("/")); + const out: Record = {}; + for (const [k, v] of Object.entries(base)) { + if (exactDeletes.has(k)) continue; + if (prefixDeletes.some((d) => k.startsWith(d))) continue; + out[k] = v; + } + for (const [k, v] of Object.entries(puts)) out[k] = v; + return out; +} + +// Seed a repo's SCOPE subtree from `base`, then produce the final tree +// two ways and assert byte-identical commit trees: +// delta — writeTreeDelta({ puts, deletes }) +// full-replace — writeTree clearing SCOPE and re-supplying the whole +// post-delta SCOPE content (the pre-narrowing shape) +async function assertDeltaMatchesFullReplace( + label: string, + base: Record, + puts: Record, + deletes: readonly string[], +) { + const finalState = applyDelta(base, puts, deletes); + const finalScopeFiles: Record = {}; + for (const [k, v] of Object.entries(finalState)) { + if (k.startsWith(SCOPE)) finalScopeFiles[k] = v; + } + + const deltaStore = await freshStore(`wtd-delta-${label}-`); + await deltaStore.initRepo(repoId); + if (Object.keys(base).length > 0) { + await deltaStore.writeTree(principal, repoId, REF, { + files: base, + message: "seed", + }); + } + await deltaStore.writeTreeDelta(principal, repoId, REF, { + changedPathPrefixes: new Set([SCOPE]), + message: "delta", + computeDelta: async () => ({ puts, deletes }), + }); + const deltaTree = await commitTreeOid(deltaStore); + + const fullStore = await freshStore(`wtd-full-${label}-`); + await fullStore.initRepo(repoId); + if (Object.keys(base).length > 0) { + await fullStore.writeTree(principal, repoId, REF, { + files: base, + message: "seed", + }); + } + await fullStore.writeTree(principal, repoId, REF, { + files: finalScopeFiles, + clearPrefix: SCOPE, + message: "full-replace", + }); + const fullTree = await commitTreeOid(fullStore); + + expect(deltaTree).toBe(fullTree); +} + +const consumedBase = { + [`${SCOPE}consumed/m1.json`]: `{"receivedAt":1}`, + [`${SCOPE}consumed/m2.json`]: `{"receivedAt":2}`, + [`${SCOPE}watermark.json`]: `{"watermark":0}`, +}; + +test("delta enqueue-add matches full-replace", async () => { + await assertDeltaMatchesFullReplace( + "enqueue", + consumedBase, + { [`${SCOPE}inbox/10-m3.json`]: `{"messageId":"m3"}` }, + [], + ); +}); + +test("delta dequeue-move matches full-replace", async () => { + await assertDeltaMatchesFullReplace( + "dequeue", + { ...consumedBase, [`${SCOPE}inbox/10-m3.json`]: `{"messageId":"m3"}` }, + { [`${SCOPE}processing/10-m3.json`]: `{"messageId":"m3"}` }, + [`${SCOPE}inbox/10-m3.json`], + ); +}); + +test("delta markConsumed-move matches full-replace", async () => { + await assertDeltaMatchesFullReplace( + "consume", + { + ...consumedBase, + [`${SCOPE}processing/10-m3.json`]: `{"messageId":"m3"}`, + }, + { + [`${SCOPE}consumed/m3.json`]: `{"receivedAt":10}`, + [`${SCOPE}watermark.json`]: `{"watermark":5}`, + }, + [`${SCOPE}processing/10-m3.json`], + ); +}); + +test("delta empty->first-entry matches full-replace", async () => { + await assertDeltaMatchesFullReplace( + "first", + {}, + { [`${SCOPE}inbox/10-m1.json`]: `{"messageId":"m1"}` }, + [], + ); +}); + +test("delta prune-to-empty matches full-replace", async () => { + await assertDeltaMatchesFullReplace( + "prune", + consumedBase, + { [`${SCOPE}watermark.json`]: `{"watermark":9}` }, + [`${SCOPE}consumed/m1.json`, `${SCOPE}consumed/m2.json`], + ); +}); diff --git a/packages/hub-sessions/src/session-service.test.ts b/packages/hub-sessions/src/session-service.test.ts index a1d02a62..35592c83 100644 --- a/packages/hub-sessions/src/session-service.test.ts +++ b/packages/hub-sessions/src/session-service.test.ts @@ -167,6 +167,7 @@ function unusedRepoStore(): RepoStore { initRepo: unused, writeTree: unused, writeTreePreservingPrefix: unused, + writeTreeDelta: unused, receivePack: unused, createPack: unused, resolveRef: unused, @@ -201,6 +202,7 @@ function createFakeRepoStore( initRepo: unused, writeTree: unused, writeTreePreservingPrefix: unused, + writeTreeDelta: unused, receivePack: unused, listRefs: unused, resolveHead: unused, diff --git a/packages/hub-sessions/src/workflow-run-kind.ts b/packages/hub-sessions/src/workflow-run-kind.ts index f596b4e8..2ca8737d 100644 --- a/packages/hub-sessions/src/workflow-run-kind.ts +++ b/packages/hub-sessions/src/workflow-run-kind.ts @@ -2557,6 +2557,10 @@ function addressSegmentFor(address: string): string { return encodeURIComponent(address); } +function addressPrefix(addressSegment: string): string { + return `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/`; +} + function inboxPath(addressSegment: string, key: string): string { return `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/${WORKFLOW_RUN_INBOX_DIR}/${key}.json`; } @@ -2565,41 +2569,60 @@ function processingPath(addressSegment: string, key: string): string { return `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/${WORKFLOW_RUN_PROCESSING_DIR}/${key}.json`; } +function consumedPath(addressSegment: string, messageId: string): string { + return `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/${WORKFLOW_RUN_CONSUMED_DIR}/${messageId}.json`; +} + +function watermarkPath(addressSegment: string): string { + return `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/${WORKFLOW_RUN_WATERMARK_FILE}`; +} + function filenameKey(receivedAt: number, messageId: string): string { return `${String(receivedAt)}-${messageId}`; } +type ClaimCheckEntry = { name: string; oid: string }; + +type AddressListing = { + inbox: ClaimCheckEntry[]; + processing: ClaimCheckEntry[]; + consumed: ClaimCheckEntry[]; + watermark: number; +}; + /** - * Read the full state of one address subtree from the ref's tip via - * `isomorphic-git`. The substrate's `writeTreePreservingPrefix` only - * surfaces direct-child blobs; the address subtree's blobs live one - * level deeper, so the merge callback uses this helper to assemble - * the full pre-image inside the per-repo lock. - * - * Returns the bytes of every entry under - * `addresses//{inbox,processing,consumed}/` keyed by - * repo-root-relative path. An empty map covers the cases where the - * repo, ref, or address subtree do not yet exist — all legitimate - * first-write states for a brand-new claim-check operation. + * Read one address's claim-check listing from the parent commit: the + * filenames and blob OIDs directly under + * `addresses//{inbox,processing,consumed}/` (NOT their + * bytes), plus the retention watermark. The bytes of the single entry a + * leg actually moves are read separately by OID via + * {@link readClaimCheckBlob}, so the unbounded consumed/ dedup index is + * enumerated (one `readTree`, filenames only) but never read + * blob-by-blob. An empty listing covers the repo/ref/address-absent + * first-write states — all legitimate for a brand-new operation. */ -async function readAddressSubtree( +async function readAddressListing( repoDir: string, - ref: string, + parentCommitSha: string | null, addressSegment: string, -): Promise> { - const out = new Map(); - let commitSha: string; - try { - commitSha = await git.resolveRef({ fs, dir: repoDir, ref }); - } catch { - return out; - } - const commit = await git.readCommit({ fs, dir: repoDir, oid: commitSha }); +): Promise { + const listing: AddressListing = { + inbox: [], + processing: [], + consumed: [], + watermark: 0, + }; + if (parentCommitSha === null) return listing; + const commit = await git.readCommit({ + fs, + dir: repoDir, + oid: parentCommitSha, + }); const addrTreeOid = await resolveSubtreeOid(repoDir, commit.commit.tree, [ WORKFLOW_RUN_ADDRESSES_PREFIX, addressSegment, ]); - if (addrTreeOid === null) return out; + if (addrTreeOid === null) return listing; const { tree: addrChildren } = await git.readTree({ fs, dir: repoDir, @@ -2607,36 +2630,40 @@ async function readAddressSubtree( }); for (const child of addrChildren) { if (child.type === "blob" && child.path === WORKFLOW_RUN_WATERMARK_FILE) { - const { blob } = await git.readBlob({ - fs, - dir: repoDir, - oid: child.oid, - }); - out.set( - `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/${WORKFLOW_RUN_WATERMARK_FILE}`, - blob, - ); + const { blob } = await git.readBlob({ fs, dir: repoDir, oid: child.oid }); + listing.watermark = parseWatermark(blob, watermarkPath(addressSegment)); continue; } if (child.type !== "tree") continue; - if (!CLAIM_CHECK_SUBDIRS.has(child.path)) continue; - const { tree: children } = await git.readTree({ + const bucket = + child.path === WORKFLOW_RUN_INBOX_DIR + ? listing.inbox + : child.path === WORKFLOW_RUN_PROCESSING_DIR + ? listing.processing + : child.path === WORKFLOW_RUN_CONSUMED_DIR + ? listing.consumed + : null; + if (bucket === null) continue; + const { tree: entries } = await git.readTree({ fs, dir: repoDir, oid: child.oid, }); - for (const blobEntry of children) { - if (blobEntry.type !== "blob") continue; - const { blob } = await git.readBlob({ - fs, - dir: repoDir, - oid: blobEntry.oid, - }); - const blobPath = `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/${child.path}/${blobEntry.path}`; - out.set(blobPath, blob); + for (const entry of entries) { + if (entry.type !== "blob") continue; + bucket.push({ name: entry.path, oid: entry.oid }); } } - return out; + return listing; +} + +/** Read a single claim-check blob by its git object id. */ +async function readClaimCheckBlob( + repoDir: string, + oid: string, +): Promise { + const { blob } = await git.readBlob({ fs, dir: repoDir, oid }); + return blob; } async function resolveSubtreeOid( @@ -2698,17 +2725,11 @@ function decodeConsumedReceivedAtOrThrow( } /** - * Read the per-address watermark from a claim-check subtree map keyed - * by repo-root-relative path. An absent watermark blob means the - * address has never pruned; treat it as watermark 0 (nothing pruned, - * nothing refused). + * Decode the per-address retention watermark from its blob bytes. The + * caller treats an absent watermark blob as 0 (the address has never + * pruned; nothing refused). */ -function readWatermarkFromMap( - existing: Map, - watermarkFull: string, -): number { - const bytes = existing.get(watermarkFull); - if (bytes === undefined) return 0; +function parseWatermark(bytes: Uint8Array, watermarkFull: string): number { let parsed: unknown; try { parsed = JSON.parse(new TextDecoder().decode(bytes)); @@ -2776,81 +2797,64 @@ export async function enqueueInbox( ...(args.rawMessage !== undefined ? { rawMessage: args.rawMessage } : {}), }; const newInboxPath = inboxPath(addressSegment, inboxKey); - const { commitSha } = await store.writeTreePreservingPrefix( - principal, - repoId, - ref, - { - preservePrefix: `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/`, - merge: async () => { - const existing = await readAddressSubtree(repoDir, ref, addressSegment); - const inboxPrefix = `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/${WORKFLOW_RUN_INBOX_DIR}/`; - const processingPrefix = `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/${WORKFLOW_RUN_PROCESSING_DIR}/`; - const consumedPrefix = `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/${WORKFLOW_RUN_CONSUMED_DIR}/`; - const watermarkFull = `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/${WORKFLOW_RUN_WATERMARK_FILE}`; - const messageIdSuffix = `-${args.messageId}.json`; - // Refuse a definitively-stale enqueue: a message whose - // receivedAt is strictly below the retention watermark could - // have had its consumed/ dedup entry pruned, so a duplicate can - // no longer be ruled out. Reject it LOUDLY rather than risk - // reprocessing. This is the second half of the exactly-once - // guarantee: above the watermark the consumed/ index is - // authoritative; below it, refuse. - const watermark = readWatermarkFromMap(existing, watermarkFull); - if (args.receivedAt < watermark) { - throw new Error( - `claim_check_stale_enqueue: address ${args.address} message ${args.messageId} receivedAt ${String(args.receivedAt)} is below the retention watermark ${String(watermark)}; its dedup entry may have been pruned, so it is refused as definitively-stale`, - ); - } - for (const [blobPath] of existing) { - if (blobPath === watermarkFull) continue; - if (blobPath === newInboxPath) { - throw new Error( - `claim_check_duplicate_inbox: ${newInboxPath} already exists`, - ); - } - if (blobPath.startsWith(consumedPrefix)) { - const fname = blobPath.slice(consumedPrefix.length); - if (fname === `${args.messageId}.json`) { - throw new Error( - `claim_check_already_consumed: address ${args.address} message ${args.messageId} is already in the consumed dedup index`, - ); - } - } - if (blobPath.startsWith(processingPrefix)) { - const fname = blobPath.slice(processingPrefix.length); - if (fname.endsWith(messageIdSuffix)) { - throw new Error( - `claim_check_already_processing: address ${args.address} message ${args.messageId} is currently in processing`, - ); - } - } - // Reject a second inbox entry for the same messageId at a - // different receivedAt. The validatePush atomicity check - // also catches this on the commit path, but the inbox - // scan here surfaces the rejection at the API boundary so - // the caller sees a precise error (rather than a generic - // tree-validation failure) and the bad tree never reaches - // the substrate. - if (blobPath.startsWith(inboxPrefix)) { - const fname = blobPath.slice(inboxPrefix.length); - if (fname.endsWith(messageIdSuffix)) { - throw new Error( - `claim_check_already_inbox: address ${args.address} message ${args.messageId} is already in the inbox at ${blobPath}`, - ); - } - } - } - const files: Record = {}; - for (const [blobPath, bytes] of existing) { - files[blobPath] = bytes; - } - files[newInboxPath] = utf8(JSON.stringify(envelope)); - return files; - }, - message: `enqueue inbox ${args.address} ${args.messageId}`, + const inboxFname = `${inboxKey}.json`; + const consumedFname = `${args.messageId}.json`; + const messageIdSuffix = `-${args.messageId}.json`; + const { commitSha } = await store.writeTreeDelta(principal, repoId, ref, { + changedPathPrefixes: new Set([addressPrefix(addressSegment)]), + message: `enqueue inbox ${args.address} ${args.messageId}`, + computeDelta: async (parentCommitSha) => { + const listing = await readAddressListing( + repoDir, + parentCommitSha, + addressSegment, + ); + // Refuse a definitively-stale enqueue: a message whose receivedAt + // is strictly below the retention watermark could have had its + // consumed/ dedup entry pruned, so a duplicate can no longer be + // ruled out. Reject it LOUDLY rather than risk reprocessing. This + // is the second half of the exactly-once guarantee: above the + // watermark the consumed/ index is authoritative; below it, refuse. + if (args.receivedAt < listing.watermark) { + throw new Error( + `claim_check_stale_enqueue: address ${args.address} message ${args.messageId} receivedAt ${String(args.receivedAt)} is below the retention watermark ${String(listing.watermark)}; its dedup entry may have been pruned, so it is refused as definitively-stale`, + ); + } + if (listing.inbox.some((e) => e.name === inboxFname)) { + throw new Error( + `claim_check_duplicate_inbox: ${newInboxPath} already exists`, + ); + } + // consumed/ is keyed by messageId alone, so this is an exact + // filename lookup against the dedup index. + if (listing.consumed.some((e) => e.name === consumedFname)) { + throw new Error( + `claim_check_already_consumed: address ${args.address} message ${args.messageId} is already in the consumed dedup index`, + ); + } + if (listing.processing.some((e) => e.name.endsWith(messageIdSuffix))) { + throw new Error( + `claim_check_already_processing: address ${args.address} message ${args.messageId} is currently in processing`, + ); + } + // Reject a second inbox entry for the same messageId at a + // different receivedAt. The validatePush atomicity check also + // catches this on the commit path, but surfacing it here gives the + // caller a precise error and keeps the bad tree off the substrate. + const inboxDup = listing.inbox.find((e) => + e.name.endsWith(messageIdSuffix), + ); + if (inboxDup !== undefined) { + throw new Error( + `claim_check_already_inbox: address ${args.address} message ${args.messageId} is already in the inbox at ${inboxPath(addressSegment, inboxDup.name.slice(0, -".json".length))}`, + ); + } + return { + puts: { [newInboxPath]: utf8(JSON.stringify(envelope)) }, + deletes: [], + }; }, - ); + }); return { commitSha, inboxKey, envelope }; } @@ -2881,68 +2885,62 @@ export async function dequeueToProcessing( const ref = claimCheckCommitRef(); const repoDir = store.getRepoDir(repoId); let dequeued: { key: string; envelope: ClaimCheckEnvelope } | null = null; - const { commitSha } = await store.writeTreePreservingPrefix( - principal, - repoId, - ref, - { - preservePrefix: `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/`, - merge: async () => { - const existing = await readAddressSubtree(repoDir, ref, addressSegment); - const inboxPrefix = `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/${WORKFLOW_RUN_INBOX_DIR}/`; - // Parse each inbox path's filename and sort by numeric - // receivedAt with a messageId tiebreak. A raw string sort - // would not agree with chronological order when receivedAt - // values have non-uniform digit widths. - type InboxCandidate = { - path: string; - receivedAt: number; - messageId: string; - }; - const candidates: InboxCandidate[] = []; - for (const p of existing.keys()) { - if (!p.startsWith(inboxPrefix)) continue; - const fname = p.slice(inboxPrefix.length); - const m = QUEUE_FILENAME_RE.exec(fname); - if (m === null || m[1] === undefined || m[2] === undefined) { - throw new Error(`claim_check_invalid_inbox_filename: ${p}`); - } - candidates.push({ - path: p, - receivedAt: Number.parseInt(m[1], 10), - messageId: m[2], - }); + const { commitSha } = await store.writeTreeDelta(principal, repoId, ref, { + changedPathPrefixes: new Set([addressPrefix(addressSegment)]), + message: `dequeue ${address}`, + computeDelta: async (parentCommitSha) => { + const listing = await readAddressListing( + repoDir, + parentCommitSha, + addressSegment, + ); + const inboxDir = `${addressPrefix(addressSegment)}${WORKFLOW_RUN_INBOX_DIR}/`; + // Sort by numeric receivedAt with a messageId tiebreak. A raw + // string sort would not agree with chronological order when + // receivedAt values have non-uniform digit widths. + type InboxCandidate = { + entry: ClaimCheckEntry; + receivedAt: number; + messageId: string; + }; + const candidates: InboxCandidate[] = []; + for (const entry of listing.inbox) { + const m = QUEUE_FILENAME_RE.exec(entry.name); + if (m === null || m[1] === undefined || m[2] === undefined) { + throw new Error( + `claim_check_invalid_inbox_filename: ${inboxDir}${entry.name}`, + ); } - candidates.sort((a, b) => { - if (a.receivedAt !== b.receivedAt) return a.receivedAt - b.receivedAt; - if (a.messageId < b.messageId) return -1; - if (a.messageId > b.messageId) return 1; - return 0; + candidates.push({ + entry, + receivedAt: Number.parseInt(m[1], 10), + messageId: m[2], }); - if (candidates.length === 0) { - dequeued = null; - return Object.fromEntries(existing); - } - const first = candidates[0]; - if (first === undefined) throw new Error("unreachable"); - const firstPath = first.path; - const bytes = existing.get(firstPath); - if (bytes === undefined) throw new Error("unreachable"); - const fname = firstPath.slice(inboxPrefix.length); - const key = fname.slice(0, -".json".length); - const envelope = decodeQueueEnvelopeOrThrow(bytes, firstPath); - dequeued = { key, envelope }; - const files: Record = {}; - for (const [blobPath, blobBytes] of existing) { - if (blobPath === firstPath) continue; - files[blobPath] = blobBytes; - } - files[processingPath(addressSegment, key)] = bytes; - return files; - }, - message: `dequeue ${address}`, + } + candidates.sort((a, b) => { + if (a.receivedAt !== b.receivedAt) return a.receivedAt - b.receivedAt; + if (a.messageId < b.messageId) return -1; + if (a.messageId > b.messageId) return 1; + return 0; + }); + const first = candidates[0]; + if (first === undefined) { + // Empty inbox: nothing to move. The commit is a no-op rewrite of + // the same tree; the caller reads `dequeued === null`. + dequeued = null; + return { puts: {}, deletes: [] }; + } + const firstPath = `${inboxDir}${first.entry.name}`; + const key = first.entry.name.slice(0, -".json".length); + const bytes = await readClaimCheckBlob(repoDir, first.entry.oid); + const envelope = decodeQueueEnvelopeOrThrow(bytes, firstPath); + dequeued = { key, envelope }; + return { + puts: { [processingPath(addressSegment, key)]: bytes }, + deletes: [firstPath], + }; }, - ); + }); if (dequeued === null) return null; const captured: { key: string; envelope: ClaimCheckEnvelope } = dequeued; return { commitSha, key: captured.key, envelope: captured.envelope }; @@ -3076,97 +3074,94 @@ export async function markConsumed( let consumedEnvelope: ConsumedEnvelope | null = null; let advancedWatermark = 0; const prunedMessageIds: string[] = []; - const { commitSha } = await store.writeTreePreservingPrefix( - principal, - repoId, - ref, - { - preservePrefix: `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/`, - merge: async () => { - const existing = await readAddressSubtree(repoDir, ref, addressSegment); - const processingPrefix = `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/${WORKFLOW_RUN_PROCESSING_DIR}/`; - const consumedPrefix = `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/${WORKFLOW_RUN_CONSUMED_DIR}/`; - const watermarkFull = `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/${WORKFLOW_RUN_WATERMARK_FILE}`; - const consumedFname = `${args.messageId}.json`; - const consumedFull = `${consumedPrefix}${consumedFname}`; - if (existing.has(consumedFull)) { - throw new Error( - `claim_check_already_consumed: ${consumedFull} already in the dedup index`, - ); - } - let processingFull: string | null = null; - let processingBytes: Uint8Array | null = null; - for (const [blobPath, bytes] of existing) { - if (!blobPath.startsWith(processingPrefix)) continue; - const fname = blobPath.slice(processingPrefix.length); - if (fname.endsWith(`-${args.messageId}.json`)) { - processingFull = blobPath; - processingBytes = bytes; - break; - } - } - if (processingFull === null || processingBytes === null) { - throw new Error( - `claim_check_processing_not_found: address ${args.address} message ${args.messageId} has no processing entry`, - ); - } - const processingEnvelope = decodeQueueEnvelopeOrThrow( - processingBytes, - processingFull, + const { commitSha } = await store.writeTreeDelta(principal, repoId, ref, { + changedPathPrefixes: new Set([addressPrefix(addressSegment)]), + message: `consume ${args.address} ${args.messageId}`, + computeDelta: async (parentCommitSha) => { + const listing = await readAddressListing( + repoDir, + parentCommitSha, + addressSegment, + ); + const consumedFull = consumedPath(addressSegment, args.messageId); + const consumedFname = `${args.messageId}.json`; + if (listing.consumed.some((e) => e.name === consumedFname)) { + throw new Error( + `claim_check_already_consumed: ${consumedFull} already in the dedup index`, ); - const envelope: ConsumedEnvelope = { - messageId: args.messageId, - receivedAt: processingEnvelope.receivedAt, - address: args.address, - runId: args.runId, - consumedAt: args.consumedAt, - mailAuditRef: processingEnvelope.mailAuditRef, - }; - consumedEnvelope = envelope; - - const priorWatermark = readWatermarkFromMap(existing, watermarkFull); - // The watermark may only advance, and never past the entry - // this commit writes (so the new entry is always retained -- - // a message consumed long after receipt may legitimately sit - // below `consumedAt - horizon`, and it is pruned on a later - // commit once the watermark passes ITS receivedAt). - const horizonBoundary = args.consumedAt - retentionHorizonMs; - const newWatermark = Math.max( - priorWatermark, - Math.min(horizonBoundary, envelope.receivedAt), + } + const processingDir = `${addressPrefix(addressSegment)}${WORKFLOW_RUN_PROCESSING_DIR}/`; + const processingEntry = listing.processing.find((e) => + e.name.endsWith(`-${args.messageId}.json`), + ); + if (processingEntry === undefined) { + throw new Error( + `claim_check_processing_not_found: address ${args.address} message ${args.messageId} has no processing entry`, ); - advancedWatermark = newWatermark; - - const files: Record = {}; - for (const [blobPath, blobBytes] of existing) { - if (blobPath === processingFull) continue; - if (blobPath === watermarkFull) continue; - if (blobPath.startsWith(consumedPrefix)) { - // Prune the oldest consumed tail: drop any retained entry - // whose receivedAt has fallen strictly below the new - // watermark. The new entry (added below) is never below - // the watermark by construction. - const consumedReceivedAt = decodeConsumedReceivedAtOrThrow( - blobBytes, - blobPath, - ); - if (consumedReceivedAt < newWatermark) { - const prunedFname = blobPath.slice(consumedPrefix.length); - prunedMessageIds.push(prunedFname.slice(0, -".json".length)); - continue; - } - } - files[blobPath] = blobBytes; - } - files[consumedFull] = utf8(JSON.stringify(envelope)); - files[watermarkFull] = utf8( - JSON.stringify({ watermark: newWatermark }), + } + const processingFull = `${processingDir}${processingEntry.name}`; + const processingBytes = await readClaimCheckBlob( + repoDir, + processingEntry.oid, + ); + const processingEnvelope = decodeQueueEnvelopeOrThrow( + processingBytes, + processingFull, + ); + const envelope: ConsumedEnvelope = { + messageId: args.messageId, + receivedAt: processingEnvelope.receivedAt, + address: args.address, + runId: args.runId, + consumedAt: args.consumedAt, + mailAuditRef: processingEnvelope.mailAuditRef, + }; + consumedEnvelope = envelope; + + // The watermark may only advance, and never past the entry this + // commit writes (so the new entry is always retained -- a message + // consumed long after receipt may legitimately sit below + // `consumedAt - horizon`, and it is pruned on a later commit once + // the watermark passes ITS receivedAt). + const horizonBoundary = args.consumedAt - retentionHorizonMs; + const newWatermark = Math.max( + listing.watermark, + Math.min(horizonBoundary, envelope.receivedAt), + ); + advancedWatermark = newWatermark; + + // Prune the oldest consumed tail: read each retained consumed + // entry's receivedAt and drop any that has fallen strictly below + // the new watermark. This is the one leg that must scan the + // consumed index — its filenames carry only the messageId, so the + // receivedAt lives in the bytes — and is the residual the + // consumed-shard lever removes. The new entry (added via puts) is + // never below the watermark by construction. + const consumedDir = `${addressPrefix(addressSegment)}${WORKFLOW_RUN_CONSUMED_DIR}/`; + const deletes: string[] = [processingFull]; + for (const entry of listing.consumed) { + const blobPath = `${consumedDir}${entry.name}`; + const bytes = await readClaimCheckBlob(repoDir, entry.oid); + const consumedReceivedAt = decodeConsumedReceivedAtOrThrow( + bytes, + blobPath, ); - return files; - }, - message: `consume ${args.address} ${args.messageId}`, + if (consumedReceivedAt < newWatermark) { + prunedMessageIds.push(entry.name.slice(0, -".json".length)); + deletes.push(blobPath); + } + } + return { + puts: { + [consumedFull]: utf8(JSON.stringify(envelope)), + [watermarkPath(addressSegment)]: utf8( + JSON.stringify({ watermark: newWatermark }), + ), + }, + deletes, + }; }, - ); + }); if (consumedEnvelope === null) throw new Error("unreachable"); const captured: ConsumedEnvelope = consumedEnvelope; return { @@ -3215,43 +3210,39 @@ export async function replayProcessingToInbox( const ref = claimCheckCommitRef(); const repoDir = store.getRepoDir(repoId); const replayedKeys: string[] = []; - const { commitSha } = await store.writeTreePreservingPrefix( - principal, - repoId, - ref, - { - preservePrefix: `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/`, - merge: async () => { - const existing = await readAddressSubtree(repoDir, ref, addressSegment); - const processingPrefix = `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/${WORKFLOW_RUN_PROCESSING_DIR}/`; - const inboxPrefix = `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/${WORKFLOW_RUN_INBOX_DIR}/`; - const files: Record = {}; - for (const [blobPath, bytes] of existing) { - if (blobPath.startsWith(processingPrefix)) { - const fname = blobPath.slice(processingPrefix.length); - const key = fname.slice(0, -".json".length); - const inboxFull = `${inboxPrefix}${fname}`; - if (existing.has(inboxFull)) { - throw new Error( - `claim_check_replay_collision: ${inboxFull} already exists; cannot replay processing entry`, - ); - } - // Re-admit the in-flight entry WITHOUT the watermark - // stale-reject enqueueInbox applies: it was already past - // dedup, so a below-watermark receivedAt is no reason to - // refuse it. Applying the stale-check here would lose a - // legitimately in-flight message after a crash. Do not - // tighten this. - files[inboxFull] = bytes; - replayedKeys.push(key); - continue; - } - files[blobPath] = bytes; + const { commitSha } = await store.writeTreeDelta(principal, repoId, ref, { + changedPathPrefixes: new Set([addressPrefix(addressSegment)]), + message: `replay processing ${address}`, + computeDelta: async (parentCommitSha) => { + const listing = await readAddressListing( + repoDir, + parentCommitSha, + addressSegment, + ); + const processingDir = `${addressPrefix(addressSegment)}${WORKFLOW_RUN_PROCESSING_DIR}/`; + const inboxDir = `${addressPrefix(addressSegment)}${WORKFLOW_RUN_INBOX_DIR}/`; + const inboxNames = new Set(listing.inbox.map((e) => e.name)); + const puts: Record = {}; + const deletes: string[] = []; + for (const entry of listing.processing) { + const inboxFull = `${inboxDir}${entry.name}`; + if (inboxNames.has(entry.name)) { + throw new Error( + `claim_check_replay_collision: ${inboxFull} already exists; cannot replay processing entry`, + ); } - return files; - }, - message: `replay processing ${address}`, + // Re-admit the in-flight entry WITHOUT the watermark stale-reject + // enqueueInbox applies: it was already past dedup, so a + // below-watermark receivedAt is no reason to refuse it. Applying + // the stale-check here would lose a legitimately in-flight + // message after a crash. Do not tighten this. + const bytes = await readClaimCheckBlob(repoDir, entry.oid); + puts[inboxFull] = bytes; + deletes.push(`${processingDir}${entry.name}`); + replayedKeys.push(entry.name.slice(0, -".json".length)); + } + return { puts, deletes }; }, - ); + }); return { commitSha, replayedKeys }; } diff --git a/packages/hub-sessions/src/workflow-run-reader.test.ts b/packages/hub-sessions/src/workflow-run-reader.test.ts index 951bd596..fddc8026 100644 --- a/packages/hub-sessions/src/workflow-run-reader.test.ts +++ b/packages/hub-sessions/src/workflow-run-reader.test.ts @@ -20,6 +20,7 @@ function repoStoreFor(dirById: Map): RepoStore { initRepo: unused, writeTree: unused, writeTreePreservingPrefix: unused, + writeTreeDelta: unused, receivePack: unused, createPack: unused, resolveRef: unused, diff --git a/packages/workflow-host/src/child/proxy-repo-store.ts b/packages/workflow-host/src/child/proxy-repo-store.ts index 64c779e6..f2001b0e 100644 --- a/packages/workflow-host/src/child/proxy-repo-store.ts +++ b/packages/workflow-host/src/child/proxy-repo-store.ts @@ -41,6 +41,7 @@ type WriteTreeArgs = Parameters[3]; type WriteTreePreservingPrefixArgs = Parameters< RepoStore["writeTreePreservingPrefix"] >[3]; +type WriteTreeDeltaArgs = Parameters[3]; type SubscribeOpts = Parameters[3]; @@ -166,6 +167,16 @@ export function createProxyWorkflowRunRepoStore( "workflow-child proxy substrate: receivePack is not supported (writes are proxied to the supervisor)", ); }, + writeTreeDelta: ( + _principal: Principal, + _repoId: RepoId, + _ref: string, + _args: WriteTreeDeltaArgs, + ): Promise => { + throw new Error( + "workflow-child proxy substrate: writeTreeDelta is not supported (claim-check writes run supervisor-side)", + ); + }, async writeTreePreservingPrefix( _principal: Principal, repoId: RepoId, From 570b06146840ee319de43deeb0b9ca93060fa666 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Wed, 1 Jul 2026 09:46:23 -0500 Subject: [PATCH 008/101] Harden the delta write against ambiguous and out-of-scope input MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The delta primitive silently resolved contradictions the way assembleTree happened to: a put and a delete of the same path, or a put under a deleted subtree, both let the put win, and a delta path outside the declared change scope would let a scoped handler skip a region the write actually mutated. Both are silent-divergence risks on the exactly-once path even though the claim-check callers never produce them. Reject them loudly. writeTreeDelta now throws on a path that is both put and deleted, or a put that lands under a subtree-prefix delete, and throws when any put or delete falls outside changedPathPrefixes. It also pins the parent tip exactly once under the lock and threads that single oid into the assembly, so computeDelta's dedup reads, validation, and the committed tree provably share one pre-image rather than relying on two resolves returning the same value. A delete of an absent path is documented as an idempotent no-op, matching the working-tree removal. Extend the guards' tests: OID-equivalence now also covers a standalone exact-file delete and a cascade delete that empties a directory and its parent, plus the two ambiguity shapes and the out-of-scope rejection. Add the two claim-check rejections that lacked a behavior test — duplicate_inbox and replay_collision. --- packages/hub-sessions/src/repo-store/store.ts | 106 ++++++++++++++++-- .../src/repo-store/write-tree-delta.test.ts | 72 +++++++++++- .../src/workflow-run-kind.test.ts | 56 +++++++++ 3 files changed, 220 insertions(+), 14 deletions(-) diff --git a/packages/hub-sessions/src/repo-store/store.ts b/packages/hub-sessions/src/repo-store/store.ts index 8664db05..d27bcf70 100644 --- a/packages/hub-sessions/src/repo-store/store.ts +++ b/packages/hub-sessions/src/repo-store/store.ts @@ -420,6 +420,66 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { } } + // Reject an ambiguous delta rather than silently pick a winner: a path + // that is both put and deleted, or a put that lands under a + // subtree-prefix delete. assembleTree would let the put win in both + // cases; making the contradiction loud keeps a caller from committing a + // tree that does not match its stated intent. + function assertDeltaUnambiguous( + puts: Record, + deletes: readonly string[], + ): void { + const deleteSet = new Set(deletes); + const deletePrefixes = deletes.filter((d) => d.endsWith("/")); + for (const p of Object.keys(puts)) { + if (deleteSet.has(p)) { + throw new Error( + `delta_ambiguous: path ${JSON.stringify(p)} is in both puts and deletes`, + ); + } + for (const dp of deletePrefixes) { + if (p.startsWith(dp)) { + throw new Error( + `delta_ambiguous: put ${JSON.stringify(p)} lands under deleted subtree ${JSON.stringify(dp)}`, + ); + } + } + } + } + + // When a delta declares a change scope, every put and delete must fall + // under it — otherwise a handler that scopes validation to + // `changedPathPrefixes` would skip a region the write actually mutated, + // an exactly-once hole. An undefined scope means validate-all, so there + // is nothing to under-cover. + function assertDeltaScoped( + puts: Record, + deletes: readonly string[], + changedPathPrefixes: ReadonlySet | undefined, + ): void { + if (changedPathPrefixes === undefined) return; + const covered = (p: string): boolean => { + for (const prefix of changedPathPrefixes) { + if (p === prefix || p.startsWith(prefix)) return true; + } + return false; + }; + for (const p of Object.keys(puts)) { + if (!covered(p)) { + throw new Error( + `delta_out_of_scope: put ${JSON.stringify(p)} is not under any changedPathPrefix`, + ); + } + } + for (const d of deletes) { + if (!covered(d)) { + throw new Error( + `delta_out_of_scope: delete ${JSON.stringify(d)} is not under any changedPathPrefix`, + ); + } + } + } + function storageOptsFor( repoId: RepoId, opts: InitRepoOpts | undefined, @@ -889,7 +949,10 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { // paths of entries that exist as blobs, or an intended clear-prefix, // so neither name-vs-type case arises; a caller that builds deletes // more freely must not rely on per-type git-rm behavior here. A `put` - // at a path overrides a delete of the same path. + // at a path overrides a delete of the same path. A delete of a path + // absent from the parent tree is an idempotent no-op (the entry is + // simply never emitted), matching `git rm --ignore-unmatch` and the + // working-tree `rm` with `force: true`. // // Recursion is scoped to the touched subtrees: a level is read and // rewritten only when a put lands under it or a delete removes within @@ -1070,6 +1133,14 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { changedPathPrefixes: ReadonlySet | undefined; message: string; }, + // When present, the parent tip already resolved under this lock. The + // delta path pins it once and hands the SAME oid to both computeDelta + // (its dedup reads) and this assembly, so the dedup snapshot, the + // committed tree, and validation are provably one pre-image. Absent, + // the write resolves its own (the writeTree / preserving-prefix path, + // which has no prior read to keep consistent with). `{ sha }` wraps + // the value so a genuinely-null pin is distinct from "not provided". + pinnedParent?: { sha: string | null }, ): Promise { const dir = repoDir(repoId); await storageInitRepo(dir, storageOptsFor(repoId, undefined)); @@ -1092,7 +1163,10 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { // that does not yet exist has no base tree (the splice starts from // empty) and parents on HEAD, matching the prior index-reset path // which seeded an empty index for a missing ref. - const parentCommitSha = await resolveRefSha(dir, ref); + const parentCommitSha = + pinnedParent !== undefined + ? pinnedParent.sha + : await resolveRefSha(dir, ref); let baseRootTreeOid: string | null = null; if (parentCommitSha !== null) { const { commit } = await git.readCommit({ @@ -1372,19 +1446,29 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { return withRepoLock(repoId, async () => { const dir = repoDir(repoId); await storageInitRepo(dir, storageOptsFor(repoId, undefined)); - // Pin the parent tip under the lock and hand it to computeDelta so - // its dedup reads and the tree assembly below observe the same - // pre-image — no lost-update window between the read and the write. + // Pin the parent tip ONCE under the lock and hand the same oid to + // computeDelta's dedup reads and to the assembly below, so the + // pre-image the delta is computed against is exactly the pre-image + // it is committed against — no lost-update window, no second + // resolve that could (in principle) observe a different tip. const parentCommitSha = await resolveRefSha(dir, ref); const delta = await args.computeDelta(parentCommitSha); for (const p of Object.keys(delta.puts)) validateDeltaPath(p, false); for (const d of delta.deletes) validateDeltaPath(d, true); - return writeTreeUnderLock(principal, repoId, ref, { - files: delta.puts, - deletes: new Set(delta.deletes), - changedPathPrefixes: args.changedPathPrefixes, - message: args.message, - }); + assertDeltaUnambiguous(delta.puts, delta.deletes); + assertDeltaScoped(delta.puts, delta.deletes, args.changedPathPrefixes); + return writeTreeUnderLock( + principal, + repoId, + ref, + { + files: delta.puts, + deletes: new Set(delta.deletes), + changedPathPrefixes: args.changedPathPrefixes, + message: args.message, + }, + { sha: parentCommitSha }, + ); }); } diff --git a/packages/hub-sessions/src/repo-store/write-tree-delta.test.ts b/packages/hub-sessions/src/repo-store/write-tree-delta.test.ts index c450abbd..cd26116a 100644 --- a/packages/hub-sessions/src/repo-store/write-tree-delta.test.ts +++ b/packages/hub-sessions/src/repo-store/write-tree-delta.test.ts @@ -4,9 +4,12 @@ // the byte-identical tree that a full-replace writeTree of the same // logical final state produces. SHA-1 tree hashing is deterministic // given identical content and structure, so equal commit-tree oids prove -// the delta path never diverges from canonical git for the claim-check -// move shapes: enqueue-add, dequeue-move, markConsumed-move, -// empty->first-entry, and prune-to-empty. +// the delta path never diverges from canonical git across the move +// shapes: enqueue-add, dequeue-move (delete+put), markConsumed-move, +// empty->first-entry, prune-to-empty, a standalone exact-file-delete, and +// a cascade-delete that empties a directory and its parent. It also pins +// the ambiguity (put and delete of the same path, or a put under a +// deleted subtree) and out-of-scope guards that fail loud. import { test, expect, afterAll, beforeAll } from "bun:test"; import fs from "node:fs"; @@ -198,3 +201,66 @@ test("delta prune-to-empty matches full-replace", async () => { [`${SCOPE}consumed/m1.json`, `${SCOPE}consumed/m2.json`], ); }); + +test("delta exact-file-delete matches full-replace", async () => { + await assertDeltaMatchesFullReplace("exactdel", consumedBase, {}, [ + `${SCOPE}consumed/m1.json`, + ]); +}); + +test("delta cascade-delete (empties dir and parent) matches full-replace", async () => { + // The only entry under SCOPE is one inbox blob; deleting it empties + // inbox/, then addresses/a/, then addresses/, so all three subtrees + // must vanish exactly as a full clearPrefix of an emptied scope does. + await assertDeltaMatchesFullReplace( + "cascade", + { [`${SCOPE}inbox/10-m1.json`]: `{"messageId":"m1"}` }, + {}, + [`${SCOPE}inbox/10-m1.json`], + ); +}); + +test("writeTreeDelta rejects a path in both puts and deletes", async () => { + const store = await freshStore("wtd-ambig-both-"); + await store.initRepo(repoId); + await expect( + store.writeTreeDelta(principal, repoId, REF, { + changedPathPrefixes: new Set([SCOPE]), + message: "ambiguous", + computeDelta: async () => ({ + puts: { [`${SCOPE}inbox/x.json`]: "v" }, + deletes: [`${SCOPE}inbox/x.json`], + }), + }), + ).rejects.toThrow(/delta_ambiguous/); +}); + +test("writeTreeDelta rejects a put under a deleted subtree", async () => { + const store = await freshStore("wtd-ambig-under-"); + await store.initRepo(repoId); + await expect( + store.writeTreeDelta(principal, repoId, REF, { + changedPathPrefixes: new Set([SCOPE]), + message: "ambiguous", + computeDelta: async () => ({ + puts: { [`${SCOPE}inbox/x.json`]: "v" }, + deletes: [`${SCOPE}inbox/`], + }), + }), + ).rejects.toThrow(/delta_ambiguous/); +}); + +test("writeTreeDelta rejects a delta path outside changedPathPrefixes", async () => { + const store = await freshStore("wtd-scope-"); + await store.initRepo(repoId); + await expect( + store.writeTreeDelta(principal, repoId, REF, { + changedPathPrefixes: new Set([SCOPE]), + message: "out of scope", + computeDelta: async () => ({ + puts: { "addresses/b/inbox/x.json": "v" }, + deletes: [], + }), + }), + ).rejects.toThrow(/delta_out_of_scope/); +}); diff --git a/packages/hub-sessions/src/workflow-run-kind.test.ts b/packages/hub-sessions/src/workflow-run-kind.test.ts index 967eb19d..2f9b6d42 100644 --- a/packages/hub-sessions/src/workflow-run-kind.test.ts +++ b/packages/hub-sessions/src/workflow-run-kind.test.ts @@ -2134,6 +2134,25 @@ describe("claim-check API — enqueueInbox per-messageId atomicity in inbox", () const entries = await fs.promises.readdir(inboxDir); expect(entries.sort()).toEqual(["100-msg-X.json"]); }); + + test("rejects a duplicate enqueue for the same messageId at the same receivedAt", async () => { + const { store, repoId, principal } = + await makeClaimCheckStore("cc-enq-dup-key-"); + await enqueueInbox(store, principal, repoId, { + address: ADDRESS, + messageId: "msg-D", + receivedAt: 100, + mailAuditRef: { store: "audit", path: "mail/D" }, + }); + await expect( + enqueueInbox(store, principal, repoId, { + address: ADDRESS, + messageId: "msg-D", + receivedAt: 100, + mailAuditRef: { store: "audit", path: "mail/D" }, + }), + ).rejects.toThrow(/claim_check_duplicate_inbox/); + }); }); // Regression at the validatePush layer for the same intra-state @@ -2154,6 +2173,43 @@ describe("workflowRunKindHandler.validatePush — claim-check intra-state atomic }); }); +describe("claim-check API — replayProcessingToInbox collision guard", () => { + test("rejects when a processing entry collides with an existing inbox entry", async () => { + // The real handler's atomicity check forbids the same key in both + // inbox/ and processing/, so seed that (impossible-in-practice) state + // through a permissive handler and confirm replay refuses rather than + // clobbering the inbox entry. + const dataDir = await makeClaimCheckTempDir("cc-replay-collide-"); + const store = createRepoStore({ + dataDir, + signingKey: claimCheckSigningKey, + handlers: { + "workflow-run": { + kind: "workflow-run", + directoryPrefix: "workflow-runs", + validatePush: () => ({ ok: true }), + onRefUpdated: () => undefined, + }, + }, + authorize: () => ({ allowed: true }), + }); + const repoId: RepoId = { kind: "workflow-run", id: "dep-replaycollide" }; + await store.initRepo(repoId); + const body = inboxBody("msg-1", 100); + await store.writeTree(HUB_PRINCIPAL, repoId, "refs/heads/events", { + files: { + [WORKFLOW_RUN_GITIGNORE_PATH]: "", + [inboxPathFor(ADDRESS_SEG, 100, "msg-1")]: body, + [processingPathFor(ADDRESS_SEG, 100, "msg-1")]: body, + }, + message: "seed colliding inbox+processing state", + }); + await expect( + replayProcessingToInbox(store, HUB_PRINCIPAL, repoId, ADDRESS), + ).rejects.toThrow(/claim_check_replay_collision/); + }); +}); + // Regression for per-commit-walk pack validation. A single pack // carrying [enqueue, dequeue] commits — produced by the supervisor's // first-mail bootstrap — must validate cleanly against a fresh target From ba1227191b300389b9f5de9df123eb9a561c0f50 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Wed, 1 Jul 2026 09:57:07 -0500 Subject: [PATCH 009/101] Surface prospective consumed OIDs from the tree listing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The delta claim-check write is O(delta), but under the delta-scope validation flag validateClaimCheckSubtree still computed each PROSPECTIVE consumed entry's OID by reading and hashing its blob, so the per-leg validation stayed O(consumed) even though the prior side already took its OIDs straight from the tree listing. buildTreeReadClosures only exposed readBlob/listDir for the prospective side — there was no prospective counterpart to priorListDirOids. Add listDirOids to buildTreeReadClosures: the assembled prospective tree's readTree listing already carries each entry's OID, so surface it exactly as the prior side does. Thread it through validatePush and route the prospective consumed-OID resolver through the listing, falling back to hashing only when a caller does not supply it (a hand-built validatePush in a test). The prior and prospective resolvers now share one listing-or-hash helper. An OID read from the listing is the same content-addressed id hashing the bytes would produce, so validation is unchanged; it just stops re-reading the retained consumed set, making the delta-scoped enqueue and dequeue legs O(delta) on validation too. --- packages/hub-sessions/src/repo-store/store.ts | 26 +++++++- packages/hub-sessions/src/repo-store/types.ts | 13 +++- .../hub-sessions/src/workflow-run-kind.ts | 65 ++++++++++++------- 3 files changed, 76 insertions(+), 28 deletions(-) diff --git a/packages/hub-sessions/src/repo-store/store.ts b/packages/hub-sessions/src/repo-store/store.ts index d27bcf70..2587591a 100644 --- a/packages/hub-sessions/src/repo-store/store.ts +++ b/packages/hub-sessions/src/repo-store/store.ts @@ -1076,6 +1076,7 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { topLevelTreePaths: () => Promise; readBlob: (relPath: string) => Promise; listDir: (relPath: string) => Promise; + listDirOids: (relPath: string) => Promise<{ name: string; oid: string }[]>; } { const topLevelTreePaths = async (): Promise => { const { tree } = await git.readTree({ @@ -1115,7 +1116,29 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { }); return tree.map((e) => e.path); }; - return { topLevelTreePaths, readBlob, listDir }; + // Same walk as `listDir` but carries each child's git object id out of + // the assembled tree's listing, mirroring the prior side's + // `priorListDirOids`. A handler validating a large retained subtree by + // its per-commit delta (workflow-run's consumed dedup index) reads the + // prospective OID straight from here instead of re-reading and hashing + // every retained blob. + const listDirOids = async ( + relPath: string, + ): Promise<{ name: string; oid: string }[]> => { + const oid = + relPath === "" + ? rootTreeOid + : await resolveTreeOid(dir, rootTreeOid, relPath, "tree"); + if (oid === null) return []; + const { tree } = await git.readTree({ + fs, + dir, + cache: cacheFor(dir), + oid, + }); + return tree.map((e) => ({ name: e.path, oid: e.oid })); + }; + return { topLevelTreePaths, readBlob, listDir, listDirOids }; } // Unlocked body of writeTree. The caller is responsible for @@ -1216,6 +1239,7 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { topLevelTreePaths: await prospective.topLevelTreePaths(), readBlob: prospective.readBlob, listDir: prospective.listDir, + listDirOids: prospective.listDirOids, priorReadBlob, priorListDir, priorListDirOids, diff --git a/packages/hub-sessions/src/repo-store/types.ts b/packages/hub-sessions/src/repo-store/types.ts index 11e93a99..4b690ed5 100644 --- a/packages/hub-sessions/src/repo-store/types.ts +++ b/packages/hub-sessions/src/repo-store/types.ts @@ -270,9 +270,15 @@ export interface KindHandler { * already carries the OID). A handler that validates a large retained * subtree by its per-commit delta uses it to prove a retained entry is * byte-unchanged by OID equality instead of re-reading the blob. It is - * `undefined` when no prior commit exists; a handler that needs OIDs - * for a prospective-side compare hashes the (usually in-memory) - * prospective bytes itself. + * `undefined` when no prior commit exists. + * + * `listDirOids` is the prospective-side mirror of `priorListDirOids`: + * each child entry's OID read straight from the prospective tree's + * listing, so a handler comparing a retained subtree by OID gets the + * prospective OID without re-reading and hashing every entry's bytes. + * It is `undefined` on paths that do not surface it (a hand-built + * validatePush in a test), in which case the handler falls back to + * hashing the prospective bytes. */ validatePush: (args: { repoId: RepoId; @@ -281,6 +287,7 @@ export interface KindHandler { topLevelTreePaths: string[]; readBlob: (path: string) => Promise; listDir: (path: string) => Promise; + listDirOids?: (path: string) => Promise<{ name: string; oid: string }[]>; priorReadBlob: (path: string) => Promise; priorListDir: (path: string) => Promise; priorListDirOids?: ( diff --git a/packages/hub-sessions/src/workflow-run-kind.ts b/packages/hub-sessions/src/workflow-run-kind.ts index 2ca8737d..e6e2ba8d 100644 --- a/packages/hub-sessions/src/workflow-run-kind.ts +++ b/packages/hub-sessions/src/workflow-run-kind.ts @@ -1435,32 +1435,32 @@ async function hashConsumedBlobOid(bytes: Uint8Array): Promise { } /** - * Build the resolver that surfaces each PRIOR consumed entry's git blob - * OID for the delta-scoped path. When the substrate supplies - * `priorListDirOids` the OID comes straight from the prior commit's tree - * listing (one `readTree` per consumed directory, cached), so the prior - * side is not re-read blob-by-blob. When it is absent (a hand-built - * validatePush in a unit test) the resolver falls back to hashing the - * prior bytes via `priorReadBlob`; that fallback is O(retained) rather - * than free but preserves identical semantics, which is all a - * correctness test needs. + * Resolve each consumed entry's git blob OID for the delta-scoped path. + * When the substrate supplies a directory OID listing (`listDirOids`) the + * OID comes straight from the tree — one `readTree` per consumed + * directory, cached — so that side is not re-read blob-by-blob. When the + * listing is absent (a hand-built validatePush in a unit test) each OID + * falls back to hashing the entry's bytes, which preserves identical + * semantics at O(retained) cost. Both the prior and prospective sides use + * this; `sideLabel` distinguishes them in the missing-OID error. */ -function makePriorConsumedOidResolver( - priorReadBlob: (path: string) => Promise, - priorListDirOids: +function makeListingOidResolver( + sideLabel: string, + listDirOids: | ((path: string) => Promise<{ name: string; oid: string }[]>) | undefined, + hashFallback: (blobPath: string) => Promise, ): (blobPath: string) => Promise { const dirOidCache = new Map>(); return async (blobPath) => { - if (priorListDirOids !== undefined) { + if (listDirOids !== undefined) { const slash = blobPath.lastIndexOf("/"); const dir = blobPath.slice(0, slash); const name = blobPath.slice(slash + 1); let byName = dirOidCache.get(dir); if (byName === undefined) { byName = new Map(); - for (const entry of await priorListDirOids(dir)) { + for (const entry of await listDirOids(dir)) { byName.set(entry.name, entry.oid); } dirOidCache.set(dir, byName); @@ -1468,11 +1468,22 @@ function makePriorConsumedOidResolver( const oid = byName.get(name); if (oid === undefined) { throw new Error( - `delta claim-check: prior tree listing has no OID for enumerated consumed entry ${blobPath}`, + `delta claim-check: ${sideLabel} tree listing has no OID for enumerated consumed entry ${blobPath}`, ); } return oid; } + return hashFallback(blobPath); + }; +} + +function makePriorConsumedOidResolver( + priorReadBlob: (path: string) => Promise, + priorListDirOids: + | ((path: string) => Promise<{ name: string; oid: string }[]>) + | undefined, +): (blobPath: string) => Promise { + return makeListingOidResolver("prior", priorListDirOids, async (blobPath) => { const bytes = await priorReadBlob(blobPath); if (bytes === null) { throw new Error( @@ -1480,7 +1491,7 @@ function makePriorConsumedOidResolver( ); } return hashConsumedBlobOid(bytes); - }; + }); } /** @@ -1496,9 +1507,10 @@ function makePriorConsumedOidResolver( * by re-walking the whole retained set: retained entries (same filename, * same blob OID) are skipped as already-validated-and-immutable, added * entries are parsed and validated, and removed entries are checked - * against the retention watermark. `priorListDirOids`, when supplied by - * the substrate, surfaces prior consumed OIDs straight from the prior - * commit's tree so the prior side is not re-read blob-by-blob. + * against the retention watermark. `priorListDirOids` and `listDirOids`, + * when supplied by the substrate, surface the prior and prospective + * consumed OIDs straight from their tree listings so neither side is + * re-read blob-by-blob. */ async function validateClaimCheckSubtree( listDir: (path: string) => Promise, @@ -1506,14 +1518,17 @@ async function validateClaimCheckSubtree( priorReadBlob: (path: string) => Promise, priorListDir: (path: string) => Promise, priorListDirOids?: (path: string) => Promise<{ name: string; oid: string }[]>, + listDirOids?: (path: string) => Promise<{ name: string; oid: string }[]>, ): Promise { // When the delta path is on, surface each consumed entry's git blob - // OID during enumeration: prospective OIDs by hashing the (in the - // measured writeTree path, in-memory) prospective bytes, prior OIDs - // from the substrate tree listing when available. The resolvers are - // omitted entirely on the exhaustive path so it stays byte-identical. + // OID during enumeration straight from the tree listing on both sides + // when the substrate provides it, falling back to hashing the bytes + // otherwise. The resolvers are omitted entirely on the exhaustive path + // so it stays byte-identical. const prospectiveConsumedOid = BENCH_DELTA_SCOPE_CLAIMCHECK - ? async (blobPath: string) => hashConsumedBlobOid(await readBlob(blobPath)) + ? makeListingOidResolver("prospective", listDirOids, async (blobPath) => + hashConsumedBlobOid(await readBlob(blobPath)), + ) : undefined; const priorConsumedOid = BENCH_DELTA_SCOPE_CLAIMCHECK ? makePriorConsumedOidResolver(priorReadBlob, priorListDirOids) @@ -2054,6 +2069,7 @@ export const workflowRunKindHandler: KindHandler = { topLevelTreePaths, readBlob, listDir, + listDirOids, priorReadBlob, priorListDir, priorListDirOids, @@ -2126,6 +2142,7 @@ export const workflowRunKindHandler: KindHandler = { priorReadBlob, priorListDir, priorListDirOids, + listDirOids, ); if (!claimCheck.ok) { logger.debug`workflow-run validatePush rejected ${repoId.kind}/${repoId.id} on ${ref}: ${claimCheck.reason}`; From f72a13b78643c3c4ced47b1eb6f441db1d008557 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Wed, 1 Jul 2026 10:12:52 -0500 Subject: [PATCH 010/101] Close the delta-write review follow-ups Reviewer should-fixes for the claim-check delta work, none changing behavior on the exercised paths: - assembleTree now throws tree_name_collision when a name is written both as a file and as a directory (a put/base `foo` plus a put or delete under `foo/`). The blob-put branch would otherwise win and drop the subtree side silently; this covers the plain writeTree and writeTreePreservingPrefix paths too, not just the delta guard. - Document the latent mode downgrade: puts are written as 100644, so a put overwriting a non-100644 base entry would lose its mode. No caller writes anything else today. - Fix the stale readProcessingEntry comment that described the old whole-working-tree reset; the delta write materializes only the touched paths. Tests: - Add the last untested claim-check rejection, invalid_inbox_filename, via a malformed inbox entry seeded through a permissive handler. - Adopt the reviewer's independent-oracle delta test: it compares writeTreeDelta's committed tree against isomorphic-git's index-based git.add + git.commit, an oracle that does not run through assembleTree (the shipped write-tree-delta.test.ts full-replace oracle does, so a shared bug would pass both sides). - Adopt the shared-cache repack-transparency test into storage-isogit: an object migrating out of a GC-pruned pack still reads under the same warm cache, and a genuinely-dropped object surfaces NotFound, not stale bytes. The delta-adversarial test references store.writeTreeDelta, which type- checks because that method is already on the RepoStore interface. --- .../src/repo-store/delta-adversarial.test.ts | 247 ++++++++++++++++++ packages/hub-sessions/src/repo-store/store.ts | 25 +- .../src/workflow-run-kind.test.ts | 35 +++ .../hub-sessions/src/workflow-run-kind.ts | 12 +- .../src/gc-cache-transparency.test.ts | 193 ++++++++++++++ 5 files changed, 505 insertions(+), 7 deletions(-) create mode 100644 packages/hub-sessions/src/repo-store/delta-adversarial.test.ts create mode 100644 packages/storage-isogit/src/gc-cache-transparency.test.ts diff --git a/packages/hub-sessions/src/repo-store/delta-adversarial.test.ts b/packages/hub-sessions/src/repo-store/delta-adversarial.test.ts new file mode 100644 index 00000000..fb206547 --- /dev/null +++ b/packages/hub-sessions/src/repo-store/delta-adversarial.test.ts @@ -0,0 +1,247 @@ +// Adversarial correctness tests for writeTreeDelta / assembleTree. The +// oracle here is isomorphic-git's INDEX-based writeTree (git.add + +// git.commit), which is genuinely independent of assembleTree — unlike +// write-tree-delta.test.ts, whose full-replace comparison also runs +// through assembleTree, so a shared bug would pass both sides. + +import { test, expect, afterAll, beforeAll } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import git from "isomorphic-git"; +import { generateKeyPair } from "@intx/crypto"; +import type { KeyPair } from "@intx/types/runtime"; +import { createRepoStore } from "./store"; +import type { AuthorizeFn, Principal, RepoId } from "./types"; + +const tempDirs: string[] = []; +let signingKey: KeyPair; + +beforeAll(async () => { + signingKey = await generateKeyPair(); +}); + +afterAll(async () => { + for (const d of tempDirs.splice(0)) { + await fs.promises + .rm(d, { recursive: true, force: true }) + .catch(() => undefined); + } +}); + +const allowAll: AuthorizeFn = () => ({ allowed: true }); +const principal: Principal = { kind: "test" }; +const REF = "refs/heads/events"; +const repoId: RepoId = { kind: "agent-state", id: "subject" }; + +function makeStore(dataDir: string) { + return createRepoStore({ + dataDir, + signingKey, + handlers: { + "agent-state": { + kind: "agent-state", + directoryPrefix: "repos-under-test", + validatePush: () => ({ ok: true }), + onRefUpdated: () => undefined, + }, + }, + authorize: allowAll, + }); +} + +async function freshStore(label: string) { + const dataDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), label)); + tempDirs.push(dataDir); + return makeStore(dataDir); +} + +async function committedTreeOid( + store: ReturnType, +): Promise { + const dir = store.getRepoDir(repoId); + const sha = await git.resolveRef({ fs, dir, ref: REF }); + const { commit } = await git.readCommit({ fs, dir, oid: sha }); + return commit.tree; +} + +// Independent oracle: materialize `files` in a scratch repo, stage each +// via git.add, and let isomorphic-git's index build the tree. This does +// not touch assembleTree at all. +async function canonicalTreeOid( + label: string, + files: Record, +): Promise { + const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), label)); + tempDirs.push(dir); + await git.init({ fs, dir, defaultBranch: "main" }); + for (const [rel, content] of Object.entries(files)) { + const full = path.join(dir, rel); + await fs.promises.mkdir(path.dirname(full), { recursive: true }); + await fs.promises.writeFile(full, content); + await git.add({ fs, dir, filepath: rel }); + } + const sha = await git.commit({ + fs, + dir, + message: "oracle", + author: { name: "o", email: "o@o" }, + }); + const { commit } = await git.readCommit({ fs, dir, oid: sha }); + return commit.tree; +} + +async function readTreePaths( + store: ReturnType, +): Promise> { + const dir = store.getRepoDir(repoId); + const sha = await git.resolveRef({ fs, dir, ref: REF }); + const { commit } = await git.readCommit({ fs, dir, oid: sha }); + const out = new Set(); + const walk = async (oid: string, prefix: string): Promise => { + const { tree } = await git.readTree({ fs, dir, oid }); + for (const e of tree) { + const p = prefix === "" ? e.path : `${prefix}/${e.path}`; + if (e.type === "tree") await walk(e.oid, p); + else out.add(p); + } + }; + await walk(commit.tree, ""); + return out; +} + +async function seed( + store: ReturnType, + base: Record, +) { + await store.initRepo(repoId); + await store.writeTree(principal, repoId, REF, { + files: base, + message: "seed", + }); +} + +test("dequeue-shaped delta removes inbox entry and adds processing, OID == canonical git", async () => { + const A = "addresses/a/"; + const base = { + [`${A}inbox/10-m3.json`]: `{"messageId":"m3"}`, + [`${A}consumed/m1.json`]: `{"receivedAt":1}`, + [`${A}consumed/m2.json`]: `{"receivedAt":2}`, + [`${A}watermark.json`]: `{"watermark":0}`, + }; + const store = await freshStore("adv-dequeue-"); + await seed(store, base); + await store.writeTreeDelta(principal, repoId, REF, { + changedPathPrefixes: new Set([A]), + message: "dequeue", + computeDelta: async () => ({ + puts: { [`${A}processing/10-m3.json`]: `{"messageId":"m3"}` }, + deletes: [`${A}inbox/10-m3.json`], + }), + }); + const paths = await readTreePaths(store); + expect(paths.has(`${A}inbox/10-m3.json`)).toBe(false); // removed, not carried + expect(paths.has(`${A}processing/10-m3.json`)).toBe(true); + expect(paths.has(`${A}consumed/m1.json`)).toBe(true); + expect(paths.has(`${A}consumed/m2.json`)).toBe(true); + + const expected = { + [`${A}processing/10-m3.json`]: `{"messageId":"m3"}`, + [`${A}consumed/m1.json`]: `{"receivedAt":1}`, + [`${A}consumed/m2.json`]: `{"receivedAt":2}`, + [`${A}watermark.json`]: `{"watermark":0}`, + }; + expect(await committedTreeOid(store)).toBe( + await canonicalTreeOid("adv-dequeue-oracle-", expected), + ); +}); + +test("cascade delete empties processing/ and the whole address subtree, OID == canonical git", async () => { + const A = "addresses/lonely/"; + const other = "addresses/keep/inbox/9-k1.json"; + const base = { + [`${A}processing/10-m3.json`]: `{"messageId":"m3"}`, + [other]: `{"messageId":"k1"}`, + }; + const store = await freshStore("adv-cascade-"); + await seed(store, base); + await store.writeTreeDelta(principal, repoId, REF, { + changedPathPrefixes: new Set([A]), + message: "prune-processing", + computeDelta: async () => ({ + puts: {}, + deletes: [`${A}processing/10-m3.json`], + }), + }); + const paths = await readTreePaths(store); + expect([...paths].some((p) => p.startsWith(A))).toBe(false); + expect(paths.has(other)).toBe(true); + + const expected = { [other]: `{"messageId":"k1"}` }; + expect(await committedTreeOid(store)).toBe( + await canonicalTreeOid("adv-cascade-oracle-", expected), + ); +}); + +test("put and delete at same exact path is rejected (delta_ambiguous)", async () => { + const A = "addresses/a/"; + const base = { + [`${A}inbox/10-m3.json`]: `{"old":true}`, + [`${A}consumed/m1.json`]: `{"receivedAt":1}`, + }; + const store = await freshStore("adv-h2a-"); + await seed(store, base); + await expect( + store.writeTreeDelta(principal, repoId, REF, { + changedPathPrefixes: new Set([A]), + message: "collide", + computeDelta: async () => ({ + puts: { [`${A}inbox/10-m3.json`]: `{"new":true}` }, + deletes: [`${A}inbox/10-m3.json`], + }), + }), + ).rejects.toThrow(/delta_ambiguous/); +}); + +test("put under a deleted subtree is rejected (delta_ambiguous)", async () => { + const A = "addresses/a/"; + const base = { + [`${A}inbox/10-m3.json`]: `{"old":true}`, + [`${A}inbox/11-m4.json`]: `{"sibling":true}`, + [`${A}consumed/m1.json`]: `{"receivedAt":1}`, + }; + const store = await freshStore("adv-h2b-"); + await seed(store, base); + await expect( + store.writeTreeDelta(principal, repoId, REF, { + changedPathPrefixes: new Set([A]), + message: "delete-subtree-with-put", + computeDelta: async () => ({ + puts: { [`${A}inbox/99-m9.json`]: `{"new":true}` }, + deletes: [`${A}inbox/`], + }), + }), + ).rejects.toThrow(/delta_ambiguous/); +}); + +test("cascade prune emptying dir and parent == canonical git", async () => { + const A = "addresses/solo/"; + const base = { + [`${A}consumed/m1.json`]: `{"receivedAt":1}`, + "addresses/other/consumed/m9.json": `{"receivedAt":9}`, + }; + const store = await freshStore("adv-cascade2-"); + await seed(store, base); + await store.writeTreeDelta(principal, repoId, REF, { + changedPathPrefixes: new Set([A]), + message: "prune-all", + computeDelta: async () => ({ + puts: {}, + deletes: [`${A}consumed/m1.json`], + }), + }); + const expected = { "addresses/other/consumed/m9.json": `{"receivedAt":9}` }; + expect(await committedTreeOid(store)).toBe( + await canonicalTreeOid("adv-cascade2-oracle-", expected), + ); +}); diff --git a/packages/hub-sessions/src/repo-store/store.ts b/packages/hub-sessions/src/repo-store/store.ts index 2587591a..1de5bd28 100644 --- a/packages/hub-sessions/src/repo-store/store.ts +++ b/packages/hub-sessions/src/repo-store/store.ts @@ -1010,6 +1010,23 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { else subtreeNames.add(rest.slice(0, slash)); } + // A name written both as a direct blob and as a directory (a put/base + // `foo` plus a put or delete under `foo/`) is contradictory. Left + // implicit, the blob-put branch below wins and the subtree side is + // silently dropped; reject it loudly instead. This covers every + // caller — writeTree's `content.files`, writeTreePreservingPrefix's + // merge output, and writeTreeDelta's puts/deletes — since all funnel + // into this same classification. + for (const name of blobPutsHere) { + if (subtreeNames.has(name)) { + throw new Error( + `tree_name_collision: ${JSON.stringify( + prefix + name, + )} is written both as a file and as a directory`, + ); + } + } + const names = new Set([ ...baseEntries.keys(), ...blobPutsHere, @@ -1021,7 +1038,13 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { const putOid = puts.get(full); if (putOid !== undefined) { // A put overrides whatever the base held and any delete of the - // same path. + // same path. Puts are always regular files (mode 100644). If a + // put ever overwrites a base entry with a different mode (an + // executable 100755 blob, a symlink 120000, or a submodule), this + // downgrades it to 100644 — the store only ever writes 100644 + // content blobs today, so no caller hits it, but a future caller + // that needs to preserve an executable bit would have to carry + // the mode through `puts` rather than assume 100644. entries.push({ mode: "100644", path: name, diff --git a/packages/hub-sessions/src/workflow-run-kind.test.ts b/packages/hub-sessions/src/workflow-run-kind.test.ts index 2f9b6d42..b01a93c7 100644 --- a/packages/hub-sessions/src/workflow-run-kind.test.ts +++ b/packages/hub-sessions/src/workflow-run-kind.test.ts @@ -2210,6 +2210,41 @@ describe("claim-check API — replayProcessingToInbox collision guard", () => { }); }); +describe("claim-check API — dequeueToProcessing filename guard", () => { + test("rejects an inbox entry whose filename is not -.json", async () => { + // Seed a malformed inbox filename via a permissive handler (the real + // handler's shape check would reject it), then confirm the FIFO + // dequeue refuses it rather than mis-parsing. + const dataDir = await makeClaimCheckTempDir("cc-bad-inbox-fname-"); + const store = createRepoStore({ + dataDir, + signingKey: claimCheckSigningKey, + handlers: { + "workflow-run": { + kind: "workflow-run", + directoryPrefix: "workflow-runs", + validatePush: () => ({ ok: true }), + onRefUpdated: () => undefined, + }, + }, + authorize: () => ({ allowed: true }), + }); + const repoId: RepoId = { kind: "workflow-run", id: "dep-badinboxfname" }; + await store.initRepo(repoId); + await store.writeTree(HUB_PRINCIPAL, repoId, "refs/heads/events", { + files: { + [WORKFLOW_RUN_GITIGNORE_PATH]: "", + [`${WORKFLOW_RUN_ADDRESSES_PREFIX}/${ADDRESS_SEG}/${WORKFLOW_RUN_INBOX_DIR}/bad.json`]: + "{}", + }, + message: "seed malformed inbox filename", + }); + await expect( + dequeueToProcessing(store, HUB_PRINCIPAL, repoId, ADDRESS), + ).rejects.toThrow(/claim_check_invalid_inbox_filename/); + }); +}); + // Regression for per-commit-walk pack validation. A single pack // carrying [enqueue, dequeue] commits — produced by the supervisor's // first-mail bootstrap — must validate cleanly against a fresh target diff --git a/packages/hub-sessions/src/workflow-run-kind.ts b/packages/hub-sessions/src/workflow-run-kind.ts index e6e2ba8d..509300d1 100644 --- a/packages/hub-sessions/src/workflow-run-kind.ts +++ b/packages/hub-sessions/src/workflow-run-kind.ts @@ -2981,12 +2981,12 @@ export type ReadProcessingEntryResult = { * input. * * The read is a flat working-tree read of - * `addresses//processing/`. The substrate materializes every - * claim-check commit into the repo's working tree (each - * `writeTreePreservingPrefix` resets the working tree to the ref tip), - * so a read issued after `dequeueToProcessing` committed -- which is - * exactly when the supervisor forwards `trigger.fired` -- observes the - * processing entry. Reading the working tree (rather than walking the + * `addresses//processing/`. The substrate materializes each + * claim-check commit's touched paths into the repo's working tree (the + * delta write removes each deleted path and writes each put after + * validation passes), so a read issued after `dequeueToProcessing` + * committed -- which is exactly when the supervisor forwards + * `trigger.fired` -- observes the processing entry. Reading the working tree (rather than walking the * committed git tree) matches the workflow-process child's sibling * reads of `workflow.json` and `runs//events/`. Because the * read issues no commit it cannot race the supervisor's `markConsumed` diff --git a/packages/storage-isogit/src/gc-cache-transparency.test.ts b/packages/storage-isogit/src/gc-cache-transparency.test.ts new file mode 100644 index 00000000..321f2673 --- /dev/null +++ b/packages/storage-isogit/src/gc-cache-transparency.test.ts @@ -0,0 +1,193 @@ +// The store threads one long-lived isomorphic-git `cache` object per repo +// through every git.* call. GC repacks the object store — consolidating +// loose objects into a fresh, differently-named pack and pruning the old +// one — behind that cache's back. These tests pin that the cache stays a +// pure accelerator across a repack: an object that migrates from a pruned +// pack into the new one still reads correctly under the SAME warm cache +// (because reads enumerate .idx files from disk, so a stranded parse of a +// pruned pack is never consulted), and an object GC genuinely drops +// surfaces NotFound rather than stale bytes. + +import { describe, test, expect, afterEach } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import git from "isomorphic-git"; +import { initAgentRepo } from "./init"; +import { runGC } from "./gc"; + +const author = { name: "Test", email: "test@test.dev" }; +const tempDirs: string[] = []; + +async function tempDir(): Promise { + const d = await fs.promises.mkdtemp(path.join(os.tmpdir(), "gc-cache-")); + tempDirs.push(d); + return d; +} + +afterEach(async () => { + const dirs = tempDirs.splice(0); + await Promise.all( + dirs.map((d) => fs.promises.rm(d, { recursive: true, force: true })), + ); +}); + +function listIdx(dir: string): string[] { + const p = path.join(dir, ".git", "objects", "pack"); + if (!fs.existsSync(p)) return []; + return fs.readdirSync(p).filter((x) => x.endsWith(".idx")); +} + +function countLoose(dir: string): number { + const base = path.join(dir, ".git", "objects"); + if (!fs.existsSync(base)) return 0; + let n = 0; + for (const d of fs.readdirSync(base)) { + if (!/^[0-9a-f]{2}$/.test(d)) continue; + n += fs.readdirSync(path.join(base, d)).length; + } + return n; +} + +async function commitFile( + dir: string, + name: string, + content: string, +): Promise { + const full = path.join(dir, name); + await fs.promises.mkdir(path.dirname(full), { recursive: true }); + await fs.promises.writeFile(full, content); + await git.add({ fs, dir, filepath: name }); + return git.commit({ fs, dir, message: `add ${name}`, author }); +} + +async function commitFileOnRef( + dir: string, + branch: string, + name: string, + content: string, +): Promise { + await git.checkout({ fs, dir, ref: branch }); + const oid = await commitFile(dir, name, content); + await git.checkout({ fs, dir, ref: "main" }); + return oid; +} + +function packfileCacheKeys(cache: object): string[] { + const packSym = Object.getOwnPropertySymbols(cache).find( + (s) => s.toString() === "Symbol(PackfileCache)", + ); + if (packSym === undefined) return []; + const map: unknown = Reflect.get(cache, packSym); + if (!(map instanceof Map)) return []; + const keys: string[] = []; + for (const k of map.keys()) { + if (typeof k === "string") keys.push(k); + } + return keys; +} + +describe("shared-cache repack transparency", () => { + test("object in a pre-GC pack reads correctly after GC prunes that pack, under the SAME warm cache", async () => { + const dir = await tempDir(); + await initAgentRepo(dir); + + const targetContent = "the-target-blob-payload-v1"; + let tip = await commitFile(dir, "state/target.txt", targetContent); + const targetOid = await git.writeBlob({ + fs, + dir, + blob: Buffer.from(targetContent), + }); + for (let i = 0; i < 5; i++) { + tip = await commitFile(dir, `state/f${i}.txt`, `payload ${i}`); + } + + // First GC consolidates all loose objects into pack #1 and prunes the + // loose copies. The target now lives only in pack #1. + await runGC(dir, { retention: "keep-history" }); + const idxBefore = listIdx(dir); + const firstIdx = idxBefore[0]; + if (firstIdx === undefined) throw new Error("expected a pack .idx"); + expect(countLoose(dir)).toBe(0); + + // One shared, long-lived cache object — the exact shape the store + // threads through every git.* call. + const cache: object = {}; + + // Warm the cache: read the target from the pre-GC pack, populating + // isomorphic-git's PackfileCache keyed by the pre-GC .idx filename. + const warm = await git.readBlob({ fs, dir, oid: targetOid, cache }); + expect(Buffer.from(warm.blob).toString()).toBe(targetContent); + expect(packfileCacheKeys(cache).some((k) => k.includes(firstIdx))).toBe( + true, + ); + + // More commits, then a second GC: repacks into a new pack #2 that + // supersedes and prunes pack #1. The target migrates to pack #2. + for (let i = 0; i < 4; i++) { + tip = await commitFile(dir, `state/g${i}.txt`, `more ${i}`); + } + await runGC(dir, { retention: "keep-history" }); + + const idxAfter = listIdx(dir); + // The pre-GC pack is gone from disk, but its parsed index is still + // stranded in the warm cache map. + expect(idxAfter).not.toContain(firstIdx); + expect(packfileCacheKeys(cache).some((k) => k.includes(firstIdx))).toBe( + true, + ); + + // The test: read the target again under the SAME warm cache, after the + // pack the cache indexed was pruned from disk. + const reread = await git.readBlob({ fs, dir, oid: targetOid, cache }); + expect(Buffer.from(reread.blob).toString()).toBe(targetContent); + + // Exercise the readTree/readCommit path through the same cache too. + const co = await git.readCommit({ fs, dir, oid: tip, cache }); + const tr = await git.readTree({ fs, dir, oid: co.commit.tree, cache }); + expect(tr.tree.length).toBeGreaterThan(0); + }); + + test("GC that drops a now-unreachable object surfaces NotFound, not stale bytes, under a warm cache", async () => { + const dir = await tempDir(); + await initAgentRepo(dir); + + for (let i = 0; i < 3; i++) { + await commitFile(dir, `state/f${i}.txt`, `keep ${i}`); + } + + // A blob reachable only from a secondary ref. + const orphanContent = "orphan-only-in-old-pack"; + const orphanOid = await git.writeBlob({ + fs, + dir, + blob: Buffer.from(orphanContent), + }); + await git.branch({ fs, dir, ref: "scratch" }); + await commitFileOnRef(dir, "scratch", "state/orphan.txt", orphanContent); + + // Pack everything reachable into pack #1. + await runGC(dir, { retention: "keep-history" }); + expect(countLoose(dir)).toBe(0); + + const cache: object = {}; + const warm = await git.readBlob({ fs, dir, oid: orphanOid, cache }); + expect(Buffer.from(warm.blob).toString()).toBe(orphanContent); + + // Drop the scratch ref so the orphan is unreachable, then GC. + await git.deleteBranch({ fs, dir, ref: "scratch" }); + await commitFile(dir, "state/f9.txt", "keep 9"); + await runGC(dir, { retention: "keep-history" }); + + // The object is genuinely gone — a NotFound throw, never stale bytes. + let threw = false; + try { + const r = await git.readBlob({ fs, dir, oid: orphanOid, cache }); + expect(Buffer.from(r.blob).toString()).not.toBe(orphanContent); + } catch { + threw = true; + } + expect(threw).toBe(true); + }); +}); From 1f65cf150dfce3c2a435e2d7ca91694eb6619460 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Wed, 1 Jul 2026 13:27:06 -0500 Subject: [PATCH 011/101] Reject delete shapes that mismatch the base entry type A no-slash delete naming a directory, or a trailing-slash delete descending into a base blob, previously dropped the mismatched base entry silently. Reject both loudly with delete_type_mismatch so a caller cannot commit a tree that contradicts its stated delete intent. A put that descends into a base blob is a file-to-directory replacement, not a delete against the wrong base type; it is left for the write path rather than rejected here. Today's claim-check callers never emit any of these shapes, so reachable writes are unchanged. --- .../src/repo-store/delta-adversarial.test.ts | 93 ++++++++++++++++ .../src/repo-store/guard-probes.test.ts | 103 ++++++++++++++++++ packages/hub-sessions/src/repo-store/store.ts | 69 +++++++++--- packages/hub-sessions/src/repo-store/types.ts | 6 +- 4 files changed, 252 insertions(+), 19 deletions(-) create mode 100644 packages/hub-sessions/src/repo-store/guard-probes.test.ts diff --git a/packages/hub-sessions/src/repo-store/delta-adversarial.test.ts b/packages/hub-sessions/src/repo-store/delta-adversarial.test.ts index fb206547..9b0e8edc 100644 --- a/packages/hub-sessions/src/repo-store/delta-adversarial.test.ts +++ b/packages/hub-sessions/src/repo-store/delta-adversarial.test.ts @@ -245,3 +245,96 @@ test("cascade prune emptying dir and parent == canonical git", async () => { await canonicalTreeOid("adv-cascade2-oracle-", expected), ); }); + +test("no-slash delete of a directory is rejected (delete_type_mismatch)", async () => { + const base = { + "foo/a.json": `{"in":"dir"}`, + "keep.json": `{"keep":true}`, + }; + const store = await freshStore("adv-del-dir-"); + await seed(store, base); + await expect( + store.writeTreeDelta(principal, repoId, REF, { + changedPathPrefixes: new Set(["foo"]), + message: "delete-dir-as-file", + computeDelta: async () => ({ + puts: {}, + deletes: ["foo"], + }), + }), + ).rejects.toThrow(/delete_type_mismatch/); +}); + +test("trailing-slash delete descending into a base blob is rejected (delete_type_mismatch)", async () => { + const base = { + foo: `{"is":"blob"}`, + "keep.json": `{"keep":true}`, + }; + const store = await freshStore("adv-del-blob-"); + await seed(store, base); + await expect( + store.writeTreeDelta(principal, repoId, REF, { + changedPathPrefixes: new Set(["foo/bar/"]), + message: "delete-blob-as-subtree", + computeDelta: async () => ({ + puts: {}, + deletes: ["foo/bar/"], + }), + }), + ).rejects.toThrow(/delete_type_mismatch/); +}); + +test("put descending into a base blob is not mislabeled a delete mismatch", async () => { + const base = { + foo: `{"is":"blob"}`, + "keep.json": `{"keep":true}`, + }; + const store = await freshStore("adv-file2dir-"); + await seed(store, base); + const run = store.writeTreeDelta(principal, repoId, REF, { + changedPathPrefixes: new Set(["foo/"]), + message: "put-under-base-blob", + computeDelta: async () => ({ + puts: { "foo/bar.json": `{"now":"dir"}` }, + deletes: [], + }), + }); + // The carve-out holds: a put-driven descent into a base blob is a + // file-to-directory replacement, not a delete against the wrong base + // type, so it must NOT raise delete_type_mismatch. The store cannot + // complete the swap end-to-end today -- working-tree materialization + // mkdir's over the base file and EEXISTs -- a separate, unreachable + // limitation. The load-bearing assertion is the absence of + // delete_type_mismatch; the EEXIST pins where the store gives out. + await expect(run).rejects.toThrow(); + await run.catch((e: unknown) => { + expect(String(e)).not.toContain("delete_type_mismatch"); + expect(String(e)).toContain("EEXIST"); + }); +}); + +test("trailing-slash delete clearing a base directory still succeeds, OID == canonical git", async () => { + const base = { + "deploy/x.json": `{"x":1}`, + "deploy/y.json": `{"y":2}`, + "keep.json": `{"keep":true}`, + }; + const store = await freshStore("adv-clear-dir-"); + await seed(store, base); + await store.writeTreeDelta(principal, repoId, REF, { + changedPathPrefixes: new Set(["deploy/"]), + message: "clear-deploy", + computeDelta: async () => ({ + puts: {}, + deletes: ["deploy/"], + }), + }); + const paths = await readTreePaths(store); + expect([...paths].some((p) => p.startsWith("deploy/"))).toBe(false); + expect(paths.has("keep.json")).toBe(true); + + const expected = { "keep.json": `{"keep":true}` }; + expect(await committedTreeOid(store)).toBe( + await canonicalTreeOid("adv-clear-dir-oracle-", expected), + ); +}); diff --git a/packages/hub-sessions/src/repo-store/guard-probes.test.ts b/packages/hub-sessions/src/repo-store/guard-probes.test.ts new file mode 100644 index 00000000..a845635b --- /dev/null +++ b/packages/hub-sessions/src/repo-store/guard-probes.test.ts @@ -0,0 +1,103 @@ +// Recursion coverage for the delete_type_mismatch guards in assembleTree. +// delta-adversarial.test.ts exercises the two mismatch shapes only at the +// tree root; these two cover paths it misses: +// +// 1. A NESTED no-slash delete of a directory -- proves the guard is +// carried through assembleTree's recursion and the thrown path +// string reflects the full nested prefix, not just the leaf name. +// 2. A single-trailing-slash delete naming a base BLOB directly +// ("foo/"), as opposed to the two-level descent into a blob +// ("foo/bar/"). This is the minimal trailing-slash-over-blob shape +// and hits the child-extraction branch at the root. + +import { test, expect, beforeAll, afterAll } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { generateKeyPair } from "@intx/crypto"; +import type { KeyPair } from "@intx/types/runtime"; +import { createRepoStore } from "./store"; +import type { AuthorizeFn, Principal, RepoId } from "./types"; + +const tempDirs: string[] = []; +let signingKey: KeyPair; +const allowAll: AuthorizeFn = () => ({ allowed: true }); +const principal: Principal = { kind: "test" }; +const REF = "refs/heads/events"; +const repoId: RepoId = { kind: "agent-state", id: "subject" }; + +beforeAll(async () => { + signingKey = await generateKeyPair(); +}); +afterAll(async () => { + for (const d of tempDirs.splice(0)) { + await fs.promises + .rm(d, { recursive: true, force: true }) + .catch(() => undefined); + } +}); + +function makeStore(dataDir: string) { + return createRepoStore({ + dataDir, + signingKey, + handlers: { + "agent-state": { + kind: "agent-state", + directoryPrefix: "repos-under-test", + validatePush: () => ({ ok: true }), + onRefUpdated: () => undefined, + }, + }, + authorize: allowAll, + }); +} +async function freshStore(label: string) { + const dataDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), label)); + tempDirs.push(dataDir); + return makeStore(dataDir); +} +async function seed( + store: ReturnType, + base: Record, +) { + await store.initRepo(repoId); + await store.writeTree(principal, repoId, REF, { + files: base, + message: "seed", + }); +} + +test("nested no-slash delete of a directory is rejected with the full path", async () => { + const base = { + "top/sub/a.json": `{"a":1}`, + "top/keep.json": `{"k":1}`, + }; + const store = await freshStore("guard-nested-deldir-"); + await seed(store, base); + let caught: unknown; + await store + .writeTreeDelta(principal, repoId, REF, { + changedPathPrefixes: new Set(["top/"]), + message: "delete-nested-dir-as-file", + computeDelta: async () => ({ puts: {}, deletes: ["top/sub"] }), + }) + .catch((e: unknown) => (caught = e)); + expect(String(caught)).toContain("delete_type_mismatch"); + expect(String(caught)).toContain("top/sub"); +}); + +test("single trailing-slash delete naming a base blob directly is rejected", async () => { + const base = { foo: `{"is":"blob"}`, "keep.json": `{"keep":true}` }; + const store = await freshStore("guard-trailblob-"); + await seed(store, base); + let caught: unknown; + await store + .writeTreeDelta(principal, repoId, REF, { + changedPathPrefixes: new Set(["foo/"]), + message: "trailing-del-blob", + computeDelta: async () => ({ puts: {}, deletes: ["foo/"] }), + }) + .catch((e: unknown) => (caught = e)); + expect(String(caught)).toContain("delete_type_mismatch"); +}); diff --git a/packages/hub-sessions/src/repo-store/store.ts b/packages/hub-sessions/src/repo-store/store.ts index 1de5bd28..2cf2b75a 100644 --- a/packages/hub-sessions/src/repo-store/store.ts +++ b/packages/hub-sessions/src/repo-store/store.ts @@ -938,21 +938,20 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { // unchanged entry by oid. The result is committed directly via // `git.commit({ tree })`, so the on-disk index is never touched. // - // `deletes` is a set of repo-root-relative paths. Removal is by NAME, - // not by base-entry type: a trailing-slash entry names a subtree - // prefix and clears it (the clear-prefix shape); a no-slash entry - // clears whatever sits at that exact path -- a blob, or an entire - // subtree if the name happens to be a directory in the base. A - // no-slash delete is therefore NOT git-rm single-file semantics, and - // a trailing-slash delete whose leading segment is a base blob drops - // that blob rather than no-op'ing. Every caller emits only exact - // paths of entries that exist as blobs, or an intended clear-prefix, - // so neither name-vs-type case arises; a caller that builds deletes - // more freely must not rely on per-type git-rm behavior here. A `put` - // at a path overrides a delete of the same path. A delete of a path - // absent from the parent tree is an idempotent no-op (the entry is - // simply never emitted), matching `git rm --ignore-unmatch` and the - // working-tree `rm` with `force: true`. + // `deletes` is a set of repo-root-relative paths, and removal must + // match the base entry's type. A no-slash entry names an exact path + // and clears the blob there; deleting a path that is a directory in + // the base is rejected with `delete_type_mismatch` (there is no + // git-rm-style recursive drop). A trailing-slash entry names a + // subtree prefix and clears it (the clear-prefix shape); a + // trailing-slash delete whose leading segment is a base blob is + // rejected with `delete_type_mismatch`, unless a `put` drives the + // same descent -- that is a legitimate file-to-directory replacement + // and is accepted (the put wins). A `put` at a path overrides a + // delete of the same path. A delete of a path absent from the parent + // tree is an idempotent no-op (the entry is simply never emitted), + // matching `git rm --ignore-unmatch` and the working-tree `rm` with + // `force: true`. // // Recursion is scoped to the touched subtrees: a level is read and // rewritten only when a put lands under it or a delete removes within @@ -992,6 +991,7 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { // deletes here, and subtrees a put or a delete descends into. const blobPutsHere = new Set(); const subtreeNames = new Set(); + const subtreePutNames = new Set(); const fileDeletesHere = new Set(); for (const full of puts.keys()) { if (prefix !== "" && !full.startsWith(prefix)) continue; @@ -999,7 +999,11 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { if (rest.length === 0) continue; const slash = rest.indexOf("/"); if (slash === -1) blobPutsHere.add(rest); - else subtreeNames.add(rest.slice(0, slash)); + else { + const child = rest.slice(0, slash); + subtreeNames.add(child); + subtreePutNames.add(child); + } } for (const del of deletes) { if (prefix !== "" && !del.startsWith(prefix)) continue; @@ -1053,9 +1057,40 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { }); continue; } - if (fileDeletesHere.has(name)) continue; // file removed + if (fileDeletesHere.has(name)) { + const base = baseEntries.get(name); + if (base !== undefined && base.type === "tree") { + throw new Error( + `delete_type_mismatch: ${JSON.stringify( + prefix + name, + )} is deleted as a file but is a directory in the base tree`, + ); + } + continue; // file removed + } if (subtreeNames.has(name)) { const base = baseEntries.get(name); + // A trailing-slash delete descending into a base blob + // contradicts the base type -- reject it. The + // `!subtreePutNames` carve-out lets a put-driven descent + // through: a `put` under `name/` is a file-to-directory + // replacement, not a delete mismatch, so it must not raise + // delete_type_mismatch. That replacement still dead-ends at + // working-tree materialization in writeTreeUnderLock (mkdir + // over the base file EEXISTs) -- a separate limitation the + // claim-check callers never reach, which must not be masked by + // mislabeling a put as a delete mismatch. + if ( + base !== undefined && + base.type === "blob" && + !subtreePutNames.has(name) + ) { + throw new Error( + `delete_type_mismatch: ${JSON.stringify( + prefix + name, + )} is deleted as a subtree but is a file in the base tree`, + ); + } const baseChildOid = base !== undefined && base.type === "tree" ? base.oid : null; const childOid = await assembleTree( diff --git a/packages/hub-sessions/src/repo-store/types.ts b/packages/hub-sessions/src/repo-store/types.ts index 4b690ed5..0409a7dc 100644 --- a/packages/hub-sessions/src/repo-store/types.ts +++ b/packages/hub-sessions/src/repo-store/types.ts @@ -189,8 +189,10 @@ export type WriteTreePreservingPrefixArgs = { * under the per-repo lock against the pinned parent tip (`parentCommitSha`, * null when the ref does not yet exist) and returns the exact files to * write and paths to delete; the substrate carries every other entry - * forward by its object id. A delete ending in `/` removes a whole - * subtree; any other delete removes a single file. + * forward by its object id. A delete ending in `/` clears that subtree; + * any other delete clears the single file at that path. A delete whose + * base entry is the wrong type -- a no-slash delete naming a directory, + * or a trailing-slash delete descending into a file -- is rejected. * * `changedPathPrefixes` is the validation scoping hint for the touched * region (e.g. `addresses//`), supplied by the caller because a From 30726faf4bdc51e63a73c843d4dc60c6697c1efa Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Wed, 1 Jul 2026 13:42:09 -0500 Subject: [PATCH 012/101] Thread the repo-store git cache through the claim-check delta reads The claim-check delta reads -- the per-address listing and the bytes of the entry each leg moves -- ran against the raw repo dir through helpers the kind had duplicated from the store, so the per-repo object cache never reached them even though they run under the same write lock as the store's own reads. Hand computeDelta cache-backed prior-tree closures (listDirOids and readBlobByOid) and route the reads through them, then delete the duplicated tree-walk and by-oid blob-read helpers. --- packages/hub-sessions/src/repo-store/index.ts | 1 + packages/hub-sessions/src/repo-store/store.ts | 26 +++- packages/hub-sessions/src/repo-store/types.ts | 16 ++- .../hub-sessions/src/workflow-run-kind.ts | 126 ++++-------------- 4 files changed, 69 insertions(+), 100 deletions(-) diff --git a/packages/hub-sessions/src/repo-store/index.ts b/packages/hub-sessions/src/repo-store/index.ts index 89eb4c54..73aeb873 100644 --- a/packages/hub-sessions/src/repo-store/index.ts +++ b/packages/hub-sessions/src/repo-store/index.ts @@ -3,6 +3,7 @@ export type { InitRepoOpts, KindHandler, NewlyTerminalRun, + PriorDeltaReads, Principal, RefEntry, RepoAction, diff --git a/packages/hub-sessions/src/repo-store/store.ts b/packages/hub-sessions/src/repo-store/store.ts index 2cf2b75a..905a7584 100644 --- a/packages/hub-sessions/src/repo-store/store.ts +++ b/packages/hub-sessions/src/repo-store/store.ts @@ -641,12 +641,27 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { priorListDirOids: ( path: string, ) => Promise<{ name: string; oid: string }[]>; + readBlobByOid: (oid: string) => Promise; } { + // Read any blob by its object id, cache-backed. Object ids are + // content addresses, so this is independent of `commitSha` and is + // available even on the no-prior-commit branch (where callers have + // no ids to read). + const readBlobByOid = async (oid: string): Promise => { + const { blob } = await git.readBlob({ + fs, + dir, + cache: cacheFor(dir), + oid, + }); + return blob; + }; if (commitSha === null) { return { priorReadBlob: async () => null, priorListDir: async () => [], priorListDirOids: async () => [], + readBlobByOid, }; } const priorReadBlob = async ( @@ -690,7 +705,7 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { }); return tree.map((e) => ({ name: e.path, oid: e.oid })); }; - return { priorReadBlob, priorListDir, priorListDirOids }; + return { priorReadBlob, priorListDir, priorListDirOids, readBlobByOid }; } // Build the `(readBlob, listDir, topLevelTreePaths)` triple a kind @@ -1534,7 +1549,14 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { // it is committed against — no lost-update window, no second // resolve that could (in principle) observe a different tip. const parentCommitSha = await resolveRefSha(dir, ref); - const delta = await args.computeDelta(parentCommitSha); + const { priorListDirOids, readBlobByOid } = buildPriorTreeClosures( + dir, + parentCommitSha, + ); + const delta = await args.computeDelta(parentCommitSha, { + readBlobByOid, + listDirOids: priorListDirOids, + }); for (const p of Object.keys(delta.puts)) validateDeltaPath(p, false); for (const d of delta.deletes) validateDeltaPath(d, true); assertDeltaUnambiguous(delta.puts, delta.deletes); diff --git a/packages/hub-sessions/src/repo-store/types.ts b/packages/hub-sessions/src/repo-store/types.ts index 0409a7dc..0382faaf 100644 --- a/packages/hub-sessions/src/repo-store/types.ts +++ b/packages/hub-sessions/src/repo-store/types.ts @@ -198,9 +198,23 @@ export type WriteTreePreservingPrefixArgs = { * region (e.g. `addresses//`), supplied by the caller because a * delta has no single clear-prefix to derive it from; `undefined` means * validate the whole tree. + * + * The `prior` argument exposes cache-backed reads of that pinned parent + * tree -- `listDirOids` for a directory's `{name, oid}` children and + * `readBlobByOid` for a blob by its object id -- so the callback reads + * through the store's per-repo object cache under the same lock rather + * than re-opening the repo. */ +export type PriorDeltaReads = { + readBlobByOid: (oid: string) => Promise; + listDirOids: (path: string) => Promise<{ name: string; oid: string }[]>; +}; + export type WriteTreeDeltaArgs = { - computeDelta: (parentCommitSha: string | null) => Promise<{ + computeDelta: ( + parentCommitSha: string | null, + prior: PriorDeltaReads, + ) => Promise<{ puts: Record; deletes: readonly string[]; }>; diff --git a/packages/hub-sessions/src/workflow-run-kind.ts b/packages/hub-sessions/src/workflow-run-kind.ts index 509300d1..76d1c0f7 100644 --- a/packages/hub-sessions/src/workflow-run-kind.ts +++ b/packages/hub-sessions/src/workflow-run-kind.ts @@ -176,6 +176,7 @@ import { type AuthorizeFn, type KindHandler, type NewlyTerminalRun, + type PriorDeltaReads, type Principal, type RepoId, type RepoStore, @@ -2613,14 +2614,15 @@ type AddressListing = { * `addresses//{inbox,processing,consumed}/` (NOT their * bytes), plus the retention watermark. The bytes of the single entry a * leg actually moves are read separately by OID via - * {@link readClaimCheckBlob}, so the unbounded consumed/ dedup index is - * enumerated (one `readTree`, filenames only) but never read - * blob-by-blob. An empty listing covers the repo/ref/address-absent - * first-write states — all legitimate for a brand-new operation. + * `prior.readBlobByOid`, so the unbounded consumed/ dedup index is + * enumerated (one `listDirOids` per bucket, names and OIDs only) but + * never read blob-by-blob. Every read goes through the store's + * cache-backed `prior` closures under the write lock. An empty listing + * covers the repo/ref/address-absent first-write states -- all + * legitimate for a brand-new operation. */ async function readAddressListing( - repoDir: string, - parentCommitSha: string | null, + prior: PriorDeltaReads, addressSegment: string, ): Promise { const listing: AddressListing = { @@ -2629,76 +2631,29 @@ async function readAddressListing( consumed: [], watermark: 0, }; - if (parentCommitSha === null) return listing; - const commit = await git.readCommit({ - fs, - dir: repoDir, - oid: parentCommitSha, - }); - const addrTreeOid = await resolveSubtreeOid(repoDir, commit.commit.tree, [ - WORKFLOW_RUN_ADDRESSES_PREFIX, - addressSegment, - ]); - if (addrTreeOid === null) return listing; - const { tree: addrChildren } = await git.readTree({ - fs, - dir: repoDir, - oid: addrTreeOid, - }); - for (const child of addrChildren) { - if (child.type === "blob" && child.path === WORKFLOW_RUN_WATERMARK_FILE) { - const { blob } = await git.readBlob({ fs, dir: repoDir, oid: child.oid }); + const addrDir = `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}`; + for (const child of await prior.listDirOids(addrDir)) { + if (child.name === WORKFLOW_RUN_WATERMARK_FILE) { + const blob = await prior.readBlobByOid(child.oid); listing.watermark = parseWatermark(blob, watermarkPath(addressSegment)); continue; } - if (child.type !== "tree") continue; const bucket = - child.path === WORKFLOW_RUN_INBOX_DIR + child.name === WORKFLOW_RUN_INBOX_DIR ? listing.inbox - : child.path === WORKFLOW_RUN_PROCESSING_DIR + : child.name === WORKFLOW_RUN_PROCESSING_DIR ? listing.processing - : child.path === WORKFLOW_RUN_CONSUMED_DIR + : child.name === WORKFLOW_RUN_CONSUMED_DIR ? listing.consumed : null; if (bucket === null) continue; - const { tree: entries } = await git.readTree({ - fs, - dir: repoDir, - oid: child.oid, - }); - for (const entry of entries) { - if (entry.type !== "blob") continue; - bucket.push({ name: entry.path, oid: entry.oid }); + for (const entry of await prior.listDirOids(`${addrDir}/${child.name}`)) { + bucket.push({ name: entry.name, oid: entry.oid }); } } return listing; } -/** Read a single claim-check blob by its git object id. */ -async function readClaimCheckBlob( - repoDir: string, - oid: string, -): Promise { - const { blob } = await git.readBlob({ fs, dir: repoDir, oid }); - return blob; -} - -async function resolveSubtreeOid( - repoDir: string, - rootTreeOid: string, - segments: readonly string[], -): Promise { - let current = rootTreeOid; - for (const segment of segments) { - const { tree } = await git.readTree({ fs, dir: repoDir, oid: current }); - const entry = tree.find((e) => e.path === segment); - if (entry === undefined) return null; - if (entry.type !== "tree") return null; - current = entry.oid; - } - return current; -} - function utf8(s: string): Uint8Array { return new TextEncoder().encode(s); } @@ -2804,7 +2759,6 @@ export async function enqueueInbox( ): Promise { const addressSegment = addressSegmentFor(args.address); const ref = claimCheckCommitRef(); - const repoDir = store.getRepoDir(repoId); const inboxKey = filenameKey(args.receivedAt, args.messageId); const envelope: ClaimCheckEnvelope = { messageId: args.messageId, @@ -2820,12 +2774,8 @@ export async function enqueueInbox( const { commitSha } = await store.writeTreeDelta(principal, repoId, ref, { changedPathPrefixes: new Set([addressPrefix(addressSegment)]), message: `enqueue inbox ${args.address} ${args.messageId}`, - computeDelta: async (parentCommitSha) => { - const listing = await readAddressListing( - repoDir, - parentCommitSha, - addressSegment, - ); + computeDelta: async (_parentCommitSha, prior) => { + const listing = await readAddressListing(prior, addressSegment); // Refuse a definitively-stale enqueue: a message whose receivedAt // is strictly below the retention watermark could have had its // consumed/ dedup entry pruned, so a duplicate can no longer be @@ -2900,17 +2850,12 @@ export async function dequeueToProcessing( ): Promise { const addressSegment = addressSegmentFor(address); const ref = claimCheckCommitRef(); - const repoDir = store.getRepoDir(repoId); let dequeued: { key: string; envelope: ClaimCheckEnvelope } | null = null; const { commitSha } = await store.writeTreeDelta(principal, repoId, ref, { changedPathPrefixes: new Set([addressPrefix(addressSegment)]), message: `dequeue ${address}`, - computeDelta: async (parentCommitSha) => { - const listing = await readAddressListing( - repoDir, - parentCommitSha, - addressSegment, - ); + computeDelta: async (_parentCommitSha, prior) => { + const listing = await readAddressListing(prior, addressSegment); const inboxDir = `${addressPrefix(addressSegment)}${WORKFLOW_RUN_INBOX_DIR}/`; // Sort by numeric receivedAt with a messageId tiebreak. A raw // string sort would not agree with chronological order when @@ -2949,7 +2894,7 @@ export async function dequeueToProcessing( } const firstPath = `${inboxDir}${first.entry.name}`; const key = first.entry.name.slice(0, -".json".length); - const bytes = await readClaimCheckBlob(repoDir, first.entry.oid); + const bytes = await prior.readBlobByOid(first.entry.oid); const envelope = decodeQueueEnvelopeOrThrow(bytes, firstPath); dequeued = { key, envelope }; return { @@ -3085,7 +3030,6 @@ export async function markConsumed( ): Promise { const addressSegment = addressSegmentFor(args.address); const ref = claimCheckCommitRef(); - const repoDir = store.getRepoDir(repoId); const retentionHorizonMs = args.retentionHorizonMs ?? DEFAULT_CONSUMED_RETENTION_MS; let consumedEnvelope: ConsumedEnvelope | null = null; @@ -3094,12 +3038,8 @@ export async function markConsumed( const { commitSha } = await store.writeTreeDelta(principal, repoId, ref, { changedPathPrefixes: new Set([addressPrefix(addressSegment)]), message: `consume ${args.address} ${args.messageId}`, - computeDelta: async (parentCommitSha) => { - const listing = await readAddressListing( - repoDir, - parentCommitSha, - addressSegment, - ); + computeDelta: async (_parentCommitSha, prior) => { + const listing = await readAddressListing(prior, addressSegment); const consumedFull = consumedPath(addressSegment, args.messageId); const consumedFname = `${args.messageId}.json`; if (listing.consumed.some((e) => e.name === consumedFname)) { @@ -3117,10 +3057,7 @@ export async function markConsumed( ); } const processingFull = `${processingDir}${processingEntry.name}`; - const processingBytes = await readClaimCheckBlob( - repoDir, - processingEntry.oid, - ); + const processingBytes = await prior.readBlobByOid(processingEntry.oid); const processingEnvelope = decodeQueueEnvelopeOrThrow( processingBytes, processingFull, @@ -3158,7 +3095,7 @@ export async function markConsumed( const deletes: string[] = [processingFull]; for (const entry of listing.consumed) { const blobPath = `${consumedDir}${entry.name}`; - const bytes = await readClaimCheckBlob(repoDir, entry.oid); + const bytes = await prior.readBlobByOid(entry.oid); const consumedReceivedAt = decodeConsumedReceivedAtOrThrow( bytes, blobPath, @@ -3225,17 +3162,12 @@ export async function replayProcessingToInbox( ): Promise { const addressSegment = addressSegmentFor(address); const ref = claimCheckCommitRef(); - const repoDir = store.getRepoDir(repoId); const replayedKeys: string[] = []; const { commitSha } = await store.writeTreeDelta(principal, repoId, ref, { changedPathPrefixes: new Set([addressPrefix(addressSegment)]), message: `replay processing ${address}`, - computeDelta: async (parentCommitSha) => { - const listing = await readAddressListing( - repoDir, - parentCommitSha, - addressSegment, - ); + computeDelta: async (_parentCommitSha, prior) => { + const listing = await readAddressListing(prior, addressSegment); const processingDir = `${addressPrefix(addressSegment)}${WORKFLOW_RUN_PROCESSING_DIR}/`; const inboxDir = `${addressPrefix(addressSegment)}${WORKFLOW_RUN_INBOX_DIR}/`; const inboxNames = new Set(listing.inbox.map((e) => e.name)); @@ -3253,7 +3185,7 @@ export async function replayProcessingToInbox( // below-watermark receivedAt is no reason to refuse it. Applying // the stale-check here would lose a legitimately in-flight // message after a crash. Do not tighten this. - const bytes = await readClaimCheckBlob(repoDir, entry.oid); + const bytes = await prior.readBlobByOid(entry.oid); puts[inboxFull] = bytes; deletes.push(`${processingDir}${entry.name}`); replayedKeys.push(entry.name.slice(0, -".json".length)); From 2688f3a9adbce00123d7a7f1f2ee16867528c9a2 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Wed, 1 Jul 2026 14:18:04 -0500 Subject: [PATCH 013/101] Make the delta-scoped claim-check validation the default Delta-scoped claim-check validation was gated behind the bench-only BENCH_DELTA_SCOPE_CLAIMCHECK flag, with an exhaustive legacy path beside it. Make the delta path unconditional and delete the flag, its OFF path, the sidecar startup marker, and the test env forwarding. The consumed dedup index is now always validated by its per-commit delta: retained entries are proven immutable by git blob OID equality without a byte re-read, added entries are parsed at the transition check, and a prune is bound to the watermark. Exactly-once rests on four pillars -- consumed immutability, the below-watermark prune bound, watermark monotonicity, and enqueue's dedup (index hit above the watermark, stale-reject below it) against the operator retention horizon. The suffix relation the legacy path also enforced is deliberately not kept: it is structural hardening, not correctness, and computing it costs the O(retained) prior read the delta path exists to avoid. On the receivePack path the substrate supplies no prospective OID listing, so consumed OIDs are hashed from bytes there rather than read from the tree listing -- correct and the same order as the legacy walk, with no delta speed-up. --- apps/sidecar/src/index.ts | 25 +- packages/hub-sessions/src/index.ts | 1 - .../src/workflow-run-kind.test.ts | 155 ++++---- .../hub-sessions/src/workflow-run-kind.ts | 337 ++++++------------ tests/hub-agent/lib/deploy-flow-env.ts | 12 - 5 files changed, 177 insertions(+), 353 deletions(-) diff --git a/apps/sidecar/src/index.ts b/apps/sidecar/src/index.ts index 934ce1fb..22762c19 100644 --- a/apps/sidecar/src/index.ts +++ b/apps/sidecar/src/index.ts @@ -1,6 +1,6 @@ import { appendFileSync } from "node:fs"; import path from "node:path"; -import { getLogger, setup } from "@intx/log"; +import { setup } from "@intx/log"; import { createInMemoryTransport } from "@intx/mail-memory"; import { createEd25519Crypto, @@ -10,10 +10,7 @@ import { } from "@intx/crypto"; import { createSidecarOrchestrator, type HubLink } from "@intx/hub-agent"; import { hexEncode } from "@intx/types"; -import { - claimCheckDeltaScopeEnabled, - createAgentRepoStore, -} from "@intx/hub-sessions"; +import { createAgentRepoStore } from "@intx/hub-sessions"; import { createTarballCache } from "@intx/tool-packaging"; import { loadAdapterRegistry } from "@intx/inference/providers"; @@ -50,24 +47,6 @@ import { loadOrMintSidecarKeypair } from "./signing-keypair"; await setup(); -const logger = getLogger(["sidecar"]); - -// One-line startup marker naming the effective delta-scope claim-check -// state for THIS process. The sidecar's supervisor owns every -// workflow-run write, so this is the process whose module const decides -// which `validateClaimCheckSubtree` path a benchmark actually exercises. -// Emitting the imported const (not a fresh env read) makes a run's live -// state observable straight from the captured sidecar output, so a -// latency bench never has to be inferred back from its per-leg slopes. -// Each branch's message is a static template so the marker prints -// verbatim (`claim-check-delta-scope=ON`) under any log renderer, rather -// than as a quoted interpolated value. -if (claimCheckDeltaScopeEnabled) { - logger.info`claim-check-delta-scope=ON`; -} else { - logger.info`claim-check-delta-scope=OFF`; -} - function requireEnv(name: string): string { const value = process.env[name]; if (value === undefined) { diff --git a/packages/hub-sessions/src/index.ts b/packages/hub-sessions/src/index.ts index 302f903e..6b48c71e 100644 --- a/packages/hub-sessions/src/index.ts +++ b/packages/hub-sessions/src/index.ts @@ -76,7 +76,6 @@ export { export { workflowRunKindHandler, workflowRunAuthorize, - claimCheckDeltaScopeEnabled, enqueueInbox, dequeueToProcessing, readProcessingEntry, diff --git a/packages/hub-sessions/src/workflow-run-kind.test.ts b/packages/hub-sessions/src/workflow-run-kind.test.ts index b01a93c7..3d8f13b7 100644 --- a/packages/hub-sessions/src/workflow-run-kind.test.ts +++ b/packages/hub-sessions/src/workflow-run-kind.test.ts @@ -2993,9 +2993,9 @@ describe("workflow-run substrate — pack-path per-run scope completeness", () = // markConsumed commit advances it and prunes consumed entries below it // (the oldest tail only); enqueueInbox refuses any inbound below it as // definitively-stale. These tests prove the gate items: the structural -// contract relaxation (validate-level: suffix-only prune, monotonic -// watermark, retained-floor) and the end-to-end exactly-once + bounded -// behaviour against a real on-disk store. +// contract relaxation (validate-level: below-watermark prune, +// monotonic watermark, retained-floor) and the end-to-end +// exactly-once + bounded behaviour against a real on-disk store. describe("workflowRunKindHandler.validatePush — retention watermark contract", () => { // Gate 3: a watermark regression is rejected. @@ -3027,12 +3027,14 @@ describe("workflowRunKindHandler.validatePush — retention watermark contract", expect(r.reason).toMatch(/watermark regressed/); }); - // Gate 2: a non-suffix deletion (drop a recent consumed entry while - // keeping an older one) is rejected. Dropping msg-recent (receivedAt - // 200) while retaining msg-old (receivedAt 100) is not a suffix of - // the age-ordered set -- the suffix guard fires (max dropped 200 > - // min retained 100), regardless of where the watermark sits. - test("rejects dropping a recent consumed entry while keeping an older one", async () => { + // A non-suffix deletion (drop a recent consumed entry while keeping + // an older one) is ACCEPTED when both entries sit below the + // watermark. The suffix relation is not enforced; a retained entry + // below the watermark only adds dedup (every resubmit in that region + // is stale-rejected at enqueue), so the prune opens no reprocess. The + // production markConsumed writer still prunes only the oldest tail, + // so a non-suffix tree never arises in practice. + test("accepts a non-suffix consumed prune below the watermark and locks the boundary", async () => { const prior = { [watermarkPathFor(ADDRESS_SEG)]: watermarkBody(0), [consumedPathFor(ADDRESS_SEG, "msg-old")]: consumedBody( @@ -3049,10 +3051,9 @@ describe("workflowRunKindHandler.validatePush — retention watermark contract", ), }; // Drop msg-recent (the younger one), keep msg-old (the older one). - // To "permit" the drop the writer must claim a watermark above 200 - // (so the dropped 200 is below it); the retained 100 is then newer - // than nothing dropped above it, but the dropped 200 is newer than - // the retained 100 -- a non-suffix prune. + // The writer claims a watermark above 200 so the dropped 200 is + // below it; the retained 100 is below it too. Both sit below the + // watermark, so every resubmit in that region is stale-rejected. const r = await validate( { [WORKFLOW_RUN_GITIGNORE_PATH]: "", @@ -3066,81 +3067,71 @@ describe("workflowRunKindHandler.validatePush — retention watermark contract", }, { priorFiles: prior }, ); - if (process.env.BENCH_DELTA_SCOPE_CLAIMCHECK === "1") { - // The delta-scoped path does NOT enforce the suffix relation: - // computing the min receivedAt over all retained entries would - // require reading every retained consumed blob, the O(retained) - // work the delta walk exists to avoid. WHY DROPPING IT IS SAFE - // HERE: both the dropped (receivedAt 200) and the retained - // (receivedAt 100) entries are strictly below the stored watermark - // (201), so `claim_check_stale_enqueue` refuses every resubmit in - // that region at the enqueue boundary regardless of the consumed/ - // shape -- the retained entry only adds dedup, so a non-suffix - // prune below the watermark cannot open a reprocess. The delta path - // therefore accepts this prune; the exhaustive path below rejects - // it. The production markConsumed writer always prunes the oldest - // tail, so it never produces a non-suffix tree on either path. - expect(r.ok).toBe(true); + // The consumed walk does NOT enforce the suffix relation: computing + // the min receivedAt over all retained entries would require + // reading every retained consumed blob, the O(retained) work the + // delta walk exists to avoid. Both the dropped (receivedAt 200) and + // retained (receivedAt 100) entries are strictly below the stored + // watermark (201), so `claim_check_stale_enqueue` refuses every + // resubmit in that region at the enqueue boundary regardless of the + // consumed/ shape -- the retained entry only adds dedup, so a + // non-suffix prune below the watermark cannot open a reprocess. + expect(r.ok).toBe(true); - // Boundary lock. The delta removed-check rejects a dropped entry at - // receivedAt >= watermark (strict, mirroring the strict - // `receivedAt < watermark` stale-reject so the entry AT the - // watermark is both retained and not stale-rejected -- no gap). - // Assert BOTH sides of the boundary so a later refactor cannot - // silently widen `>=` to `>` and drop the entry sitting exactly at - // the watermark, which would let a resubmit at that receivedAt miss - // dedup. - // (a) strictly above the watermark -> rejected. - const droppedAbove = await validate( - { - [WORKFLOW_RUN_GITIGNORE_PATH]: "", + // Boundary lock. The removed-check rejects a dropped entry at + // receivedAt >= watermark (strict, mirroring the strict + // `receivedAt < watermark` stale-reject so the entry AT the + // watermark is both retained and not stale-rejected -- no gap). + // Assert BOTH sides of the boundary so a later refactor cannot + // silently widen `>=` to `>` and drop the entry sitting exactly at + // the watermark, which would let a resubmit at that receivedAt miss + // dedup. + // (a) strictly above the watermark -> rejected. + const droppedAbove = await validate( + { + [WORKFLOW_RUN_GITIGNORE_PATH]: "", + [watermarkPathFor(ADDRESS_SEG)]: watermarkBody(200), + }, + { + priorFiles: { [watermarkPathFor(ADDRESS_SEG)]: watermarkBody(200), + [consumedPathFor(ADDRESS_SEG, "msg-above")]: consumedBody( + "msg-above", + 300, + "run-above", + 350, + ), }, - { - priorFiles: { - [watermarkPathFor(ADDRESS_SEG)]: watermarkBody(200), - [consumedPathFor(ADDRESS_SEG, "msg-above")]: consumedBody( - "msg-above", - 300, - "run-above", - 350, - ), - }, - }, - ); - expect(droppedAbove.ok).toBe(false); - if (droppedAbove.ok) throw new Error("unreachable"); - expect(droppedAbove.reason).toMatch(/not below the retention watermark/); + }, + ); + expect(droppedAbove.ok).toBe(false); + if (droppedAbove.ok) throw new Error("unreachable"); + expect(droppedAbove.reason).toMatch(/not below the retention watermark/); - // (b) exactly equal to the watermark -> rejected (the off-by-one - // that widening `>=` to `>` would open). - const droppedAtBoundary = await validate( - { - [WORKFLOW_RUN_GITIGNORE_PATH]: "", + // (b) exactly equal to the watermark -> rejected (the off-by-one + // that widening `>=` to `>` would open). + const droppedAtBoundary = await validate( + { + [WORKFLOW_RUN_GITIGNORE_PATH]: "", + [watermarkPathFor(ADDRESS_SEG)]: watermarkBody(200), + }, + { + priorFiles: { [watermarkPathFor(ADDRESS_SEG)]: watermarkBody(200), + [consumedPathFor(ADDRESS_SEG, "msg-at")]: consumedBody( + "msg-at", + 200, + "run-at", + 250, + ), }, - { - priorFiles: { - [watermarkPathFor(ADDRESS_SEG)]: watermarkBody(200), - [consumedPathFor(ADDRESS_SEG, "msg-at")]: consumedBody( - "msg-at", - 200, - "run-at", - 250, - ), - }, - }, - ); - expect(droppedAtBoundary.ok).toBe(false); - if (droppedAtBoundary.ok) throw new Error("unreachable"); - expect(droppedAtBoundary.reason).toMatch( - /not below the retention watermark/, - ); - return; - } - expect(r.ok).toBe(false); - if (r.ok) throw new Error("unreachable"); - expect(r.reason).toMatch(/prune is not a suffix/); + }, + ); + expect(droppedAtBoundary.ok).toBe(false); + if (droppedAtBoundary.ok) throw new Error("unreachable"); + expect(droppedAtBoundary.reason).toMatch( + /not below the retention watermark/, + ); }); // A dropped entry that the watermark has NOT passed is rejected (you diff --git a/packages/hub-sessions/src/workflow-run-kind.ts b/packages/hub-sessions/src/workflow-run-kind.ts index 76d1c0f7..047ba888 100644 --- a/packages/hub-sessions/src/workflow-run-kind.ts +++ b/packages/hub-sessions/src/workflow-run-kind.ts @@ -122,23 +122,19 @@ // DELETED only as a watermark-consistent retention prune (see the // watermark invariants below); any other deletion is rejected. // - Retention prune (the bounded-`consumed/` contract): the consumed -// dedup index may shrink only by a watermark-passed SUFFIX prune. -// A consumed entry present in the prior tree may be absent from the +// dedup index may shrink only by a watermark-passed prune. A +// consumed entry present in the prior tree may be absent from the // prospective tree only when (a) its `receivedAt` is strictly // below the prospective `watermark.json` value (you may prune only -// what the watermark passed), (b) the watermark did not regress -// (`prospective watermark >= prior watermark`), and (c) it is older -// than every retained prior consumed entry (`max receivedAt of -// dropped <= min receivedAt of retained`) -- the suffix relation, -// so a writer can trim only the oldest age-ordered tail and can -// never selectively delete a recent messageId to sneak a reprocess -// past dedup. A RETAINED entry is NOT required to sit at or above -// the watermark: a message consumed long after receipt (or one -// replayed back in-flight after a crash) may legitimately carry a -// below-watermark `receivedAt` and survive until a later commit -// prunes it. Retaining it gives only EXTRA dedup -- a re-submission -// at or above the watermark still hits the entry, one below it is -// stale-rejected at enqueue -- so it never weakens exactly-once. +// what the watermark passed) and (b) the watermark did not regress +// (`prospective watermark >= prior watermark`). A RETAINED entry is +// NOT required to sit at or above the watermark: a message consumed +// long after receipt (or one replayed back in-flight after a crash) +// may legitimately carry a below-watermark `receivedAt` and survive +// until a later commit prunes it. Retaining it gives only EXTRA +// dedup -- a re-submission at or above the watermark still hits the +// entry, one below it is stale-rejected at enqueue -- so it never +// weakens exactly-once. // - Inbox→processing transition: a processing entry that is newly // added (not present in the prior tree) must be backed by a // matching inbox entry in the prior tree at the same @@ -190,33 +186,6 @@ import { const logger = getLogger(["hub-sessions", "workflow-run-kind"]); -/** - * Experiment flag (latency gate, default OFF). When set, the claim-check - * validation walk in `validateClaimCheckSubtree` validates only the - * per-commit DELTA of the consumed dedup index against the prior tree -- - * keyed by (filename, blob OID) -- instead of re-reading, re-parsing and - * re-validating the ENTIRE retained consumed set on every commit. Read - * once at module load; OFF reproduces the exhaustive walk byte-for-byte, - * so nothing changes unless the flag is explicitly set. - * - * A second, independent experiment flag (commit batching) is intended to - * live alongside this one; it is deliberately NOT defined here yet. - */ -const BENCH_DELTA_SCOPE_CLAIMCHECK = - process.env.BENCH_DELTA_SCOPE_CLAIMCHECK === "1"; - -/** - * The resolved delta-scope claim-check state for the process that loaded - * this module. Exported so the process actually running `validatePush` - * (the sidecar's supervisor, which owns every workflow-run write) can - * emit an unambiguous startup marker reflecting the ACTUAL module const, - * rather than a fresh `process.env` read that could diverge from what the - * validation walk observes. This is the observable that proves whether - * the delta path is live for a given process, so a benchmark run can be - * read directly rather than inferred from its timing slopes. - */ -export const claimCheckDeltaScopeEnabled = BENCH_DELTA_SCOPE_CLAIMCHECK; - export type WorkflowRunHubPrincipal = { readonly kind: "hub" }; export type WorkflowRunSidecarPrincipal = { @@ -1062,13 +1031,13 @@ type ClaimCheckBlob = { messageIdFromFilename: string; blobPath: string; /** - * Git blob object id of the entry, populated only for `consumed` - * entries and only when the delta-scoped claim-check path is enabled - * (`BENCH_DELTA_SCOPE_CLAIMCHECK`). Two entries at the same path whose - * OIDs match are byte-identical (git trees are content-addressed), so - * the delta path skips the immutability byte-comparison for retained - * consumed entries. Left `undefined` on the exhaustive (flag-OFF) path - * and on inbox/processing entries, which the delta path does not scope. + * Git blob object id of the entry, resolved for `consumed` entries + * when the enumeration is given an OID resolver. Two entries at the + * same path whose OIDs match are byte-identical (git trees are + * content-addressed), so the consumed immutability check compares + * OIDs instead of re-reading both blobs for retained entries. Left + * `undefined` on inbox/processing entries, which the resolver does + * not cover. */ oid?: string; }; @@ -1396,32 +1365,6 @@ async function parseQueueBlob( return { ok: true, body: validated }; } -async function checkPriorBytesImmutable( - blobPath: string, - readBlob: (path: string) => Promise, - priorReadBlob: (path: string) => Promise, - label: string, -): Promise { - const prior = await priorReadBlob(blobPath); - if (prior === null) return { ok: true }; - const prospective = await readBlob(blobPath); - if (prior.byteLength !== prospective.byteLength) { - return { - ok: false, - reason: `${label} ${blobPath} bytes diverge from the prior tree (lengths ${String(prior.byteLength)} vs ${String(prospective.byteLength)}); ${label} entries are immutable once written`, - }; - } - for (let i = 0; i < prior.byteLength; i++) { - if (prior[i] !== prospective[i]) { - return { - ok: false, - reason: `${label} ${blobPath} bytes diverge from the prior tree at offset ${String(i)}; ${label} entries are immutable once written`, - }; - } - } - return { ok: true }; -} - /** * Compute the git blob OID of a consumed entry from a byte reader, * used only when the delta-scoped path lacks a substrate-provided prior @@ -1499,19 +1442,18 @@ function makePriorConsumedOidResolver( * Validate the `addresses//{inbox,processing,consumed}` * subtree as a whole. The walk enforces filename shape, JSON envelope * structure, address round-trip, per-messageId atomicity across the - * three queue states, consumed-blob immutability via prior-bytes - * equality, and the inbox→processing / processing→consumed - * transition invariants against the prior tree. + * three queue states, consumed-blob immutability, and the + * inbox→processing / processing→consumed transition invariants against + * the prior tree. * - * When `BENCH_DELTA_SCOPE_CLAIMCHECK` is set the consumed dedup index is - * validated by its per-commit DELTA against the prior tree rather than - * by re-walking the whole retained set: retained entries (same filename, - * same blob OID) are skipped as already-validated-and-immutable, added - * entries are parsed and validated, and removed entries are checked - * against the retention watermark. `priorListDirOids` and `listDirOids`, - * when supplied by the substrate, surface the prior and prospective - * consumed OIDs straight from their tree listings so neither side is - * re-read blob-by-blob. + * The consumed dedup index is validated by its per-commit DELTA against + * the prior tree rather than by re-walking the whole retained set: + * retained entries (same filename, same blob OID) are skipped as + * already-validated-and-immutable, added entries are parsed and + * validated, and removed entries are checked against the retention + * watermark. `priorListDirOids` and `listDirOids`, when supplied by the + * substrate, surface the prior and prospective consumed OIDs straight + * from their tree listings so neither side is re-read blob-by-blob. */ async function validateClaimCheckSubtree( listDir: (path: string) => Promise, @@ -1521,19 +1463,18 @@ async function validateClaimCheckSubtree( priorListDirOids?: (path: string) => Promise<{ name: string; oid: string }[]>, listDirOids?: (path: string) => Promise<{ name: string; oid: string }[]>, ): Promise { - // When the delta path is on, surface each consumed entry's git blob - // OID during enumeration straight from the tree listing on both sides - // when the substrate provides it, falling back to hashing the bytes - // otherwise. The resolvers are omitted entirely on the exhaustive path - // so it stays byte-identical. - const prospectiveConsumedOid = BENCH_DELTA_SCOPE_CLAIMCHECK - ? makeListingOidResolver("prospective", listDirOids, async (blobPath) => - hashConsumedBlobOid(await readBlob(blobPath)), - ) - : undefined; - const priorConsumedOid = BENCH_DELTA_SCOPE_CLAIMCHECK - ? makePriorConsumedOidResolver(priorReadBlob, priorListDirOids) - : undefined; + // Surface each consumed entry's git blob OID during enumeration + // straight from the tree listing on both sides when the substrate + // provides it, falling back to hashing the bytes otherwise. + const prospectiveConsumedOid = makeListingOidResolver( + "prospective", + listDirOids, + async (blobPath) => hashConsumedBlobOid(await readBlob(blobPath)), + ); + const priorConsumedOid = makePriorConsumedOidResolver( + priorReadBlob, + priorListDirOids, + ); const enumerated = await enumerateClaimCheckBlobs( listDir, @@ -1602,16 +1543,11 @@ async function validateClaimCheckSubtree( } for (const entry of bucket.consumed) { // Cross-state atomicity needs each consumed messageId in the map; - // the messageId is the filename stem, so on the delta path this - // needs no blob read. Retained consumed entries are not re-parsed - // (their envelope was validated when first written and their bytes - // are proven immutable by the OID compare below); added consumed - // entries are parsed and validated by the transition check further - // down. The exhaustive path re-parses every consumed entry here. - if (!BENCH_DELTA_SCOPE_CLAIMCHECK) { - const parsed = await parseConsumedBlob(entry, readBlob); - if (!parsed.ok) return parsed; - } + // the messageId is the filename stem, so this needs no blob read. + // Retained consumed entries are not re-parsed (their envelope was + // validated when first written and their bytes are proven + // immutable by the OID compare below); added consumed entries are + // parsed and validated by the transition check further down. const list = messageIdToLocations.get(entry.messageIdFromFilename) ?? []; list.push({ kind: entry.kind, filename: entry.filename }); messageIdToLocations.set(entry.messageIdFromFilename, list); @@ -1640,51 +1576,37 @@ async function validateClaimCheckSubtree( } } - // Consumed entries are immutable. The exhaustive path re-reads both - // the prospective and the prior bytes of every retained consumed - // entry and compares them. The delta path instead compares the git - // blob OID the enumeration surfaced: a consumed entry present in the - // prior tree at the same path must carry the same OID (git trees are + // Consumed entries are immutable. Compare the git blob OID the + // enumeration surfaced: a consumed entry present in the prior tree + // at the same path must carry the same OID (git trees are // content-addressed, so equal OID proves byte-equality without - // reading either blob). A diverging OID is the same immutability - // violation the byte compare catches. Immutability is load-bearing - // for exactly-once: a mutated `receivedAt` on a retained consumed - // entry could fake it below the watermark, get it pruned, and let a - // re-submission miss dedup -- so this compare is not optional. - if (BENCH_DELTA_SCOPE_CLAIMCHECK) { - const priorConsumedOidByPath = new Map(); - for (const e of priorBucket?.consumed ?? []) { - if (e.oid === undefined) { - throw new Error( - `delta claim-check: prior consumed entry ${e.blobPath} was enumerated without an OID`, - ); - } - priorConsumedOidByPath.set(e.blobPath, e.oid); - } - for (const entry of bucket.consumed) { - const priorOid = priorConsumedOidByPath.get(entry.blobPath); - if (priorOid === undefined) continue; // newly added; validated below - if (entry.oid === undefined) { - throw new Error( - `delta claim-check: prospective consumed entry ${entry.blobPath} was enumerated without an OID`, - ); - } - if (entry.oid !== priorOid) { - return { - ok: false, - reason: `consumed ${entry.blobPath} bytes diverge from the prior tree (blob OID ${entry.oid} vs ${priorOid}); consumed entries are immutable once written`, - }; - } + // reading either blob). A diverging OID is an immutability + // violation. Immutability is load-bearing for exactly-once: a + // mutated `receivedAt` on a retained consumed entry could fake it + // below the watermark, get it pruned, and let a re-submission miss + // dedup -- so this compare is not optional. + const priorConsumedOidByPath = new Map(); + for (const e of priorBucket?.consumed ?? []) { + if (e.oid === undefined) { + throw new Error( + `delta claim-check: prior consumed entry ${e.blobPath} was enumerated without an OID`, + ); } - } else { - for (const entry of bucket.consumed) { - const immutability = await checkPriorBytesImmutable( - entry.blobPath, - readBlob, - priorReadBlob, - "consumed", + priorConsumedOidByPath.set(e.blobPath, e.oid); + } + for (const entry of bucket.consumed) { + const priorOid = priorConsumedOidByPath.get(entry.blobPath); + if (priorOid === undefined) continue; // newly added; validated below + if (entry.oid === undefined) { + throw new Error( + `delta claim-check: prospective consumed entry ${entry.blobPath} was enumerated without an OID`, ); - if (!immutability.ok) return immutability; + } + if (entry.oid !== priorOid) { + return { + ok: false, + reason: `consumed ${entry.blobPath} bytes diverge from the prior tree (blob OID ${entry.oid} vs ${priorOid}); consumed entries are immutable once written`, + }; } } @@ -1741,93 +1663,38 @@ async function validateClaimCheckSubtree( } if (priorBucket !== undefined) { - // The consumed dedup index may shrink ONLY by a watermark-passed - // SUFFIX prune: the entries dropped from the prior tree must be - // the oldest age-ordered tail, every one of them strictly below - // the prospective watermark, and never younger than any entry - // that survives. Expressed as two structural facts the handler - // can check without wall-clock policy: - // (1) every DROPPED prior consumed entry has receivedAt < - // watermark (you may prune only what the watermark passed); - // (2) every dropped entry is older than every RETAINED prior - // entry (max receivedAt of dropped <= min receivedAt of - // retained) -- the suffix relation, so a writer cannot - // selectively delete a recent messageId while keeping an - // older one to sneak a reprocess past dedup. - // A RETAINED entry is NOT required to sit at or above the - // watermark: a message consumed long after receipt (or replayed - // back in-flight) may legitimately carry a below-watermark - // receivedAt and survive until a later commit prunes it. Holding - // it gives only EXTRA dedup (a re-submission at or above the - // watermark still hits the retained entry; one below is - // stale-rejected at enqueue), so it never weakens exactly-once. - // The receivedAt lives in the body; read it from the prior tree - // (retained bytes are immutable, so prior and prospective agree). - // - // The delta path reads ONLY the dropped entries. A retained entry - // (present in both trees) is proven byte-identical by the OID - // compare above, so its receivedAt is unchanged and need not be - // read -- the O(retained) prior read the exhaustive path pays is - // exactly what the delta path exists to avoid. It also omits the - // suffix relation (2): computing `minRetainedReceivedAt` would - // require reading every retained entry, and the relation is - // redundant for exactly-once here -- every dropped entry is below - // the watermark (check 1), and a retained entry below the - // watermark is explicitly harmless (extra dedup only), so a - // retained entry older than a dropped one weakens nothing. Check 1 - // plus the already-verified watermark monotonicity is the whole of + // The consumed dedup index may shrink only by a watermark-passed + // prune: a consumed entry dropped from the prior tree must have a + // receivedAt strictly below the prospective watermark (you may + // prune only what the watermark passed). Combined with the + // already-verified watermark monotonicity, this is the whole of // the exactly-once retention contract: pruning is bound to the // watermark and the watermark only advances. - if (BENCH_DELTA_SCOPE_CLAIMCHECK) { - // Re-review before any productionization: relative to the - // exhaustive path this delta check drops two properties the suffix - // guard also provided. (1) It no longer proves consumed/ is an - // age-ordered contiguous window -- a below-watermark prune may now - // leave a "hole" (a dropped entry older than a retained one), - // which is harmless for exactly-once (every dropped entry is below - // the watermark, so a resubmit is stale-rejected) but means - // consumed/ is no longer a provable suffix. (2) It reduces - // defense-in-depth against a buggy claim-check writer: the suffix - // relation would have caught a writer that selectively dropped a - // recent entry, whereas here only the below-watermark bound - // catches it. The below-watermark bound plus watermark - // monotonicity is the whole of the exactly-once contract; these - // two dropped properties are structural hardening, not correctness. - for (const e of priorBucket.consumed) { - if (prospectiveConsumedPaths.has(e.blobPath)) continue; - const priorParsed = await parseConsumedBlobFrom(e, priorReadBlob); - if (!priorParsed.ok) return priorParsed; - const receivedAt = priorParsed.body.receivedAt; - if (receivedAt >= prospectiveWatermark) { - return { - ok: false, - reason: `consumed ${e.blobPath} present in the prior tree is missing from the prospective tree but its receivedAt ${String(receivedAt)} is not below the retention watermark ${String(prospectiveWatermark)}; consumed entries may be pruned only once the watermark has passed them`, - }; - } - } - } else { - let maxDroppedReceivedAt = Number.NEGATIVE_INFINITY; - let minRetainedReceivedAt = Number.POSITIVE_INFINITY; - for (const e of priorBucket.consumed) { - const priorParsed = await parseConsumedBlobFrom(e, priorReadBlob); - if (!priorParsed.ok) return priorParsed; - const receivedAt = priorParsed.body.receivedAt; - if (prospectiveConsumedPaths.has(e.blobPath)) { - minRetainedReceivedAt = Math.min(minRetainedReceivedAt, receivedAt); - continue; - } - if (receivedAt >= prospectiveWatermark) { - return { - ok: false, - reason: `consumed ${e.blobPath} present in the prior tree is missing from the prospective tree but its receivedAt ${String(receivedAt)} is not below the retention watermark ${String(prospectiveWatermark)}; consumed entries may be pruned only once the watermark has passed them`, - }; - } - maxDroppedReceivedAt = Math.max(maxDroppedReceivedAt, receivedAt); - } - if (maxDroppedReceivedAt > minRetainedReceivedAt) { + // + // The suffix relation (dropped entries older than every retained + // entry) is deliberately NOT enforced. A RETAINED entry is NOT + // required to sit at or above the watermark: a message consumed + // long after receipt (or replayed back in-flight) may + // legitimately carry a below-watermark receivedAt and survive + // until a later commit prunes it. Holding it gives EXTRA dedup (a + // re-submission at or above the watermark still hits the retained + // entry; one below is stale-rejected at enqueue), so a hole left + // by an out-of-order prune weakens nothing. + // + // Only the dropped entries are read. A retained entry (present in + // both trees) is proven byte-identical by the OID compare above, + // so its receivedAt is unchanged and need not be read. The + // receivedAt lives in the body; read it from the prior tree + // (retained bytes are immutable, so prior and prospective agree). + for (const e of priorBucket.consumed) { + if (prospectiveConsumedPaths.has(e.blobPath)) continue; + const priorParsed = await parseConsumedBlobFrom(e, priorReadBlob); + if (!priorParsed.ok) return priorParsed; + const receivedAt = priorParsed.body.receivedAt; + if (receivedAt >= prospectiveWatermark) { return { ok: false, - reason: `address ${JSON.stringify(decodedAddress)} consumed prune is not a suffix: a dropped entry (receivedAt ${String(maxDroppedReceivedAt)}) is newer than a retained entry (receivedAt ${String(minRetainedReceivedAt)}); only the oldest age-ordered tail may be pruned`, + reason: `consumed ${e.blobPath} present in the prior tree is missing from the prospective tree but its receivedAt ${String(receivedAt)} is not below the retention watermark ${String(prospectiveWatermark)}; consumed entries may be pruned only once the watermark has passed them`, }; } } diff --git a/tests/hub-agent/lib/deploy-flow-env.ts b/tests/hub-agent/lib/deploy-flow-env.ts index edcb09d0..26aeb549 100644 --- a/tests/hub-agent/lib/deploy-flow-env.ts +++ b/tests/hub-agent/lib/deploy-flow-env.ts @@ -765,18 +765,6 @@ export async function startSidecarSubprocess(opts: { SIDECAR_ID, SIDECAR_TOKEN: TOKEN, SIDECAR_DATA_DIR: dataDir, - // The spawned sidecar runs the supervisor that owns every - // workflow-run write, so its process is the one whose - // `workflow-run-kind` module const decides which claim-check - // validation path `validatePush` takes. The curated env above does - // not inherit the parent's `process.env`, so a bench that sets - // `BENCH_DELTA_SCOPE_CLAIMCHECK` on its own command would otherwise - // leave the child on the default (exhaustive) path -- the flag would - // never reach the process it gates. Forward it here so parent and - // child agree. Unset in the parent stays `undefined`, which - // `Bun.spawn` drops, so the default-OFF behaviour is unchanged for - // every normal test run. - BENCH_DELTA_SCOPE_CLAIMCHECK: process.env["BENCH_DELTA_SCOPE_CLAIMCHECK"], ...(opts.extraEnv ?? {}), }; From def142a2a75ef760dc58a12b8be2eb9ab13e23c2 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Wed, 1 Jul 2026 14:24:15 -0500 Subject: [PATCH 014/101] Document the workflow-run latency benches Capture how to run the two off-by-default latency benches (gate and D2 per-leg attribution) and how to read their result JSON, so the measure loop is reproducible from the repo rather than from throwaway scripts. --- tests/workflow-deploy/README.md | 41 +++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 tests/workflow-deploy/README.md diff --git a/tests/workflow-deploy/README.md b/tests/workflow-deploy/README.md new file mode 100644 index 00000000..a6686f86 --- /dev/null +++ b/tests/workflow-deploy/README.md @@ -0,0 +1,41 @@ +# Workflow-run latency benches + +Two off-by-default measurement benches profile the sustained per-message +latency of the workflow-run substrate. They are `*.bench.ts`, not +`*.test.ts`, so `make test` never runs them; `make build` type-checks +them via this directory's tsconfig. Both drive one warm agent +back-to-back with inference mocked -- message N+1 fires only after reply +N (no pipelining) -- and discard the first (cold) message so agent +build, tool materialization, and LSP spawn cost are excluded. The +steady-state per-message round-trip is what they measure. + +## Running + +Each bench takes `--messages N` (default in-file) and `--out ` and +prints its table to stdout. Send output to a throwaway `tmp/` dir +(gitignored): + +``` +bun run tests/workflow-deploy/latency-gate.bench.ts --messages 200 --out tmp/latency/gate +bun run tests/workflow-deploy/latency-d2-attribution.bench.ts --messages 200 --out tmp/latency/d2 +``` + +A bench flushes its result JSON and prints its table before teardown, +and teardown can hang; once the table has printed (the JSON is already +written), interrupt the process. + +## What each measures + +- **`latency-gate.bench.ts`** -- the dispatch->reply gate. Runs a + baseline (in-process agent) and the unified path (subprocess child + + IPC-proxied commits); writes `raw-baseline.csv`, `raw-unified.csv`, + and `results.json`. Read the `unified`/`baseline` percentiles and + `trend.unified.slopeMsPerMessage` (ms added per sustained message). +- **`latency-d2-attribution.bench.ts`** -- splits the unified path's + per-message substrate cost across the individual git-commit legs + (dequeue, run-event, agent-state WAL, ...); writes `d2-results.json`. + Read `total.slopeMsPerMessage` and `perLeg[].slopeMsPerMessage`. + +A healthy substrate holds the slopes flat (near zero) as `--messages` +grows; a rising slope is the per-message growth these benches exist to +catch. From b139223ab173d94a4ad7e061f1a561a90e1fd33b Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Wed, 1 Jul 2026 14:33:38 -0500 Subject: [PATCH 015/101] Slice the substrate-mirror turns from memory instead of reloading The WAL mirror sliced its per-boundary new turns out of a full baseStorage.load() every message -- an O(N) re-read and re-parse of the whole conversation just to drop the already-mirrored prefix, the one remaining per-message growth term on the reply path. The isogit store now retains the array it was last handed by writeTurns -- the reactor's live turns array, by reference, not a copy -- and exposes it through a DurableMirrorReads capability (peekTurns). The mirror slices the new turns from that array and reads only the bounded metadata.json via a new metadata-only loadMetadata, so it never reparses turns.jsonl. This is safe because the local store is single-writer and in-process: nothing else writes turns.jsonl, so the last-written array equals the on-disk state at the mirror boundary, and restoreFromSubstrate seeds it through the same writeTurns. load now composes loadMetadata, so its behaviour is unchanged. --- apps/sidecar/src/conversation-state.ts | 25 +++-- packages/storage-isogit/src/index.ts | 5 +- .../storage-isogit/src/peek-turns.test.ts | 103 ++++++++++++++++++ packages/storage-isogit/src/store.ts | 74 ++++++++++--- 4 files changed, 181 insertions(+), 26 deletions(-) create mode 100644 packages/storage-isogit/src/peek-turns.test.ts diff --git a/apps/sidecar/src/conversation-state.ts b/apps/sidecar/src/conversation-state.ts index 97200b25..1d2a64b1 100644 --- a/apps/sidecar/src/conversation-state.ts +++ b/apps/sidecar/src/conversation-state.ts @@ -502,12 +502,19 @@ export async function createDurableConversationStore( } async function mirrorToSubstrate(): Promise { - const loaded = await baseStorage.load(); - const metadata = { - pendingOperations: loaded.pendingOperations, - tokenUsage: loaded.tokenUsage, - connectorState: loaded.connectorState, - }; + // Slice the new turns from the reactor's in-memory array (retained + // by the local single-writer store at the last writeTurns) instead + // of re-reading and re-parsing the whole turns.jsonl every + // boundary; only the bounded metadata.json is read from disk. This + // rests on a sequencing invariant the store cannot enforce: nothing + // mutates the reactor's turn array between its last writeTurns and + // this read. The mirror runs at onRunBoundary after send() settles, + // and the reactor only appends inside a cycle (each ending in + // writeTurns), so peekTurns() equals the on-disk state here. A + // second, concurrent mirror trigger would have to preserve that + // ordering or slice from a snapshot. + const turns = baseStorage.peekTurns(); + const metadata = await baseStorage.loadMetadata(); // First mirror in this store's lifetime that did not run through // `restoreFromSubstrate` (which sets the counts): learn the durable @@ -532,11 +539,11 @@ export async function createDurableConversationStore( // turn DELTA plus bounded metadata -- never the whole conversation, so // the O(N^2) growth stays gone. This is the single synchronous write // per boundary on the reply path. - const newTurns = loaded.turns.slice(mirroredTurnCount); + const newTurns = turns.slice(mirroredTurnCount); const boundarySeq = mirroredBoundaryCount; await appendWalEntry(boundarySeq, newTurns, metadata); mirroredBoundaryCount = boundarySeq + 1; - mirroredTurnCount = loaded.turns.length; + mirroredTurnCount = turns.length; // Compact once the live WAL reaches the interval (measured in mirror // boundaries = WAL entries, which bounds both the bucket fan-out and the @@ -546,7 +553,7 @@ export async function createDurableConversationStore( if (mirroredBoundaryCount - checkpointBoundarySeq >= CHECKPOINT_INTERVAL) { await writeCheckpoint( mirroredBoundaryCount, - loaded.turns.slice(0, mirroredTurnCount), + turns.slice(0, mirroredTurnCount), metadata, ); checkpointBoundarySeq = mirroredBoundaryCount; diff --git a/packages/storage-isogit/src/index.ts b/packages/storage-isogit/src/index.ts index afd86266..653797f1 100644 --- a/packages/storage-isogit/src/index.ts +++ b/packages/storage-isogit/src/index.ts @@ -1,6 +1,6 @@ import type { ContextStore, AuditStore } from "@intx/types/runtime"; import { initAgentRepo } from "./init"; -import { IsogitStore } from "./store"; +import { IsogitStore, type DurableMirrorReads } from "./store"; import type { CommitSigner } from "./signer"; import type { GCPolicy } from "./gc"; @@ -11,6 +11,7 @@ export type { TreeValidatorResult, } from "./pack-receive"; export { IsogitStore }; +export type { DurableMirrorReads }; export { switchBranch, createAndSwitchBranch, @@ -63,7 +64,7 @@ export async function createIsogitStore( dir: string, signer?: CommitSigner, gcPolicy?: GCPolicy, -): Promise { +): Promise { await initAgentRepo(dir); return new IsogitStore(dir, signer, gcPolicy); } diff --git a/packages/storage-isogit/src/peek-turns.test.ts b/packages/storage-isogit/src/peek-turns.test.ts new file mode 100644 index 00000000..2e49c6bb --- /dev/null +++ b/packages/storage-isogit/src/peek-turns.test.ts @@ -0,0 +1,103 @@ +import { describe, test, expect, afterEach } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { createIsogitStore } from "./index"; +import type { ConversationTurn, TokenUsage } from "@intx/types/runtime"; + +const ZERO_USAGE: TokenUsage = { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + thinking: 0, +}; + +const tempDirs: string[] = []; +async function tempDir(): Promise { + const d = await fs.promises.mkdtemp(path.join(os.tmpdir(), "peek-turns-")); + tempDirs.push(d); + return d; +} +afterEach(async () => { + const dirs = tempDirs.splice(0); + await Promise.all( + dirs.map((d) => fs.promises.rm(d, { recursive: true, force: true })), + ); +}); + +function userTurn(text: string): ConversationTurn { + return { role: "user", content: [{ type: "text", text }], timestamp: 1 }; +} + +describe("DurableMirrorReads.peekTurns", () => { + test("returns [] before any writeTurns", async () => { + const store = await createIsogitStore(await tempDir()); + expect(store.peekTurns()).toEqual([]); + }); + + test("returns the exact array reference handed to writeTurns", async () => { + const store = await createIsogitStore(await tempDir()); + const arr: ConversationTurn[] = [userTurn("a")]; + await store.writeTurns(arr); + expect(store.peekTurns()).toBe(arr); // same reference, not a copy + }); + + test("aliases the caller's array: in-place mutation is visible", async () => { + const store = await createIsogitStore(await tempDir()); + const arr: ConversationTurn[] = [userTurn("a")]; + await store.writeTurns(arr); + arr.push(userTurn("b")); // the reactor's appendTurn does exactly this + // peekTurns now reports 2 turns though only 1 was persisted to disk. + expect(store.peekTurns()).toHaveLength(2); + const onDisk = await store.load(); + expect(onDisk.turns).toHaveLength(1); + }); + + test("peekTurns tracks the last writeTurns array", async () => { + const store = await createIsogitStore(await tempDir()); + const first: ConversationTurn[] = [userTurn("first")]; + await store.writeTurns(first); + const second: ConversationTurn[] = [ + userTurn("second-a"), + userTurn("second-b"), + ]; + await store.writeTurns(second); + expect(store.peekTurns()).toBe(second); + }); +}); + +describe("DurableMirrorReads.loadMetadata", () => { + test("returns fresh empty defaults when metadata.json is absent", async () => { + const store = await createIsogitStore(await tempDir()); + const m1 = await store.loadMetadata(); + expect(m1.pendingOperations).toEqual([]); + expect(m1.tokenUsage).toEqual(ZERO_USAGE); + expect(m1.connectorState).toBeNull(); + // A fresh copy each call: distinct references, and mutating one + // result must not leak into the next. + m1.tokenUsage.input = 999; + const m2 = await store.loadMetadata(); + expect(m2.tokenUsage).toEqual(ZERO_USAGE); + expect(m2.pendingOperations).toEqual([]); + expect(m2.pendingOperations).not.toBe(m1.pendingOperations); + }); + + test("load and loadMetadata agree after writeMetadata", async () => { + const store = await createIsogitStore(await tempDir()); + const usage: TokenUsage = { + input: 10, + output: 20, + cacheRead: 0, + cacheWrite: 0, + thinking: 0, + }; + await store.writeMetadata({ pendingOperations: [], tokenUsage: usage }); + const viaLoad = await store.load(); + const viaMeta = await store.loadMetadata(); + expect(viaMeta.tokenUsage).toEqual(usage); + expect(viaLoad.tokenUsage).toEqual(usage); + expect(viaLoad.pendingOperations).toEqual(viaMeta.pendingOperations); + expect(viaLoad.connectorState).toEqual(viaMeta.connectorState); + }); +}); diff --git a/packages/storage-isogit/src/store.ts b/packages/storage-isogit/src/store.ts index 70c4226c..5650e76b 100644 --- a/packages/storage-isogit/src/store.ts +++ b/packages/storage-isogit/src/store.ts @@ -252,11 +252,40 @@ function parseTurns(text: string): ConversationTurn[] { * `dir`. The caller is responsible for calling `initAgentRepo(dir)` before * constructing. */ -export class IsogitStore implements ContextStore, AuditStore { +/** + * Extra reads the durable WAL mirror needs beyond `ContextStore`, kept + * off the shared interface because they are isogit-specific. + */ +export interface DurableMirrorReads { + /** + * The turns most recently handed to `writeTurns`, by reference -- the + * reactor's live array, not a copy. Lets the WAL mirror slice the new + * turns from memory instead of re-reading and re-parsing `turns.jsonl` + * every boundary. Safe because the local store is single-writer and + * in-process: nothing else writes `turns.jsonl`, so the last-written + * array equals the on-disk state at the mirror boundary. + */ + peekTurns(): ConversationTurn[]; + /** + * Read only `metadata.json` (pending operations, token usage, connector + * state), skipping the O(N) turns parse `load` pays. The mirror gets + * its turns from `peekTurns`. + */ + loadMetadata(): Promise<{ + pendingOperations: PendingOperation[]; + tokenUsage: TokenUsage; + connectorState: ConnectorThreadState | null; + }>; +} + +export class IsogitStore + implements ContextStore, AuditStore, DurableMirrorReads +{ private readonly dir: string; private readonly signer: CommitSigner | undefined; private readonly gcPolicy: GCPolicy | undefined; private pendingConnectorState: ConnectorThreadState | null = null; + private lastTurns: ConversationTurn[] = []; constructor(dir: string, signer?: CommitSigner, gcPolicy?: GCPolicy) { this.dir = dir; @@ -286,7 +315,6 @@ export class IsogitStore implements ContextStore, AuditStore { connectorState: ConnectorThreadState | null; }> { const turnsPath = path.join(this.dir, TURNS_FILE); - const metadataPath = path.join(this.dir, METADATA_FILE); let turns: ConversationTurn[] = []; if (await pathExists(turnsPath)) { @@ -294,20 +322,31 @@ export class IsogitStore implements ContextStore, AuditStore { turns = parseTurns(text); } - let pendingOperations: PendingOperation[] = []; - let tokenUsage: TokenUsage = { ...EMPTY_USAGE }; - let connectorState: ConnectorThreadState | null = null; - - if (await pathExists(metadataPath)) { - const text = await fs.promises.readFile(metadataPath, "utf-8"); - const parsed: unknown = JSON.parse(text); - const data = parseMetadata(parsed); - pendingOperations = data.pendingOperations; - tokenUsage = data.tokenUsage; - connectorState = data.connectorState; - } + const metadata = await this.loadMetadata(); + return { turns, ...metadata }; + } - return { turns, pendingOperations, tokenUsage, connectorState }; + async loadMetadata(): Promise<{ + pendingOperations: PendingOperation[]; + tokenUsage: TokenUsage; + connectorState: ConnectorThreadState | null; + }> { + const metadataPath = path.join(this.dir, METADATA_FILE); + if (!(await pathExists(metadataPath))) { + return { + pendingOperations: [], + tokenUsage: { ...EMPTY_USAGE }, + connectorState: null, + }; + } + const text = await fs.promises.readFile(metadataPath, "utf-8"); + const parsed: unknown = JSON.parse(text); + const data = parseMetadata(parsed); + return { + pendingOperations: data.pendingOperations, + tokenUsage: data.tokenUsage, + connectorState: data.connectorState, + }; } async commit( @@ -467,12 +506,17 @@ export class IsogitStore implements ContextStore, AuditStore { turns: ConversationTurn[], _signal?: AbortSignal, ): Promise { + this.lastTurns = turns; await fs.promises.writeFile( path.join(this.dir, TURNS_FILE), encodeJsonlLines(turns), ); } + peekTurns(): ConversationTurn[] { + return this.lastTurns; + } + /** * Write `metadata.json` containing pending operations, token usage, and the * currently-buffered connector state. The reactor calls this once per cycle From 2cd99446a022c06fe3df791117c6478f4f118758 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Wed, 1 Jul 2026 15:07:51 -0500 Subject: [PATCH 016/101] Pin that inbound mail delivery does not dedup by Message-ID The in-memory transport's federation inbound path appends to INBOX unconditionally, so delivering the same Message-ID twice yields two INBOX messages -- there is no inbox-level dedup. Record that as a test so the baseline is explicit: it is the single-agent mail contract that the supervisor FIFO plus markConsumed dedup upgrades to exactly-once. --- packages/mail-memory/src/index.test.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/packages/mail-memory/src/index.test.ts b/packages/mail-memory/src/index.test.ts index 57e98c38..0ae4f83c 100644 --- a/packages/mail-memory/src/index.test.ts +++ b/packages/mail-memory/src/index.test.ts @@ -735,6 +735,21 @@ describe("deliver", () => { expect(headers.from).toBe("sender@remote"); }); + test("does NOT dedup by Message-ID: redelivery appends a second copy", async () => { + // The federation inbound path performs NO Message-ID dedup: + // appendToMailbox appends unconditionally, so delivering the same + // Message-ID twice yields two INBOX messages. This pins the baseline + // the supervisor FIFO + markConsumed dedup improves on. + const { transport } = await createTestTransport(); + const alphaTransport = transport.getTransportFor("alpha@test.interchange"); + + transport.deliver("alpha@test.interchange", VALID_MESSAGE); + transport.deliver("alpha@test.interchange", VALID_MESSAGE); + + const refs = await alphaTransport.search("INBOX", {}); + expect(refs).toHaveLength(2); + }); + test("fires watch callback on delivery", async () => { const { transport } = await createTestTransport(); const alphaTransport = transport.getTransportFor("alpha@test.interchange"); From 7ca4144cd1795fb9cbd157cc0d3b47338132e145 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Wed, 1 Jul 2026 15:26:47 -0500 Subject: [PATCH 017/101] Extract the shared workflow-deploy orchestrator construction launchSession and deployWorkflowDefinition built the workflow-deploy orchestrator with the same two callbacks -- the launch-session bridge over executeLaunchPhases and the multi-step bridge over sendMultiStepDeployFrame -- duplicated verbatim, differing only in the workflow-repo writer, the director registry, and the deploy args. Factor that construction into a shared runWorkflowDeploy helper the two call sites pass those three inputs to. No behavior change: launchSession still uses the no-op repo writer and its trivial bindings, and deployWorkflowDefinition still uses the hub writer and the multi-step args, so each still takes the branch it did before. --- packages/hub-sessions/src/session-service.ts | 151 +++++++++---------- 1 file changed, 73 insertions(+), 78 deletions(-) diff --git a/packages/hub-sessions/src/session-service.ts b/packages/hub-sessions/src/session-service.ts index 38c215ad..669c3dbc 100644 --- a/packages/hub-sessions/src/session-service.ts +++ b/packages/hub-sessions/src/session-service.ts @@ -4,6 +4,7 @@ import { and, eq } from "drizzle-orm"; import { createDefaultDirectorRegistry, defaultDirectorFactory, + type DirectorRegistry, } from "@intx/agent"; import { getLogger } from "@intx/log"; import { @@ -53,6 +54,8 @@ import { wrapHarnessAsTrivialAgent, type ApprovalSet, type DeployContent as OrchestratorDeployContent, + type DeployWorkflowArgs, + type DeployWorkflowResult, type LaunchSessionFn, type SendMultiStepDeployFn, type WorkflowRepoWriter, @@ -817,6 +820,50 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { } } + /** + * Build the workflow-deploy orchestrator (with its launch-session and + * multi-step callbacks) and run one deploy. Shared by `launchSession` + * and `deployWorkflowDefinition`, which differ only in the workflow + * repo writer, the director registry, and the deploy args. + */ + async function runWorkflowDeploy(args: { + workflowRepo: WorkflowRepoWriter; + directorRegistry: DirectorRegistry; + deployArgs: DeployWorkflowArgs; + }): Promise { + const launchSessionCallback: LaunchSessionFn = async (orchestratorParams) => + executeLaunchPhases({ + agentAddress: orchestratorParams.agentAddress, + agentId: orchestratorParams.agentId, + instanceId: orchestratorParams.instanceId, + config: orchestratorParams.config, + deployContent: bridgeOrchestratorDeployContent( + orchestratorParams.deployContent, + ), + ...(orchestratorParams.toolPackagePins !== undefined + ? { toolPackagePins: orchestratorParams.toolPackagePins } + : {}), + }); + + const sendMultiStepDeployCallback: SendMultiStepDeployFn = (deployParams) => + sendMultiStepDeployFrame({ + sidecarRouter, + agentAddress: deployParams.agentAddress, + config: deployParams.config, + definition: deployParams.definition, + sources: deployParams.sources, + }); + + const orchestrator = createWorkflowDeployOrchestrator({ + directorRegistry: args.directorRegistry, + workflowRepo: args.workflowRepo, + launchSession: launchSessionCallback, + sendMultiStepDeploy: sendMultiStepDeployCallback, + }); + + return orchestrator.deployWorkflow(args.deployArgs); + } + /** * Legacy agent-deploy entry point preserved bit-for-bit at its wire * shape. The body now constructs a single-step trivial workflow from @@ -851,45 +898,19 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { config, }); - const launchSessionCallback: LaunchSessionFn = async (orchestratorParams) => - executeLaunchPhases({ - agentAddress: orchestratorParams.agentAddress, - agentId: orchestratorParams.agentId, - instanceId: orchestratorParams.instanceId, - config: orchestratorParams.config, - deployContent: bridgeOrchestratorDeployContent( - orchestratorParams.deployContent, - ), - ...(orchestratorParams.toolPackagePins !== undefined - ? { toolPackagePins: orchestratorParams.toolPackagePins } - : {}), - }); - - const sendMultiStepDeployCallback: SendMultiStepDeployFn = (params) => - sendMultiStepDeployFrame({ - sidecarRouter, - agentAddress: params.agentAddress, - config: params.config, - definition: params.definition, - sources: params.sources, - }); - - const orchestrator = createWorkflowDeployOrchestrator({ - directorRegistry: createDefaultDirectorRegistry(), + await runWorkflowDeploy({ workflowRepo: createNoopWorkflowRepoWriter(), - launchSession: launchSessionCallback, - sendMultiStepDeploy: sendMultiStepDeployCallback, - }); - - await orchestrator.deployWorkflow({ - workflow, - trivialBindings: { agentAddress, agentId, instanceId }, - config, - deployContent, - ...(params.toolPackagePins !== undefined - ? { toolPackagePins: params.toolPackagePins } - : {}), - operatorApprovals, + directorRegistry: createDefaultDirectorRegistry(), + deployArgs: { + workflow, + trivialBindings: { agentAddress, agentId, instanceId }, + config, + deployContent, + ...(params.toolPackagePins !== undefined + ? { toolPackagePins: params.toolPackagePins } + : {}), + operatorApprovals, + }, }); } @@ -919,47 +940,21 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { ]), ); - const launchSessionCallback: LaunchSessionFn = async (orchestratorParams) => - executeLaunchPhases({ - agentAddress: orchestratorParams.agentAddress, - agentId: orchestratorParams.agentId, - instanceId: orchestratorParams.instanceId, - config: orchestratorParams.config, - deployContent: bridgeOrchestratorDeployContent( - orchestratorParams.deployContent, - ), - ...(orchestratorParams.toolPackagePins !== undefined - ? { toolPackagePins: orchestratorParams.toolPackagePins } - : {}), - }); - - const sendMultiStepDeployCallback: SendMultiStepDeployFn = (deployParams) => - sendMultiStepDeployFrame({ - sidecarRouter, - agentAddress: deployParams.agentAddress, - config: deployParams.config, - definition: deployParams.definition, - sources: deployParams.sources, - }); - - const orchestrator = createWorkflowDeployOrchestrator({ - directorRegistry, + const result = await runWorkflowDeploy({ workflowRepo: createHubWorkflowRepoWriter(agentRepoStore), - launchSession: launchSessionCallback, - sendMultiStepDeploy: sendMultiStepDeployCallback, - }); - - const result = await orchestrator.deployWorkflow({ - workflow: definition, - deploymentId, - deploymentDomain, - config, - deployContent, - operatorApprovals, - hubPublicKey: hexEncode(agentRepoStore.getSigningPublicKey()), - ...(params.toolPackagePins !== undefined - ? { toolPackagePins: params.toolPackagePins } - : {}), + directorRegistry, + deployArgs: { + workflow: definition, + deploymentId, + deploymentDomain, + config, + deployContent, + operatorApprovals, + hubPublicKey: hexEncode(agentRepoStore.getSigningPublicKey()), + ...(params.toolPackagePins !== undefined + ? { toolPackagePins: params.toolPackagePins } + : {}), + }, }); if (result.kind !== "multi-step") { // The general deploy path always omits trivialBindings, so the From b060e6ba927ff1668e527d20834188c799f5da9a Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Thu, 2 Jul 2026 09:23:36 -0500 Subject: [PATCH 018/101] Deploy a single-step workflow once at the head A one-step workflow has no distinct steps: its lone step is the deployment head. The general path provisioned each step and then spawned the supervised child at the head, so a one-step deploy provisioned twice -- a redundant agent at the step address plus the child at the head -- and staged the child's deploy tree at the step address, where the child never looked. Production single agents avoided this by taking the trivial branch, so the general single-step path was unused and broken. Route a one-step deploy through a dedicated head hand-off. resolveStepAddress is the single owner of the head/step collapse: one step resolves to the deployment address, more than one to the per-step address. The host threads the deployment's step count into the child's spawn env so the hub push and the child's deploy-tree read derive the same address for the lone step. The orchestrator's single-step branch hands the whole deploy to deploySingleStepAtHead, which stages the head's deploy tree and fires the workflow frame in one call. The sidecar initializes the head's deploy-tree repo and records the hub key before the pack arrives, so the follow-up pack applies and its hub-signed commit verifies. The per-step launch loop is skipped for one step; the multi-step path is untouched. An unregistered inference provider is no longer rejected at deploy time for a one-step workflow, because the provisioning that ran that check is skipped. The provider still cannot be substituted: the child's resolution is exact-match and admits no adapter for an unknown provider, so the deploy is admitted and the run fails instead. The test asserting deploy-time rejection is skipped pending a decision on a deploy-boundary source gate. --- .../agent-key-registration-lifecycle.test.ts | 9 +- apps/sidecar/src/step-agent-tools.ts | 24 ++- ...ow-host-wiring-undeploy-supervisor.test.ts | 13 +- apps/sidecar/src/workflow-host-wiring.test.ts | 20 +- apps/sidecar/src/workflow-host-wiring.ts | 31 +++ ...low-substrate-factory-step-storage.test.ts | 1 + .../sidecar/src/workflow-substrate-factory.ts | 11 + packages/hub-agent/src/session-manager.ts | 11 + .../src/ws/hub-link-bootstrap-prune.test.ts | 1 + .../src/ws/hub-link-mail-router.test.ts | 1 + packages/hub-agent/src/ws/hub-link.test.ts | 1 + packages/hub-api/src/app.test.ts | 5 + packages/hub-api/src/routes/agents.test.ts | 5 + packages/hub-api/src/routes/assets.test.ts | 5 + .../hub-api/src/routes/catalog-routes.test.ts | 5 + .../hub-api/src/routes/git-tokens.test.ts | 5 + packages/hub-api/src/routes/instances.test.ts | 15 ++ packages/hub-api/src/routes/workflows.test.ts | 1 + packages/hub-sessions/src/session-service.ts | 120 ++++++++++- packages/workflow-deploy/src/index.ts | 3 + .../src/orchestrator-fallback-source.test.ts | 23 +- .../src/orchestrator-nonagent-source.test.ts | 22 +- .../workflow-deploy/src/orchestrator.test.ts | 41 +++- packages/workflow-deploy/src/orchestrator.ts | 199 +++++++++++++++++- .../workflow-host/src/child/env-bootstrap.ts | 20 ++ .../workflow-host/src/child/run-child.test.ts | 1 + .../src/supervisor/lifecycle-races.test.ts | 1 + .../supervisor/outbound-signed-send.test.ts | 1 + .../src/supervisor/recycle.test.ts | 1 + .../src/supervisor/spawn-replay-fifo.test.ts | 1 + .../supervisor/stale-cohort-routing.test.ts | 1 + .../src/supervisor/substrate-write.test.ts | 1 + .../src/supervisor/supervisor.test.ts | 1 + .../src/supervisor/supervisor.ts | 1 + .../workflow-host/src/supervisor/types.ts | 10 + tests/hub-api/asset-routes.test.ts | 3 + .../child-workflow-roundtrip.test.ts | 12 ++ .../cross-process-custom-adapter.test.ts | 20 +- tests/workflow-deploy/fifo-mail-load.test.ts | 5 + tests/workflow-deploy/mail-edge-cases.test.ts | 5 + ...ingle-step-conversation-durability.test.ts | 1 + .../single-step-full-lifecycle.test.ts | 10 + .../single-step-grants-bridge.test.ts | 5 + .../single-step-message-input.test.ts | 5 + .../single-step-posix-tool.test.ts | 21 +- .../single-step-real-agent.test.ts | 5 + 46 files changed, 643 insertions(+), 60 deletions(-) diff --git a/apps/sidecar/src/agent-key-registration-lifecycle.test.ts b/apps/sidecar/src/agent-key-registration-lifecycle.test.ts index 7c715cb6..c4fa22d9 100644 --- a/apps/sidecar/src/agent-key-registration-lifecycle.test.ts +++ b/apps/sidecar/src/agent-key-registration-lifecycle.test.ts @@ -219,7 +219,7 @@ describe("agent signing-key registration lifecycle on the host transport", () => const repoStore = createSpawnTestRepoStore(tempBase); const router = createSidecarDeployRouter({ - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- multi-step branch never invokes sessions + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the single-step branch invokes only initRepo (head deploy-tree repo); provisionAgent/persistHubPublicKey stay unused (the supervised child mints its own key and persists no hub-agent config) sessions: { provisionAgent: async () => { throw new Error("must not invoke provisionAgent"); @@ -227,14 +227,13 @@ describe("agent signing-key registration lifecycle on the host transport", () => persistHubPublicKey: async () => { throw new Error("must not invoke persistHubPublicKey"); }, + initRepo: async () => undefined, } as unknown as Parameters< typeof createSidecarDeployRouter >[0]["sessions"], - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- only loadOrGenerateKey is exercised (the single-step branch registers the agent's signing key on the transport before spawn) + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the single-step branch registers the agent's signing key (loadOrGenerateKey) and records the hub key (recordHubKey) at the head before spawn keyStore: { - recordHubKey: () => { - throw new Error("must not invoke recordHubKey"); - }, + recordHubKey: () => undefined, loadOrGenerateKey: async () => ({ keyPair: await generateKeyPair(), isNew: false, diff --git a/apps/sidecar/src/step-agent-tools.ts b/apps/sidecar/src/step-agent-tools.ts index ef7d5934..79141bf1 100644 --- a/apps/sidecar/src/step-agent-tools.ts +++ b/apps/sidecar/src/step-agent-tools.ts @@ -44,7 +44,7 @@ import { } from "@intx/hub-agent/paths"; import { getLogger } from "@intx/log"; import type { LoadedToolFactory } from "@intx/tool-packaging"; -import { deriveStepAddress } from "@intx/workflow-deploy"; +import { resolveStepAddress } from "@intx/workflow-deploy"; import { parseAgentAddress } from "@intx/types"; import { materializeToolPackages } from "./tool-materialization"; @@ -133,18 +133,23 @@ function isStepToolMaterialization( * substrate's `agent-state/` layout -- the multi-step deploy path * never pushes step `agent-state` packs to the child's substrate. * - * The step's mail address is the orchestrator's - * `deriveStepAddress(deploymentId, stepId, deploymentDomain)`. The - * orchestrator's `deploymentId`/`deploymentDomain` are recovered from - * the deployment mailbox address the supervisor threaded into the + * The step's mail address is `resolveStepAddress(...)`, the single owner + * of the head/step collapse: for a single-step deployment the lone step + * IS the head (the deployment mailbox itself), so the tree is read at the + * head; for multi-step it is `deriveStepAddress(deploymentId, stepId, + * deploymentDomain)`. The `deploymentId`/`deploymentDomain` are recovered + * from the deployment mailbox address the supervisor threaded into the * child as `MAILBOX_ADDRESS` (`ins_@`): the - * instance-id local part minus the `ins_` prefix is the deploymentId, - * and the address domain is the deploymentDomain. + * instance-id local part minus the `ins_` prefix is the deploymentId, and + * the address domain is the deploymentDomain. `stepCount` is sourced from + * the host (via `substrateEnv`) so producer and consumer never derive + * divergent addresses. */ export function stepDeployTreeDir(args: { dataDir: string; mailboxAddress: string; stepId: string; + stepCount: number; }): string { const parsed = parseAgentAddress(args.mailboxAddress); if (parsed === null) { @@ -158,10 +163,11 @@ export function stepDeployTreeDir(args: { ); } const deploymentId = parsed.instanceId.slice(INSTANCE_PREFIX.length); - const stepAddress = deriveStepAddress({ + const stepAddress = resolveStepAddress({ deploymentId, stepId: args.stepId, deploymentDomain: parsed.domain, + stepCount: args.stepCount, }); return path.join(args.dataDir, sanitizeAddress(stepAddress)); } @@ -183,6 +189,7 @@ export async function materializeStepTools(args: { dataDir: string; mailboxAddress: string; stepId: string; + stepCount: number; /** Per-step state root; cache + instance dir + workspace live under it. */ storeDir: string; cache: StepToolCacheConfig; @@ -192,6 +199,7 @@ export async function materializeStepTools(args: { dataDir: args.dataDir, mailboxAddress: args.mailboxAddress, stepId: args.stepId, + stepCount: args.stepCount, }); const deployTree = await readDeployTree(deployTreeDir); diff --git a/apps/sidecar/src/workflow-host-wiring-undeploy-supervisor.test.ts b/apps/sidecar/src/workflow-host-wiring-undeploy-supervisor.test.ts index 031db701..dd5ee834 100644 --- a/apps/sidecar/src/workflow-host-wiring-undeploy-supervisor.test.ts +++ b/apps/sidecar/src/workflow-host-wiring-undeploy-supervisor.test.ts @@ -234,24 +234,23 @@ describe("createSidecarDeployRouter multi-step undeploy shuts the supervisor dow const drainRouter = createMultistepDrainRouter(); const router = createSidecarDeployRouter({ - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- multi-step branch never invokes sessions + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the single-step branch invokes only initRepo (head deploy-tree repo); provisionAgent/persistHubPublicKey stay unused (the supervised child mints its own key and persists no hub-agent config) sessions: { provisionAgent: async () => { - throw new Error("multi-step branch must not invoke provisionAgent"); + throw new Error("single-step branch must not invoke provisionAgent"); }, persistHubPublicKey: async () => { throw new Error( - "multi-step branch must not invoke persistHubPublicKey", + "single-step branch must not invoke persistHubPublicKey", ); }, + initRepo: async () => undefined, } as unknown as Parameters< typeof createSidecarDeployRouter >[0]["sessions"], - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- only loadOrGenerateKey is exercised (the single-step branch registers the agent's signing key on the transport before spawn) + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the single-step branch registers the agent's signing key (loadOrGenerateKey) and records the hub key (recordHubKey) at the head before spawn keyStore: { - recordHubKey: () => { - throw new Error("multi-step branch must not invoke recordHubKey"); - }, + recordHubKey: () => undefined, loadOrGenerateKey: async () => ({ keyPair: await generateKeyPair(), isNew: false, diff --git a/apps/sidecar/src/workflow-host-wiring.test.ts b/apps/sidecar/src/workflow-host-wiring.test.ts index 021c8093..716e1e51 100644 --- a/apps/sidecar/src/workflow-host-wiring.test.ts +++ b/apps/sidecar/src/workflow-host-wiring.test.ts @@ -79,6 +79,7 @@ describe("createSidecarWorkflowSupervisor", () => { workflowRunRepoId: { kind: "workflow-run", id: "wire-test" }, workflowRunRef: "refs/heads/main", deploymentId: "wire-test", + stepCount: 1, deploymentMailAddress: "wire-test@example.com", deriveStepAddress: ({ deploymentId, stepId }) => `${deploymentId}-${stepId}@example.com`, @@ -114,6 +115,7 @@ describe("createSidecarWorkflowSupervisor", () => { workflowRunRepoId: { kind: "workflow-run", id: "inbound" }, workflowRunRef: "refs/heads/main", deploymentId: "inbound", + stepCount: 1, deploymentMailAddress: "inbound@example.com", deriveStepAddress: ({ deploymentId, stepId }) => `${deploymentId}-${stepId}@example.com`, @@ -338,7 +340,7 @@ describe("createSidecarDeployRouter wires the InferenceEvent subscription to rec }; const router = createSidecarDeployRouter({ - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the deploy-router test exercises only provisionAgent + persistHubPublicKey + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the single-step branch exercises initRepo; provisionAgent/persistHubPublicKey remain stubbed for the trivial-branch cases in this file sessions: { provisionAgent: async (_config: unknown) => ({ publicKey: "pk-trivial", @@ -350,6 +352,9 @@ describe("createSidecarDeployRouter wires the InferenceEvent subscription to rec persistHubPublicKey: async (_a: string, _h: string) => { /* no-op */ }, + initRepo: async (_a: string) => { + /* no-op */ + }, } as unknown as Parameters< typeof createSidecarDeployRouter >[0]["sessions"], @@ -1148,24 +1153,23 @@ describe("createSidecarDeployRouter multi-step branch", () => { ...(opts.multistepSubstrateEnv ?? {}), }; const router = createSidecarDeployRouter({ - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the multi-step branch never invokes provisionAgent; the stub throws if it does + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the workflow path never invokes provisionAgent/persistHubPublicKey (single-step uses the narrow initRepo; the child mints its own key); the stubs throw if it does. initRepo is a no-op for the single-step head repo. sessions: { provisionAgent: async () => { - throw new Error("multi-step branch must not invoke provisionAgent"); + throw new Error("workflow branch must not invoke provisionAgent"); }, persistHubPublicKey: async () => { throw new Error( - "multi-step branch must not invoke persistHubPublicKey", + "workflow branch must not invoke persistHubPublicKey", ); }, + initRepo: async () => undefined, } as unknown as Parameters< typeof createSidecarDeployRouter >[0]["sessions"], - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub; the single-step head deploy records the hub key for pack verification keyStore: { - recordHubKey: () => { - throw new Error("multi-step branch must not invoke recordHubKey"); - }, + recordHubKey: () => undefined, loadOrGenerateKey: async () => ({ keyPair: await generateKeyPair(), isNew: false, diff --git a/apps/sidecar/src/workflow-host-wiring.ts b/apps/sidecar/src/workflow-host-wiring.ts index e61d00a0..0978795b 100644 --- a/apps/sidecar/src/workflow-host-wiring.ts +++ b/apps/sidecar/src/workflow-host-wiring.ts @@ -402,6 +402,12 @@ export type CreateSidecarWorkflowSupervisorOpts = { workflowRunRef: string; /** Deployment id baked into principal claims and address derivation. */ deploymentId: string; + /** + * Step count of the deployed `WorkflowDefinition` (`stepOrder.length`). + * Threaded into the child's spawn-time env so its deploy-tree read + * collapses onto the head for a single-step deployment. + */ + stepCount: number; /** Deployment's mail address. */ deploymentMailAddress: string; /** Per-step mail-address derivation. */ @@ -1059,6 +1065,7 @@ export function createSidecarDeployRouter(deps: { }, workflowRunRef: "refs/heads/main", deploymentId, + stepCount: projection.definition.stepOrder.length, deploymentMailAddress: frame.agentAddress, deriveStepAddress: stepStrategy.deriveStepAddress, deriveStepRepoId: stepStrategy.deriveStepRepoId, @@ -1135,6 +1142,26 @@ export function createSidecarDeployRouter(deps: { deps.createAgentCrypto(keyPair), ); agentTransportRegistered = true; + + // A single-step workflow stages its deploy tree at the head (the + // lone step IS the head). Initialize the head's on-disk + // deploy-tree repo now, before this frame's ack, so the hub's + // follow-up deploy-pack push has a repo to `applyDeployPack` into. + // The narrow `initRepo` (not `provisionAgent`) is deliberate: the + // supervised child mints its own keypair at boot and persists no + // hub-agent config, and `provisionAgent`'s duplicate-address guard + // would reject the head the deployment machinery already tracks. + await deps.sessions.initRepo(frame.agentAddress); + + // Record the hub's public key at the head so the follow-up + // deploy-pack apply can verify the hub-signed deploy-tree commit. + // The verifier resolves the key from the in-memory key store's + // `recordHubKey` map, not the persisted `agent.json`, so this alone + // satisfies verification -- `persistHubPublicKey` (agent.json + // durability) is skipped along with `persistConfig`, which the + // workflow head has no agent.json to write into. Without this + // `applyDeployPack` rejects the pack as `signature_invalid`. + deps.keyStore.recordHubKey(frame.agentAddress, frame.hubPublicKey); } const stepOrder = [...projection.definition.stepOrder]; @@ -1304,6 +1331,9 @@ export function createSidecarDeployRouter(deps: { }, workflowRunRef: "refs/heads/main", deploymentId, + // A trivial deploy is a single agent: one step, so the child's + // deploy-tree read collapses onto the head. + stepCount: 1, deploymentMailAddress: frame.agentAddress, deriveStepAddress: ({ deploymentId: dep, stepId }) => `${dep}-${stepId}`, @@ -1622,6 +1652,7 @@ export function createSidecarWorkflowSupervisor( workflowRunRepoId: opts.workflowRunRepoId, workflowRunRef: opts.workflowRunRef, deploymentId: opts.deploymentId, + stepCount: opts.stepCount, deploymentMailAddress: opts.deploymentMailAddress, readPrincipal: supervisorPrincipal, deriveStepAddress: opts.deriveStepAddress, diff --git a/apps/sidecar/src/workflow-substrate-factory-step-storage.test.ts b/apps/sidecar/src/workflow-substrate-factory-step-storage.test.ts index f74f669f..2f863d15 100644 --- a/apps/sidecar/src/workflow-substrate-factory-step-storage.test.ts +++ b/apps/sidecar/src/workflow-substrate-factory-step-storage.test.ts @@ -93,6 +93,7 @@ function buildDeps(opts: { workflowRunRepoId: WORKFLOW_RUN_REPO_ID, signer: (payload: string) => Promise.resolve(`sig:${payload.length}`), mailboxAddress: "ins_deployment-keying@example.com", + stepCount: 1, outboundMailBridge: stubOutboundMailBridge(), cache: { cacheMaxBytes: 1_000_000, registryMaxTarballBytes: 1_000_000 }, adapters: createBuiltinRegistry(), diff --git a/apps/sidecar/src/workflow-substrate-factory.ts b/apps/sidecar/src/workflow-substrate-factory.ts index 2decc305..ecec5a94 100644 --- a/apps/sidecar/src/workflow-substrate-factory.ts +++ b/apps/sidecar/src/workflow-substrate-factory.ts @@ -442,6 +442,15 @@ export interface SidecarStepBuildEnvDeps { * the agent's `CryptoProvider` against. */ mailboxAddress: string; + /** + * Step count of the deployed `WorkflowDefinition` (`stepOrder.length`), + * threaded from the host through the spawn-time env. Selects the + * head/step collapse when locating a step's deploy tree + * (`stepDeployTreeDir` -> `resolveStepAddress`): a single-step + * deployment reads at the head, a multi-step deployment at the per-step + * address, matching the host's producer push. + */ + stepCount: number; /** * Child-side outbound-mail bridge over the upstream control channel * (OUTBOUND half of mailbox ownership, §3a). The per-step env builder @@ -573,6 +582,7 @@ export function createSidecarStepBuildEnv( dataDir: deps.dataDir, mailboxAddress: deps.mailboxAddress, stepId, + stepCount: deps.stepCount, storeDir, cache: deps.cache, }); @@ -1016,6 +1026,7 @@ export function createSidecarSubstrateFactory( workflowRunRepoId, signer: conversationSigner, mailboxAddress: env.spawn.mailboxAddress, + stepCount: env.spawn.stepCount, outboundMailBridge: env.outboundMailBridge, cache: stepToolCache, adapters: childAdapterRegistry, diff --git a/packages/hub-agent/src/session-manager.ts b/packages/hub-agent/src/session-manager.ts index db0a14f1..9bccf406 100644 --- a/packages/hub-agent/src/session-manager.ts +++ b/packages/hub-agent/src/session-manager.ts @@ -131,6 +131,16 @@ export type AgentEventListener = (event: InferenceEvent) => void; export type SessionManager = { provisionAgent(config: AgentConfig): Promise; + /** + * Initialize the on-disk deploy-tree repo for an address without + * minting a keypair, persisting config, or entering the + * provisioned/pending bookkeeping. A single-step workflow deploy uses + * this at the head so the follow-up deploy-pack apply has a repo to + * apply into; the supervised workflow-process child mints its own + * keypair at boot, so `provisionAgent`'s keypair, `persistConfig`, and + * duplicate-address guard are neither needed nor wanted on that path. + */ + initRepo(address: string): Promise; startSession(agentAddress: string): Promise; destroySession(agentAddress: string): Promise; abortSession(agentAddress: string, reason: string): Promise; @@ -812,6 +822,7 @@ export function createSessionManager( return { provisionAgent, + initRepo: (address: string) => repoStore.initRepo(address), startSession, destroySession, abortSession, diff --git a/packages/hub-agent/src/ws/hub-link-bootstrap-prune.test.ts b/packages/hub-agent/src/ws/hub-link-bootstrap-prune.test.ts index 4f6da257..69e73a33 100644 --- a/packages/hub-agent/src/ws/hub-link-bootstrap-prune.test.ts +++ b/packages/hub-agent/src/ws/hub-link-bootstrap-prune.test.ts @@ -123,6 +123,7 @@ function createMockSessionManager(): SessionManager & { }, }; }, + initRepo: (_address: string) => Promise.resolve(), async startSession(_agentAddress: string): Promise { /* unused */ }, diff --git a/packages/hub-agent/src/ws/hub-link-mail-router.test.ts b/packages/hub-agent/src/ws/hub-link-mail-router.test.ts index de24b281..c2538616 100644 --- a/packages/hub-agent/src/ws/hub-link-mail-router.test.ts +++ b/packages/hub-agent/src/ws/hub-link-mail-router.test.ts @@ -155,6 +155,7 @@ function createMockSessionManager(): SessionManager & { }, }; }, + initRepo: (_address: string) => Promise.resolve(), async startSession(agentAddress: string): Promise { if (mock.shouldThrow !== null) throw new Error(mock.shouldThrow); mock.started.push(agentAddress); diff --git a/packages/hub-agent/src/ws/hub-link.test.ts b/packages/hub-agent/src/ws/hub-link.test.ts index b1a2ca86..f08222ac 100644 --- a/packages/hub-agent/src/ws/hub-link.test.ts +++ b/packages/hub-agent/src/ws/hub-link.test.ts @@ -170,6 +170,7 @@ function createMockSessionManager(): SessionManager & { }, }; }, + initRepo: (_address: string) => Promise.resolve(), async startSession(agentAddress: string): Promise { if (mock.shouldThrow !== null) throw new Error(mock.shouldThrow); mock.started.push(agentAddress); diff --git a/packages/hub-api/src/app.test.ts b/packages/hub-api/src/app.test.ts index 08fc0957..282b4298 100644 --- a/packages/hub-api/src/app.test.ts +++ b/packages/hub-api/src/app.test.ts @@ -28,6 +28,11 @@ const sessionService: SessionService = { "mock: sessionService.deployWorkflowDefinition not implemented", ); }, + deploySingleStepAtHead(_params) { + throw new Error( + "mock: sessionService.deploySingleStepAtHead not implemented", + ); + }, sendUserMessage(_params) { throw new Error("mock: sessionService.sendUserMessage not implemented"); }, diff --git a/packages/hub-api/src/routes/agents.test.ts b/packages/hub-api/src/routes/agents.test.ts index 340f6a51..69e56815 100644 --- a/packages/hub-api/src/routes/agents.test.ts +++ b/packages/hub-api/src/routes/agents.test.ts @@ -201,6 +201,11 @@ function createMockSessionService(): SessionService { "mock: sessionService.deployWorkflowDefinition not implemented", ); }, + deploySingleStepAtHead: () => { + throw new Error( + "mock: sessionService.deploySingleStepAtHead not implemented", + ); + }, sendUserMessage: () => { throw new Error("mock: sessionService.sendUserMessage not implemented"); }, diff --git a/packages/hub-api/src/routes/assets.test.ts b/packages/hub-api/src/routes/assets.test.ts index 7414f3e2..3d6b572f 100644 --- a/packages/hub-api/src/routes/assets.test.ts +++ b/packages/hub-api/src/routes/assets.test.ts @@ -317,6 +317,11 @@ function createMockSessionService(): SessionService { "mock: sessionService.deployWorkflowDefinition not implemented", ); }, + deploySingleStepAtHead: () => { + throw new Error( + "mock: sessionService.deploySingleStepAtHead not implemented", + ); + }, sendUserMessage: () => { throw new Error("mock: sessionService.sendUserMessage not implemented"); }, diff --git a/packages/hub-api/src/routes/catalog-routes.test.ts b/packages/hub-api/src/routes/catalog-routes.test.ts index e32fbdd3..7b2716af 100644 --- a/packages/hub-api/src/routes/catalog-routes.test.ts +++ b/packages/hub-api/src/routes/catalog-routes.test.ts @@ -216,6 +216,11 @@ function createMockSessionService(): SessionService { "mock: sessionService.deployWorkflowDefinition not implemented", ); }, + deploySingleStepAtHead: () => { + throw new Error( + "mock: sessionService.deploySingleStepAtHead not implemented", + ); + }, sendUserMessage: () => { throw new Error("mock: sessionService.sendUserMessage not implemented"); }, diff --git a/packages/hub-api/src/routes/git-tokens.test.ts b/packages/hub-api/src/routes/git-tokens.test.ts index 94965876..4c921403 100644 --- a/packages/hub-api/src/routes/git-tokens.test.ts +++ b/packages/hub-api/src/routes/git-tokens.test.ts @@ -240,6 +240,11 @@ function createMockSessionService(): SessionService { "mock: sessionService.deployWorkflowDefinition not implemented", ); }, + deploySingleStepAtHead: () => { + throw new Error( + "mock: sessionService.deploySingleStepAtHead not implemented", + ); + }, sendUserMessage: () => { throw new Error("mock: sessionService.sendUserMessage not implemented"); }, diff --git a/packages/hub-api/src/routes/instances.test.ts b/packages/hub-api/src/routes/instances.test.ts index 4a2367f2..4cba4467 100644 --- a/packages/hub-api/src/routes/instances.test.ts +++ b/packages/hub-api/src/routes/instances.test.ts @@ -290,6 +290,9 @@ function createMockSessionService(): SessionService { deployWorkflowDefinition(_params) { return notImpl("deployWorkflowDefinition"); }, + deploySingleStepAtHead(_params) { + return notImpl("deploySingleStepAtHead"); + }, sendUserMessage(_params) { return notImpl("sendUserMessage"); }, @@ -660,6 +663,9 @@ describe("POST /agents/instances/:instanceId/mail", () => { deployWorkflowDefinition() { throw new Error("not implemented"); }, + deploySingleStepAtHead() { + throw new Error("not implemented"); + }, endSession() { throw new Error("not implemented"); }, @@ -808,6 +814,9 @@ describe("POST /agents/instances/:instanceId/mail attachments", () => { deployWorkflowDefinition() { throw new Error("not implemented"); }, + deploySingleStepAtHead() { + throw new Error("not implemented"); + }, endSession() { throw new Error("not implemented"); }, @@ -1238,6 +1247,9 @@ describe("POST /agents/instances seeds creator agent-state grant", () => { deployWorkflowDefinition: () => { throw new Error("mock: deployWorkflowDefinition not implemented"); }, + deploySingleStepAtHead: () => { + throw new Error("mock: deploySingleStepAtHead not implemented"); + }, sendUserMessage: () => { throw new Error("mock: sendUserMessage not implemented"); }, @@ -1454,6 +1466,9 @@ describe("POST /agents/instances seeds creator agent-state grant", () => { deployWorkflowDefinition: () => { throw new Error("mock: deployWorkflowDefinition not implemented"); }, + deploySingleStepAtHead: () => { + throw new Error("mock: deploySingleStepAtHead not implemented"); + }, sendUserMessage: () => { throw new Error("mock: sendUserMessage not implemented"); }, diff --git a/packages/hub-api/src/routes/workflows.test.ts b/packages/hub-api/src/routes/workflows.test.ts index 606e03d7..abb0485c 100644 --- a/packages/hub-api/src/routes/workflows.test.ts +++ b/packages/hub-api/src/routes/workflows.test.ts @@ -231,6 +231,7 @@ function createMockSessionService( } return Promise.resolve(result); }, + deploySingleStepAtHead: () => notImpl("deploySingleStepAtHead"), sendUserMessage: () => notImpl("sendUserMessage"), endSession: () => notImpl("endSession"), }; diff --git a/packages/hub-sessions/src/session-service.ts b/packages/hub-sessions/src/session-service.ts index 669c3dbc..b4c8d690 100644 --- a/packages/hub-sessions/src/session-service.ts +++ b/packages/hub-sessions/src/session-service.ts @@ -56,6 +56,7 @@ import { type DeployContent as OrchestratorDeployContent, type DeployWorkflowArgs, type DeployWorkflowResult, + type DeploySingleStepFn, type LaunchSessionFn, type SendMultiStepDeployFn, type WorkflowRepoWriter, @@ -128,6 +129,17 @@ export type SessionService = { toolPackagePins?: readonly ToolPackagePin[]; }): Promise; + /** + * Deploy a one-step workflow once at the head through the deploy core, + * without the DB-backed `workflow_deployment` projection row. Stages the + * head's deploy tree (deploy-tree write, pack, asset fan-out), fires the + * deployment `agent.deploy` frame carrying the workflow definition + + * source pin (the sidecar initializes the head repo and spawns the + * workflow-process child), then delivers the pack to the head. Returns + * the sidecar supervisor's principal public key. See `DeploySingleStepFn`. + */ + deploySingleStepAtHead: DeploySingleStepFn; + /** * Deploy a multi-step `WorkflowDefinition` through the workflow-deploy * orchestrator's multi-step branch. This is the general workflow @@ -597,9 +609,23 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { config: HarnessConfig; deployContent: DeployContent; toolPackagePins?: readonly ToolPackagePin[]; - }): Promise { + /** + * Single-step workflow deploy. When present, Phase 1 fires the + * deployment `agent.deploy` frame carrying the workflow definition + + * source pins (the sidecar initializes the head repo on receipt and + * spawns the workflow-process child) instead of the plain provision + * frame, and Phase 3's warm-harness `sendSessionStart` is skipped (the + * supervised child drives the agent, not a warm hub-agent harness). + * The returned supervisor public key comes from that frame's ack. + */ + workflowFrame?: { + definition: WorkflowDefinition; + sources: Record; + }; + }): Promise<{ publicKey: string } | undefined> { const { agentAddress, agentId, instanceId, config, deployContent } = params; const toolPackagePins = params.toolPackagePins ?? []; + const workflowFrame = params.workflowFrame; // Phase 0: Resolve attached assets first so the skill index is in // hand before the deploy tree is written. The `` @@ -749,9 +775,29 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { throw new SessionLaunchError("write", err, false); } - // Phase 1: Provision on sidecar. + // Phase 1: Provision on sidecar. A single-step workflow deploy sends + // the deployment `agent.deploy` frame carrying the workflow definition + // + source pins INSTEAD of the plain provision frame: the sidecar's + // deploy router initializes the head repo on receipt (so the Phase 2 + // pack has a repo to apply into) and spawns the workflow-process + // child. The plain-provision frame stays for the legacy agent-deploy + // passthrough. Firing the frame before the Phase 2 pack is the + // ordering barrier -- the head repo must exist before the pack + // applies. The ack's supervisor public key is surfaced to the caller. + let deployAckPublicKey: string | undefined; try { - await sidecarRouter.sendAgentDeploy(agentAddress, config); + if (workflowFrame !== undefined) { + const ack = await sendMultiStepDeployFrame({ + sidecarRouter, + agentAddress, + config, + definition: workflowFrame.definition, + sources: workflowFrame.sources, + }); + deployAckPublicKey = ack.publicKey; + } else { + await sidecarRouter.sendAgentDeploy(agentAddress, config); + } } catch (err) { throw new SessionLaunchError("provision", err, false); } @@ -812,14 +858,61 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { } } - try { - await sidecarRouter.sendSessionStart(agentAddress); - } catch (err) { - await attemptCleanup(agentAddress, "start", err); - throw new SessionLaunchError("start", err, false); + // Phase 3: Warm-harness session start. A single-step workflow deploy + // skips this: the supervised workflow-process child drives the agent + // per message through the supervisor, so there is no warm hub-agent + // harness to start. The legacy agent-deploy path starts its warm + // harness here. + if (workflowFrame === undefined) { + try { + await sidecarRouter.sendSessionStart(agentAddress); + } catch (err) { + await attemptCleanup(agentAddress, "start", err); + throw new SessionLaunchError("start", err, false); + } } + + return deployAckPublicKey === undefined + ? undefined + : { publicKey: deployAckPublicKey }; } + /** + * Deploy a one-step workflow once at the head. Reuses the full + * launch-phase machinery (deploy-tree write, pack, asset fan-out) via + * `executeLaunchPhases`, swapping the Phase 1 provision frame for the + * workflow frame and skipping the Phase 3 warm-harness start. The + * workflow frame makes the sidecar initialize the head repo and spawn + * the workflow-process child; the follow-up pack lands the head's deploy + * tree. Returns the supervisor's principal public key from the frame's + * ack. A workflow-frame launch always yields a deploy-ack key; its + * absence is a wiring bug, not a tolerable case. + */ + const deploySingleStepAtHead: DeploySingleStepFn = async (deployParams) => { + const result = await executeLaunchPhases({ + agentAddress: deployParams.agentAddress, + agentId: deployParams.agentId, + instanceId: deployParams.instanceId, + config: deployParams.config, + deployContent: bridgeOrchestratorDeployContent( + deployParams.deployContent, + ), + workflowFrame: { + definition: deployParams.definition, + sources: deployParams.sources, + }, + ...(deployParams.toolPackagePins !== undefined + ? { toolPackagePins: deployParams.toolPackagePins } + : {}), + }); + if (result === undefined) { + throw new Error( + "single-step deploy at head: executeLaunchPhases returned no deploy-ack public key for a workflow-frame deploy", + ); + } + return result; + }; + /** * Build the workflow-deploy orchestrator (with its launch-session and * multi-step callbacks) and run one deploy. Shared by `launchSession` @@ -831,8 +924,12 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { directorRegistry: DirectorRegistry; deployArgs: DeployWorkflowArgs; }): Promise { - const launchSessionCallback: LaunchSessionFn = async (orchestratorParams) => - executeLaunchPhases({ + const launchSessionCallback: LaunchSessionFn = async ( + orchestratorParams, + ) => { + // The legacy launch path (no `workflowFrame`) provisions a warm + // agent and returns no deploy-ack key; the void return is discarded. + await executeLaunchPhases({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, @@ -844,6 +941,7 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { ? { toolPackagePins: orchestratorParams.toolPackagePins } : {}), }); + }; const sendMultiStepDeployCallback: SendMultiStepDeployFn = (deployParams) => sendMultiStepDeployFrame({ @@ -859,6 +957,7 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { workflowRepo: args.workflowRepo, launchSession: launchSessionCallback, sendMultiStepDeploy: sendMultiStepDeployCallback, + deploySingleStepAtHead, }); return orchestrator.deployWorkflow(args.deployArgs); @@ -1396,6 +1495,7 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { return { launchSession, + deploySingleStepAtHead, deployWorkflowDefinition, sendUserMessage, endSession, diff --git a/packages/workflow-deploy/src/index.ts b/packages/workflow-deploy/src/index.ts index 0310b3f7..f1ffa2f4 100644 --- a/packages/workflow-deploy/src/index.ts +++ b/packages/workflow-deploy/src/index.ts @@ -28,6 +28,7 @@ export { createWorkflowDeployOrchestrator, deriveDeploymentAddress, deriveStepAddress, + resolveStepAddress, deriveStepAgentId, deriveWorkflowRunRepoId, isWorkflowDerivedAddress, @@ -35,8 +36,10 @@ export { CapabilityApprovalDeniedError, MultiStepDeployHandoffMissingError, MultiStepDeploymentArgsMissingError, + SingleStepDeployHandoffMissingError, WorkflowDefinitionInvalidError, type DeployContent, + type DeploySingleStepFn, type DeployWorkflowArgs, type DeployWorkflowResult, type LaunchSessionFn, diff --git a/packages/workflow-deploy/src/orchestrator-fallback-source.test.ts b/packages/workflow-deploy/src/orchestrator-fallback-source.test.ts index aedd9788..1510a90e 100644 --- a/packages/workflow-deploy/src/orchestrator-fallback-source.test.ts +++ b/packages/workflow-deploy/src/orchestrator-fallback-source.test.ts @@ -31,6 +31,7 @@ import { createWorkflowDeployOrchestrator, WorkflowDefinitionInvalidError, type DeployContent, + type DeploySingleStepFn, type LaunchSessionFn, type SendMultiStepDeployFn, type WorkflowRepoWriter, @@ -106,13 +107,29 @@ function createRecordingDeps() { sources.push(params.sources); return { publicKey: "00".repeat(32) }; }; + // A one-step workflow deploys once at the head via this hand-off; it + // records the pinned sources exactly as the multi-step hand-off does so + // the source-pin assertions read the same `sources` array regardless of + // which branch the deploy took. + const singleStep: DeploySingleStepFn = async (params) => { + sources.push(params.sources); + return { publicKey: "00".repeat(32) }; + }; const repoWrites: { workflowRepoId: string }[] = []; const workflowRepo: WorkflowRepoWriter = { async writeWorkflowRepo(params) { repoWrites.push({ workflowRepoId: params.workflowRepoId }); }, }; - return { launches, launch, sources, multiStep, workflowRepo, repoWrites }; + return { + launches, + launch, + sources, + multiStep, + singleStep, + workflowRepo, + repoWrites, + }; } describe("pickStepInferenceSource (agent step)", () => { @@ -126,6 +143,7 @@ describe("pickStepInferenceSource (agent step)", () => { workflowRepo: deps.workflowRepo, launchSession: deps.launch, sendMultiStepDeploy: deps.multiStep, + deploySingleStepAtHead: deps.singleStep, }); // HarnessConfig has only a default-pinned source for openai:default-model, @@ -180,6 +198,7 @@ describe("pickStepInferenceSource (agent step)", () => { workflowRepo: deps.workflowRepo, launchSession: deps.launch, sendMultiStepDeploy: deps.multiStep, + deploySingleStepAtHead: deps.singleStep, }); const config = makeConfig({ @@ -240,6 +259,7 @@ describe("pickStepInferenceSource (agent step)", () => { workflowRepo: deps.workflowRepo, launchSession: deps.launch, sendMultiStepDeploy: deps.multiStep, + deploySingleStepAtHead: deps.singleStep, }); const config = makeConfig({ @@ -307,6 +327,7 @@ describe("pickStepInferenceSource (agent step)", () => { workflowRepo: deps.workflowRepo, launchSession: deps.launch, sendMultiStepDeploy: deps.multiStep, + deploySingleStepAtHead: deps.singleStep, }); const config = makeConfig({ sources: [ diff --git a/packages/workflow-deploy/src/orchestrator-nonagent-source.test.ts b/packages/workflow-deploy/src/orchestrator-nonagent-source.test.ts index 607c5d6b..dd9435c8 100644 --- a/packages/workflow-deploy/src/orchestrator-nonagent-source.test.ts +++ b/packages/workflow-deploy/src/orchestrator-nonagent-source.test.ts @@ -35,6 +35,7 @@ import { createWorkflowDeployOrchestrator, WorkflowDefinitionInvalidError, type DeployContent, + type DeploySingleStepFn, type LaunchSessionFn, type SendMultiStepDeployFn, type WorkflowRepoWriter, @@ -113,13 +114,29 @@ function createRecordingDeps() { sources.push(params.sources); return { publicKey: "00".repeat(32) }; }; + // A one-step workflow deploys once at the head via this hand-off; it + // records the pinned sources exactly as the multi-step hand-off does so + // the source-pin assertions read the same `sources` array regardless of + // which branch the deploy took. + const singleStep: DeploySingleStepFn = async (params) => { + sources.push(params.sources); + return { publicKey: "00".repeat(32) }; + }; const repoWrites: { workflowRepoId: string }[] = []; const workflowRepo: WorkflowRepoWriter = { async writeWorkflowRepo(params) { repoWrites.push({ workflowRepoId: params.workflowRepoId }); }, }; - return { launches, launch, sources, multiStep, workflowRepo, repoWrites }; + return { + launches, + launch, + sources, + multiStep, + singleStep, + workflowRepo, + repoWrites, + }; } describe("pickStepInferenceSource (non-agent step)", () => { @@ -133,6 +150,7 @@ describe("pickStepInferenceSource (non-agent step)", () => { workflowRepo: deps.workflowRepo, launchSession: deps.launch, sendMultiStepDeploy: deps.multiStep, + deploySingleStepAtHead: deps.singleStep, }); // HarnessConfig carries TWO sources: the agent's preferred and a @@ -209,6 +227,7 @@ describe("pickStepInferenceSource (non-agent step)", () => { workflowRepo: deps.workflowRepo, launchSession: deps.launch, sendMultiStepDeploy: deps.multiStep, + deploySingleStepAtHead: deps.singleStep, }); const config = makeConfig({ @@ -256,6 +275,7 @@ describe("pickStepInferenceSource (non-agent step)", () => { workflowRepo: deps.workflowRepo, launchSession: deps.launch, sendMultiStepDeploy: deps.multiStep, + deploySingleStepAtHead: deps.singleStep, }); const config = makeConfig({ diff --git a/packages/workflow-deploy/src/orchestrator.test.ts b/packages/workflow-deploy/src/orchestrator.test.ts index fe64d5f4..f53dd67b 100644 --- a/packages/workflow-deploy/src/orchestrator.test.ts +++ b/packages/workflow-deploy/src/orchestrator.test.ts @@ -31,6 +31,7 @@ import { WorkflowDefinitionInvalidError, wrapHarnessAsTrivialAgent, type DeployContent, + type DeploySingleStepFn, type LaunchSessionFn, type SendMultiStepDeployFn, type WorkflowRepoWriter, @@ -204,6 +205,20 @@ function createRecordingMultiStepDeploy(publicKey = "ff".repeat(32)): { return { fn, calls }; } +type RecordedSingleStepDeploy = Parameters[0]; + +function createRecordingSingleStepDeploy(publicKey = "ff".repeat(32)): { + fn: DeploySingleStepFn; + calls: RecordedSingleStepDeploy[]; +} { + const calls: RecordedSingleStepDeploy[] = []; + const fn: DeploySingleStepFn = async (params) => { + calls.push(params); + return { publicKey }; + }; + return { fn, calls }; +} + describe("createWorkflowDeployOrchestrator", () => { describe("trivial branch", () => { test("preserves the existing agent address and launches once", async () => { @@ -578,22 +593,24 @@ describe("createWorkflowDeployOrchestrator", () => { ).rejects.toBeInstanceOf(MultiStepDeploymentArgsMissingError); }); - test("single-step workflow without trivialBindings takes the derived path", async () => { + test("single-step workflow without trivialBindings deploys once at the head", async () => { const agent = makeAgent("only"); const workflow = makeTrivialWorkflow(agent); const directorRegistry = createDefaultDirectorRegistry(); const workflowRepo = createRecordingWorkflowRepoWriter(); const launch = createRecordingLaunch(); const multiStep = createRecordingMultiStepDeploy(); + const singleStep = createRecordingSingleStepDeploy(); const orchestrator = createWorkflowDeployOrchestrator({ directorRegistry, workflowRepo, launchSession: launch.fn, sendMultiStepDeploy: multiStep.fn, + deploySingleStepAtHead: singleStep.fn, }); const approvals = approvedGrantsForWorkflow(workflow, [agent]); - await orchestrator.deployWorkflow({ + const result = await orchestrator.deployWorkflow({ workflow, deploymentId: "dep_xyz", deploymentDomain: "workflow.interchange", @@ -603,17 +620,23 @@ describe("createWorkflowDeployOrchestrator", () => { operatorApprovals: approvals, }); - expect(launch.launches).toHaveLength(1); - const launched = launch.launches[0]; - if (launched === undefined) throw new Error("missing launch"); + // A one-step workflow deploys ONCE at the head: the lone step + // collapses onto the deployment (head) address, so there is no + // per-step `launchSession` and no separate multi-step frame. + expect(launch.launches).toHaveLength(0); + expect(multiStep.calls).toHaveLength(0); + expect(singleStep.calls).toHaveLength(1); + const call = singleStep.calls[0]; + if (call === undefined) throw new Error("missing single-step deploy"); + expect(call.agentAddress).toBe("ins_dep_xyz@workflow.interchange"); + expect(call.agentId).toBe("ins_dep_xyz"); + expect(call.instanceId).toBe("ins_dep_xyz"); const expectedStepId = workflow.stepOrder[0]; if (expectedStepId === undefined) { throw new Error("missing step id"); } - expect(launched.agentAddress).toBe( - `ins_dep_xyz-${expectedStepId}@workflow.interchange`, - ); - expect(multiStep.calls).toHaveLength(1); + expect(call.sources[expectedStepId]).toBeDefined(); + expect(result.kind).toBe("multi-step"); }); test("source-pin failure carries workflow.id and names the offending provider+model", async () => { diff --git a/packages/workflow-deploy/src/orchestrator.ts b/packages/workflow-deploy/src/orchestrator.ts index 7afac64e..74e3086e 100644 --- a/packages/workflow-deploy/src/orchestrator.ts +++ b/packages/workflow-deploy/src/orchestrator.ts @@ -119,6 +119,35 @@ export type SendMultiStepDeployFn = (params: { hubPublicKey: string; }) => Promise; +/** + * Single-step deploy hand-off. A one-step workflow has no distinct steps + * (the lone step IS the head), so it does NOT take the per-step + * provisioning loop: it deploys once at the head, staging the head's + * deploy tree AND firing the deployment `agent.deploy` frame that carries + * the workflow definition and the sole step's source pin. The caller-site + * closure produces the deploy pack, sends the workflow frame (the sidecar + * initializes the head repo on receipt), then delivers the pack to the + * head; it waits on the `agent.deploy.ack` and surfaces the supervisor's + * principal public key back through the result. + * + * This carries the head deploy content and tool pins (which the head-tree + * staging needs) alongside the definition + sources (which the frame + * needs) -- the union of what `LaunchSessionFn` and `SendMultiStepDeployFn` + * carry, because for one step the tree staging and the frame collapse onto + * a single head deploy. + */ +export type DeploySingleStepFn = (params: { + agentAddress: string; + agentId: string; + instanceId: string; + config: HarnessConfig; + deployContent: DeployContent; + definition: WorkflowDefinition; + sources: Record; + hubPublicKey: string; + toolPackagePins?: readonly ToolPackagePin[]; +}) => Promise; + /** * Result returned by `sendMultiStepDeploy`. Surfaces the sidecar * supervisor's principal public key (hex-encoded Ed25519) from the @@ -187,6 +216,17 @@ export interface WorkflowDeployOrchestratorDeps { * `MultiStepDeployHandoffMissingError` if the dep is absent. */ readonly sendMultiStepDeploy?: SendMultiStepDeployFn; + /** + * Deploys a single-step workflow once at the head: stages the head's + * deploy tree and fires the deployment `agent.deploy` frame in one + * hand-off (see `DeploySingleStepFn`). The single-step branch calls + * this exactly once and never runs the per-step `launchSession` loop. + * + * Optional for the same reason as `sendMultiStepDeploy`; the single-step + * branch fails fast with `SingleStepDeployHandoffMissingError` if the + * dep is absent. + */ + readonly deploySingleStepAtHead?: DeploySingleStepFn; } export interface DeployWorkflowArgs { @@ -298,6 +338,21 @@ export class MultiStepDeployHandoffMissingError extends Error { } } +/** + * Error thrown when the single-step branch is reached but the + * `deploySingleStepAtHead` dependency was not wired. Parallel to + * `MultiStepDeployHandoffMissingError`; the trivial branch does not + * consult this dep, so it is optional on the deps record. + */ +export class SingleStepDeployHandoffMissingError extends Error { + constructor() { + super( + "single-step deploy requires deploySingleStepAtHead dep; wire it on the orchestrator's WorkflowDeployOrchestratorDeps record", + ); + this.name = "SingleStepDeployHandoffMissingError"; + } +} + /** * Error thrown when the capability-approval gate rejects the deploy. * Carries the per-step `pending` delta and the unresolvable director @@ -341,8 +396,13 @@ function formatApprovalDeniedMessage( export function createWorkflowDeployOrchestrator( deps: WorkflowDeployOrchestratorDeps, ): WorkflowDeployOrchestrator { - const { directorRegistry, workflowRepo, launchSession, sendMultiStepDeploy } = - deps; + const { + directorRegistry, + workflowRepo, + launchSession, + sendMultiStepDeploy, + deploySingleStepAtHead, + } = deps; return { async deployWorkflow( @@ -380,6 +440,19 @@ export function createWorkflowDeployOrchestrator( return { kind: "trivial" }; } + // A one-step workflow has no distinct steps: the lone step IS the + // head. It deploys once at the head (no per-step provisioning loop), + // so it routes through the dedicated single-step hand-off rather + // than `runMultiStepBranch`. The multi-step branch is reached only + // for `stepOrder.length >= 2`. + if (args.workflow.stepOrder.length === 1) { + const result = await runSingleStepAtHead({ + args, + deploySingleStepAtHead, + }); + return { kind: "multi-step", publicKey: result.publicKey }; + } + const result = await runMultiStepBranch({ args, launchSession, @@ -407,6 +480,97 @@ function isTrivialDeploy( return { bindings: args.trivialBindings }; } +/** + * Deploy a one-step workflow once at the head. The lone step has no + * distinct per-step address -- it IS the head (`deriveDeploymentAddress`) + * -- so this pins the sole step's inference source, builds the head + * config + deploy content, and hands the whole thing to + * `deploySingleStepAtHead` in a single call. There is no per-step + * `launchSession` loop and no separate deployment frame: the tree staging + * and the `agent.deploy` frame collapse onto one head deploy. The result + * surfaces the sidecar supervisor's principal public key, same as the + * multi-step branch. + */ +async function runSingleStepAtHead(args: { + args: DeployWorkflowArgs; + deploySingleStepAtHead: DeploySingleStepFn | undefined; +}): Promise { + const { args: deploy, deploySingleStepAtHead } = args; + const deploymentId = deploy.deploymentId; + const deploymentDomain = deploy.deploymentDomain; + if (deploymentId === undefined) { + throw new MultiStepDeploymentArgsMissingError("deploymentId"); + } + if (deploymentDomain === undefined) { + throw new MultiStepDeploymentArgsMissingError("deploymentDomain"); + } + if (deploySingleStepAtHead === undefined) { + throw new SingleStepDeployHandoffMissingError(); + } + if (deploy.hubPublicKey === undefined) { + throw new MultiStepDeploymentArgsMissingError("hubPublicKey"); + } + + // The sole step. `validateWorkflowDefinition` already guaranteed + // `stepOrder` is non-empty and every entry has a matching `steps` + // primitive; the index access is re-narrowed here for the compiler. + const stepId = deploy.workflow.stepOrder[0]; + if (stepId === undefined) { + throw new WorkflowDefinitionInvalidError( + deploy.workflow.id, + "single-step deploy requires a non-empty stepOrder", + ); + } + const primitive = deploy.workflow.steps[stepId]; + if (primitive === undefined) { + throw new WorkflowDefinitionInvalidError( + deploy.workflow.id, + `step ${stepId} listed in stepOrder is missing from steps`, + ); + } + const stepAgent = extractAgent(primitive); + const source = pickStepInferenceSource({ + stepAgent, + stepId, + workflowId: deploy.workflow.id, + config: deploy.config, + operatorApprovals: deploy.operatorApprovals, + }); + + // The lone step IS the head: one deploy at the deployment address, no + // per-step derivation. The head's agentId and instanceId are the same + // `ins_` identity. + const headAddress = deriveDeploymentAddress({ + deploymentId, + deploymentDomain, + }); + const headId = deriveDeploymentAgentId({ deploymentId }); + const headConfig: HarnessConfig = { + ...deploy.config, + agentAddress: headAddress, + agentId: headId, + ...(stepAgent !== null ? { systemPrompt: stepAgent.systemPrompt } : {}), + }; + const headDeployContent: DeployContent = + stepAgent !== null + ? { ...deploy.deployContent, systemPrompt: stepAgent.systemPrompt } + : deploy.deployContent; + + return deploySingleStepAtHead({ + agentAddress: headAddress, + agentId: headId, + instanceId: headId, + config: headConfig, + deployContent: headDeployContent, + definition: deploy.workflow, + sources: { [stepId]: source }, + hubPublicKey: deploy.hubPublicKey, + ...(deploy.toolPackagePins !== undefined + ? { toolPackagePins: deploy.toolPackagePins } + : {}), + }); +} + async function runMultiStepBranch(args: { args: DeployWorkflowArgs; launchSession: LaunchSessionFn; @@ -628,6 +792,37 @@ export function deriveDeploymentAddress(args: { return `ins_${args.deploymentId}@${args.deploymentDomain}`; } +/** + * Resolve where a step's deploy tree lives, given the deployment's step + * count. This is the single owner of the head/step collapse DECISION for + * a consumer that must choose the address without knowing the deploy + * shape: a one-step workflow has no distinct steps, so its lone step IS + * the head (`deriveDeploymentAddress`); a multi-step deployment keeps the + * head distinct from its per-step addresses (`deriveStepAddress`). The + * sidecar child reads its deploy tree from the address this returns, + * keyed only off the deployment mailbox and the host-sourced `stepCount`. + * + * The producers do not route through here -- each handles one shape + * unconditionally: the single-step deploy stages the tree at the head, + * the multi-step deploy at each per-step address. Because `stepCount` is + * the deployed definition's `stepOrder.length`, sourced from the host, + * the consumer's collapse always agrees with whichever producer staged + * the tree; the two processes never derive divergent addresses. + */ +export function resolveStepAddress(args: { + deploymentId: string; + stepId: string; + deploymentDomain: string; + stepCount: number; +}): string { + return args.stepCount === 1 + ? deriveDeploymentAddress({ + deploymentId: args.deploymentId, + deploymentDomain: args.deploymentDomain, + }) + : deriveStepAddress(args); +} + /** * Derive the deployment-level agent id used on the `agent.deploy` * frame's `agentId` field. Pure function of `(deploymentId)`. diff --git a/packages/workflow-host/src/child/env-bootstrap.ts b/packages/workflow-host/src/child/env-bootstrap.ts index ef25b7da..551fd462 100644 --- a/packages/workflow-host/src/child/env-bootstrap.ts +++ b/packages/workflow-host/src/child/env-bootstrap.ts @@ -32,6 +32,12 @@ const SpawnTimeEnvShape = type({ DEPLOYMENT_ID: "string > 0", DEFINITION_HASH: "string > 0", MAILBOX_ADDRESS: "string > 0", + // Step count of the deployed `WorkflowDefinition` (`stepOrder.length`), + // stringified by the supervisor. The child's deploy-tree read collapses + // onto the head for a single-step deployment (`resolveStepAddress`), so + // producer and consumer never derive divergent step addresses. Parsed to + // a positive integer below; a non-integer or non-positive value throws. + STEP_COUNT: "string > 0", // Warm-keep signal (design §3b). The supervisor sets this to the // string `"true"` only for the single-step long-lived deployment the // deploy projection marked a warm candidate; any other value (or the @@ -60,6 +66,13 @@ export interface SpawnTimeEnv { definitionHash: string; /** Mail address the deployment registered on the bus. */ mailboxAddress: string; + /** + * Number of steps in the deployed `WorkflowDefinition` + * (`stepOrder.length`). Selects the head/step collapse in the sidecar's + * `resolveStepAddress`: a single-step deployment reads its deploy tree + * at the head, a multi-step deployment at the per-step address. + */ + stepCount: number; /** * Whether this deployment's agent is warm-kept across messages (design * §3b). True only for the single-step long-lived deployment the deploy @@ -113,6 +126,12 @@ export function parseSpawnTimeEnv( `workflow-child IPC_CHANNEL_ID must be ${String(expectedChannelIdHex)} hex chars; got ${String(validated.IPC_CHANNEL_ID.length)}`, ); } + const stepCount = Number(validated.STEP_COUNT); + if (!Number.isInteger(stepCount) || stepCount <= 0) { + throw new Error( + `workflow-child STEP_COUNT must be a positive integer; got ${JSON.stringify(validated.STEP_COUNT)}`, + ); + } return { channelId: validated.IPC_CHANNEL_ID, hmacKey, @@ -120,6 +139,7 @@ export function parseSpawnTimeEnv( deploymentId: validated.DEPLOYMENT_ID, definitionHash: validated.DEFINITION_HASH, mailboxAddress: validated.MAILBOX_ADDRESS, + stepCount, // Strict `=== "true"` so any other value (including the key's // absence) reads false. Warm-keep is opt-in and deterministic; a // typo'd or partial value must not silently enable it. diff --git a/packages/workflow-host/src/child/run-child.test.ts b/packages/workflow-host/src/child/run-child.test.ts index d5d57032..e31db82f 100644 --- a/packages/workflow-host/src/child/run-child.test.ts +++ b/packages/workflow-host/src/child/run-child.test.ts @@ -347,6 +347,7 @@ function makeSpawnEnv(opts: { DEPLOYMENT_ID: "deployment-x", DEFINITION_HASH: "definition-hash-abc", MAILBOX_ADDRESS: "deployment-x@example.com", + STEP_COUNT: "1", }; } diff --git a/packages/workflow-host/src/supervisor/lifecycle-races.test.ts b/packages/workflow-host/src/supervisor/lifecycle-races.test.ts index a78be829..0593480d 100644 --- a/packages/workflow-host/src/supervisor/lifecycle-races.test.ts +++ b/packages/workflow-host/src/supervisor/lifecycle-races.test.ts @@ -346,6 +346,7 @@ async function buildBindings(opts: { workflowRunRepoId: { kind: "workflow-run", id: "deployment-x" }, workflowRunRef: "refs/heads/main", deploymentId: "deployment-x", + stepCount: 1, deploymentMailAddress: "deployment-x@example.com", readPrincipal: { kind: "supervisor" }, deriveStepAddress: ({ deploymentId, stepId }) => diff --git a/packages/workflow-host/src/supervisor/outbound-signed-send.test.ts b/packages/workflow-host/src/supervisor/outbound-signed-send.test.ts index d9ddaed3..8dff7d56 100644 --- a/packages/workflow-host/src/supervisor/outbound-signed-send.test.ts +++ b/packages/workflow-host/src/supervisor/outbound-signed-send.test.ts @@ -264,6 +264,7 @@ describe("supervisor-backed outbound signed send (Phase 4.3)", () => { workflowRunRepoId: { kind: "workflow-run", id: "outbound-dep" }, workflowRunRef: "refs/heads/main", deploymentId: "outbound-dep", + stepCount: 1, deploymentMailAddress: AGENT_ADDRESS, readPrincipal: { kind: "supervisor" }, deriveStepAddress: () => AGENT_ADDRESS, diff --git a/packages/workflow-host/src/supervisor/recycle.test.ts b/packages/workflow-host/src/supervisor/recycle.test.ts index e53db2af..b830a062 100644 --- a/packages/workflow-host/src/supervisor/recycle.test.ts +++ b/packages/workflow-host/src/supervisor/recycle.test.ts @@ -471,6 +471,7 @@ async function buildBindings(opts: { workflowRunRepoId: { kind: "workflow-run", id: "deployment-x" }, workflowRunRef: "refs/heads/main", deploymentId: "deployment-x", + stepCount: 1, deploymentMailAddress: "deployment-x@example.com", readPrincipal: { kind: "supervisor" }, deriveStepAddress: ({ deploymentId, stepId }) => diff --git a/packages/workflow-host/src/supervisor/spawn-replay-fifo.test.ts b/packages/workflow-host/src/supervisor/spawn-replay-fifo.test.ts index a26252f0..653df60f 100644 --- a/packages/workflow-host/src/supervisor/spawn-replay-fifo.test.ts +++ b/packages/workflow-host/src/supervisor/spawn-replay-fifo.test.ts @@ -548,6 +548,7 @@ async function boot(opts: { prefix: string }): Promise< workflowRunRepoId, workflowRunRef: "refs/heads/main", deploymentId: "deployment-x", + stepCount: 1, deploymentMailAddress, readPrincipal: { kind: "supervisor" }, deriveStepAddress: ({ deploymentId, stepId }) => diff --git a/packages/workflow-host/src/supervisor/stale-cohort-routing.test.ts b/packages/workflow-host/src/supervisor/stale-cohort-routing.test.ts index 9a6f3fb5..1c895571 100644 --- a/packages/workflow-host/src/supervisor/stale-cohort-routing.test.ts +++ b/packages/workflow-host/src/supervisor/stale-cohort-routing.test.ts @@ -445,6 +445,7 @@ describe("H-S2 stale-cohort routing pinch-point", () => { workflowRunRepoId: { kind: "workflow-run", id: "deployment-x" }, workflowRunRef: "refs/heads/main", deploymentId: "deployment-x", + stepCount: 1, deploymentMailAddress: "deployment-x@example.com", readPrincipal: { kind: "supervisor" }, deriveStepAddress: ({ deploymentId, stepId }) => diff --git a/packages/workflow-host/src/supervisor/substrate-write.test.ts b/packages/workflow-host/src/supervisor/substrate-write.test.ts index 3516a75d..87f29d71 100644 --- a/packages/workflow-host/src/supervisor/substrate-write.test.ts +++ b/packages/workflow-host/src/supervisor/substrate-write.test.ts @@ -520,6 +520,7 @@ async function bootSupervisor(opts: { workflowRunRepoId: { kind: "workflow-run", id: "deployment-x" }, workflowRunRef: "refs/heads/main", deploymentId: "deployment-x", + stepCount: 1, deploymentMailAddress: "deployment-x@example.com", readPrincipal: { kind: "supervisor" }, deriveStepAddress: ({ deploymentId, stepId }) => diff --git a/packages/workflow-host/src/supervisor/supervisor.test.ts b/packages/workflow-host/src/supervisor/supervisor.test.ts index 1c6ef424..280af860 100644 --- a/packages/workflow-host/src/supervisor/supervisor.test.ts +++ b/packages/workflow-host/src/supervisor/supervisor.test.ts @@ -509,6 +509,7 @@ async function buildBindings(opts: { workflowRunRepoId: { kind: "workflow-run", id: "deployment-x" }, workflowRunRef: "refs/heads/main", deploymentId: "deployment-x", + stepCount: 1, deploymentMailAddress: "deployment-x@example.com", readPrincipal: { kind: "supervisor" }, deriveStepAddress: ({ deploymentId, stepId }) => diff --git a/packages/workflow-host/src/supervisor/supervisor.ts b/packages/workflow-host/src/supervisor/supervisor.ts index c67730d6..daaff2ad 100644 --- a/packages/workflow-host/src/supervisor/supervisor.ts +++ b/packages/workflow-host/src/supervisor/supervisor.ts @@ -1358,6 +1358,7 @@ export function createWorkflowSupervisor( DEPLOYMENT_ID: bindings.deploymentId, DEFINITION_HASH: opts.definitionHash, MAILBOX_ADDRESS: bindings.deploymentMailAddress, + STEP_COUNT: String(bindings.stepCount), WARM_KEEP: opts.warmKeep ? "true" : "false", }; diff --git a/packages/workflow-host/src/supervisor/types.ts b/packages/workflow-host/src/supervisor/types.ts index b9cf3b66..27e4f598 100644 --- a/packages/workflow-host/src/supervisor/types.ts +++ b/packages/workflow-host/src/supervisor/types.ts @@ -401,6 +401,16 @@ export interface WorkflowSupervisorBindings { workflowRunRef: string; /** Deployment id baked into the supervisor's principal claims. */ deploymentId: string; + /** + * Number of steps in the deployed `WorkflowDefinition` + * (`stepOrder.length`). The supervisor threads this into the child's + * spawn-time env (`STEP_COUNT`) so the child's deploy-tree read + * (`resolveStepAddress` in the sidecar step tools) collapses onto the + * head for a single-step deployment exactly as the host's producer + * push does -- one source of truth for the head/step collapse across + * the two processes. Fixed for the deployment's lifetime. + */ + stepCount: number; /** * Mail address the deployment registers on the bus. The * supervisor registers this on spawn and unregisters on teardown; diff --git a/tests/hub-api/asset-routes.test.ts b/tests/hub-api/asset-routes.test.ts index 3f459b0c..c22ab3c1 100644 --- a/tests/hub-api/asset-routes.test.ts +++ b/tests/hub-api/asset-routes.test.ts @@ -165,6 +165,9 @@ function createMockSessionService(): SessionService { deployWorkflowDefinition: notImplemented( "sessionService.deployWorkflowDefinition", ), + deploySingleStepAtHead: notImplemented( + "sessionService.deploySingleStepAtHead", + ), sendUserMessage: notImplemented("sessionService.sendUserMessage"), endSession: notImplemented("sessionService.endSession"), }; diff --git a/tests/workflow-deploy/child-workflow-roundtrip.test.ts b/tests/workflow-deploy/child-workflow-roundtrip.test.ts index 5d4133c8..5134b4c3 100644 --- a/tests/workflow-deploy/child-workflow-roundtrip.test.ts +++ b/tests/workflow-deploy/child-workflow-roundtrip.test.ts @@ -245,6 +245,10 @@ describe("parent -> child workflow round-trip", () => { workflowRepo, launchSession, sendMultiStepDeploy, + // A single-step (child-spawning) workflow deploys once at the head + // through the deploy core's single-step hand-off. + deploySingleStepAtHead: (params) => + env.hub.sessionService.deploySingleStepAtHead(params), }); const childResult = await orchestrator.deployWorkflow({ @@ -620,6 +624,10 @@ describe("parent -> child workflow round-trip", () => { workflowRepo, launchSession, sendMultiStepDeploy, + // A single-step (child-spawning) workflow deploys once at the head + // through the deploy core's single-step hand-off. + deploySingleStepAtHead: (params) => + env.hub.sessionService.deploySingleStepAtHead(params), }); // Deploy grandchild and child as workflow assets so the parent's @@ -975,6 +983,10 @@ describe("parent -> child workflow round-trip", () => { workflowRepo, launchSession, sendMultiStepDeploy, + // A single-step (child-spawning) workflow deploys once at the head + // through the deploy core's single-step hand-off. + deploySingleStepAtHead: (params) => + env.hub.sessionService.deploySingleStepAtHead(params), }); for (let i = 0; i < SIBLINGS_CHILD_COUNT; i += 1) { diff --git a/tests/workflow-deploy/cross-process-custom-adapter.test.ts b/tests/workflow-deploy/cross-process-custom-adapter.test.ts index 76a9ab56..aa4376ff 100644 --- a/tests/workflow-deploy/cross-process-custom-adapter.test.ts +++ b/tests/workflow-deploy/cross-process-custom-adapter.test.ts @@ -35,6 +35,7 @@ import { createWorkflowDeployOrchestrator, deriveDeploymentAddress, type ApprovalSet, + type DeploySingleStepFn, type LaunchSessionFn, type SendMultiStepDeployFn, type WorkflowRepoWriter, @@ -183,6 +184,9 @@ async function deployCustomProviderWorkflow( sources: params.sources, }); + const deploySingleStepAtHead: DeploySingleStepFn = (params) => + env.hub.sessionService.deploySingleStepAtHead(params); + const workflowRepo: WorkflowRepoWriter = { async writeWorkflowRepo(args) { const repoId: RepoId = { kind: "workflow", id: args.workflowRepoId }; @@ -208,6 +212,7 @@ async function deployCustomProviderWorkflow( workflowRepo, launchSession, sendMultiStepDeploy, + deploySingleStepAtHead, }); const result = await orchestrator.deployWorkflow({ @@ -286,7 +291,20 @@ describe("cross-process custom inference adapter (INTR-233)", () => { expect(reply).toContain(body); }); - test("a provider absent from the manifest is rejected at the source gate", async () => { + // PENDING an operator security-posture decision: a single-step workflow + // now deploys at the head, which skips the hub's provision/session-start + // path where `canBuildSource` used to reject an unregistered provider at + // deploy time. The no-conjure invariant still holds (the child's + // exact-match `registry.resolve` throws with no adapter substitution), + // but the rejection is deferred to run-time (`RunFailed`) instead of + // synchronous at deploy. The intended fix is a deploy-core source- + // admission gate owned by the instance-routing work, covering single- + // and multi-step uniformly; the trivial/warm-path cleanup depends on + // that gate existing first (multi-step's gate currently rides the same + // per-step warm provisioning). This test asserts deploy-time rejection + // and is held (not rewritten) until the operator rules whether to + // restore the deploy-time gate or accept run-time-only enforcement. + test.skip("a provider absent from the manifest is rejected at the source gate", async () => { // The firewall: the operator registry holds only the built-ins plus the // manifest's "custom-x". A provider id that no manifest entry and no // built-in supplies is rejected by `canBuildSource` against that same diff --git a/tests/workflow-deploy/fifo-mail-load.test.ts b/tests/workflow-deploy/fifo-mail-load.test.ts index 1b559c96..4c1bf4c3 100644 --- a/tests/workflow-deploy/fifo-mail-load.test.ts +++ b/tests/workflow-deploy/fifo-mail-load.test.ts @@ -31,6 +31,7 @@ import { createWorkflowDeployOrchestrator, deriveDeploymentAddress, type ApprovalSet, + type DeploySingleStepFn, type LaunchSessionFn, type SendMultiStepDeployFn, type WorkflowRepoWriter, @@ -224,6 +225,9 @@ describe("FIFO mail-trigger serialization under load", () => { sources: params.sources, }); + const deploySingleStepAtHead: DeploySingleStepFn = (params) => + env.hub.sessionService.deploySingleStepAtHead(params); + const workflowRepo: WorkflowRepoWriter = { async writeWorkflowRepo(args) { const repoId: RepoId = { kind: "workflow", id: args.workflowRepoId }; @@ -249,6 +253,7 @@ describe("FIFO mail-trigger serialization under load", () => { workflowRepo, launchSession, sendMultiStepDeploy, + deploySingleStepAtHead, }); let result: Awaited>; diff --git a/tests/workflow-deploy/mail-edge-cases.test.ts b/tests/workflow-deploy/mail-edge-cases.test.ts index 30e5573c..6ded567a 100644 --- a/tests/workflow-deploy/mail-edge-cases.test.ts +++ b/tests/workflow-deploy/mail-edge-cases.test.ts @@ -60,6 +60,7 @@ import { createWorkflowDeployOrchestrator, deriveDeploymentAddress, type ApprovalSet, + type DeploySingleStepFn, type LaunchSessionFn, type SendMultiStepDeployFn, type WorkflowRepoWriter, @@ -430,6 +431,9 @@ async function deployEdgeWorkflow( sources: params.sources, }); + const deploySingleStepAtHead: DeploySingleStepFn = (params) => + env.hub.sessionService.deploySingleStepAtHead(params); + const workflowRepo: WorkflowRepoWriter = { async writeWorkflowRepo(args) { const repoId: RepoId = { kind: "workflow", id: args.workflowRepoId }; @@ -455,6 +459,7 @@ async function deployEdgeWorkflow( workflowRepo, launchSession, sendMultiStepDeploy, + deploySingleStepAtHead, }); const result = await orchestrator.deployWorkflow({ diff --git a/tests/workflow-deploy/single-step-conversation-durability.test.ts b/tests/workflow-deploy/single-step-conversation-durability.test.ts index 06f233a0..69e6d9cf 100644 --- a/tests/workflow-deploy/single-step-conversation-durability.test.ts +++ b/tests/workflow-deploy/single-step-conversation-durability.test.ts @@ -622,6 +622,7 @@ describe("single-step conversation durability across respawn (Phase 4.5)", () => DEPLOYMENT_ID, DEFINITION_HASH: "definition-hash", MAILBOX_ADDRESS: MAILBOX, + STEP_COUNT: "1", WARM_KEEP: "true", }); diff --git a/tests/workflow-deploy/single-step-full-lifecycle.test.ts b/tests/workflow-deploy/single-step-full-lifecycle.test.ts index 8d57df6d..56a5b43d 100644 --- a/tests/workflow-deploy/single-step-full-lifecycle.test.ts +++ b/tests/workflow-deploy/single-step-full-lifecycle.test.ts @@ -78,6 +78,7 @@ import { deriveDeploymentAddress, isWorkflowDerivedAddress, type ApprovalSet, + type DeploySingleStepFn, type LaunchSessionFn, type SendMultiStepDeployFn, type WorkflowRepoWriter, @@ -263,6 +264,14 @@ describe("single-step full lifecycle on the unified child path (Phase 4.6)", () sources: params.sources, }); + // A single-step workflow routes through the deploy core's single-step + // hand-off, which deploys once at the head: the service stages the + // head's deploy tree and fires the workflow frame in one call. The + // per-step `launchSession`/`sendMultiStepDeploy` callbacks above are + // reached only by the multi-step branch (>= 2 steps). + const deploySingleStepAtHead: DeploySingleStepFn = (params) => + env.hub.sessionService.deploySingleStepAtHead(params); + const workflowRepo: WorkflowRepoWriter = { async writeWorkflowRepo(args) { const repoId: RepoId = { kind: "workflow", id: args.workflowRepoId }; @@ -288,6 +297,7 @@ describe("single-step full lifecycle on the unified child path (Phase 4.6)", () workflowRepo, launchSession, sendMultiStepDeploy, + deploySingleStepAtHead, }); let result: Awaited>; diff --git a/tests/workflow-deploy/single-step-grants-bridge.test.ts b/tests/workflow-deploy/single-step-grants-bridge.test.ts index d6630c5b..2bd36bf4 100644 --- a/tests/workflow-deploy/single-step-grants-bridge.test.ts +++ b/tests/workflow-deploy/single-step-grants-bridge.test.ts @@ -53,6 +53,7 @@ import { deriveDeploymentAddress, isWorkflowDerivedAddress, type ApprovalSet, + type DeploySingleStepFn, type LaunchSessionFn, type SendMultiStepDeployFn, type WorkflowRepoWriter, @@ -225,6 +226,9 @@ describe("single-step launched-agent grants bridge via spawned child", () => { sources: params.sources, }); + const deploySingleStepAtHead: DeploySingleStepFn = (params) => + env.hub.sessionService.deploySingleStepAtHead(params); + const workflowRepo: WorkflowRepoWriter = { async writeWorkflowRepo(args) { const repoId: RepoId = { kind: "workflow", id: args.workflowRepoId }; @@ -250,6 +254,7 @@ describe("single-step launched-agent grants bridge via spawned child", () => { workflowRepo, launchSession, sendMultiStepDeploy, + deploySingleStepAtHead, }); let result: Awaited>; diff --git a/tests/workflow-deploy/single-step-message-input.test.ts b/tests/workflow-deploy/single-step-message-input.test.ts index 52e51ace..14769124 100644 --- a/tests/workflow-deploy/single-step-message-input.test.ts +++ b/tests/workflow-deploy/single-step-message-input.test.ts @@ -29,6 +29,7 @@ import { createWorkflowDeployOrchestrator, deriveDeploymentAddress, type ApprovalSet, + type DeploySingleStepFn, type LaunchSessionFn, type SendMultiStepDeployFn, type WorkflowRepoWriter, @@ -152,6 +153,9 @@ describe("single-step message-input round-trip", () => { sources: params.sources, }); + const deploySingleStepAtHead: DeploySingleStepFn = (params) => + env.hub.sessionService.deploySingleStepAtHead(params); + const workflowRepo: WorkflowRepoWriter = { async writeWorkflowRepo(args) { const repoId: RepoId = { kind: "workflow", id: args.workflowRepoId }; @@ -177,6 +181,7 @@ describe("single-step message-input round-trip", () => { workflowRepo, launchSession, sendMultiStepDeploy, + deploySingleStepAtHead, }); let result: Awaited>; diff --git a/tests/workflow-deploy/single-step-posix-tool.test.ts b/tests/workflow-deploy/single-step-posix-tool.test.ts index 01754d07..d7c60866 100644 --- a/tests/workflow-deploy/single-step-posix-tool.test.ts +++ b/tests/workflow-deploy/single-step-posix-tool.test.ts @@ -31,8 +31,8 @@ import { defineWorkflow, step, type WorkflowDefinition } from "@intx/workflow"; import { createWorkflowDeployOrchestrator, deriveDeploymentAddress, - deriveStepAddress, type ApprovalSet, + type DeploySingleStepFn, type LaunchSessionFn, type SendMultiStepDeployFn, type WorkflowRepoWriter, @@ -184,6 +184,9 @@ describe("single-step posix-tool in-child execution", () => { sources: params.sources, }); + const deploySingleStepAtHead: DeploySingleStepFn = (params) => + env.hub.sessionService.deploySingleStepAtHead(params); + const workflowRepo: WorkflowRepoWriter = { async writeWorkflowRepo(args) { const repoId: RepoId = { kind: "workflow", id: args.workflowRepoId }; @@ -209,6 +212,7 @@ describe("single-step posix-tool in-child execution", () => { workflowRepo, launchSession, sendMultiStepDeploy, + deploySingleStepAtHead, }); let result: Awaited>; @@ -312,20 +316,21 @@ describe("single-step posix-tool in-child execution", () => { expect(fs.readFileSync(sentinelPath, "utf-8")).toBe(SENTINEL_CONTENT); // The step's deploy tree (carrying the resolved tool-package - // manifest) landed at the step's legacy agent dir on the child -- - // the on-disk source the child read for materialization. - const stepAddress = deriveStepAddress({ + // manifest) landed at the HEAD's legacy agent dir on the child -- the + // on-disk source the child read for materialization. A single-step + // workflow collapses its lone step onto the head, so the deploy tree + // is staged at the deployment (head) address, not a per-step address. + const headAddress = deriveDeploymentAddress({ deploymentId: DEPLOYMENT_ID, - stepId: STEP_ID, deploymentDomain: DEPLOYMENT_DOMAIN, }); - const stepDeployDir = path.join( + const headDeployDir = path.join( env.sidecar.dataDir, - sanitizeAddress(stepAddress), + sanitizeAddress(headAddress), "deploy", ); expect( - fs.existsSync(path.join(stepDeployDir, "tool-packages-manifest.json")), + fs.existsSync(path.join(headDeployDir, "tool-packages-manifest.json")), ).toBe(true); }); }); diff --git a/tests/workflow-deploy/single-step-real-agent.test.ts b/tests/workflow-deploy/single-step-real-agent.test.ts index 4a4a5894..2699bd7d 100644 --- a/tests/workflow-deploy/single-step-real-agent.test.ts +++ b/tests/workflow-deploy/single-step-real-agent.test.ts @@ -33,6 +33,7 @@ import { createWorkflowDeployOrchestrator, deriveDeploymentAddress, type ApprovalSet, + type DeploySingleStepFn, type LaunchSessionFn, type SendMultiStepDeployFn, type WorkflowRepoWriter, @@ -160,6 +161,9 @@ describe("single-step real-agent round-trip", () => { sources: params.sources, }); + const deploySingleStepAtHead: DeploySingleStepFn = (params) => + env.hub.sessionService.deploySingleStepAtHead(params); + const workflowRepo: WorkflowRepoWriter = { async writeWorkflowRepo(args) { const repoId: RepoId = { kind: "workflow", id: args.workflowRepoId }; @@ -185,6 +189,7 @@ describe("single-step real-agent round-trip", () => { workflowRepo, launchSession, sendMultiStepDeploy, + deploySingleStepAtHead, }); let result: Awaited>; From 056fc6c9513c25ed4206877f61ab031b9fb1b3d5 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Thu, 2 Jul 2026 10:14:46 -0500 Subject: [PATCH 019/101] Reject a workflow deploy that names an unbuildable provider The single-step head deploy skips the hub provisioning that ran canBuildSource, so a workflow whose step pins an inference provider the sidecar cannot build was admitted and failed only when the step's inference first resolved, at run time. The no-conjure invariant held -- the child's exact-match resolution admits no substitute adapter -- but deploy-time admission control was lost, and a misconfigured source produced a false "deploy succeeded" signal. Add a source-admission gate at the sidecar deploy router, where the buildable-provider set (the adapter registry) is known; the hub is a different process and cannot own this check. The gate validates every step's pinned source before the workflow-process child is spawned, reusing the harness builder's canBuildSource predicate against the one registry. A rejected provider throws back through the deploy frame, so the deploy rejects synchronously at deploy time. It covers single- and multi-step, and every step's source, not just the default. This is distinct from the orchestrator's operator-approval check: that gates on whether the operator approved a provider:model pair; this gates on whether the provider is buildable at all. A source can be approved yet unbuildable. --- .../agent-key-registration-lifecycle.test.ts | 1 + apps/sidecar/src/index.ts | 20 ++++--- ...orkflow-host-wiring-deploy-failure.test.ts | 2 + ...ow-host-wiring-undeploy-supervisor.test.ts | 1 + apps/sidecar/src/workflow-host-wiring.test.ts | 54 +++++++++++++++++++ apps/sidecar/src/workflow-host-wiring.ts | 29 ++++++++++ .../cross-process-custom-adapter.test.ts | 24 +++------ 7 files changed, 107 insertions(+), 24 deletions(-) diff --git a/apps/sidecar/src/agent-key-registration-lifecycle.test.ts b/apps/sidecar/src/agent-key-registration-lifecycle.test.ts index c4fa22d9..7766a5d9 100644 --- a/apps/sidecar/src/agent-key-registration-lifecycle.test.ts +++ b/apps/sidecar/src/agent-key-registration-lifecycle.test.ts @@ -246,6 +246,7 @@ describe("agent signing-key registration lifecycle on the host transport", () => repoStore, signingKeySeed: keyPair.privateKey, createAgentCrypto: createEd25519Crypto, + assertSourceBuildable: () => undefined, registerDeployment: () => undefined, unregisterDeployment: () => undefined, multistepSubprocessSpawner: spawner, diff --git a/apps/sidecar/src/index.ts b/apps/sidecar/src/index.ts index 22762c19..4e12e357 100644 --- a/apps/sidecar/src/index.ts +++ b/apps/sidecar/src/index.ts @@ -320,19 +320,24 @@ if (hostTmpdir !== undefined) { multistepSubstrateEnv["TMPDIR"] = hostTmpdir; } +// Hoisted so the deploy router's source-admission gate reuses the exact +// same `canBuildSource` predicate the harness builder enforces, against +// the one adapter registry, rather than a second copy of the check. +const buildHarness = createDefaultHarnessBuilder({ + cacheRoot, + cacheMaxBytes, + registryMaxTarballBytes, + adapters, + gcPolicy: agentGCPolicy, +}); + const orchestrator = createSidecarOrchestrator({ hubURL: hubWsUrl, sidecarId, token: sidecarToken, dataDir, transport, - buildHarness: createDefaultHarnessBuilder({ - cacheRoot, - cacheMaxBytes, - registryMaxTarballBytes, - adapters, - gcPolicy: agentGCPolicy, - }), + buildHarness, createAgentCrypto: createEd25519Crypto, cryptoOps: { generateKeyPair, @@ -356,6 +361,7 @@ const orchestrator = createSidecarOrchestrator({ repoStore: wrappedRepoStore, signingKeySeed: sidecarSigningKey.privateKey, createAgentCrypto: createEd25519Crypto, + assertSourceBuildable: buildHarness.canBuildSource, registerDeployment: ({ deploymentId, agentAddress }) => { deploymentAddressRegistry.record(deploymentId, agentAddress); }, diff --git a/apps/sidecar/src/workflow-host-wiring-deploy-failure.test.ts b/apps/sidecar/src/workflow-host-wiring-deploy-failure.test.ts index 3414031a..b4c36de4 100644 --- a/apps/sidecar/src/workflow-host-wiring-deploy-failure.test.ts +++ b/apps/sidecar/src/workflow-host-wiring-deploy-failure.test.ts @@ -104,6 +104,7 @@ describe("deploy-failure registry leak", () => { >[0]["repoStore"], signingKeySeed: new Uint8Array(32), createAgentCrypto: createEd25519Crypto, + assertSourceBuildable: () => undefined, registerDeployment: ({ deploymentId, agentAddress }) => { registry.record(deploymentId, agentAddress); }, @@ -188,6 +189,7 @@ describe("deploy-failure registry leak", () => { repoStore: repoStoreStub as RepoStore, signingKeySeed: new Uint8Array(32), createAgentCrypto: createEd25519Crypto, + assertSourceBuildable: () => undefined, registerDeployment: ({ deploymentId, agentAddress }) => { registry.record(deploymentId, agentAddress); }, diff --git a/apps/sidecar/src/workflow-host-wiring-undeploy-supervisor.test.ts b/apps/sidecar/src/workflow-host-wiring-undeploy-supervisor.test.ts index dd5ee834..fbd86af0 100644 --- a/apps/sidecar/src/workflow-host-wiring-undeploy-supervisor.test.ts +++ b/apps/sidecar/src/workflow-host-wiring-undeploy-supervisor.test.ts @@ -265,6 +265,7 @@ describe("createSidecarDeployRouter multi-step undeploy shuts the supervisor dow repoStore, signingKeySeed: keyPair.privateKey, createAgentCrypto: createEd25519Crypto, + assertSourceBuildable: () => undefined, registerDeployment: () => { /* no-op */ }, diff --git a/apps/sidecar/src/workflow-host-wiring.test.ts b/apps/sidecar/src/workflow-host-wiring.test.ts index 716e1e51..4506f01d 100644 --- a/apps/sidecar/src/workflow-host-wiring.test.ts +++ b/apps/sidecar/src/workflow-host-wiring.test.ts @@ -375,6 +375,7 @@ describe("createSidecarDeployRouter wires the InferenceEvent subscription to rec repoStore, signingKeySeed: keyPair.privateKey, createAgentCrypto: createEd25519Crypto, + assertSourceBuildable: () => undefined, registerDeployment: () => { /* the in-test repoStore is a stub; the pack-push facade is exercised separately */ }, @@ -758,6 +759,7 @@ describe("createSidecarDeployRouter trivial-frame regression", () => { repoStore, signingKeySeed: keyPair.privateKey, createAgentCrypto: createEd25519Crypto, + assertSourceBuildable: () => undefined, registerDeployment: () => { /* no-op */ }, @@ -831,6 +833,7 @@ describe("createSidecarDeployRouter trivial-frame regression", () => { repoStore, signingKeySeed: keyPair.privateKey, createAgentCrypto: createEd25519Crypto, + assertSourceBuildable: () => undefined, registerDeployment: () => { /* no-op */ }, @@ -919,6 +922,7 @@ describe("createSidecarDeployRouter trivial-frame regression", () => { repoStore, signingKeySeed: keyPair.privateKey, createAgentCrypto: createEd25519Crypto, + assertSourceBuildable: () => undefined, registerDeployment: () => undefined, unregisterDeployment: () => undefined, multistepSubprocessSpawner: () => { @@ -994,6 +998,7 @@ describe("createSidecarDeployRouter trivial-frame regression", () => { repoStore, signingKeySeed: keyPair.privateKey, createAgentCrypto: createEd25519Crypto, + assertSourceBuildable: () => undefined, registerDeployment: () => undefined, unregisterDeployment: () => undefined, multistepSubprocessSpawner: () => { @@ -1079,6 +1084,7 @@ describe("createSidecarDeployRouter trivial-frame regression", () => { repoStore, signingKeySeed: keyPair.privateKey, createAgentCrypto: createEd25519Crypto, + assertSourceBuildable: () => undefined, registerDeployment: () => undefined, unregisterDeployment: () => undefined, multistepSubprocessSpawner: () => { @@ -1133,6 +1139,9 @@ describe("createSidecarDeployRouter multi-step branch", () => { deploymentId: string; agentAddress: string; }) => void; + assertSourceBuildable?: Parameters< + typeof createSidecarDeployRouter + >[0]["assertSourceBuildable"]; }) { const transport = createInMemoryTransport(); const keyPair = await generateKeyPair(); @@ -1184,6 +1193,7 @@ describe("createSidecarDeployRouter multi-step branch", () => { repoStore, signingKeySeed: keyPair.privateKey, createAgentCrypto: createEd25519Crypto, + assertSourceBuildable: opts.assertSourceBuildable ?? (() => undefined), registerDeployment: opts.registerDeployment ?? (() => undefined), unregisterDeployment: () => { /* no-op */ @@ -1444,6 +1454,50 @@ describe("createSidecarDeployRouter multi-step branch", () => { ); }); + test("rejects a deploy whose step pins an unbuildable provider before spawning", async () => { + // The source-admission gate runs before any state is claimed or the + // child is spawned. A step whose pinned source names a provider the + // sidecar cannot build must reject the whole deploy synchronously -- + // the admission control property -- rather than spawning a child that + // fails when the step's inference first resolves. + let spawnCount = 0; + const trackingSpawner: SubprocessSpawner = () => { + spawnCount++; + throw new Error("spawn must not be reached for an inadmissible source"); + }; + const { router } = await buildMultistepFixture({ + spawner: trackingSpawner, + assertSourceBuildable: (source) => { + if (source.provider === "ghost-provider") { + throw new Error( + `Source provider "${source.provider}" is not registered`, + ); + } + }, + }); + + const frame = makeMultistepFrame({ + agentAddress: "ins_unbuildable@example.com", + definition: { + id: "wf-unbuildable", + triggers: [{ type: "manual" }], + stepOrder: ["step-1"], + steps: { "step-1": { kind: "step" } }, + }, + sources: { + "step-1": { + ...makeInferenceSource("step-1"), + provider: "ghost-provider", + }, + }, + }); + + await expect(router.deploy(frame)).rejects.toThrow( + /ghost-provider.*not registered/, + ); + expect(spawnCount).toBe(0); + }); + test("a spawner that throws synchronously surfaces a structured rejection rather than hanging in starting", async () => { // Simulates `Bun.spawn` failing to launch (binary missing, // permissions error). The router must surface the rejection diff --git a/apps/sidecar/src/workflow-host-wiring.ts b/apps/sidecar/src/workflow-host-wiring.ts index 0978795b..02e55ece 100644 --- a/apps/sidecar/src/workflow-host-wiring.ts +++ b/apps/sidecar/src/workflow-host-wiring.ts @@ -55,6 +55,7 @@ import { parseInferenceEvent, type CryptoProvider, type InferenceEvent, + type InferenceSource, type KeyPair, } from "@intx/types/runtime"; import type { AgentDeployFrame } from "@intx/types/sidecar"; @@ -674,6 +675,21 @@ export function createSidecarDeployRouter(deps: { * than emit unsigned mail. */ createAgentCrypto: (keyPair: KeyPair) => CryptoProvider; + /** + * Source-admission gate: throws if a step's pinned inference source + * names a provider this sidecar cannot build. The buildable-provider + * set is sidecar config (the boot edge's adapter registry), so this + * admission control lives at the sidecar -- the hub is a different + * process and cannot know a given sidecar's providers. Production wires + * the default harness builder's `canBuildSource` verbatim, so a rejected + * provider carries the same `"... is not registered"` message. + * + * Distinct from the orchestrator's operator-approval check + * (`pickStepInferenceSource`): that gates on whether the operator + * approved a `provider:model`; this gates on whether the provider is + * buildable at all. A source can be approved yet unbuildable. + */ + assertSourceBuildable: (source: InferenceSource) => void; /** * Record a `(deploymentId -> agentAddress)` mapping the boot edge's * workflow-run pack push facade consults when it must address an @@ -905,6 +921,19 @@ export function createSidecarDeployRouter(deps: { // supervisor. validateWorkflowProjection(projection); + // Source-admission gate: reject a deploy whose any step pins an + // inference provider this sidecar cannot build, BEFORE any state is + // claimed or the child is spawned. The throw propagates back through + // the deploy frame so the hub's `deployWorkflow` rejects synchronously + // at deploy time, rather than the child failing the run when the + // step's inference first resolves. Covers single- and multi-step: the + // projection's `narrow` guarantees every stepOrder entry has a + // `sources` entry. + for (const stepId of projection.definition.stepOrder) { + const source = projection.sources[stepId]; + if (source !== undefined) deps.assertSourceBuildable(source); + } + const deploymentId = deriveTrivialDeploymentId(frame.agentAddress); // Single-step launched-agent deploy vs. derived multi-step deploy. diff --git a/tests/workflow-deploy/cross-process-custom-adapter.test.ts b/tests/workflow-deploy/cross-process-custom-adapter.test.ts index aa4376ff..ffd58a94 100644 --- a/tests/workflow-deploy/cross-process-custom-adapter.test.ts +++ b/tests/workflow-deploy/cross-process-custom-adapter.test.ts @@ -291,25 +291,15 @@ describe("cross-process custom inference adapter (INTR-233)", () => { expect(reply).toContain(body); }); - // PENDING an operator security-posture decision: a single-step workflow - // now deploys at the head, which skips the hub's provision/session-start - // path where `canBuildSource` used to reject an unregistered provider at - // deploy time. The no-conjure invariant still holds (the child's - // exact-match `registry.resolve` throws with no adapter substitution), - // but the rejection is deferred to run-time (`RunFailed`) instead of - // synchronous at deploy. The intended fix is a deploy-core source- - // admission gate owned by the instance-routing work, covering single- - // and multi-step uniformly; the trivial/warm-path cleanup depends on - // that gate existing first (multi-step's gate currently rides the same - // per-step warm provisioning). This test asserts deploy-time rejection - // and is held (not rewritten) until the operator rules whether to - // restore the deploy-time gate or accept run-time-only enforcement. - test.skip("a provider absent from the manifest is rejected at the source gate", async () => { + test("a provider absent from the manifest is rejected at the source gate", async () => { // The firewall: the operator registry holds only the built-ins plus the // manifest's "custom-x". A provider id that no manifest entry and no - // built-in supplies is rejected by `canBuildSource` against that same - // registry, before the agent ever launches -- so a provider string (which - // deploy/tenant config controls) cannot conjure an adapter. + // built-in supplies is rejected by the sidecar deploy router's + // source-admission gate (which reuses `canBuildSource` against that same + // registry) before the workflow-process child is spawned -- so a + // provider string (which deploy/tenant config controls) cannot conjure + // an adapter, and the deploy is rejected synchronously at deploy time + // rather than failing the first run. await expect( deployCustomProviderWorkflow( "cross-process-custom-negative", From a758d7d5cb4620f2ff2003885b2cfff3311fd76e Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Thu, 2 Jul 2026 11:26:11 -0500 Subject: [PATCH 020/101] Extract the workflow-deployment spawn sequence into a core `deployMultiStep` did everything inline in one `try/finally`: claimed the deployment slug, materialized the workflow definition and grants on disk, constructed the supervisor, did the single-step key/repo/hub-key registration, spawned the child, registered the live deployment, and unwound partial state on failure. Split it into two owners. `deployMultiStep` claims the deployment slug, materializes the deploy-only durable state -- `workflow.json` (via a new `materializeWorkflowJson` helper) and the step grants -- and hands a `WorkflowDeploySpec` to `spawnWorkflowDeployment`, releasing the slug if any of that throws. `spawnWorkflowDeployment` is the single owner of the spawn sequence: supervisor construction, the single-step key/repo/hub-key registration, the spawn, the live mail/signal/drain and address registrations, and the unwind of exactly those. It derives every per-deployment value from the spec rather than from a deploy frame, so a future caller can drive the same spawn from a source other than a live frame. The slug claim stays ahead of every durable write, preserving the router's guarantee that a colliding deploymentId is rejected before any repo state is touched. `writeStepGrants` runs before supervisor construction -- it needs only the repo store and step strategy -- so the materialize phase stays contiguous. No behavior change. --- apps/sidecar/src/workflow-host-wiring.ts | 599 +++++++++++------------ 1 file changed, 292 insertions(+), 307 deletions(-) diff --git a/apps/sidecar/src/workflow-host-wiring.ts b/apps/sidecar/src/workflow-host-wiring.ts index 02e55ece..0147fe83 100644 --- a/apps/sidecar/src/workflow-host-wiring.ts +++ b/apps/sidecar/src/workflow-host-wiring.ts @@ -911,180 +911,143 @@ export function createSidecarDeployRouter(deps: { if (existing === agentAddress) slugClaims.delete(deploymentId); } - async function deployMultiStep( - frame: AgentDeployFrame, - projection: NonNullable, - ): Promise { - // Boundary validation: a malformed projection is rejected at the - // router edge before the supervisor is constructed so the link - // surfaces a structured failure rather than a hung `starting` - // supervisor. - validateWorkflowProjection(projection); - - // Source-admission gate: reject a deploy whose any step pins an - // inference provider this sidecar cannot build, BEFORE any state is - // claimed or the child is spawned. The throw propagates back through - // the deploy frame so the hub's `deployWorkflow` rejects synchronously - // at deploy time, rather than the child failing the run when the - // step's inference first resolves. Covers single- and multi-step: the - // projection's `narrow` guarantees every stepOrder entry has a - // `sources` entry. - for (const stepId of projection.definition.stepOrder) { - const source = projection.sources[stepId]; - if (source !== undefined) deps.assertSourceBuildable(source); + /** + * Materialize the workflow definition on the sidecar's local substrate so + * the workflow-process child's `loadWorkflowDefinition` can read + * `workflow.json` out of the workflow-asset repo's working tree. The + * destination mirrors the bare RepoStore's `getRepoDir` for + * `{ kind: "workflow", id }`: + * `${SIDECAR_DATA_DIR}/assets/workflow//workflow.json`. The child reads + * via `fs.readFile`, so writing the bytes outside git suffices. This is + * deploy-only durable state; the restore path finds it already on disk. + */ + async function materializeWorkflowJson( + sidecarDataDir: string | undefined, + definition: NonNullable["definition"], + ): Promise { + if (typeof sidecarDataDir !== "string" || sidecarDataDir.length === 0) { + throw new Error( + "sidecar deploy router: SIDECAR_DATA_DIR must be present in the multi-step substrate env; the workflow-process child resolves the workflow-asset repo dir against this data dir", + ); + } + const workflowAssetPath = pathJoin( + sidecarDataDir, + "assets", + "workflow", + definition.id, + "workflow.json", + ); + const workflowAssetBytes = JSON.stringify(definition, null, 2); + try { + await mkdir(dirname(workflowAssetPath), { recursive: true }); + // Idempotent: only rewrite when the on-disk content differs. Treats a + // missing file as different. + let existing: string | null = null; + try { + existing = await readFile(workflowAssetPath, "utf8"); + } catch (cause) { + if ( + !( + cause instanceof Error && + "code" in cause && + (cause as { code: unknown }).code === "ENOENT" + ) + ) { + throw cause; + } + } + if (existing !== workflowAssetBytes) { + await writeFile(workflowAssetPath, workflowAssetBytes, "utf8"); + } + } catch (cause) { + const reason = cause instanceof Error ? cause.message : String(cause); + throw new Error( + `sidecar deploy router: failed to materialize workflow.json at ${workflowAssetPath}: ${reason}`, + { cause }, + ); } + } - const deploymentId = deriveTrivialDeploymentId(frame.agentAddress); + /** + * The per-deployment inputs the shared spawn core needs to stand up a + * workflow deployment, independent of the live deploy frame. The live + * deploy path builds this from `frame`/`projection`; a boot-time restore + * path builds the same shape from the persisted deployment record. + */ + interface WorkflowDeploySpec { + agentAddress: string; + definition: NonNullable["definition"]; + sources: NonNullable["sources"]; + /** Correlates the child's inference events to the deploy's session. */ + sessionId: string | undefined; + /** + * Hub public key recorded at the head for deploy-pack verification and + * inbound hub-frame verification. Required for a single-step + * deployment (whose head IS the agent identity); undefined for a + * genuine multi-step deployment, which derives per-step addresses and + * records no head key. + */ + hubPublicKey: string | undefined; + } - // Single-step launched-agent deploy vs. derived multi-step deploy. - // - // A one-step projection is the agent-launch identity path: the sole - // step keeps the deployment's own (legacy) mail address, and its - // grants live in the legacy agent-state repo keyed by the legacy - // instance id (`parseAgentId(frame.agentAddress)`). This preserves - // the identity the legacy agent-deploy path established -- the - // workflow-run repo stays keyed by `deriveWorkflowRunRepoId(legacy)` - // and `agent_instance.address` remains the `ins_` legacy shape. - // - // A multi-step projection derives `-` per step - // for both the mail address and the agent-state repo id, isolating - // each step's grants in its own repo. + /** + * The single owner of the workflow-deployment spawn sequence: construct + * the supervisor, register the single-step agent's outbound key + head + * repo + hub key, spawn the workflow-process child, then register the + * live deployment (supervisor, mail/signal/drain routers, address + * mapping). Its `try/finally` unwinds every piece of partial state if any + * step throws, so a failed spawn leaks nothing. Both the live deploy path + * and the boot-time restore path route through here so the two can never + * diverge on how a deployment is stood up. Callers materialize the + * deploy-only durable state (`workflow.json`, step grants) before calling. + */ + async function spawnWorkflowDeployment( + spec: WorkflowDeploySpec, + ): Promise { + const deploymentId = deriveTrivialDeploymentId(spec.agentAddress); + + // Single-step launched-agent deploy vs. derived multi-step deploy. A + // one-step deployment keeps the deployment's own (legacy) mail address + // and its grants in the legacy agent-state repo keyed by the legacy + // instance id. A multi-step deployment derives `-` + // per step for both the mail address and the agent-state repo id. const stepStrategy = createStepStrategy({ - legacyAddress: frame.agentAddress, - stepOrder: projection.definition.stepOrder, + legacyAddress: spec.agentAddress, + stepOrder: spec.definition.stepOrder, multistepDeriveStepAddress, }); - claimSlug(deploymentId, frame.agentAddress); - // Release every piece of partial state if any step between here - // and `registerDeployment` throws. The undeploy hook -- the only - // other release caller -- never fires for failed deploys, so - // without this finally the slug, the freshly-spawned - // workflow-process child, the `activeSupervisors` entry, and the - // three multistep router registrations would all leak. The + // Unwind every piece of spawn state if any step between here and + // `registerDeployment` throws, so a failed spawn leaks no + // freshly-spawned workflow-process child, `activeSupervisors` entry, + // transport registration, or multistep router registration. The // ordering inside the finally is the reverse of the success-path - // registration order so each release runs against state the - // success path has already established. - let claimedSlugSucceeded = false; + // registration order. The caller owns the deployment slug: it must + // claim the collision guard before any durable write and release it on + // failure, so the slug is not touched here. + let succeeded = false; let wiredForUnwind: SidecarWorkflowSupervisor | undefined; let supervisorRegistered = false; let routersRegistered = false; let agentTransportRegistered = false; try { - const definitionHash = await computeWireDefinitionHash( - projection.definition, - ); + const definitionHash = await computeWireDefinitionHash(spec.definition); // Per-deployment substrate-config keys the workflow-substrate-factory - // validator requires (`SIDECAR_SUBSTRATE_CONFIG_KEYS` / - // `SubstrateConfig` in `workflow-substrate-factory.ts`). The boot - // edge's `multistepSubstrateEnv` only carries the boot-edge constants - // (`SIDECAR_DATA_DIR`, signing keys, hub link anchors); the four - // workflow-definition / workflow-run identity keys must be derived - // per-deploy here so the child's substrate-config validator passes - // at startup. - // - // `WORKFLOW_RUN_REPO_ID` mirrors `workflowRunRepoId.id` (the - // substrate-safe slug of the deployment address) and - // `WORKFLOW_RUN_REF` mirrors `workflowRunRef`, so the child resolves - // the same workflow-run repo the supervisor is writing into. - // - // `WORKFLOW_DEFINITION_REPO_ID` is set to `projection.definition.id` - // (the workflow asset's repo id; see the orchestrator's - // `WorkflowRepoWriter.writeWorkflowRepo` call, which writes the - // asset repo keyed by `workflow.id`). The child's - // `loadWorkflowDefinition` in - // `packages/workflow-host/src/child/run-child.ts` reads - // `workflow.json` out of this repo's working tree. The current - // Phase I multi-step test path does not yet stage the workflow - // asset on the sidecar's substrate (the orchestrator writes the - // asset to the hub repo store, not the sidecar's data dir), so the - // child's `loadWorkflowDefinition` will not actually find a - // `workflow.json` until the workflow-asset deploy lands on the - // sidecar. The value is structurally correct -- a consistent, - // deterministic id derived from the definition -- so the substrate- - // config validator passes and the next gap (if any) surfaces - // structurally rather than at the env-keys boundary. - // - // `WORKFLOW_DEFINITION_REF` is `"refs/heads/main"` to mirror the - // hub's `DEFAULT_ASSET_REF` and the workflow-run ref the supervisor - // uses. + // validator requires. The boot edge's `multistepSubstrateEnv` carries + // the boot-edge constants; the four workflow-definition / workflow-run + // identity keys are derived per-deploy here. `STEP_INFERENCE_SOURCES` + // threads the pinned per-step sources into the child. const substrateEnv: Record = { ...multistepSubstrateEnv, - WORKFLOW_DEFINITION_REPO_ID: projection.definition.id, + WORKFLOW_DEFINITION_REPO_ID: spec.definition.id, WORKFLOW_DEFINITION_REF: "refs/heads/main", WORKFLOW_RUN_REPO_ID: deploymentId, WORKFLOW_RUN_REF: "refs/heads/main", - [STEP_INFERENCE_SOURCES_ENV_KEY]: JSON.stringify(projection.sources), + [STEP_INFERENCE_SOURCES_ENV_KEY]: JSON.stringify(spec.sources), }; - // Materialize the workflow asset on the sidecar's local substrate - // so the workflow-process child's `loadWorkflowDefinition` can read - // `workflow.json` out of the workflow-asset repo's working tree - // (`packages/workflow-host/src/child/run-child.ts`). The hub's - // orchestrator writes the asset to the hub's repo store; the - // sidecar's substrate is a separate data dir on disk, and nothing - // replicates between the two today. - // - // The frame inlines the validated `WorkflowDefinition` already - // (see `AgentDeployFrame.workflow.definition`), so the router has - // the bytes in scope. The destination path mirrors the bare - // RepoStore's `getRepoDir` for `{ kind: "workflow", id }`: - // `${SIDECAR_DATA_DIR}/assets/workflow//workflow.json`. The - // workflow-asset repo kind handler's `directoryPrefix` is - // `assets/workflow` (see `packages/hub-sessions/src/workflow-kind.ts`). - // - // The child reads via `fs.readFile`, not via a git ref resolution, - // so writing the bytes outside any git operation is sufficient. - const sidecarDataDir = substrateEnv.SIDECAR_DATA_DIR; - if (typeof sidecarDataDir !== "string" || sidecarDataDir.length === 0) { - throw new Error( - "sidecar deploy router: SIDECAR_DATA_DIR must be present in the multi-step substrate env for the multi-step branch; the workflow-process child resolves the workflow-asset repo dir against this data dir", - ); - } - const workflowAssetPath = pathJoin( - sidecarDataDir, - "assets", - "workflow", - projection.definition.id, - "workflow.json", - ); - const workflowAssetBytes = JSON.stringify(projection.definition, null, 2); - try { - await mkdir(dirname(workflowAssetPath), { recursive: true }); - // Idempotent: only rewrite when the on-disk content differs from - // the projection. Treats a missing file as different. - let existing: string | null = null; - try { - existing = await readFile(workflowAssetPath, "utf8"); - } catch (cause) { - if ( - !( - cause instanceof Error && - "code" in cause && - (cause as { code: unknown }).code === "ENOENT" - ) - ) { - throw cause; - } - } - if (existing !== workflowAssetBytes) { - await writeFile(workflowAssetPath, workflowAssetBytes, "utf8"); - } - } catch (cause) { - const reason = cause instanceof Error ? cause.message : String(cause); - throw new Error( - `sidecar deploy router: failed to materialize workflow.json at ${workflowAssetPath}: ${reason}`, - { cause }, - ); - } - const wired = createSidecarWorkflowSupervisor({ - // Hold the constructed supervisor in the outer scope so the - // failure-path finally can `shutdown()` it; assignment - // happens immediately so even a synchronous throw inside - // the closure's call sites below is covered. - // (See the wiredForUnwind declaration above the try.) transport: deps.transport, repoStore: deps.repoStore, signingKeySeed: deps.signingKeySeed, @@ -1094,8 +1057,8 @@ export function createSidecarDeployRouter(deps: { }, workflowRunRef: "refs/heads/main", deploymentId, - stepCount: projection.definition.stepOrder.length, - deploymentMailAddress: frame.agentAddress, + stepCount: spec.definition.stepOrder.length, + deploymentMailAddress: spec.agentAddress, deriveStepAddress: stepStrategy.deriveStepAddress, deriveStepRepoId: stepStrategy.deriveStepRepoId, substrateEnv, @@ -1105,8 +1068,7 @@ export function createSidecarDeployRouter(deps: { : {}), // The multi-step branch never invokes trivialLaunch, but the // supervisor's constructor requires the binding. Wire a sentinel - // that throws so a stray invocation surfaces loudly rather than - // silently succeeding. + // that throws so a stray invocation surfaces loudly. trivialLaunch: () => { throw new Error( "sidecar deploy router: trivialLaunch invoked on the multi-step branch; this is a programming bug", @@ -1123,151 +1085,89 @@ export function createSidecarDeployRouter(deps: { : {}), }); - // Grants bridge (the sharp edge of the always-spawn convergence). - // - // The legacy agent-deploy path shipped grants in-band on the - // deploy frame; the in-process runtime loaded them straight into - // its grant store and never wrote them to disk. The spawned child - // does not see the frame: it reads each step's grants out of - // `state/grants.json` in the step's agent-state repo while the - // supervisor assembles the credentialsSnapshot inside `spawn()`. - // - // Write every step's grants to the step's agent-state repo BEFORE - // `spawn()` so the supervisor's read sees them. The hub is the - // source of truth -- `frame.config.grants` is the operator-approved - // grant set the hub shipped -- and the same `deriveStepRepoId` the - // supervisor reads with keys the write so the read and the write - // address the same repo. A missing or empty file would make every - // authorize fail closed, so the write must surface its failure. - await writeStepGrants({ - repoStore: deps.repoStore, - deploymentId, - stepOrder: projection.definition.stepOrder, - deriveStepRepoId: stepStrategy.deriveStepRepoId, - grants: frame.config.grants, - }); - // OUTBOUND half of mailbox ownership (§3a): register the spawned - // agent's signing key on the host transport so the supervisor's - // outbound mail path (`MailBusBindings.sendOutbound`) signs the - // agent's replies as the AGENT's identity, with parity to the - // in-process path's `transport.register(address, crypto)`. - // - // Gated on the single-step launched-agent deploy: there the - // deployment mail address IS the legacy agent identity whose - // keypair lives in the keyStore, so the supervisor can sign - // outbound mail as that address. A genuine multi-step deploy - // derives a distinct per-step address with no keypair on the host; - // per-step outbound signing for multi-step is out of 4.3 scope - // (the unified single-agent path 4.3 targets is the single-step - // case). The registration happens before `spawn()` so the agent's - // address is live the instant the first reply routes outbound. - if (projection.definition.stepOrder.length === 1) { + // agent's signing key on the host transport so the supervisor signs + // the agent's replies as the AGENT's identity. Gated on the + // single-step deploy, where the deployment mail address IS the legacy + // agent identity whose keypair lives in the keyStore. Registration + // happens before `spawn()` so the address is live the instant the + // first reply routes outbound. + if (spec.definition.stepOrder.length === 1) { const { keyPair } = await deps.keyStore.loadOrGenerateKey( - frame.agentAddress, + spec.agentAddress, ); deps.transport.register( - frame.agentAddress, + spec.agentAddress, deps.createAgentCrypto(keyPair), ); agentTransportRegistered = true; // A single-step workflow stages its deploy tree at the head (the - // lone step IS the head). Initialize the head's on-disk - // deploy-tree repo now, before this frame's ack, so the hub's - // follow-up deploy-pack push has a repo to `applyDeployPack` into. - // The narrow `initRepo` (not `provisionAgent`) is deliberate: the - // supervised child mints its own keypair at boot and persists no - // hub-agent config, and `provisionAgent`'s duplicate-address guard - // would reject the head the deployment machinery already tracks. - await deps.sessions.initRepo(frame.agentAddress); + // lone step IS the head). Initialize the head's on-disk deploy-tree + // repo (idempotent) so the hub's deploy-pack push has a repo to + // apply into. The narrow `initRepo` (not `provisionAgent`) is + // deliberate: the supervised child mints its own keypair and + // persists no hub-agent config. + await deps.sessions.initRepo(spec.agentAddress); - // Record the hub's public key at the head so the follow-up - // deploy-pack apply can verify the hub-signed deploy-tree commit. - // The verifier resolves the key from the in-memory key store's - // `recordHubKey` map, not the persisted `agent.json`, so this alone - // satisfies verification -- `persistHubPublicKey` (agent.json - // durability) is skipped along with `persistConfig`, which the - // workflow head has no agent.json to write into. Without this - // `applyDeployPack` rejects the pack as `signature_invalid`. - deps.keyStore.recordHubKey(frame.agentAddress, frame.hubPublicKey); + // Record the hub's public key at the head so the deploy-pack apply + // (and any inbound hub-signed frame) verifies against it. The + // verifier resolves the key from the in-memory key store's + // `recordHubKey` map, so a single-step deployment cannot stand up + // without it. + if (spec.hubPublicKey === undefined) { + throw new Error( + "sidecar deploy router: a single-step workflow deployment requires a hubPublicKey to record at the head; none was supplied", + ); + } + deps.keyStore.recordHubKey(spec.agentAddress, spec.hubPublicKey); } - const stepOrder = [...projection.definition.stepOrder]; - // Warm-keep is the single-step launched-agent deploy (design §3b): - // the sole step IS the long-lived agent, so the child warm-keeps it - // across messages. A genuine multi-step deploy keeps - // instantiate-send-teardown per step -- warm-keeping N steps would - // hold N agents and N LSP subprocesses for no benefit. The signal - // is carried explicitly from this projection-level recognition down - // through the spawn env to the child's run-loop, never re-derived - // heuristically there. - const warmKeep = projection.definition.stepOrder.length === 1; + const stepOrder = [...spec.definition.stepOrder]; + // Warm-keep is the single-step launched-agent deploy: the sole step + // IS the long-lived agent, so the child warm-keeps it across + // messages. A multi-step deploy keeps instantiate-send-teardown per + // step. The signal is carried explicitly down through the spawn env. + const warmKeep = spec.definition.stepOrder.length === 1; const spawnOpts: SpawnOpts = { stepOrder, definitionHash, warmKeep, onInferenceEvent: (event) => { - // The event arrives already HMAC-verified and validated as an - // `EventPayload` over the child's event channel. Re-narrow it - // to the manually-defined `InferenceEvent` union the hub's - // `agent.event` sink consumes (the arktype-inferred - // `EventPayload` widens a few discriminants the hand-written - // type narrows). A parse failure here means the channel - // delivered something the validator rejects, which would be a - // corruption upstream -- drop it loudly rather than forwarding - // an unvalidated payload onto the hub timeline. + // The event arrives HMAC-verified over the child's event channel. + // Re-narrow it to the hub's `InferenceEvent` union; a parse + // failure means upstream corruption, so drop it loudly rather + // than forwarding an unvalidated payload onto the hub timeline. const validated = parseInferenceEvent(event); if (validated instanceof type.errors) { - logger.warn`dropping workflow inference event for ${frame.agentAddress}: ${validated.summary}`; + logger.warn`dropping workflow inference event for ${spec.agentAddress}: ${validated.summary}`; return; } - publishInferenceEvent( - frame.agentAddress, - validated, - frame.config.sessionId, - ); + publishInferenceEvent(spec.agentAddress, validated, spec.sessionId); }, }; - // Surface spawn-time errors structurally: if the subprocess - // spawner crashes immediately (binary missing, env malformed, - // EXEC error) the supervisor's `wireChild` races the child's - // `exited` against `readyPromise` and the rejection propagates - // here. The router lets it surface; the link's deploy handler - // converts the rejection into a structured failure frame. The - // supervisor is registered against the deployment address only - // after spawn succeeds; a spawn-time rejection leaves the - // registry untouched so the undeploy hook does not chase a - // supervisor that never owned a child. + // Surface spawn-time errors structurally: a subprocess spawner that + // crashes immediately rejects here, and the caller converts the + // rejection into a structured failure frame. The supervisor is + // registered against the deployment address only after spawn succeeds, + // so a spawn-time rejection leaves the registry untouched. await wired.supervisor.spawn(spawnOpts); - // Child process is live after `spawn` resolves; the failure - // unwind needs the supervisor handle from here on. wiredForUnwind = wired; - activeSupervisors.set(frame.agentAddress, wired); + activeSupervisors.set(spec.agentAddress, wired); supervisorRegistered = true; // Bind the deployment's mail address to this supervisor's - // `routeInbound` so the sidecar's hub-link dispatches inbound - // mail for the deployment address into the supervisor's mail-bus - // subscription rather than the legacy session path. The legacy - // path is the wrong receiver for multi-step deployments: the - // deployment address is never registered on `transport` (no - // `startSession` runs against it) and there is no `sessions` - // entry to satisfy `commitInboundMail`. Registration happens - // after `spawn` succeeds so a spawn-time rejection leaves the - // registry untouched. - deps.multistepMailRouter?.register(frame.agentAddress, (message) => { + // `routeInbound` so the hub-link dispatches inbound mail into the + // supervisor's mail-bus subscription. Registration happens after + // `spawn` succeeds so a spawn-time rejection leaves the registry + // untouched. + deps.multistepMailRouter?.register(spec.agentAddress, (message) => { wired.routeInbound(message); }); - // Register the signal-delivery handler against the deployment - // address so a hub-side `signal.deliver` frame dispatches through - // the supervisor's `deliverSignal`, which forwards a control IPC - // `signal.deliver` to the workflow-process child. The child writes - // the resulting `SignalReceived` event through its own substrate; - // the workflow-run pack-push pipeline then propagates the commit - // to the hub with no concurrent writer at the workflow-run ref. - deps.multistepSignalRouter?.register(frame.agentAddress, async (args) => { + // Register the signal-delivery handler so a hub `signal.deliver` frame + // dispatches through the supervisor's `deliverSignal`. + deps.multistepSignalRouter?.register(spec.agentAddress, async (args) => { await wired.supervisor.deliverSignal({ runId: args.runId, signalName: args.signalName, @@ -1275,47 +1175,40 @@ export function createSidecarDeployRouter(deps: { payload: args.payload, }); }); - // Register the drain handler against the deployment address so a - // hub-side `drain.deliver` frame dispatches through the - // supervisor's `drain`, which forwards a `drain` control IPC frame - // to the workflow-process child and arms one `drainTimeout` - // accumulator per in-flight run. Cancel-mode in-flight steps abort - // on the child side; wait-mode steps continue. Each accumulator - // commits a signed `CancelRequested{origin: "supervisor-drain"}` - // against the workflow-run repo when the deadline expires. - deps.multistepDrainRouter?.register(frame.agentAddress, async (args) => { + // Register the drain handler so a hub `drain.deliver` frame dispatches + // through the supervisor's `drain`. + deps.multistepDrainRouter?.register(spec.agentAddress, async (args) => { await wired.supervisor.drain({ deadlineMs: args.deadlineMs }); }); routersRegistered = true; // Register the deployment-address mapping last so a failure in any - // earlier step (asset materialization, supervisor.spawn) leaves the - // boot-edge `DeploymentAddressRegistry` untouched. The link's - // `handleAgentDeploy` catches a rejection here and surfaces - // `agent.error` without invoking the undeploy hook; a partial - // registration would persist a `(deploymentId -> agentAddress)` - // entry for a deployment that never finished standing up. + // earlier step leaves the boot-edge `DeploymentAddressRegistry` + // untouched. deps.registerDeployment({ deploymentId, - agentAddress: frame.agentAddress, + agentAddress: spec.agentAddress, }); - claimedSlugSucceeded = true; - return { - publicKey: await derivePrincipalPublicKeyHex(deps.signingKeySeed), - }; + // Derive the ack public key BEFORE marking the spawn succeeded so an + // (unreachable, deterministic) derivation failure unwinds the spawn + // rather than leaving a live-but-unacked deployment whose slug the + // caller then frees. Once `succeeded` is set the finally is a no-op + // and the deployment is retained. + const publicKey = await derivePrincipalPublicKeyHex(deps.signingKeySeed); + succeeded = true; + return { publicKey }; } finally { - if (!claimedSlugSucceeded) { - // Unwind in reverse registration order so each step undoes - // state the success path has confirmed; ordering matches - // the `undeploy` hook for consistency. + if (!succeeded) { + // Unwind in reverse registration order so each step undoes state + // the success path confirmed; ordering matches the `undeploy` hook. if (routersRegistered) { - deps.multistepMailRouter?.unregister(frame.agentAddress); - deps.multistepSignalRouter?.unregister(frame.agentAddress); - deps.multistepDrainRouter?.unregister(frame.agentAddress); + deps.multistepMailRouter?.unregister(spec.agentAddress); + deps.multistepSignalRouter?.unregister(spec.agentAddress); + deps.multistepDrainRouter?.unregister(spec.agentAddress); } if (supervisorRegistered) { - activeSupervisors.delete(frame.agentAddress); + activeSupervisors.delete(spec.agentAddress); } if (wiredForUnwind !== undefined) { await wiredForUnwind.supervisor.shutdown().catch((cause) => { @@ -1325,17 +1218,109 @@ export function createSidecarDeployRouter(deps: { }); } if (agentTransportRegistered) { - // Drop the agent's transport registration so a failed deploy - // does not leave the address live on the host transport with a - // dangling `CryptoProvider`. `unregister` is safe to call even - // if the address was never registered. - deps.transport.unregister(frame.agentAddress); + // Drop the agent's transport registration so a failed deploy does + // not leave the address live with a dangling `CryptoProvider`. + deps.transport.unregister(spec.agentAddress); } - releaseSlug(deploymentId, frame.agentAddress); } } } + async function deployMultiStep( + frame: AgentDeployFrame, + projection: NonNullable, + ): Promise { + // Boundary validation: a malformed projection is rejected at the + // router edge before the supervisor is constructed so the link + // surfaces a structured failure rather than a hung `starting` + // supervisor. + validateWorkflowProjection(projection); + + // Source-admission gate: reject a deploy whose any step pins an + // inference provider this sidecar cannot build, BEFORE any state is + // claimed or the child is spawned. The throw propagates back through + // the deploy frame so the hub's `deployWorkflow` rejects synchronously + // at deploy time, rather than the child failing the run when the + // step's inference first resolves. Covers single- and multi-step: the + // projection's `narrow` guarantees every stepOrder entry has a + // `sources` entry. + for (const stepId of projection.definition.stepOrder) { + const source = projection.sources[stepId]; + if (source !== undefined) deps.assertSourceBuildable(source); + } + + const deploymentId = deriveTrivialDeploymentId(frame.agentAddress); + + // Single-step launched-agent deploy vs. derived multi-step deploy. + // + // A one-step projection is the agent-launch identity path: the sole + // step keeps the deployment's own (legacy) mail address, and its + // grants live in the legacy agent-state repo keyed by the legacy + // instance id (`parseAgentId(frame.agentAddress)`). This preserves + // the identity the legacy agent-deploy path established -- the + // workflow-run repo stays keyed by `deriveWorkflowRunRepoId(legacy)` + // and `agent_instance.address` remains the `ins_` legacy shape. + // + // A multi-step projection derives `-` per step + // for both the mail address and the agent-state repo id, isolating + // each step's grants in its own repo. + const stepStrategy = createStepStrategy({ + legacyAddress: frame.agentAddress, + stepOrder: projection.definition.stepOrder, + multistepDeriveStepAddress, + }); + + // Claim the deployment slug BEFORE any durable write so a colliding + // deploymentId (two distinct addresses projecting to the same slug) is + // rejected before `workflow.json`, the step grants, or the supervisor + // touch disk -- the router's "no repo state touched before rejection" + // guarantee. The claim is released on any failure below; a successful + // deploy keeps it (the undeploy hook releases it at teardown). The + // spawn core owns unwinding the supervisor and registrations it stands + // up; the slug is the caller's. + claimSlug(deploymentId, frame.agentAddress); + try { + // Materialize the deploy-only durable state the spawned child and the + // supervisor read from disk: the workflow definition (`workflow.json`) + // and each step's grants. The restore path finds both already on disk + // and skips this; both land before the shared spawn core runs. + await materializeWorkflowJson( + multistepSubstrateEnv.SIDECAR_DATA_DIR, + projection.definition, + ); + + // Grants bridge: the spawned child does not see the frame; it reads + // each step's grants out of `state/grants.json` in the step's + // agent-state repo while the supervisor assembles the + // credentialsSnapshot. Write the operator-approved + // `frame.config.grants` to the same repo the supervisor reads via + // `deriveStepRepoId`, before the spawn core, so the read sees them. + await writeStepGrants({ + repoStore: deps.repoStore, + deploymentId, + stepOrder: projection.definition.stepOrder, + deriveStepRepoId: stepStrategy.deriveStepRepoId, + grants: frame.config.grants, + }); + + // Hand off to the shared spawn core. A single-step deployment records + // its hub key at the head; a multi-step deployment carries none. + return await spawnWorkflowDeployment({ + agentAddress: frame.agentAddress, + definition: projection.definition, + sources: projection.sources, + sessionId: frame.config.sessionId, + hubPublicKey: + projection.definition.stepOrder.length === 1 + ? frame.hubPublicKey + : undefined, + }); + } catch (cause) { + releaseSlug(deploymentId, frame.agentAddress); + throw cause; + } + } + return { async deploy(frame): Promise { if (frame.workflow !== undefined) { @@ -1344,10 +1329,10 @@ export function createSidecarDeployRouter(deps: { let publicKey: string | undefined; const deploymentId = deriveTrivialDeploymentId(frame.agentAddress); claimSlug(deploymentId, frame.agentAddress); - // See deployMultiStep's `claimedSlugSucceeded` guard: without - // it, any failure between `claimSlug` and the - // `registerDeployment` call below leaves the slug claimed - // forever (the undeploy hook never fires for failed deploys). + // Same slug-release discipline as deployMultiStep's claim/try/catch: + // without it, any failure between `claimSlug` and the + // `registerDeployment` call below leaves the slug claimed forever + // (the undeploy hook never fires for failed deploys). let trivialClaimedSlugSucceeded = false; try { const wired = createSidecarWorkflowSupervisor({ From bc2f88f625228d88197f249ee95485a5f1feac1e Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Thu, 2 Jul 2026 12:00:57 -0500 Subject: [PATCH 021/101] Persist a per-deployment record for restore A workflow deployment holds three inputs only in the frame and in memory: its pinned per-step sources, its session id, and (single-step) its head hub key. Nothing on the sidecar's disk carries them, and there is no enumerable record of which deployments are active -- so a deployment cannot be re-established after a sidecar process restart. Write a `deployment.json` beside each deployment's workflow-run substrate at `workflow-runs//`, carrying those three inputs plus the head address and the definition id. The definition itself stays in `workflow.json`, referenced by id, and the grants stay in the step repos, so neither is duplicated. The record is written after the slug is claimed and before the child is spawned, so a crash mid-spawn leaves a record a later boot can re-drive. A soft-failed deploy deletes it in the failure unwind, and undeploy deletes it at teardown, so only a live or crash-interrupted deployment ever has one. The set of active workflow deployments is now a durable, inspectable on-disk fact rather than in-memory-only state. --- .../agent-key-registration-lifecycle.test.ts | 37 +++++- .../src/workflow-deployment-record.test.ts | 125 ++++++++++++++++++ .../sidecar/src/workflow-deployment-record.ts | 80 +++++++++++ apps/sidecar/src/workflow-host-wiring.test.ts | 43 ++++++ apps/sidecar/src/workflow-host-wiring.ts | 76 ++++++++--- 5 files changed, 344 insertions(+), 17 deletions(-) create mode 100644 apps/sidecar/src/workflow-deployment-record.test.ts create mode 100644 apps/sidecar/src/workflow-deployment-record.ts diff --git a/apps/sidecar/src/agent-key-registration-lifecycle.test.ts b/apps/sidecar/src/agent-key-registration-lifecycle.test.ts index 7766a5d9..fe680fba 100644 --- a/apps/sidecar/src/agent-key-registration-lifecycle.test.ts +++ b/apps/sidecar/src/agent-key-registration-lifecycle.test.ts @@ -29,7 +29,13 @@ import { } from "@intx/workflow-host"; import type { AgentDeployFrame } from "@intx/types/sidecar"; -import { createSidecarDeployRouter } from "./workflow-host-wiring"; +import { type } from "arktype"; + +import { + createSidecarDeployRouter, + deriveTrivialDeploymentId, +} from "./workflow-host-wiring"; +import { WorkflowDeploymentRecord } from "./workflow-deployment-record"; import { createMultistepDrainRouter, createMultistepMailRouter, @@ -316,6 +322,26 @@ describe("agent signing-key registration lifecycle on the host transport", () => // (b) Registered after deploy -- the supervisor can now sign agent mail. expect(isRegistered(transport)).toBe(true); + // (b') The deploy persisted a schema-valid restore record for the + // deployment, carrying the head address so a boot-time restore can + // re-establish it. + const deploymentId = deriveTrivialDeploymentId(AGENT_ADDRESS); + const recordFile = path.join( + dataDir, + "workflow-runs", + deploymentId, + "deployment.json", + ); + const parsedRecord = WorkflowDeploymentRecord( + JSON.parse(await fs.readFile(recordFile, "utf8")), + ); + if (parsedRecord instanceof type.errors) { + throw new Error( + `deployment record failed validation: ${parsedRecord.summary}`, + ); + } + expect(parsedRecord.agentAddress).toBe(AGENT_ADDRESS); + const undeploy = router.undeploy; if (undeploy === undefined) throw new Error("router.undeploy undefined"); await undeploy({ @@ -327,6 +353,15 @@ describe("agent signing-key registration lifecycle on the host transport", () => // (c) Unregistered after undeploy -- no leaked key for a torn-down agent. expect(isRegistered(transport)).toBe(false); + // (c') Undeploy dropped the restore record so a boot-time restore will + // not re-spawn the torn-down deployment. + expect( + await fs.access(recordFile).then( + () => true, + () => false, + ), + ).toBe(false); + await fs.rm(tempBase, { recursive: true, force: true }); await fs.rm(dataDir, { recursive: true, force: true }); }); diff --git a/apps/sidecar/src/workflow-deployment-record.test.ts b/apps/sidecar/src/workflow-deployment-record.test.ts new file mode 100644 index 00000000..80c25c45 --- /dev/null +++ b/apps/sidecar/src/workflow-deployment-record.test.ts @@ -0,0 +1,125 @@ +import { describe, test, expect } from "bun:test"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import { type } from "arktype"; + +import { + WorkflowDeploymentRecord, + writeWorkflowDeploymentRecord, + deleteWorkflowDeploymentRecord, +} from "./workflow-deployment-record"; + +async function makeDataDir(): Promise { + return fs.mkdtemp(path.join(os.tmpdir(), "wdr-")); +} + +function recordPath(dataDir: string, deploymentId: string): string { + return path.join(dataDir, "workflow-runs", deploymentId, "deployment.json"); +} + +async function fileExists(p: string): Promise { + try { + await fs.access(p); + return true; + } catch { + return false; + } +} + +const SINGLE_STEP: WorkflowDeploymentRecord = { + version: 1, + agentAddress: "ins_abc123@tenant.example", + definitionId: "wf_abc123", + sources: { + "step-1": { + id: "anthropic:mock", + provider: "anthropic", + baseURL: "https://api.example/anthropic", + apiKey: "sk-x", + model: "claude-mock", + }, + }, + sessionId: "ses_1", + hubPublicKey: "deadbeef", +}; + +// A multi-step deployment records no head hub key and may carry no session +// id -- both optional fields absent. +const MULTI_STEP: WorkflowDeploymentRecord = { + version: 1, + agentAddress: "ins_dep_xyz@tenant.example", + definitionId: "wf_xyz", + sources: { + plan: { + id: "anthropic:mock", + provider: "anthropic", + baseURL: "https://api.example/anthropic", + apiKey: "sk-y", + model: "claude-mock", + }, + execute: { + id: "openai:mock", + provider: "openai", + baseURL: "https://api.example/openai", + apiKey: "sk-z", + model: "gpt-mock", + }, + }, +}; + +describe("workflow deployment record store", () => { + test("round-trips a schema-valid record through disk (single-step)", async () => { + const dataDir = await makeDataDir(); + const deploymentId = "abc123-tenant-example"; + await writeWorkflowDeploymentRecord(dataDir, deploymentId, SINGLE_STEP); + + // The record embeds source apiKeys, so it must not be group/world + // readable on a shared host. + const stat = await fs.stat(recordPath(dataDir, deploymentId)); + expect(stat.mode & 0o077).toBe(0); + + const raw = await fs.readFile(recordPath(dataDir, deploymentId), "utf8"); + const parsed = WorkflowDeploymentRecord(JSON.parse(raw)); + if (parsed instanceof type.errors) { + throw new Error(`record failed validation: ${parsed.summary}`); + } + expect(parsed).toEqual(SINGLE_STEP); + + await fs.rm(dataDir, { recursive: true, force: true }); + }); + + test("round-trips a record with the optional fields absent (multi-step)", async () => { + const dataDir = await makeDataDir(); + const deploymentId = "dep_xyz-tenant-example"; + await writeWorkflowDeploymentRecord(dataDir, deploymentId, MULTI_STEP); + + const raw = await fs.readFile(recordPath(dataDir, deploymentId), "utf8"); + const parsed = WorkflowDeploymentRecord(JSON.parse(raw)); + if (parsed instanceof type.errors) { + throw new Error(`record failed validation: ${parsed.summary}`); + } + expect(parsed).toEqual(MULTI_STEP); + expect("hubPublicKey" in parsed).toBe(false); + expect("sessionId" in parsed).toBe(false); + + await fs.rm(dataDir, { recursive: true, force: true }); + }); + + test("delete removes the record and is a no-op when absent", async () => { + const dataDir = await makeDataDir(); + const deploymentId = "gone-1"; + + // No-op when the record was never written. + await deleteWorkflowDeploymentRecord(dataDir, deploymentId); + + await writeWorkflowDeploymentRecord(dataDir, deploymentId, SINGLE_STEP); + expect(await fileExists(recordPath(dataDir, deploymentId))).toBe(true); + + await deleteWorkflowDeploymentRecord(dataDir, deploymentId); + expect(await fileExists(recordPath(dataDir, deploymentId))).toBe(false); + + await fs.rm(dataDir, { recursive: true, force: true }); + }); +}); diff --git a/apps/sidecar/src/workflow-deployment-record.ts b/apps/sidecar/src/workflow-deployment-record.ts new file mode 100644 index 00000000..d86c018c --- /dev/null +++ b/apps/sidecar/src/workflow-deployment-record.ts @@ -0,0 +1,80 @@ +// Sidecar-local persistence of the per-deployment record needed to +// re-establish a workflow deployment across a sidecar PROCESS restart. The +// record is co-located with the deployment's workflow-run substrate at +// `${dataDir}/workflow-runs//deployment.json`, so a single +// teardown reclaims both and a boot scan can enumerate the active +// deployments beside the run state they resume. +// +// It carries only the inputs that are otherwise frame/in-memory only: +// `sources` (pinned per-step inference sources, threaded to the child via +// the spawn env and durable nowhere else), `sessionId` (inference-event +// correlation), and `hubPublicKey` (the head's deploy-pack / inbound +// verification key, recorded only in memory today). The definition itself +// lives in `assets/workflow//workflow.json`, referenced by +// `definitionId`, and each step's grants live in its agent-state repo, so +// neither is duplicated here. + +import { mkdir, rm, writeFile } from "node:fs/promises"; +import { dirname, join as pathJoin } from "node:path"; + +import { type } from "arktype"; + +import { InferenceSource } from "@intx/types/runtime"; + +const RECORD_FILENAME = "deployment.json"; + +/** + * The on-disk deployment record. `version` guards the schema shape so a + * future reader can reject or migrate a stale record rather than parse it + * blindly. Validated at read time (the boot scan) at the trust boundary. + */ +export const WorkflowDeploymentRecord = type({ + version: "1", + agentAddress: "string > 0", + definitionId: "string > 0", + sources: { + "[string]": InferenceSource, + }, + "sessionId?": "string > 0", + "hubPublicKey?": "string > 0", +}); +export type WorkflowDeploymentRecord = typeof WorkflowDeploymentRecord.infer; + +function recordPath(dataDir: string, deploymentId: string): string { + return pathJoin(dataDir, "workflow-runs", deploymentId, RECORD_FILENAME); +} + +/** + * Persist a deployment record. Written after the deployment's slug is + * claimed and before the child is spawned, so a crash mid-spawn leaves a + * record the boot scan re-drives. Idempotent: it overwrites any existing + * record for the same deployment. + */ +export async function writeWorkflowDeploymentRecord( + dataDir: string, + deploymentId: string, + record: WorkflowDeploymentRecord, +): Promise { + const path = recordPath(dataDir, deploymentId); + await mkdir(dirname(path), { recursive: true }); + // Owner-only (0o600): the record embeds each source's `apiKey`, so it must + // not be world-readable on a shared host. This matches the private-key + // writes elsewhere on the sidecar and is stricter than the legacy + // `agent.json`, which persists the same credentials at the default mode. + await writeFile(path, JSON.stringify(record, null, 2), { + encoding: "utf8", + mode: 0o600, + }); +} + +/** + * Remove a deployment record. Called on undeploy and on a soft-failed + * deploy so a torn-down or never-completed deployment is not restored on + * the next boot. A missing record is not an error (`force`). + */ +export async function deleteWorkflowDeploymentRecord( + dataDir: string, + deploymentId: string, +): Promise { + await rm(recordPath(dataDir, deploymentId), { force: true }); +} diff --git a/apps/sidecar/src/workflow-host-wiring.test.ts b/apps/sidecar/src/workflow-host-wiring.test.ts index 4506f01d..7cdf312c 100644 --- a/apps/sidecar/src/workflow-host-wiring.test.ts +++ b/apps/sidecar/src/workflow-host-wiring.test.ts @@ -25,6 +25,7 @@ import { computeWireDefinitionHash, createSidecarDeployRouter, createSidecarWorkflowSupervisor, + deriveTrivialDeploymentId, driveTrivialRunChain, STEP_INFERENCE_SOURCES_ENV_KEY, validateWorkflowProjection, @@ -1454,6 +1455,48 @@ describe("createSidecarDeployRouter multi-step branch", () => { ); }); + test("a soft-failed deploy (spawn rejects) leaves no restore record", async () => { + const crashSpawner: SubprocessSpawner = () => { + throw new Error("ENOENT: binary missing"); + }; + const { router, substrateEnv } = await buildMultistepFixture({ + spawner: crashSpawner, + }); + const agentAddress = "ins_softfail@example.com"; + const frame = makeMultistepFrame({ + agentAddress, + definition: { + id: "wf-softfail", + triggers: [{ type: "manual" }], + stepOrder: ["step-1"], + steps: { "step-1": { kind: "step" } }, + }, + sources: { "step-1": makeInferenceSource("step-1") }, + }); + + await expect(router.deploy(frame)).rejects.toThrow(/ENOENT/); + + // The record is written before the spawn, so the soft-failure catch must + // delete it -- a boot-time restore must not re-spawn a deploy that never + // completed. (A hard crash mid-spawn, by contrast, deliberately leaves + // the record for the restore to re-drive.) + const dataDir = substrateEnv.SIDECAR_DATA_DIR; + if (dataDir === undefined) + throw new Error("fixture SIDECAR_DATA_DIR unset"); + const recordFile = path.join( + dataDir, + "workflow-runs", + deriveTrivialDeploymentId(agentAddress), + "deployment.json", + ); + expect( + await fs.access(recordFile).then( + () => true, + () => false, + ), + ).toBe(false); + }); + test("rejects a deploy whose step pins an unbuildable provider before spawning", async () => { // The source-admission gate runs before any state is claimed or the // child is spawned. A step whose pinned source names a provider the diff --git a/apps/sidecar/src/workflow-host-wiring.ts b/apps/sidecar/src/workflow-host-wiring.ts index 0147fe83..df2e1477 100644 --- a/apps/sidecar/src/workflow-host-wiring.ts +++ b/apps/sidecar/src/workflow-host-wiring.ts @@ -67,6 +67,11 @@ import type { MultistepMailRouter, MultistepSignalRouter, } from "./workflow-run-pack-client"; +import { + deleteWorkflowDeploymentRecord, + writeWorkflowDeploymentRecord, + type WorkflowDeploymentRecord, +} from "./workflow-deployment-record"; const logger = getLogger(["interchange", "sidecar", "workflow-host-wiring"]); @@ -1278,16 +1283,54 @@ export function createSidecarDeployRouter(deps: { // deploy keeps it (the undeploy hook releases it at teardown). The // spawn core owns unwinding the supervisor and registrations it stands // up; the slug is the caller's. + // Resolve the sidecar data dir once: the deployment record, workflow.json, + // and the per-step scratch all root under it. Required for any deployment + // that spawns a child. + const dataDir = stepStateDataDir; + if (typeof dataDir !== "string" || dataDir.length === 0) { + throw new Error( + "sidecar deploy router: SIDECAR_DATA_DIR must be present in the multi-step substrate env; the deployment record and workflow-process child root under it", + ); + } + + // The spec the shared spawn core consumes, and the durable record that + // lets a boot-time restore rebuild the SAME spec (definition re-read from + // workflow.json by id, grants from the step repos, and the record's + // frame/in-memory-only inputs: sources, session id, single-step hub key). + const spec: WorkflowDeploySpec = { + agentAddress: frame.agentAddress, + definition: projection.definition, + sources: projection.sources, + sessionId: frame.config.sessionId, + hubPublicKey: + projection.definition.stepOrder.length === 1 + ? frame.hubPublicKey + : undefined, + }; + const record: WorkflowDeploymentRecord = { + version: 1, + agentAddress: spec.agentAddress, + definitionId: spec.definition.id, + sources: spec.sources, + ...(spec.sessionId !== undefined ? { sessionId: spec.sessionId } : {}), + ...(spec.hubPublicKey !== undefined + ? { hubPublicKey: spec.hubPublicKey } + : {}), + }; + claimSlug(deploymentId, frame.agentAddress); try { + // Persist the deployment record BEFORE the spawn so a crash mid-spawn + // leaves a record the boot scan re-drives (an idempotent re-spawn; the + // child's in-flight-run discovery resumes any run). A soft-failed deploy + // deletes it below, so only a crash-interrupted deploy leaves one. + await writeWorkflowDeploymentRecord(dataDir, deploymentId, record); + // Materialize the deploy-only durable state the spawned child and the // supervisor read from disk: the workflow definition (`workflow.json`) // and each step's grants. The restore path finds both already on disk // and skips this; both land before the shared spawn core runs. - await materializeWorkflowJson( - multistepSubstrateEnv.SIDECAR_DATA_DIR, - projection.definition, - ); + await materializeWorkflowJson(dataDir, projection.definition); // Grants bridge: the spawned child does not see the frame; it reads // each step's grants out of `state/grants.json` in the step's @@ -1303,19 +1346,13 @@ export function createSidecarDeployRouter(deps: { grants: frame.config.grants, }); - // Hand off to the shared spawn core. A single-step deployment records - // its hub key at the head; a multi-step deployment carries none. - return await spawnWorkflowDeployment({ - agentAddress: frame.agentAddress, - definition: projection.definition, - sources: projection.sources, - sessionId: frame.config.sessionId, - hubPublicKey: - projection.definition.stepOrder.length === 1 - ? frame.hubPublicKey - : undefined, - }); + // Hand off to the shared spawn core. + return await spawnWorkflowDeployment(spec); } catch (cause) { + // Soft failure (this process survived, the deploy threw): drop the + // record and release the slug so the failed deploy is neither restored + // nor leaks its slug. + await deleteWorkflowDeploymentRecord(dataDir, deploymentId); releaseSlug(deploymentId, frame.agentAddress); throw cause; } @@ -1496,6 +1533,13 @@ export function createSidecarDeployRouter(deps: { ); } } + // Drop the deployment record so a boot-time restore does not re-spawn a + // torn-down deployment. Runs on every undeploy -- not only when a + // supervisor was active -- so a record left behind by a + // crash-interrupted deploy is reclaimed too. + if (stepStateDataDir !== undefined) { + await deleteWorkflowDeploymentRecord(stepStateDataDir, deploymentId); + } releaseSlug(deploymentId, frame.agentAddress); deps.unregisterDeployment({ deploymentId, From 4f3545ae05389d3f212eef4b394e21726c363127 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Thu, 2 Jul 2026 12:41:25 -0500 Subject: [PATCH 022/101] Restore persisted workflow deployments on sidecar restart Workflow deployments run in a supervised child process the sidecar spawns, tracked only in memory. A process restart lost them: nothing re-spawned the child or re-registered the head's mailbox, so a deployed agent went silent until it was redeployed. The legacy single-agent path already survived a restart by scanning its on-disk config; workflow deployments had no equivalent. Drive a restore pass at boot, before the hub connection opens, so a single-step head's transport registration is live before the hub can route to it. The pass scans the persisted per-deployment records and, for each, re-reads the stored definition, re-validates it through the same wire and structural gates a fresh deploy frame clears, re-runs the source-admission gate, and routes through the shared spawn core the live deploy path uses. The core gains a guard that refuses to spawn a second supervisor for an address already live, so a restore pass and the legacy restore cannot both stand up one address. Restore is soft-fail per record: an unbuildable provider, a corrupt definition, or a spawn failure is logged and the record is left on disk for a later boot to retry -- it is never deleted, unlike the deploy path's cleanup of a record it just wrote. A record whose address no longer derives its own directory name is skipped. The mechanism is step-count agnostic: it rebuilds the deployment spec from the record and routes through the same core the live deploy path uses, so a genuine multi-step deployment restores exactly the way a single-step one does. --- apps/sidecar/src/index.ts | 33 +- .../src/workflow-deployment-record.test.ts | 54 +++ .../sidecar/src/workflow-deployment-record.ts | 88 +++- apps/sidecar/src/workflow-host-wiring.test.ts | 403 +++++++++++++++++- apps/sidecar/src/workflow-host-wiring.ts | 186 +++++++- 5 files changed, 756 insertions(+), 8 deletions(-) diff --git a/apps/sidecar/src/index.ts b/apps/sidecar/src/index.ts index 4e12e357..5e7f5813 100644 --- a/apps/sidecar/src/index.ts +++ b/apps/sidecar/src/index.ts @@ -34,6 +34,7 @@ import type { DispatchTimingMark } from "@intx/workflow-host"; import { createSidecarDeployRouter, createSidecarWorkflowSupervisor, + type SidecarDeployRouter, } from "./workflow-host-wiring"; import { createDeploymentAddressRegistry, @@ -331,6 +332,11 @@ const buildHarness = createDefaultHarnessBuilder({ gcPolicy: agentGCPolicy, }); +// Set by the `createDeployRouter` callback below (invoked synchronously +// during construction) so the boot edge can drive the router's restore pass +// before `orchestrator.start()` connects to the hub. +let sidecarDeployRouter: SidecarDeployRouter | undefined; + const orchestrator = createSidecarOrchestrator({ hubURL: hubWsUrl, sidecarId, @@ -352,8 +358,8 @@ const orchestrator = createSidecarOrchestrator({ keyStore, onAgentEvent, publishWorkflowInferenceEvent, - }) => - createSidecarDeployRouter({ + }) => { + const router = createSidecarDeployRouter({ sessions, keyStore, onAgentEvent, @@ -376,11 +382,32 @@ const orchestrator = createSidecarOrchestrator({ ...(onDispatchTiming !== undefined ? { onDispatchTiming } : {}), ...(repackEveryMessages !== undefined ? { repackEveryMessages } : {}), ...(consumedRetentionMs !== undefined ? { consumedRetentionMs } : {}), - }), + }); + // Capture the router so the boot edge can drive its restore pass before + // connecting. `createDeployRouter` runs synchronously during + // `createSidecarOrchestrator` construction (exactly once, before the + // handle returns), so `sidecarDeployRouter` is populated by the time the + // restore call below runs. + sidecarDeployRouter = router; + return router; + }, }); resolvedHubLink = orchestrator.hubLink; +// Re-establish the workflow deployments a prior sidecar process persisted, +// BEFORE opening the hub connection: each single-step head must have its +// mailbox/transport registration live before the hub can route to it. +// Assert the router was captured rather than optional-chaining it, so a +// future refactor that made `createDeployRouter` fire lazily would fail loud +// here instead of silently skipping restore. +if (sidecarDeployRouter === undefined) { + throw new Error( + "sidecar boot: deploy router was not constructed before workflow-deployment restore", + ); +} +await sidecarDeployRouter.restoreWorkflowDeployments(); + orchestrator.start(); // Keep `createSidecarWorkflowSupervisor` reachable from this entry diff --git a/apps/sidecar/src/workflow-deployment-record.test.ts b/apps/sidecar/src/workflow-deployment-record.test.ts index 80c25c45..258a9282 100644 --- a/apps/sidecar/src/workflow-deployment-record.test.ts +++ b/apps/sidecar/src/workflow-deployment-record.test.ts @@ -9,6 +9,7 @@ import { WorkflowDeploymentRecord, writeWorkflowDeploymentRecord, deleteWorkflowDeploymentRecord, + scanWorkflowDeploymentRecords, } from "./workflow-deployment-record"; async function makeDataDir(): Promise { @@ -123,3 +124,56 @@ describe("workflow deployment record store", () => { await fs.rm(dataDir, { recursive: true, force: true }); }); }); + +describe("scanWorkflowDeploymentRecords", () => { + test("returns an empty list when the workflow-runs directory is absent", async () => { + const dataDir = await makeDataDir(); + // First boot: nothing has been deployed, so `workflow-runs/` does not + // exist. That is the legitimate empty case, not an error. + expect(await scanWorkflowDeploymentRecords(dataDir)).toEqual([]); + await fs.rm(dataDir, { recursive: true, force: true }); + }); + + test("returns every schema-valid record keyed by its directory name", async () => { + const dataDir = await makeDataDir(); + await writeWorkflowDeploymentRecord(dataDir, "dep-a", SINGLE_STEP); + await writeWorkflowDeploymentRecord(dataDir, "dep-b", MULTI_STEP); + + const scanned = await scanWorkflowDeploymentRecords(dataDir); + const byId = new Map(scanned.map((s) => [s.deploymentId, s.record])); + expect(byId.size).toBe(2); + expect(byId.get("dep-a")).toEqual(SINGLE_STEP); + expect(byId.get("dep-b")).toEqual(MULTI_STEP); + + await fs.rm(dataDir, { recursive: true, force: true }); + }); + + test("soft-fails a corrupt or schema-invalid record while returning the valid ones", async () => { + const dataDir = await makeDataDir(); + await writeWorkflowDeploymentRecord(dataDir, "dep-valid", SINGLE_STEP); + + // A directory whose record is not valid JSON. + const corruptDir = path.join(dataDir, "workflow-runs", "dep-corrupt"); + await fs.mkdir(corruptDir, { recursive: true }); + await fs.writeFile(path.join(corruptDir, "deployment.json"), "{ not json"); + + // A directory whose record parses but fails the schema (missing fields). + const invalidDir = path.join(dataDir, "workflow-runs", "dep-invalid"); + await fs.mkdir(invalidDir, { recursive: true }); + await fs.writeFile( + path.join(invalidDir, "deployment.json"), + JSON.stringify({ version: 1 }), + ); + + // A bare run directory with no record at all. + await fs.mkdir(path.join(dataDir, "workflow-runs", "dep-empty"), { + recursive: true, + }); + + const scanned = await scanWorkflowDeploymentRecords(dataDir); + expect(scanned.map((s) => s.deploymentId)).toEqual(["dep-valid"]); + expect(scanned[0]?.record).toEqual(SINGLE_STEP); + + await fs.rm(dataDir, { recursive: true, force: true }); + }); +}); diff --git a/apps/sidecar/src/workflow-deployment-record.ts b/apps/sidecar/src/workflow-deployment-record.ts index d86c018c..9d1a5c4f 100644 --- a/apps/sidecar/src/workflow-deployment-record.ts +++ b/apps/sidecar/src/workflow-deployment-record.ts @@ -14,15 +14,31 @@ // `definitionId`, and each step's grants live in its agent-state repo, so // neither is duplicated here. -import { mkdir, rm, writeFile } from "node:fs/promises"; +import { mkdir, readdir, readFile, rm, writeFile } from "node:fs/promises"; import { dirname, join as pathJoin } from "node:path"; import { type } from "arktype"; +import { getLogger } from "@intx/log"; import { InferenceSource } from "@intx/types/runtime"; +const logger = getLogger([ + "interchange", + "sidecar", + "workflow-deployment-record", +]); + const RECORD_FILENAME = "deployment.json"; +/** True for a `node:fs` rejection whose `code` is `ENOENT`. */ +function isENOENT(cause: unknown): boolean { + return ( + cause instanceof Error && + "code" in cause && + (cause as { code: unknown }).code === "ENOENT" + ); +} + /** * The on-disk deployment record. `version` guards the schema shape so a * future reader can reject or migrate a stale record rather than parse it @@ -78,3 +94,73 @@ export async function deleteWorkflowDeploymentRecord( ): Promise { await rm(recordPath(dataDir, deploymentId), { force: true }); } + +/** A restorable deployment: its directory-derived id plus the validated record. */ +export interface ScannedWorkflowDeployment { + /** The `workflow-runs/` directory name the record was found under. */ + deploymentId: string; + record: WorkflowDeploymentRecord; +} + +/** + * Enumerate the persisted deployment records under `workflow-runs/` so a + * boot-time restore can re-establish each deployment. Soft-fails per record: + * a missing `deployment.json`, unparseable JSON, or a record that fails schema + * validation is logged and skipped rather than wedging the whole boot -- one + * corrupt record must not strand every other deployment. An absent + * `workflow-runs/` directory is the legitimate first-boot case and yields an + * empty list, not an error. + * + * The returned `deploymentId` is the directory name; the caller cross-checks it + * against the record's own address before trusting it. + */ +export async function scanWorkflowDeploymentRecords( + dataDir: string, +): Promise { + const runsDir = pathJoin(dataDir, "workflow-runs"); + let entries; + try { + entries = await readdir(runsDir, { withFileTypes: true }); + } catch (cause) { + if (isENOENT(cause)) return []; + throw cause; + } + + const scanned: ScannedWorkflowDeployment[] = []; + for (const entry of entries) { + if (!entry.isDirectory()) continue; + const deploymentId = entry.name; + const path = recordPath(dataDir, deploymentId); + + let raw: string; + try { + raw = await readFile(path, "utf8"); + } catch (cause) { + // A run directory with no record: a crash between mkdir and the record + // write, or a run whose record was already reclaimed. Nothing to + // restore from -- skip. + if (isENOENT(cause)) { + logger.warn`skipping workflow-runs/${deploymentId}: no ${RECORD_FILENAME} to restore from`; + continue; + } + throw cause; + } + + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (cause) { + const reason = cause instanceof Error ? cause.message : String(cause); + logger.warn`skipping workflow-runs/${deploymentId}: ${RECORD_FILENAME} is not valid JSON: ${reason}`; + continue; + } + + const record = WorkflowDeploymentRecord(parsed); + if (record instanceof type.errors) { + logger.warn`skipping workflow-runs/${deploymentId}: ${RECORD_FILENAME} failed validation: ${record.summary}`; + continue; + } + scanned.push({ deploymentId, record }); + } + return scanned; +} diff --git a/apps/sidecar/src/workflow-host-wiring.test.ts b/apps/sidecar/src/workflow-host-wiring.test.ts index 7cdf312c..590272c2 100644 --- a/apps/sidecar/src/workflow-host-wiring.test.ts +++ b/apps/sidecar/src/workflow-host-wiring.test.ts @@ -35,6 +35,10 @@ import { createMultistepMailRouter, type MultistepMailRouter, } from "./workflow-run-pack-client"; +import { + writeWorkflowDeploymentRecord, + type WorkflowDeploymentRecord, +} from "./workflow-deployment-record"; function createMinimalStubRepoStore(): RepoStore { const stub: Partial = { @@ -1143,8 +1147,16 @@ describe("createSidecarDeployRouter multi-step branch", () => { assertSourceBuildable?: Parameters< typeof createSidecarDeployRouter >[0]["assertSourceBuildable"]; + /** + * Reuse an existing transport instead of a fresh one. The restore + * tests deploy through one fixture, then build a SECOND fixture over + * the same on-disk data dir with a FRESH transport to model a sidecar + * process restart (the in-memory transport is process-local, so a + * restart starts with an empty registration table). + */ + transport?: ReturnType; }) { - const transport = createInMemoryTransport(); + const transport = opts.transport ?? createInMemoryTransport(); const keyPair = await generateKeyPair(); const tempBase = await createTempBaseDir("sidecar-multistep-"); const repoStore = createSpawnTestRepoStore(tempBase); @@ -1213,7 +1225,13 @@ describe("createSidecarDeployRouter multi-step branch", () => { ? { multistepMailRouter: opts.multistepMailRouter } : {}), }); - return { router, tempBase, keyPair, substrateEnv: mergedSubstrateEnv }; + return { + router, + tempBase, + keyPair, + substrateEnv: mergedSubstrateEnv, + transport, + }; } test("validates the projection, constructs SpawnOpts from the frame, drives spawn, and surfaces the supervisor's principal pubkey", async () => { @@ -1971,4 +1989,385 @@ describe("createSidecarDeployRouter multi-step branch", () => { expect(secondResult.publicKey).toMatch(/^[0-9a-f]{64}$/); expect(registerCallCount).toBe(2); }); + + // ------------------------------------------------------------------ + // Boot-time restore of persisted workflow deployments + // ------------------------------------------------------------------ + + // A mock spawner that serves a fresh control/event channel per spawn and + // lets the test complete each child's `ready` handshake. Both `deploy` and + // `restoreWorkflowDeployments` block on `supervisor.spawn` until `ready` + // lands, so every spawned child needs its handshake driven. + function makeReadyDrivingSpawner(pidBase: number) { + type Spawn = { + env: Record; + childToSupervisor: ReturnType; + eventChildToSupervisor: ReturnType; + }; + const spawns: Spawn[] = []; + const spawner: SubprocessSpawner = ({ env }) => { + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const eventChildToSupervisor = createMemoryFrameStream(); + let resolveExit: ((code: number) => void) | undefined; + const exited = new Promise((resolve) => { + resolveExit = resolve; + }); + spawns.push({ env, childToSupervisor, eventChildToSupervisor }); + const handle: SubprocessHandle = { + pid: pidBase + spawns.length, + controlWriter: supervisorToChild.writer, + controlReader: childToSupervisor.reader, + eventReader: eventChildToSupervisor.reader, + kill: () => { + childToSupervisor.close(); + eventChildToSupervisor.close(); + resolveExit?.(0); + }, + exited, + }; + return handle; + }; + async function driveReadyFor(index: number): Promise { + while (spawns.length <= index) { + await new Promise((r) => setTimeout(r, 1)); + } + const spawn = spawns[index]; + if (spawn === undefined) { + throw new Error(`spawn ${String(index)} missing`); + } + const channelId = spawn.env.IPC_CHANNEL_ID; + if (channelId === undefined) { + throw new Error("IPC_CHANNEL_ID missing in spawn env"); + } + const childIpcKeyPair = await generateKeyPair(); + const childSender = createControlChannelSender({ + privateKeySeed: childIpcKeyPair.privateKey, + channelId, + writer: { + write(line: string) { + spawn.childToSupervisor.inject(line); + return Promise.resolve(); + }, + }, + }); + await childSender.send({ + type: "ready", + data: { + childPid: pidBase + index, + childPublicKey: Buffer.from(childIpcKeyPair.publicKey).toString( + "hex", + ), + }, + }); + } + return { spawner, driveReadyFor, spawnCount: () => spawns.length }; + } + + function isRegistered( + transport: ReturnType, + address: string, + ): boolean { + try { + transport.getTransportFor(address); + return true; + } catch { + return false; + } + } + + function recordExists( + dataDir: string, + deploymentId: string, + ): Promise { + return fs + .access( + path.join(dataDir, "workflow-runs", deploymentId, "deployment.json"), + ) + .then( + () => true, + () => false, + ); + } + + function singleStepFrame( + agentAddress: string, + definitionId: string, + ): AgentDeployFrame { + return makeMultistepFrame({ + agentAddress, + definition: { + id: definitionId, + triggers: [{ type: "manual" }], + stepOrder: ["step-1"], + steps: { "step-1": { kind: "step" } }, + }, + sources: { "step-1": makeInferenceSource("step-1") }, + }); + } + + test("restore re-spawns a persisted single-step deployment and re-registers its head on a fresh transport", async () => { + const dataDir = await createTempBaseDir("sidecar-restore-restart-data-"); + const head = "ins_restart@example.com"; + + // First process: deploy a single-step workflow. The deploy persists a + // restore record under `dataDir` and materializes its `workflow.json`. + const first = makeReadyDrivingSpawner(9100); + const { router: routerA } = await buildMultistepFixture({ + spawner: first.spawner, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + const deployPromise = routerA.deploy(singleStepFrame(head, "wf-restart")); + await first.driveReadyFor(0); + await deployPromise; + + // Second process (simulated restart): a FRESH transport (empty + // registration table) and fresh in-memory router state over the SAME + // on-disk data dir. + const second = makeReadyDrivingSpawner(9200); + const freshTransport = createInMemoryTransport(); + const { router: routerB } = await buildMultistepFixture({ + spawner: second.spawner, + transport: freshTransport, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + + // Nothing is registered before restore -- the restart started clean. + expect(isRegistered(freshTransport, head)).toBe(false); + + const restorePromise = routerB.restoreWorkflowDeployments(); + await second.driveReadyFor(0); + await restorePromise; + + // The deployment was re-spawned exactly once and its head is live again. + expect(second.spawnCount()).toBe(1); + expect(isRegistered(freshTransport, head)).toBe(true); + }); + + test("restore soft-fails a record whose workflow.json is missing and restores the rest", async () => { + const dataDir = await createTempBaseDir("sidecar-restore-softfail-data-"); + const goodHead = "ins_good@example.com"; + const badHead = "ins_bad@example.com"; + + const first = makeReadyDrivingSpawner(9300); + const { router: routerA } = await buildMultistepFixture({ + spawner: first.spawner, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + const deployGood = routerA.deploy(singleStepFrame(goodHead, "wf-good")); + await first.driveReadyFor(0); + await deployGood; + const deployBad = routerA.deploy(singleStepFrame(badHead, "wf-bad")); + await first.driveReadyFor(1); + await deployBad; + + // Remove the bad deployment's definition so its restore read faults. + await fs.rm( + path.join(dataDir, "assets", "workflow", "wf-bad", "workflow.json"), + ); + + const second = makeReadyDrivingSpawner(9400); + const freshTransport = createInMemoryTransport(); + const { router: routerB } = await buildMultistepFixture({ + spawner: second.spawner, + transport: freshTransport, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + + // The good deployment re-spawns (exactly one handshake to drive); + // scan order is filesystem-dependent, but only the good record spawns. + const restorePromise = routerB.restoreWorkflowDeployments(); + await second.driveReadyFor(0); + await restorePromise; + + expect(second.spawnCount()).toBe(1); + expect(isRegistered(freshTransport, goodHead)).toBe(true); + expect(isRegistered(freshTransport, badHead)).toBe(false); + // The failed record is KEPT on disk -- never deleted, unlike a + // soft-failed deploy -- so a later boot can retry it. + expect( + await recordExists(dataDir, deriveTrivialDeploymentId(badHead)), + ).toBe(true); + }); + + test("restore applies validateWorkflowProjection: a stepOrder entry with no matching steps is skipped", async () => { + const dataDir = await createTempBaseDir("sidecar-restore-validator-data-"); + const head = "ins_validator@example.com"; + const deploymentId = deriveTrivialDeploymentId(head); + + // Hand-write a record plus a workflow.json whose `stepOrder` names a step + // `steps` does not define. This clears the wire arktype + // (`AgentDeployWorkflow` only checks that `sources` cover `stepOrder`) but + // MUST be rejected by `validateWorkflowProjection`, the second gate the + // deploy path applies. If restore ran only the arktype it would spawn a + // child for a structurally invalid definition. + const record: WorkflowDeploymentRecord = { + version: 1, + agentAddress: head, + definitionId: "wf-missing-step", + sources: { "step-1": makeInferenceSource("step-1") }, + hubPublicKey: "hub-pk", + }; + await writeWorkflowDeploymentRecord(dataDir, deploymentId, record); + const workflowJsonPath = path.join( + dataDir, + "assets", + "workflow", + "wf-missing-step", + "workflow.json", + ); + await fs.mkdir(path.dirname(workflowJsonPath), { recursive: true }); + await fs.writeFile( + workflowJsonPath, + JSON.stringify({ + id: "wf-missing-step", + triggers: [{ type: "manual" }], + stepOrder: ["step-1"], + steps: {}, + }), + "utf8", + ); + + const spawner = makeReadyDrivingSpawner(9500); + const freshTransport = createInMemoryTransport(); + const { router } = await buildMultistepFixture({ + spawner: spawner.spawner, + transport: freshTransport, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + + await router.restoreWorkflowDeployments(); + + expect(spawner.spawnCount()).toBe(0); + expect(isRegistered(freshTransport, head)).toBe(false); + }); + + test("restore is a no-op for a deployment already live in this process", async () => { + const dataDir = await createTempBaseDir("sidecar-restore-guard-data-"); + const head = "ins_guard@example.com"; + + const spawner = makeReadyDrivingSpawner(9600); + const { router, transport } = await buildMultistepFixture({ + spawner: spawner.spawner, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + + const deployPromise = router.deploy(singleStepFrame(head, "wf-guard")); + await spawner.driveReadyFor(0); + await deployPromise; + expect(spawner.spawnCount()).toBe(1); + + // The record is on disk and the address is live in this same process. A + // restore pass must NOT spawn a second child for an address the core's + // double-spawn guard already owns (the transition guard the B-reroute + // follow-up leans on). + await router.restoreWorkflowDeployments(); + + expect(spawner.spawnCount()).toBe(1); + expect(isRegistered(transport, head)).toBe(true); + }); + + test("a second deploy for a live address is rejected without orphaning its restore record", async () => { + const dataDir = await createTempBaseDir("sidecar-restore-dup-data-"); + const head = "ins_dup@example.com"; + const deploymentId = deriveTrivialDeploymentId(head); + + const spawner = makeReadyDrivingSpawner(9700); + const { router, transport } = await buildMultistepFixture({ + spawner: spawner.spawner, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + + const deployPromise = router.deploy(singleStepFrame(head, "wf-dup")); + await spawner.driveReadyFor(0); + await deployPromise; + expect(await recordExists(dataDir, deploymentId)).toBe(true); + + // A second deploy for the already-live address must be rejected WITHOUT + // touching the running deployment's durable state. The reject fires + // before any overwrite; without it, deployMultiStep's catch would delete + // the live deployment's record and release its slug, silently breaking + // the next restart for a still-running agent. + await expect( + router.deploy(singleStepFrame(head, "wf-dup")), + ).rejects.toThrow(/already deployed/); + expect(spawner.spawnCount()).toBe(1); + expect(await recordExists(dataDir, deploymentId)).toBe(true); + expect(isRegistered(transport, head)).toBe(true); + }); + + test("restore skips a record whose address does not derive its directory name", async () => { + const dataDir = await createTempBaseDir("sidecar-restore-mismatch-data-"); + const head = "ins_mismatch@example.com"; + // A record filed under a directory that is NOT its own derived slug -- + // a corrupt or misplaced record that must not be restored under the + // wrong slug. + const wrongDir = "not-the-right-slug"; + const record: WorkflowDeploymentRecord = { + version: 1, + agentAddress: head, + definitionId: "wf-mismatch", + sources: { "step-1": makeInferenceSource("step-1") }, + hubPublicKey: "hub-pk", + }; + await writeWorkflowDeploymentRecord(dataDir, wrongDir, record); + + const spawner = makeReadyDrivingSpawner(9800); + const freshTransport = createInMemoryTransport(); + const { router } = await buildMultistepFixture({ + spawner: spawner.spawner, + transport: freshTransport, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + + await router.restoreWorkflowDeployments(); + + expect(spawner.spawnCount()).toBe(0); + expect(isRegistered(freshTransport, head)).toBe(false); + // The record is kept on a skip, not deleted. + expect(await recordExists(dataDir, wrongDir)).toBe(true); + }); + + test("restore soft-fails and keeps the record when the pinned source is no longer buildable", async () => { + const dataDir = await createTempBaseDir( + "sidecar-restore-unbuildable-data-", + ); + const head = "ins_unbuildable_restore@example.com"; + const deploymentId = deriveTrivialDeploymentId(head); + + // First process: a permissive gate lets the deploy through, persisting + // the record and its workflow.json. + const first = makeReadyDrivingSpawner(9900); + const { router: routerA } = await buildMultistepFixture({ + spawner: first.spawner, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + const deployPromise = routerA.deploy( + singleStepFrame(head, "wf-unbuildable-restore"), + ); + await first.driveReadyFor(0); + await deployPromise; + + // Restart with a gate that now rejects the pinned provider. + const second = makeReadyDrivingSpawner(10000); + const freshTransport = createInMemoryTransport(); + const { router: routerB } = await buildMultistepFixture({ + spawner: second.spawner, + transport: freshTransport, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + assertSourceBuildable: (source) => { + throw new Error( + `Source provider "${source.provider}" is not registered`, + ); + }, + }); + + await routerB.restoreWorkflowDeployments(); + + expect(second.spawnCount()).toBe(0); + expect(isRegistered(freshTransport, head)).toBe(false); + // The record survives so a later boot, once the provider is buildable + // again, can retry it. + expect(await recordExists(dataDir, deploymentId)).toBe(true); + }); }); diff --git a/apps/sidecar/src/workflow-host-wiring.ts b/apps/sidecar/src/workflow-host-wiring.ts index df2e1477..2e41e930 100644 --- a/apps/sidecar/src/workflow-host-wiring.ts +++ b/apps/sidecar/src/workflow-host-wiring.ts @@ -58,7 +58,10 @@ import { type InferenceSource, type KeyPair, } from "@intx/types/runtime"; -import type { AgentDeployFrame } from "@intx/types/sidecar"; +import { + AgentDeployWorkflow, + type AgentDeployFrame, +} from "@intx/types/sidecar"; import { STEP_ID_PATTERN } from "@intx/workflow"; import { deriveWorkflowRunRepoId } from "@intx/workflow-deploy"; @@ -69,6 +72,7 @@ import type { } from "./workflow-run-pack-client"; import { deleteWorkflowDeploymentRecord, + scanWorkflowDeploymentRecords, writeWorkflowDeploymentRecord, type WorkflowDeploymentRecord, } from "./workflow-deployment-record"; @@ -659,6 +663,26 @@ async function derivePrincipalPublicKeyHex( return hexEncode(await derivePublicKeyBytes(signingKeySeed)); } +/** + * The sidecar's `DeployRouter` plus the boot-time restore driver. The link + * routes `agent.deploy`/`agent.undeploy` through the `DeployRouter` surface; + * the sidecar boot edge additionally calls `restoreWorkflowDeployments` once, + * before connecting to the hub, to re-establish the deployments a prior + * process persisted. The extra method is sidecar-app-only, so it rides on the + * concrete router type rather than the shared `DeployRouter` contract. + */ +export interface SidecarDeployRouter extends DeployRouter { + /** + * Re-establish every persisted workflow deployment on this sidecar's local + * substrate. Runs once at boot, before `hubLink.connect()`, so a single-step + * head's mailbox/transport registration is live before the hub routes to it. + * Soft-fails per deployment: a record that cannot be restored (unbuildable + * provider, corrupt `workflow.json`, spawn failure) is logged and left on + * disk for a later boot to retry -- it is never deleted here. + */ + restoreWorkflowDeployments(): Promise; +} + export function createSidecarDeployRouter(deps: { sessions: SessionManager; keyStore: AgentKeyStore; @@ -845,7 +869,7 @@ export function createSidecarDeployRouter(deps: { * handler for the operator-owned horizon invariant. */ consumedRetentionMs?: number; -}): DeployRouter { +}): SidecarDeployRouter { // Validate the signing seed at construction so a malformed key fails // sidecar boot rather than the first multi-step deploy, where the // public key is derived from it (`derivePrincipalPublicKeyHex`). The @@ -973,6 +997,31 @@ export function createSidecarDeployRouter(deps: { } } + /** + * Read a workflow definition back off the sidecar's local substrate for a + * boot-time restore. Mirrors `materializeWorkflowJson`'s path derivation + * (`${dataDir}/assets/workflow//workflow.json`). Returns the + * parsed-but-unvalidated JSON: the on-disk file is untrusted at restore + * (partial write, corruption, tamper), so the caller re-validates it through + * the same wire + structural gates the deploy path applies. A missing file + * or unparseable JSON throws; the restore loop's per-record catch converts + * that into a warn-and-skip. + */ + async function readWorkflowJson( + sidecarDataDir: string, + definitionId: string, + ): Promise { + const workflowAssetPath = pathJoin( + sidecarDataDir, + "assets", + "workflow", + definitionId, + "workflow.json", + ); + const raw = await readFile(workflowAssetPath, "utf8"); + return JSON.parse(raw); + } + /** * The per-deployment inputs the shared spawn core needs to stand up a * workflow deployment, independent of the live deploy frame. The live @@ -1009,6 +1058,19 @@ export function createSidecarDeployRouter(deps: { async function spawnWorkflowDeployment( spec: WorkflowDeploySpec, ): Promise { + // Fail loud if this address already has a live supervisor. A single-step + // deploy would fault anyway (its `transport.register` throws on a + // duplicate), but a genuine multi-step head never registers on the + // transport and the `activeSupervisors.set` below would silently clobber + // the running deployment's handle. Both the deploy path and the boot + // restore path route through here, so this is the single transition guard + // against a double-spawn -- notably a boot restore racing a legacy + // restore for the same address (the B-reroute follow-up relies on it). + if (activeSupervisors.has(spec.agentAddress)) { + throw new Error( + `sidecar deploy router: a supervisor is already active for ${spec.agentAddress}; refusing to spawn a second`, + ); + } const deploymentId = deriveTrivialDeploymentId(spec.agentAddress); // Single-step launched-agent deploy vs. derived multi-step deploy. A @@ -1254,6 +1316,27 @@ export function createSidecarDeployRouter(deps: { if (source !== undefined) deps.assertSourceBuildable(source); } + // Reject a re-deploy of an address already live in this process BEFORE + // touching any durable state. The durable writes below (the restore + // record, workflow.json, step grants) are destructive overwrites of state + // owned by whatever deployment currently holds the address; overwriting is + // only legal when this deploy owns the address. `spawnWorkflowDeployment` + // carries the same guard as its single-owner backstop, but it throws only + // after this method has already overwritten -- so the guard must also sit + // ahead of the writes, or a duplicate frame would clobber a live + // deployment's record (and the catch below would then delete it). A + // re-deploy after `undeploy` passes: `undeploy` drops the + // `activeSupervisors` entry; a failed deploy that unwound is not in the + // map either. Known boundary: the trivial branch (`frame.workflow` + // undefined) never enters `activeSupervisors`, so a trivial re-deploy + // against a live multi-step address is not caught here; that path is + // slated for removal, taking the boundary with it. + if (activeSupervisors.has(frame.agentAddress)) { + throw new Error( + `sidecar deploy router: ${frame.agentAddress} is already deployed; undeploy it before redeploying`, + ); + } + const deploymentId = deriveTrivialDeploymentId(frame.agentAddress); // Single-step launched-agent deploy vs. derived multi-step deploy. @@ -1546,6 +1629,105 @@ export function createSidecarDeployRouter(deps: { agentAddress: frame.agentAddress, }); }, + async restoreWorkflowDeployments(): Promise { + const dataDir = stepStateDataDir; + if (dataDir === undefined) { + // No substrate config was wired (a trivial-only or test router that + // never spawns a child): nothing was ever persisted under this data + // dir, so there is nothing to restore. + return; + } + + const scanned = await scanWorkflowDeploymentRecords(dataDir); + // Restore serially, not in parallel: deterministic boot-log ordering, + // one isolable warning per failed record, and no concurrent + // child-spawn / transport-register storm. Restore runs before + // `hubLink.connect()`, so there are no concurrent deploys to contend + // with. Each record's failure is caught so one bad deployment cannot + // strand the rest. + for (const { deploymentId, record } of scanned) { + try { + // Integrity: the stored address must re-derive to its own directory + // name. A mismatch means a corrupt or misplaced record; skip it + // rather than restore a deployment under the wrong slug. + const derived = deriveTrivialDeploymentId(record.agentAddress); + if (derived !== deploymentId) { + logger.warn`skipping workflow deployment restore: ${record.agentAddress} derives slug ${derived}, not its directory ${deploymentId}`; + continue; + } + + // Re-read and RE-VALIDATE the definition off disk with the exact + // gates the deploy path applies: the wire arktype + // (`AgentDeployWorkflow`) to narrow the untrusted on-disk shape, + // then `validateWorkflowProjection` for the structural invariants + // the arktype does not cover (non-empty stepOrder, every stepOrder + // entry backed by a `steps` entry). The on-disk `workflow.json` is + // untrusted at restore, so it must clear the same bar a fresh + // deploy frame clears -- no weaker. + const definitionRaw = await readWorkflowJson( + dataDir, + record.definitionId, + ); + const projection = AgentDeployWorkflow({ + definition: definitionRaw, + sources: record.sources, + }); + if (projection instanceof type.errors) { + logger.warn`skipping workflow deployment restore for ${record.agentAddress}: workflow.json failed validation: ${projection.summary}`; + continue; + } + validateWorkflowProjection(projection); + + // Re-run the source-admission gate: refuse to restore a deployment + // whose pinned provider this sidecar can no longer build. The + // record is KEPT (not deleted) so a later boot with the provider + // restored retries it. + for (const stepId of projection.definition.stepOrder) { + const source = projection.sources[stepId]; + if (source !== undefined) deps.assertSourceBuildable(source); + } + + const spec: WorkflowDeploySpec = { + agentAddress: record.agentAddress, + definition: projection.definition, + sources: projection.sources, + sessionId: record.sessionId, + hubPublicKey: record.hubPublicKey, + }; + + // The slug is the caller's, matching `deployMultiStep`: claim before + // the spawn, release on failure. Unlike deploy's soft-fail, restore + // does NOT delete the record and does NOT re-materialize + // `workflow.json` or the step grants -- all of that is already on + // disk from the original deploy. A failed restore just warns and + // leaves the record for the next boot; there is deliberately no GC + // of a permanently-unrestorable record here (an operator reclaims it + // by undeploying the address). + // + // Release only a slug THIS pass newly claimed: if the address is + // already live (its slug still held by the running deployment), the + // core's double-spawn guard throws, and freeing the slug then would + // strand a live deployment's collision guard. `claimSlug` is a + // no-op for an already-held (deploymentId, address) pair, so the + // pre-claim check distinguishes the two. + const slugNewlyClaimed = + slugClaims.get(deploymentId) !== record.agentAddress; + claimSlug(deploymentId, record.agentAddress); + try { + await spawnWorkflowDeployment(spec); + logger.info`Restored workflow deployment for ${record.agentAddress}`; + } catch (cause) { + if (slugNewlyClaimed) { + releaseSlug(deploymentId, record.agentAddress); + } + throw cause; + } + } catch (cause) { + const reason = cause instanceof Error ? cause.message : String(cause); + logger.warn`Failed to restore workflow deployment ${deploymentId}: ${reason}`; + } + } + }, }; } From b78efb05d77925f8d316009a4ed0e1acc0095fe9 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Thu, 2 Jul 2026 13:33:51 -0500 Subject: [PATCH 023/101] Promote child-termination helpers out of the recycle path The SIGTERM-then-SIGKILL child termination the recycle path uses (killChildHandle) and its resolve-only deadline (waitDeadline, plus the default timer functions) are the exact machinery the spawn path needs to bound its ready handshake and force down a child that never signals ready. Both were module-private to the recycle module. Move them into a shared supervisor-internal module so the spawn path can reuse one implementation rather than growing a second copy. killChildHandle takes an explicit dependency object -- optional injectable timers plus the caller's logger -- instead of the recycle context, so its SIGKILL-escalation warning is attributed to whichever path initiated the kill. The recycle path passes its own logger and timers. The one observable change is that warning's prefix, now "child termination" rather than "recycle" to match the shared home. DEFAULT_KILL_TIMEOUT_MS moves with the helpers and is re-exported from the supervisor barrel at its existing name, so the public surface is identical. --- .../src/supervisor/child-termination.test.ts | 144 ++++++++++++++++++ .../src/supervisor/child-termination.ts | 97 ++++++++++++ .../workflow-host/src/supervisor/index.ts | 3 +- .../workflow-host/src/supervisor/recycle.ts | 68 +-------- 4 files changed, 249 insertions(+), 63 deletions(-) create mode 100644 packages/workflow-host/src/supervisor/child-termination.test.ts create mode 100644 packages/workflow-host/src/supervisor/child-termination.ts diff --git a/packages/workflow-host/src/supervisor/child-termination.test.ts b/packages/workflow-host/src/supervisor/child-termination.test.ts new file mode 100644 index 00000000..7f9a7c32 --- /dev/null +++ b/packages/workflow-host/src/supervisor/child-termination.test.ts @@ -0,0 +1,144 @@ +import { describe, test, expect } from "bun:test"; + +import { getLogger } from "@intx/log"; + +import { + killChildHandle, + waitDeadline, + defaultSetTimer, +} from "./child-termination"; +import type { SubprocessHandle } from "./types"; + +const logger = getLogger(["workflow-host", "supervisor", "child-termination"]); + +function emptyControlReader(): AsyncIterableIterator { + return (async function* () { + /* no control frames in these unit tests */ + })(); +} + +function emptyEventReader(): AsyncIterableIterator { + return (async function* () { + /* no event frames in these unit tests */ + })(); +} + +/** + * A `SubprocessHandle` whose only live surface is `kill` (recording the + * signals it receives) and `exited`. When `sigtermExits` is false the child + * ignores SIGTERM and settles `exited` only on SIGKILL -- the wedged-child + * shape the escalation branch exists for. The stream fields are inert + * stubs; `killChildHandle` never touches them. + */ +function makeHandle(opts: { sigtermExits: boolean }): { + handle: SubprocessHandle; + killSignals: string[]; +} { + const killSignals: string[] = []; + let resolveExit: ((code: number) => void) | undefined; + const exited = new Promise((resolve) => { + resolveExit = resolve; + }); + const handle: SubprocessHandle = { + pid: 1234, + controlWriter: { write: () => Promise.resolve() }, + controlReader: { read: emptyControlReader }, + eventReader: { read: emptyEventReader }, + kill: (signal?: number | string) => { + const sig = typeof signal === "string" ? signal : String(signal ?? ""); + killSignals.push(sig); + if (opts.sigtermExits || sig === "SIGKILL") { + resolveExit?.(0); + } + }, + exited, + }; + return { handle, killSignals }; +} + +describe("killChildHandle", () => { + test("a child that exits on SIGTERM is not escalated to SIGKILL", async () => { + const { handle, killSignals } = makeHandle({ sigtermExits: true }); + let cleared = 0; + + await killChildHandle(handle, 5_000, { + logger, + // A child that exits before the deadline means the timer callback + // never fires; the deadline handle is still cleared exactly once. + setTimer: () => Symbol("timer"), + clearTimer: () => { + cleared += 1; + }, + }); + + expect(killSignals).toEqual(["SIGTERM"]); + expect(cleared).toBe(1); + }); + + test("a child that ignores SIGTERM is escalated to SIGKILL when the deadline wins", async () => { + const { handle, killSignals } = makeHandle({ sigtermExits: false }); + let cleared = 0; + const scheduled: (() => void)[] = []; + + // waitDeadline calls setTimer synchronously, before killChildHandle's + // first await, so the callback is captured by the time this call returns + // its pending promise. Firing it makes the deadline win the race. + const pending = killChildHandle(handle, 5_000, { + logger, + setTimer: (cb) => { + scheduled.push(cb); + return Symbol("timer"); + }, + clearTimer: () => { + cleared += 1; + }, + }); + expect(scheduled).toHaveLength(1); + scheduled[0]?.(); + await pending; + + expect(killSignals).toEqual(["SIGTERM", "SIGKILL"]); + expect(cleared).toBe(1); + }); + + test("omitting the timer deps falls back to real timers and still escalates", async () => { + const { handle, killSignals } = makeHandle({ sigtermExits: false }); + + // No setTimer/clearTimer keys -- the omit-key path the recycle call site + // takes when its context timers are undefined. A tiny real timeout drives + // the escalation without a fake timer. + await killChildHandle(handle, 5, { logger }); + + expect(killSignals).toEqual(["SIGTERM", "SIGKILL"]); + }); +}); + +describe("waitDeadline", () => { + test("resolves via the injected timer and surfaces the timer handle", async () => { + const scheduled: (() => void)[] = []; + const sentinel = Symbol("timer"); + const { promise, handle } = waitDeadline((cb) => { + scheduled.push(cb); + return sentinel; + }, 1_000); + + expect(handle).toBe(sentinel); + let resolved = false; + void promise.then(() => { + resolved = true; + }); + expect(resolved).toBe(false); + scheduled[0]?.(); + await promise; + expect(resolved).toBe(true); + }); + + test("defaultSetTimer schedules the callback via the real event loop", async () => { + let fired = false; + defaultSetTimer(() => { + fired = true; + }, 1); + await new Promise((r) => setTimeout(r, 15)); + expect(fired).toBe(true); + }); +}); diff --git a/packages/workflow-host/src/supervisor/child-termination.ts b/packages/workflow-host/src/supervisor/child-termination.ts new file mode 100644 index 00000000..f05e5524 --- /dev/null +++ b/packages/workflow-host/src/supervisor/child-termination.ts @@ -0,0 +1,97 @@ +// Child termination plus the resolve-only deadline / injectable-timer +// primitives its escalation drives. Two supervisor paths need to bound a +// wait on a workflow-process child with a timer and then force the child +// down: the recycle path (SIGTERM -> deadline -> SIGKILL between cohorts) +// and the spawn path's ready-handshake timeout (a child that spawns but +// never emits `ready`). Both need the same `killChildHandle` escalation +// and the same resolve-only `waitDeadline` raced against a child event, +// and both need injectable timers so tests can drive the deadline +// deterministically. Factoring them here keeps one implementation instead +// of a copy per path. + +import { getLogger } from "@intx/log"; + +import type { SubprocessHandle } from "./types"; + +/** + * Default kill-timeout between SIGTERM and SIGKILL. Used when a caller + * supplies no override; the recycle path exposes it as a per-deployment + * override and the spawn ready-timeout teardown uses it directly. + */ +export const DEFAULT_KILL_TIMEOUT_MS = 5_000; + +/** + * Injected dependencies for `killChildHandle`. `setTimer`/`clearTimer` + * default to the real `setTimeout`/`clearTimeout` when omitted so tests + * can substitute a deterministic timer. `logger` is supplied by the + * caller so the SIGKILL-escalation warning is attributed to the path that + * initiated the kill (recycle vs. spawn) rather than a single shared + * namespace. + */ +export interface KillChildHandleDeps { + setTimer?: (cb: () => void, ms: number) => unknown; + clearTimer?: (handle: unknown) => void; + logger: ReturnType; +} + +/** + * Issue SIGTERM and wait for the child to exit. If the exit does not land + * within `killTimeoutMs`, escalate to SIGKILL and wait again. SIGKILL is + * unignorable, so `exited` is guaranteed to settle -- a child that traps + * or never services SIGTERM cannot wedge this call. The supervisor's + * spawner returns the `exited` promise; this helper does not consult OS + * primitives directly. + */ +export async function killChildHandle( + handle: SubprocessHandle, + killTimeoutMs: number, + deps: KillChildHandleDeps, +): Promise { + const setTimer = deps.setTimer ?? defaultSetTimer; + const clearTimer = deps.clearTimer ?? defaultClearTimer; + + handle.kill("SIGTERM"); + const sigTermDeadline = waitDeadline(setTimer, killTimeoutMs); + const exitedFirst = await Promise.race([ + handle.exited.then(() => "exited" as const), + sigTermDeadline.promise.then(() => "deadline" as const), + ]); + if (exitedFirst === "exited") { + clearTimer(sigTermDeadline.handle); + return; + } + clearTimer(sigTermDeadline.handle); + deps.logger + .warn`child termination: SIGTERM did not land within ${String(killTimeoutMs)}ms; escalating to SIGKILL`; + handle.kill("SIGKILL"); + await handle.exited.catch(() => { + /* swallowed: a non-zero exit on SIGKILL is the expected outcome; + termination treats handle exit as success regardless of code. */ + }); +} + +/** + * A resolve-only deadline: a promise that resolves after `ms` via the + * injected `setTimer`, plus the timer handle so the caller can cancel it + * with the matching `clearTimer` once the race settles. Never rejects, so + * racing it against a rejecting promise leaves no unhandled rejection. + */ +export function waitDeadline( + setTimer: (cb: () => void, ms: number) => unknown, + ms: number, +): { promise: Promise; handle: unknown } { + let h: unknown; + const promise = new Promise((resolve) => { + h = setTimer(() => resolve(), ms); + }); + return { promise, handle: h }; +} + +export function defaultSetTimer(cb: () => void, ms: number): unknown { + return setTimeout(cb, ms); +} + +export function defaultClearTimer(handle: unknown): void { + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- production wiring; the handle is the value `setTimeout` returned, narrowed back at the boundary + clearTimeout(handle as ReturnType); +} diff --git a/packages/workflow-host/src/supervisor/index.ts b/packages/workflow-host/src/supervisor/index.ts index 9ef4574b..7b42e0cc 100644 --- a/packages/workflow-host/src/supervisor/index.ts +++ b/packages/workflow-host/src/supervisor/index.ts @@ -46,10 +46,11 @@ export { type DrainTimeoutOpts, } from "./drain-timeout"; +export { DEFAULT_KILL_TIMEOUT_MS } from "./child-termination"; + export { createRecyclePolicy, triggerRecycle, - DEFAULT_KILL_TIMEOUT_MS, DEFAULT_POLICY_INTERVAL_MS, MAX_BUFFERED_MAIL, type ChildWiring, diff --git a/packages/workflow-host/src/supervisor/recycle.ts b/packages/workflow-host/src/supervisor/recycle.ts index 0da6521c..f0ce145b 100644 --- a/packages/workflow-host/src/supervisor/recycle.ts +++ b/packages/workflow-host/src/supervisor/recycle.ts @@ -101,6 +101,7 @@ import { type CredentialsSnapshot, } from "./credentials"; import type { SubprocessHandle, WorkflowSupervisorBindings } from "./types"; +import { DEFAULT_KILL_TIMEOUT_MS, killChildHandle } from "./child-termination"; const logger = getLogger(["workflow-host", "supervisor", "recycle"]); @@ -112,13 +113,6 @@ const logger = getLogger(["workflow-host", "supervisor", "recycle"]); */ export const MAX_BUFFERED_MAIL = 256; -/** - * Default kill-timeout between SIGTERM and SIGKILL during step 2. - * Operator-overridable per deployment via the supervisor's recycle - * bindings; this is the value used when no override is supplied. - */ -export const DEFAULT_KILL_TIMEOUT_MS = 5_000; - /** * Default supervisor-policy check interval. The policy thread wakes * roughly every minute, evaluates the configured bounds against the @@ -304,7 +298,11 @@ export async function triggerRecycle( // `killTimeoutMs`, the handle's hard kill lands. The injected // spawner's `SubprocessHandle.kill()` is the surface the recycle // path touches; the spawner owns the Node primitives. - await killChildHandle(ctx.current.handle, killTimeoutMs, ctx); + await killChildHandle(ctx.current.handle, killTimeoutMs, { + logger, + ...(ctx.setTimer !== undefined ? { setTimer: ctx.setTimer } : {}), + ...(ctx.clearTimer !== undefined ? { clearTimer: ctx.clearTimer } : {}), + }); // Step 3: respawn. Fresh channelId, fresh HMAC key, fresh Ed25519 // IPC keypair. Per-step credentials are re-read so a grants update @@ -403,60 +401,6 @@ export async function triggerRecycle( }; } -/** - * Issue SIGTERM and wait for the child to exit. If the exit does not - * land within `killTimeoutMs`, escalate to SIGKILL and wait again. - * The supervisor's spawner returns the `exited` promise; the recycle - * path does not consult OS primitives directly. - */ -async function killChildHandle( - handle: SubprocessHandle, - killTimeoutMs: number, - ctx: RecycleContext, -): Promise { - const setTimer = ctx.setTimer ?? defaultSetTimer; - const clearTimer = ctx.clearTimer ?? defaultClearTimer; - - handle.kill("SIGTERM"); - const sigTermDeadline = waitDeadline(setTimer, killTimeoutMs); - const exitedFirst = await Promise.race([ - handle.exited.then(() => "exited" as const), - sigTermDeadline.promise.then(() => "deadline" as const), - ]); - if (exitedFirst === "exited") { - clearTimer(sigTermDeadline.handle); - return; - } - clearTimer(sigTermDeadline.handle); - logger.warn`recycle: SIGTERM did not land within ${String(killTimeoutMs)}ms; escalating to SIGKILL`; - handle.kill("SIGKILL"); - await handle.exited.catch(() => { - /* swallowed: a non-zero exit on SIGKILL is the expected outcome; - the recycle path treats handle exit as success regardless of - code. */ - }); -} - -function waitDeadline( - setTimer: (cb: () => void, ms: number) => unknown, - ms: number, -): { promise: Promise; handle: unknown } { - let h: unknown; - const promise = new Promise((resolve) => { - h = setTimer(() => resolve(), ms); - }); - return { promise, handle: h }; -} - -function defaultSetTimer(cb: () => void, ms: number): unknown { - return setTimeout(cb, ms); -} - -function defaultClearTimer(handle: unknown): void { - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- production wiring; the handle is the value `setTimeout` returned, narrowed back at the boundary - clearTimeout(handle as ReturnType); -} - /** * Iterate the new child's control-receive iterator until the `ready` * frame lands. Identical shape to the supervisor's spawn-time helper; From e991c565a6b80c69dcee62f8598df17f1fc6c13e Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Thu, 2 Jul 2026 14:02:15 -0500 Subject: [PATCH 024/101] Bound the child spawn ready handshake with a timeout The supervisor's spawn awaited the child's `ready` handshake with no bound: a child that spawned but neither signalled ready nor exited blocked spawn forever. The sidecar awaits restore spawns serially at boot before it connects to the hub, so one wedged child took the whole sidecar dark on a restart, silently. Race the ready wait against a deadline. On expiry, kill the child -- SIGTERM then SIGKILL, since a wedged child may ignore SIGTERM, and SIGKILL guarantees its exit settles -- and reject the spawn. The ready promise's three outcomes (ready, child-exit failure, timeout) are folded into values so the race never rejects and the single deadline-timer clear runs on every path before we act on the result; a race that could reject would skip the clear on the child-exit path and leak an armed deadline that keeps the event loop alive. Callers need no change. A live deploy surfaces the rejection as a deploy failure; boot-time restore logs it, keeps the record, and moves to the next deployment -- a wedged child no longer hangs either. The timeout is operator config: `CHILD_READY_TIMEOUT_MS` resolved at the sidecar boot edge and threaded to every supervisor, defaulting to 30s. The kill escalation reuses the existing 5s kill-timeout default. --- apps/sidecar/src/index.ts | 18 +++ apps/sidecar/src/workflow-host-wiring.test.ts | 36 +++++ apps/sidecar/src/workflow-host-wiring.ts | 25 ++++ .../src/supervisor/child-termination.ts | 4 +- .../src/supervisor/supervisor.test.ts | 124 ++++++++++++++++++ .../src/supervisor/supervisor.ts | 72 ++++++++-- .../workflow-host/src/supervisor/types.ts | 20 ++- 7 files changed, 283 insertions(+), 16 deletions(-) diff --git a/apps/sidecar/src/index.ts b/apps/sidecar/src/index.ts index 5e7f5813..6509efec 100644 --- a/apps/sidecar/src/index.ts +++ b/apps/sidecar/src/index.ts @@ -166,6 +166,23 @@ if (consumedRetentionRaw !== undefined && consumedRetentionRaw.trim() !== "") { consumedRetentionMs = parsed; } +// Bound on the child's spawn-time `ready` handshake. Threaded to every +// per-deployment supervisor; on expiry the supervisor kills the child and +// rejects the spawn, so a child that spawns but never signals ready fails +// the deploy (or is skipped by boot-time restore) instead of hanging it. +// Absent, the supervisor applies its 30s default. +const readyTimeoutRaw = process.env["CHILD_READY_TIMEOUT_MS"]; +let readyTimeoutMs: number | undefined; +if (readyTimeoutRaw !== undefined && readyTimeoutRaw.trim() !== "") { + const parsed = Number.parseInt(readyTimeoutRaw, 10); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error( + `CHILD_READY_TIMEOUT_MS must be a positive integer (milliseconds), got ${readyTimeoutRaw}`, + ); + } + readyTimeoutMs = parsed; +} + // Sweep any tmp staging directories left behind by a `put` or // `extractTarball` that crashed between staging and the final rename // on a previous boot. Running here, before the orchestrator starts @@ -382,6 +399,7 @@ const orchestrator = createSidecarOrchestrator({ ...(onDispatchTiming !== undefined ? { onDispatchTiming } : {}), ...(repackEveryMessages !== undefined ? { repackEveryMessages } : {}), ...(consumedRetentionMs !== undefined ? { consumedRetentionMs } : {}), + ...(readyTimeoutMs !== undefined ? { readyTimeoutMs } : {}), }); // Capture the router so the boot edge can drive its restore pass before // connecting. `createDeployRouter` runs synchronously during diff --git a/apps/sidecar/src/workflow-host-wiring.test.ts b/apps/sidecar/src/workflow-host-wiring.test.ts index 590272c2..81b87b6c 100644 --- a/apps/sidecar/src/workflow-host-wiring.test.ts +++ b/apps/sidecar/src/workflow-host-wiring.test.ts @@ -1155,6 +1155,13 @@ describe("createSidecarDeployRouter multi-step branch", () => { * restart starts with an empty registration table). */ transport?: ReturnType; + /** + * Spawn ready-handshake timeout (ms) threaded to every supervisor the + * router constructs. The ready-timeout test uses a small value with a + * spawner that never drives `ready`, asserting the deploy rejects with + * the threaded value echoed in the message. + */ + readyTimeoutMs?: number; }) { const transport = opts.transport ?? createInMemoryTransport(); const keyPair = await generateKeyPair(); @@ -1224,6 +1231,9 @@ describe("createSidecarDeployRouter multi-step branch", () => { ...(opts.multistepMailRouter !== undefined ? { multistepMailRouter: opts.multistepMailRouter } : {}), + ...(opts.readyTimeoutMs !== undefined + ? { readyTimeoutMs: opts.readyTimeoutMs } + : {}), }); return { router, @@ -2370,4 +2380,30 @@ describe("createSidecarDeployRouter multi-step branch", () => { // again, can retry it. expect(await recordExists(dataDir, deploymentId)).toBe(true); }); + + test("a deploy whose child never signals ready times out and rejects", async () => { + const dataDir = await createTempBaseDir("sidecar-ready-timeout-data-"); + const head = "ins_readytimeout@example.com"; + const deploymentId = deriveTrivialDeploymentId(head); + + // A spawner whose child is created but never driven through the `ready` + // handshake. With a small threaded readyTimeoutMs the supervisor times + // out, kills the child, and rejects the spawn. The message echoes the + // threaded value, so this also proves readyTimeoutMs reaches the + // supervisor across the router's forwarding. + const spawner = makeReadyDrivingSpawner(10100); + const { router } = await buildMultistepFixture({ + spawner: spawner.spawner, + readyTimeoutMs: 40, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + + await expect( + router.deploy(singleStepFrame(head, "wf-readytimeout")), + ).rejects.toThrow(/did not emit ready within 40ms/); + + // The deploy soft-failed, so its restore record was cleaned up -- a + // wedged deploy leaves nothing for a later boot to re-spawn. + expect(await recordExists(dataDir, deploymentId)).toBe(false); + }); }); diff --git a/apps/sidecar/src/workflow-host-wiring.ts b/apps/sidecar/src/workflow-host-wiring.ts index 2e41e930..fe457810 100644 --- a/apps/sidecar/src/workflow-host-wiring.ts +++ b/apps/sidecar/src/workflow-host-wiring.ts @@ -471,6 +471,13 @@ export type CreateSidecarWorkflowSupervisorOpts = { * supervisor applies `DEFAULT_CONSUMED_RETENTION_MS` (24h). */ consumedRetentionMs?: number; + /** + * Spawn ready-handshake timeout (ms), forwarded to the supervisor's + * `readyTimeoutMs` binding. The boot edge resolves the operator's + * `CHILD_READY_TIMEOUT_MS` config; absent, the supervisor applies + * `DEFAULT_READY_TIMEOUT_MS` (30s). + */ + readyTimeoutMs?: number; }; export type SidecarWorkflowSupervisor = { @@ -869,6 +876,15 @@ export function createSidecarDeployRouter(deps: { * handler for the operator-owned horizon invariant. */ consumedRetentionMs?: number; + /** + * Spawn ready-handshake timeout (ms) forwarded to every supervisor the + * router constructs. The sidecar boot edge resolves the operator's + * `CHILD_READY_TIMEOUT_MS` config; absent, the supervisor applies + * `DEFAULT_READY_TIMEOUT_MS` (30s). A child that spawns but never + * signals ready is killed and its spawn rejected rather than hanging + * the deploy or boot-time restore. + */ + readyTimeoutMs?: number; }): SidecarDeployRouter { // Validate the signing seed at construction so a malformed key fails // sidecar boot rather than the first multi-step deploy, where the @@ -1150,6 +1166,9 @@ export function createSidecarDeployRouter(deps: { ...(deps.consumedRetentionMs !== undefined ? { consumedRetentionMs: deps.consumedRetentionMs } : {}), + ...(deps.readyTimeoutMs !== undefined + ? { readyTimeoutMs: deps.readyTimeoutMs } + : {}), }); // OUTBOUND half of mailbox ownership (§3a): register the spawned @@ -1527,6 +1546,9 @@ export function createSidecarDeployRouter(deps: { ...(deps.consumedRetentionMs !== undefined ? { consumedRetentionMs: deps.consumedRetentionMs } : {}), + ...(deps.readyTimeoutMs !== undefined + ? { readyTimeoutMs: deps.readyTimeoutMs } + : {}), }); await wired.supervisor.deploy({ agentAddress: frame.agentAddress, @@ -1910,6 +1932,9 @@ export function createSidecarWorkflowSupervisor( ...(opts.consumedRetentionMs !== undefined ? { consumedRetentionMs: opts.consumedRetentionMs } : {}), + ...(opts.readyTimeoutMs !== undefined + ? { readyTimeoutMs: opts.readyTimeoutMs } + : {}), }); return { supervisor, diff --git a/packages/workflow-host/src/supervisor/child-termination.ts b/packages/workflow-host/src/supervisor/child-termination.ts index f05e5524..90aeef90 100644 --- a/packages/workflow-host/src/supervisor/child-termination.ts +++ b/packages/workflow-host/src/supervisor/child-termination.ts @@ -73,8 +73,8 @@ export async function killChildHandle( /** * A resolve-only deadline: a promise that resolves after `ms` via the * injected `setTimer`, plus the timer handle so the caller can cancel it - * with the matching `clearTimer` once the race settles. Never rejects, so - * racing it against a rejecting promise leaves no unhandled rejection. + * with the matching `clearTimer` once the race settles. It only ever + * resolves, so it contributes no rejection of its own to a race. */ export function waitDeadline( setTimer: (cb: () => void, ms: number) => unknown, diff --git a/packages/workflow-host/src/supervisor/supervisor.test.ts b/packages/workflow-host/src/supervisor/supervisor.test.ts index 280af860..378abdd6 100644 --- a/packages/workflow-host/src/supervisor/supervisor.test.ts +++ b/packages/workflow-host/src/supervisor/supervisor.test.ts @@ -740,6 +740,130 @@ describe("createWorkflowSupervisor", () => { expect(mailBus.registered()).not.toContain("deployment-x@example.com"); }); + // Harness for the ready-timeout tests: an injected FakeTimer registry + // (deterministic, per greybeard's ruling against real timers) plus a + // controllable child whose control reader the test can close to model a + // child that exits before signalling ready. `createdTimers` retains every + // timer even after it is cleared, so a test can capture the ready deadline + // and later assert it was cancelled. + async function makeReadyTimeoutHarness(readyTimeoutMs: number) { + type FakeTimer = { cb: () => void; ms: number; cancelled: boolean }; + const timers = new Set(); + const createdTimers: FakeTimer[] = []; + + const baseDir = await makeTempDir("supervisor-ready-timeout-"); + const supervisorIpcKeyPair = await generateKeyPair(); + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const eventChildToSupervisor = createMemoryFrameStream(); + let resolveExit: ((code: number) => void) | undefined; + const exited = new Promise((resolve) => { + resolveExit = resolve; + }); + const killSignals: string[] = []; + + const spawner: SubprocessSpawner = ({ env: _env }) => ({ + pid: 5150, + controlWriter: supervisorToChild.writer, + controlReader: childToSupervisor.reader, + eventReader: eventChildToSupervisor.reader, + kill: (signal) => { + killSignals.push( + typeof signal === "string" ? signal : String(signal ?? ""), + ); + childToSupervisor.close(); + eventChildToSupervisor.close(); + resolveExit?.(0); + }, + exited, + }); + + const mailBus = createMockMailBus(); + const baseBindings = await buildBindings({ + baseDir, + spawner, + signSpy: () => ({ sig: new Uint8Array(64), principalKind: "supervisor" }), + mailBus, + }); + const bindings: WorkflowSupervisorBindings = { + ...baseBindings, + ipcKeyPairFactory: () => Promise.resolve(supervisorIpcKeyPair), + readyTimeoutMs, + setTimer: (cb, ms) => { + const t: FakeTimer = { cb, ms, cancelled: false }; + timers.add(t); + createdTimers.push(t); + return t; + }, + clearTimer: (handle) => { + if (handle === null || typeof handle !== "object") return; + for (const t of timers) { + if (t === handle) { + t.cancelled = true; + timers.delete(t); + return; + } + } + }, + }; + const supervisor = createWorkflowSupervisor(bindings); + + // Resolve once the spawn has armed its ready deadline (which happens + // after the spawner is invoked, so this also confirms the child spawned). + async function waitForReadyDeadline(): Promise { + for (;;) { + const t = createdTimers.find((x) => x.ms === readyTimeoutMs); + if (t !== undefined) return t; + await new Promise((r) => setTimeout(r, 1)); + } + } + + return { supervisor, killSignals, childToSupervisor, waitForReadyDeadline }; + } + + const readyTimeoutSpawnOpts = { + stepOrder: ["step-1"], + definitionHash: "def-hash-abc", + warmKeep: false, + onInferenceEvent: () => { + /* unused in the ready-timeout tests */ + }, + }; + + test("spawn times out, kills the child, rejects, and clears the ready deadline", async () => { + const h = await makeReadyTimeoutHarness(7_777); + // Never send `ready`. Spawn blocks on the handshake until the deadline. + const spawnPromise = h.supervisor.spawn(readyTimeoutSpawnOpts); + const readyDeadline = await h.waitForReadyDeadline(); + readyDeadline.cb(); + + await expect(spawnPromise).rejects.toThrow( + /child did not emit ready within 7777ms; killed/, + ); + expect(h.killSignals).toContain("SIGTERM"); + // The unconditional deadline-timer clear ran on the timeout path. + expect(readyDeadline.cancelled).toBe(true); + }); + + test("spawn clears the ready deadline when the child exits before ready", async () => { + const h = await makeReadyTimeoutHarness(8_888); + const spawnPromise = h.supervisor.spawn(readyTimeoutSpawnOpts); + const readyDeadline = await h.waitForReadyDeadline(); + + // The child exits before signalling ready: closing the control reader + // ends `waitForReady`, rejecting the ready promise. Because the outcomes + // are folded to values, the race resolves to the failed outcome rather + // than rejecting, so the unconditional deadline-timer clear still runs. + // A race that rejected here would skip the clear and leak an armed + // deadline that keeps the event loop alive for up to readyTimeoutMs. + h.childToSupervisor.close(); + + await expect(spawnPromise).rejects.toThrow( + /control channel ended before child emitted ready/, + ); + expect(readyDeadline.cancelled).toBe(true); + }); + test("drain() forwards the `drain` control frame and arms a drainTimeout accumulator per in-flight run", async () => { const baseDir = await makeTempDir("supervisor-drain-arm-"); await seedStepGrants( diff --git a/packages/workflow-host/src/supervisor/supervisor.ts b/packages/workflow-host/src/supervisor/supervisor.ts index daaff2ad..7bfa8624 100644 --- a/packages/workflow-host/src/supervisor/supervisor.ts +++ b/packages/workflow-host/src/supervisor/supervisor.ts @@ -115,9 +115,28 @@ import { createTerminalBroadcaster, type TerminalBroadcaster, } from "./terminal-broadcaster"; +import { + DEFAULT_KILL_TIMEOUT_MS, + defaultClearTimer, + defaultSetTimer, + killChildHandle, + waitDeadline, +} from "./child-termination"; const logger = getLogger(["workflow-host", "supervisor"]); +/** + * Default bound on the child's spawn-time `ready` handshake. A spawned + * child that neither emits `ready` nor exits would otherwise block + * `spawn` forever; on expiry the supervisor kills the child and rejects + * the spawn. Generous enough for a real child to boot, open its IPC + * channels, and sign `ready` under load; short enough that a wedged child + * does not stall the spawn (and, on the sidecar, boot-time restore) + * indefinitely. Operator-overridable via `WorkflowSupervisorBindings. + * readyTimeoutMs`. + */ +const DEFAULT_READY_TIMEOUT_MS = 30_000; + /** * Default watchdog timeout for the supervisor's * `synchronouslyDispatchTerminalWrite`. The handler holds the @@ -554,6 +573,14 @@ export function createWorkflowSupervisor( // consumedRetentionMs` for the operator-owned invariant. const consumedRetentionMs = bindings.consumedRetentionMs ?? DEFAULT_CONSUMED_RETENTION_MS; + // Resolve the spawn ready-handshake timeout and its timers once at the + // bindings edge. The timers reuse the same injectable pair the drain + // path resolves (`bindings.setTimer`/`clearTimer`); the ready-timeout + // race and its kill-escalation drive them, and tests substitute a + // deterministic timer through the same bindings. + const readyTimeoutMs = bindings.readyTimeoutMs ?? DEFAULT_READY_TIMEOUT_MS; + const readySetTimer = bindings.setTimer ?? defaultSetTimer; + const readyClearTimer = bindings.clearTimer ?? defaultClearTimer; /** * Resolved on every successful `enqueueInbox`; the dispatch loop * awaits this promise after a null dequeue so it returns to @@ -1463,7 +1490,41 @@ export function createWorkflowSupervisor( ); state.mailUnsubscribe = mailUnsubscribe; - const readyInfo = await wired.readyPromise; + // Bound the `ready` handshake. `wired.readyPromise` resolves on `ready` + // and rejects when the control channel ends (the child exited); a child + // that neither readies nor exits would block here forever. Fold all three + // outcomes into values so the single `readyClearTimer` below runs on every + // path -- ready, child-exit failure, and timeout -- before we act on the + // result. A `Promise.race` that could reject would skip the clear on the + // child-exit path and leak an armed deadline that keeps the event loop + // alive for up to `readyTimeoutMs`. The deadline is resolve-only, so it + // contributes no rejection of its own. Kill on timeout uses the + // SIGTERM->SIGKILL escalation because a wedged child may ignore SIGTERM; + // SIGKILL guarantees `exited` settles. + const readyOutcome = wired.readyPromise.then( + (info) => ({ kind: "ready" as const, info }), + (err: unknown) => ({ kind: "failed" as const, err }), + ); + const readyDeadline = waitDeadline(readySetTimer, readyTimeoutMs); + const readyRace = await Promise.race([ + readyOutcome, + readyDeadline.promise.then(() => ({ kind: "timeout" as const })), + ]); + readyClearTimer(readyDeadline.handle); + if (readyRace.kind === "timeout") { + await killChildHandle(wired.wiring.handle, DEFAULT_KILL_TIMEOUT_MS, { + setTimer: readySetTimer, + clearTimer: readyClearTimer, + logger, + }); + throw new Error( + `workflow-host supervisor: child did not emit ready within ${readyTimeoutMs}ms; killed`, + ); + } + if (readyRace.kind === "failed") { + throw readyRace.err; + } + const readyInfo = readyRace.info; // Push the assembled credentialsSnapshot to the child before the // mail buffer drains. Without this, the child's @@ -2510,15 +2571,6 @@ function defaultNow(): number { return Date.now(); } -function defaultSetTimer(cb: () => void, ms: number): unknown { - return setTimeout(cb, ms); -} - -function defaultClearTimer(handle: unknown): void { - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the handle is the value `setTimeout` returned, narrowed back at the boundary - clearTimeout(handle as ReturnType); -} - /** * Drain the event-channel receive iterator into the host-supplied * sink. The function resolves when the iterator ends (child exit or diff --git a/packages/workflow-host/src/supervisor/types.ts b/packages/workflow-host/src/supervisor/types.ts index 27e4f598..a3864310 100644 --- a/packages/workflow-host/src/supervisor/types.ts +++ b/packages/workflow-host/src/supervisor/types.ts @@ -478,10 +478,11 @@ export interface WorkflowSupervisorBindings { */ now?: () => number; /** - * Scheduling primitive the supervisor threads into the drainTimeout - * accumulator. Production wires `(cb, ms) => setTimeout(cb, ms)`; - * tests inject a deterministic timer host. Defaults to - * `setTimeout` when omitted. + * General scheduling primitive the supervisor threads into its timed + * waits: the drainTimeout accumulator, the spawn ready-handshake + * timeout, and that timeout's SIGTERM->SIGKILL kill escalation. + * Production wires `(cb, ms) => setTimeout(cb, ms)`; tests inject a + * deterministic timer host. Defaults to `setTimeout` when omitted. */ setTimer?: (cb: () => void, ms: number) => unknown; /** @@ -571,6 +572,17 @@ export interface WorkflowSupervisorBindings { * refused stale enqueue, not silent double-processing). */ consumedRetentionMs?: number; + /** + * Bound on the child's spawn-time `ready` handshake, in milliseconds. + * A spawned child that neither emits `ready` nor exits would block + * `spawn` forever; on expiry the supervisor kills the child (SIGTERM, + * then SIGKILL) and rejects the spawn. The boot edge resolves the + * operator's config and supplies it; absent, `DEFAULT_READY_TIMEOUT_MS` + * (30s) applies. Callers surface the rejection through their existing + * spawn-failure path, so a wedged child fails the deploy (or, on the + * sidecar, is skipped by boot-time restore) instead of hanging it. + */ + readyTimeoutMs?: number; /** * Watchdog timeout (ms) for the supervisor's substrate-write * handler's wait on the dispatch loop's `markConsumed` when a From bcf40abb866340eeb0d0aad87da7dfe159c12ab8 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Thu, 2 Jul 2026 21:50:59 -0500 Subject: [PATCH 025/101] Re-register workflow deployment routes on sidecar reconnect The hub routes mail purely on an in-memory address index. A workflow deployment address enters that index only when the deployment is first sent to the sidecar; on a WS close the hub removes it, and nothing ever re-adds it. Single-agent sessions come back through the reconnect frame's challenge, but workflow deployments are on a separate, sidecar-local restore path the hub never hears about. So after a WS reconnect or a sidecar restart the hub silently drops mail to a live workflow deployment -- it lands in the undelivered queue until the deployment is redeployed. Have the sidecar announce the workflow-substrate addresses it currently hosts on every connect, and have the hub re-register them for routing. These addresses are hub-minted and carry no per-address key, so they re-register the same unchallenged way they first entered the index at deploy time -- the challenge is for session addresses, which have keys. The sidecar reads them live from its active-supervisor set (so a deploy arriving during the restore window is included) and carries them in a new frame field on both the register and reconnect frames; the hub tracks them in a per-connection set so a close removes exactly this connection's routes, and a later connection reclaiming an address is not clobbered by the prior owner's close. Keep that set physically distinct from the challenged session set -- they differ on the challenge/re-add path -- but give the connection's ownership readers a unified view. Pack-transfer authorization, in-flight cancellation, and disconnect teardown ask "does this connection own the address," which must be true for a reconnected workflow deployment too; otherwise its workflow-run pack pushes -- which keep the hub's read-only run-observation mirror current -- would be rejected as unrouted after a reconnect while its mail resumed. The disconnect event now carries every owned address, so its field is renamed to match. This is a pre-existing gap, independent of the disk-restore feature: it bites even on a bare reconnect with the sidecar process still alive. --- apps/sidecar/src/index.ts | 14 ++ apps/sidecar/src/workflow-host-wiring.ts | 18 ++ .../hub-agent/src/sidecar-orchestrator.ts | 10 + packages/hub-agent/src/ws/hub-link.ts | 42 +++- .../src/hub-session-orchestrator.test.ts | 4 +- .../src/hub-session-orchestrator.ts | 4 +- .../src/ws/sidecar-events.test.ts | 8 +- .../hub-sessions/src/ws/sidecar-events.ts | 7 +- .../src/ws/sidecar-handler.test.ts | 238 +++++++++++++++++- .../hub-sessions/src/ws/sidecar-handler.ts | 107 +++++++- packages/types/src/sidecar.ts | 10 + 11 files changed, 429 insertions(+), 33 deletions(-) diff --git a/apps/sidecar/src/index.ts b/apps/sidecar/src/index.ts index 6509efec..1725de09 100644 --- a/apps/sidecar/src/index.ts +++ b/apps/sidecar/src/index.ts @@ -370,6 +370,20 @@ const orchestrator = createSidecarOrchestrator({ mailInboundRouter: multistepMailRouter, signalInboundRouter: multistepSignalRouter, drainInboundRouter: multistepDrainRouter, + // The hub link calls this on every (re)connect to announce the workflow + // deployments this sidecar hosts so the hub re-registers their routes. + // `createDeployRouter` runs synchronously during construction (below), so + // the router is captured before the link ever connects; assert rather than + // optional-chain so a wiring regression fails loud instead of silently + // announcing no deployments. + getWorkflowAddresses: () => { + if (sidecarDeployRouter === undefined) { + throw new Error( + "sidecar boot: deploy router was not constructed before the hub link requested workflow addresses", + ); + } + return sidecarDeployRouter.activeAddresses(); + }, createDeployRouter: ({ sessions, keyStore, diff --git a/apps/sidecar/src/workflow-host-wiring.ts b/apps/sidecar/src/workflow-host-wiring.ts index fe457810..d81ced80 100644 --- a/apps/sidecar/src/workflow-host-wiring.ts +++ b/apps/sidecar/src/workflow-host-wiring.ts @@ -688,6 +688,17 @@ export interface SidecarDeployRouter extends DeployRouter { * disk for a later boot to retry -- it is never deleted here. */ restoreWorkflowDeployments(): Promise; + /** + * The workflow-substrate deployment addresses (`ins_dep_...`) this router + * currently hosts a live supervisor for -- the set of addresses this + * sidecar can route mail to. The boot edge announces these to the hub on + * (re)connect so the hub re-registers them for routing: they are hub-minted + * and carry no per-address key, so unlike single-agent sessions they are + * not re-established by the challenge flow, and without this announcement + * the hub drops their route on a WS reconnect. Reflects `deploy`/`undeploy` + * and boot-time restore live, so a caller re-reads it per connect. + */ + activeAddresses(): string[]; } export function createSidecarDeployRouter(deps: { @@ -1750,6 +1761,13 @@ export function createSidecarDeployRouter(deps: { } } }, + activeAddresses(): string[] { + // `activeSupervisors` is keyed by deployment agent address and holds + // exactly the deployments with a live supervisor (deploy and restore + // add; undeploy and spawn-unwind remove), so its keys are the addresses + // this sidecar can currently route mail to. + return [...activeSupervisors.keys()]; + }, }; } diff --git a/packages/hub-agent/src/sidecar-orchestrator.ts b/packages/hub-agent/src/sidecar-orchestrator.ts index 435a342a..9d9c6735 100644 --- a/packages/hub-agent/src/sidecar-orchestrator.ts +++ b/packages/hub-agent/src/sidecar-orchestrator.ts @@ -127,6 +127,14 @@ export type SidecarOrchestratorConfig = { * orchestrator forwards the binding unchanged to `createHubLink`. */ drainInboundRouter?: DrainInboundRouter; + /** + * Returns the workflow-substrate deployment addresses this sidecar + * currently hosts. Forwarded to the hub link, which announces them on + * every (re)connect so the hub re-registers them for routing without a + * challenge. Production wires this to the deploy router's + * `activeAddresses`; omitted, the link announces none. + */ + getWorkflowAddresses?: () => string[]; pingIntervalMs?: number; reconnectDelayMs?: number; scheduleReconnect?: ReconnectScheduler; @@ -160,6 +168,7 @@ export function createSidecarOrchestrator( mailInboundRouter, signalInboundRouter, drainInboundRouter, + getWorkflowAddresses, pingIntervalMs, reconnectDelayMs, scheduleReconnect, @@ -249,6 +258,7 @@ export function createSidecarOrchestrator( ...(mailInboundRouter !== undefined ? { mailInboundRouter } : {}), ...(signalInboundRouter !== undefined ? { signalInboundRouter } : {}), ...(drainInboundRouter !== undefined ? { drainInboundRouter } : {}), + ...(getWorkflowAddresses !== undefined ? { getWorkflowAddresses } : {}), ...(pingIntervalMs !== undefined ? { pingIntervalMs } : {}), ...(reconnectDelayMs !== undefined ? { reconnectDelayMs } : {}), ...(scheduleReconnect !== undefined ? { scheduleReconnect } : {}), diff --git a/packages/hub-agent/src/ws/hub-link.ts b/packages/hub-agent/src/ws/hub-link.ts index 5eb2544f..9ab66595 100644 --- a/packages/hub-agent/src/ws/hub-link.ts +++ b/packages/hub-agent/src/ws/hub-link.ts @@ -228,6 +228,16 @@ export type HubLinkConfig = { * observable rather than silent. */ drainInboundRouter?: DrainInboundRouter; + /** + * Returns the workflow-substrate deployment addresses this sidecar + * currently hosts a live supervisor for. Called on every (re)connect to + * announce them to the hub for routing: they are hub-minted with no + * per-address key, so they re-register WITHOUT the challenge flow, and + * without this announcement the hub drops their route on a WS reconnect. + * Defaults to none when omitted (tests / deployments with no workflow + * substrate). + */ + getWorkflowAddresses?: () => string[]; pingIntervalMs?: number; reconnectDelayMs?: number; scheduleReconnect?: ReconnectScheduler; @@ -280,6 +290,7 @@ export function createHubLink(config: HubLinkConfig): HubLink { mailInboundRouter, signalInboundRouter, drainInboundRouter, + getWorkflowAddresses = () => [], pingIntervalMs = DEFAULT_PING_INTERVAL_MS, reconnectDelayMs = DEFAULT_RECONNECT_DELAY_MS, scheduleReconnect = defaultScheduleReconnect, @@ -1038,6 +1049,15 @@ export function createHubLink(config: HubLinkConfig): HubLink { try { const { restored, failed } = await sessions.restoreSessions(); + // The live workflow-substrate deployment addresses this sidecar + // hosts, re-read AFTER restore so any deploy that arrived during the + // restore window is included. These are hub-minted (no per-address + // key) and re-register for routing WITHOUT a challenge -- the hub + // otherwise drops their route on a WS reconnect. Carried on whichever + // terminal frame fires below; the frame carries the complete current + // set, so a plain register does not wipe them. + const workflowAddresses = getWorkflowAddresses(); + if (restored.length > 0) { const deployRefs: Record = {}; for (const entry of restored) { @@ -1058,6 +1078,7 @@ export function createHubLink(config: HubLinkConfig): HubLink { sidecarId, token, agentAddresses: restored.map((e) => e.address), + ...(workflowAddresses.length > 0 ? { workflowAddresses } : {}), deployRefs, }); if (failed.length > 0) { @@ -1066,22 +1087,23 @@ export function createHubLink(config: HubLinkConfig): HubLink { logger.info`Sent reconnect with ${String(restored.length)} agent(s)`; } } else { - // Reached only when nothing was restored from disk. The - // empty register on open already established routability, so - // skip a second empty frame. Any address present here belongs - // to an agent provisioned during the restore window — - // provisions route on that open register — which the hub - // already placed in addressIndex when it routed the deploy. - // Re-announcing those is consistent with that entry, not an - // unchallenged write of a disk-restored address; those take - // the reconnect branch above. + // Nothing was restored from disk. The empty register on open + // already established routability for session addresses. Send a + // register when there are session addresses to re-announce OR + // workflow deployments to (re-)register -- a workflow-only sidecar + // restores nothing here yet must still announce its deployments. + // Session addresses present here belong to agents provisioned + // during the restore window (already in addressIndex from their + // deploy); disk-restored sessions take the challenged reconnect + // branch above. const addresses = sessions.getAddresses(); - if (addresses.length > 0) { + if (addresses.length > 0 || workflowAddresses.length > 0) { send({ type: "register", sidecarId, token, agentAddresses: addresses, + ...(workflowAddresses.length > 0 ? { workflowAddresses } : {}), }); } } diff --git a/packages/hub-sessions/src/hub-session-orchestrator.test.ts b/packages/hub-sessions/src/hub-session-orchestrator.test.ts index 1c8d3055..5c3aeebb 100644 --- a/packages/hub-sessions/src/hub-session-orchestrator.test.ts +++ b/packages/hub-sessions/src/hub-session-orchestrator.test.ts @@ -346,7 +346,7 @@ describe("createHubSessionOrchestrator", () => { describe("sidecar.disconnect", () => { test("abandons every collector for the closed connection", () => { harness.events.emit("sidecar.disconnect", { - agentAddresses: ["a@x", "b@x", "c@x"], + ownedAddresses: ["a@x", "b@x", "c@x"], }); const abandonedAddrs = harness.collectors.calls .filter((c) => c.kind === "abandon") @@ -522,7 +522,7 @@ describe("createHubSessionOrchestrator", () => { test("removes all subscriptions so later emits are inert", () => { harness.dispose(); harness.events.emit("sidecar.disconnect", { - agentAddresses: [AGENT_ADDRESS], + ownedAddresses: [AGENT_ADDRESS], }); const abandoned = harness.collectors.calls.find( (c) => c.kind === "abandon", diff --git a/packages/hub-sessions/src/hub-session-orchestrator.ts b/packages/hub-sessions/src/hub-session-orchestrator.ts index 0bda152c..e913c07e 100644 --- a/packages/hub-sessions/src/hub-session-orchestrator.ts +++ b/packages/hub-sessions/src/hub-session-orchestrator.ts @@ -91,8 +91,8 @@ export function createHubSessionOrchestrator( ); unsubscribers.push( - events.on("sidecar.disconnect", ({ agentAddresses }) => { - for (const addr of agentAddresses) { + events.on("sidecar.disconnect", ({ ownedAddresses }) => { + for (const addr of ownedAddresses) { eventCollectors.abandon(addr); } }), diff --git a/packages/hub-sessions/src/ws/sidecar-events.test.ts b/packages/hub-sessions/src/ws/sidecar-events.test.ts index 54388a77..d2714afd 100644 --- a/packages/hub-sessions/src/ws/sidecar-events.test.ts +++ b/packages/hub-sessions/src/ws/sidecar-events.test.ts @@ -28,7 +28,7 @@ describe("createSidecarEmitter", () => { order.push("b"); }); - emitter.emit("sidecar.disconnect", { agentAddresses: [] }); + emitter.emit("sidecar.disconnect", { ownedAddresses: [] }); expect(order).toEqual(["a", "b"]); }); @@ -40,9 +40,9 @@ describe("createSidecarEmitter", () => { count++; }); - emitter.emit("sidecar.disconnect", { agentAddresses: [] }); + emitter.emit("sidecar.disconnect", { ownedAddresses: [] }); unsubscribe(); - emitter.emit("sidecar.disconnect", { agentAddresses: [] }); + emitter.emit("sidecar.disconnect", { ownedAddresses: [] }); expect(count).toBe(1); }); @@ -57,7 +57,7 @@ describe("createSidecarEmitter", () => { seen.push("ran"); }); - emitter.emit("sidecar.disconnect", { agentAddresses: [] }); + emitter.emit("sidecar.disconnect", { ownedAddresses: [] }); expect(seen).toEqual(["ran"]); }); diff --git a/packages/hub-sessions/src/ws/sidecar-events.ts b/packages/hub-sessions/src/ws/sidecar-events.ts index 450b4dd7..44553cc4 100644 --- a/packages/hub-sessions/src/ws/sidecar-events.ts +++ b/packages/hub-sessions/src/ws/sidecar-events.ts @@ -53,10 +53,11 @@ export type SidecarEventMap = { }; /** Notification. Emitted once when a sidecar's connection closes, - * carrying the set of agent addresses that were registered on that - * connection. */ + * carrying every address the connection owned -- challenged session + * addresses and hub-minted workflow-substrate deployment addresses + * alike -- so lifecycle teardown covers both. */ "sidecar.disconnect": { - agentAddresses: string[]; + ownedAddresses: string[]; }; /** Notification. Emitted when a mail.outbound frame from a sidecar diff --git a/packages/hub-sessions/src/ws/sidecar-handler.test.ts b/packages/hub-sessions/src/ws/sidecar-handler.test.ts index 7854df0a..998516c0 100644 --- a/packages/hub-sessions/src/ws/sidecar-handler.test.ts +++ b/packages/hub-sessions/src/ws/sidecar-handler.test.ts @@ -171,6 +171,124 @@ describe("SidecarRouter", () => { }); }); + describe("workflow-address re-registration", () => { + test("register routes workflow addresses without a challenge", () => { + // The restart scenario: a sidecar hosting only workflow deployments + // reconnects and announces them via `workflowAddresses`. They must + // become routable directly -- they are hub-minted with no per-address + // key to challenge. Before this, such a sidecar sent no terminal frame + // carrying them, so the hub never re-learned the route. + const ws = createMockWs(); + router.handleOpen(ws); + router.handleMessage( + ws, + JSON.stringify({ + type: "register", + sidecarId: "sc-1", + token: "tok", + agentAddresses: [], + workflowAddresses: ["ins_dep_abc@local"], + }), + ); + + expect(router.getRoutableAddresses()).toEqual(["ins_dep_abc@local"]); + expect(router.routeMail("ins_dep_abc@local", "dGVzdA==")).toBe(true); + }); + + test("reconnect preserves a workflow address not in the challenged list", () => { + // A reconnect lists its workflow addresses only in `workflowAddresses`, + // never in the challenged `agentAddresses`. The internal empty-register + // clears the connection's addresses; the reconnect re-adds the workflow + // set, so it stays routable across the reconnect. + const ws = createMockWs(); + router.handleOpen(ws); + router.handleMessage( + ws, + JSON.stringify({ + type: "register", + sidecarId: "sc-1", + token: "tok", + agentAddresses: [], + workflowAddresses: ["ins_dep_abc@local"], + }), + ); + expect(router.routeMail("ins_dep_abc@local", "dGVzdA==")).toBe(true); + + router.handleMessage( + ws, + JSON.stringify({ + type: "reconnect", + sidecarId: "sc-1", + token: "tok", + agentAddresses: [], + workflowAddresses: ["ins_dep_abc@local"], + }), + ); + + expect(router.routeMail("ins_dep_abc@local", "dGVzdA==")).toBe(true); + }); + + test("disconnect removes workflow addresses from the routing table", () => { + // Guards the leak: without removing the connection's workflow addresses + // on close, a stale `addr -> dead ws` entry would survive in the routing + // table and the next sidecar could not cleanly reclaim it. + const ws = createMockWs(); + router.handleOpen(ws); + router.handleMessage( + ws, + JSON.stringify({ + type: "register", + sidecarId: "sc-1", + token: "tok", + agentAddresses: [], + workflowAddresses: ["ins_dep_abc@local"], + }), + ); + expect(router.routeMail("ins_dep_abc@local", "dGVzdA==")).toBe(true); + + router.handleClose(ws); + + expect(router.getRoutableAddresses()).toEqual([]); + expect(router.routeMail("ins_dep_abc@local", "dGVzdA==")).toBe(false); + }); + + test("a fresh ws reclaims a workflow address after an abrupt disconnect", () => { + // Restart without a clean close: the old ws still holds the route. A + // fresh ws re-announcing the same workflow address takes over routing, + // and the stale ws's later close must not clobber the new owner (the + // ownership guard in handleClose). + const oldWs = createMockWs(); + router.handleOpen(oldWs); + router.handleMessage( + oldWs, + JSON.stringify({ + type: "register", + sidecarId: "sc-1", + token: "tok", + agentAddresses: [], + workflowAddresses: ["ins_dep_abc@local"], + }), + ); + + const newWs = createMockWs(); + router.handleOpen(newWs); + router.handleMessage( + newWs, + JSON.stringify({ + type: "register", + sidecarId: "sc-1", + token: "tok", + agentAddresses: [], + workflowAddresses: ["ins_dep_abc@local"], + }), + ); + + router.handleClose(oldWs); + + expect(router.routeMail("ins_dep_abc@local", "dGVzdA==")).toBe(true); + }); + }); + describe("mail routing", () => { test("routes mail between two sidecars", () => { const ws1 = createMockWs(); @@ -925,8 +1043,8 @@ describe("SidecarRouter", () => { test("sidecar.disconnect is emitted on router.events", () => { const router = createSidecarRouter({}); const seen: string[][] = []; - router.events.on("sidecar.disconnect", ({ agentAddresses }) => { - seen.push(agentAddresses); + router.events.on("sidecar.disconnect", ({ ownedAddresses }) => { + seen.push(ownedAddresses); }); const ws = createMockWs(); @@ -2564,6 +2682,122 @@ describe("SidecarRouter", () => { expect(ack.repoId).toEqual(repoId); }); + test("a workflow deployment announced via workflowAddresses can still push workflow-run packs", async () => { + // The reconnect/restart shape: the deployment address is announced via + // `workflowAddresses`, not `agentAddresses`. Pack-push authorization must + // still recognize it as owned by this connection -- otherwise the hub's + // workflow-run observation mirror silently stops updating after a + // reconnect (mail routing resumes, but pack push would reject the + // address as unrouted). + const { router: r, calls } = buildPackRouter(); + const ws = createMockWs(); + const addr = "ins_dep_wfr@local"; + r.handleOpen(ws); + r.handleMessage( + ws, + JSON.stringify({ + type: "register", + sidecarId: "sc-wfr", + token: "tok", + agentAddresses: [], + workflowAddresses: [addr], + }), + ); + + const repoId: RepoId = { kind: "workflow-run", id: "dep-wfr-2" }; + pushPack(r, ws, { + agentAddress: addr, + repoId, + transferId: "t-wfr-2", + pack: new Uint8Array([9, 8, 7]), + ref: "refs/heads/events", + commitSha: "e".repeat(40), + }); + + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(calls.length).toBe(1); + expect(calls[0]?.method).toBe("receiveWorkflowRunPack"); + const ack = lastSent(ws); + expect(ack.type).toBe("repo.pack.ack"); + }); + + test("a superseded connection's close does not cancel the new owner's workflow-run transfer", async () => { + // Abrupt restart: a fresh ws reclaims a workflow address a still-open + // stale ws owns, then starts a pack transfer. When the stale ws finally + // closes, its teardown must not cancel the new owner's in-flight + // transfer (cancelByAgent is keyed by address, not connection). The + // ghost cleanup on the fresh register evicts the address from the stale + // connection so its close leaves the new owner alone. + const { router: r, calls } = buildPackRouter(); + const addr = "ins_dep_reclaim@local"; + const repoId: RepoId = { kind: "workflow-run", id: "dep-reclaim" }; + const pack = new Uint8Array([4, 5, 6, 7]); + const transferId = "t-reclaim"; + + const oldWs = createMockWs(); + r.handleOpen(oldWs); + r.handleMessage( + oldWs, + JSON.stringify({ + type: "register", + sidecarId: "sc", + token: "tok", + agentAddresses: [], + workflowAddresses: [addr], + }), + ); + + const newWs = createMockWs(); + r.handleOpen(newWs); + r.handleMessage( + newWs, + JSON.stringify({ + type: "register", + sidecarId: "sc", + token: "tok", + agentAddresses: [], + workflowAddresses: [addr], + }), + ); + + // The new owner starts (but does not finish) a workflow-run transfer. + for (const chunk of chunkPack(pack)) { + r.handleMessage( + newWs, + JSON.stringify({ + type: "repo.pack.push", + agentAddress: addr, + repoId, + transferId, + seq: chunk.seq, + data: chunk.data, + }), + ); + } + + // The superseded connection closes mid-transfer. + r.handleClose(oldWs); + + // The new owner completes the transfer; it must not have been cancelled. + r.handleMessage( + newWs, + JSON.stringify({ + type: "repo.pack.done", + agentAddress: addr, + repoId, + transferId, + ref: "refs/heads/events", + commitSha: "a".repeat(40), + }), + ); + + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(calls.length).toBe(1); + expect(calls[0]?.method).toBe("receiveWorkflowRunPack"); + }); + test("agent-state and workflow-run packs use independent receivers (concurrent transferIds)", async () => { const { router: r, calls } = buildPackRouter(); const ws = createMockWs(); diff --git a/packages/hub-sessions/src/ws/sidecar-handler.ts b/packages/hub-sessions/src/ws/sidecar-handler.ts index 1f5bee0c..0057f874 100644 --- a/packages/hub-sessions/src/ws/sidecar-handler.ts +++ b/packages/hub-sessions/src/ws/sidecar-handler.ts @@ -36,9 +36,35 @@ const logger = getLogger(["hub", "ws", "sidecar"]); export type SidecarConnection = { sidecarId: string; agentAddresses: Set; + // Workflow-substrate deployment addresses (ins_dep_...) this connection + // hosts. Kept separate from `agentAddresses`: these are hub-minted and + // registered for routing WITHOUT the per-address challenge, so they must + // not be dragged through the challenge/re-add dance the session addresses + // take. `handleClose` cleans both sets out of `addressIndex`. + workflowAddresses: Set; send(frame: HubFrame): void; }; +/** + * Whether this connection owns `address` for routing/lifecycle purposes -- + * as a challenged session address OR a hub-minted workflow-substrate address. + * The two sets are kept physically distinct (they differ on the challenge / + * re-add path), but ownership readers -- pack-transfer authorization, + * in-flight cancellation, disconnect teardown -- must see the union, or a + * reconnected workflow deployment (which lives only in `workflowAddresses`) + * is silently treated as unowned even though its mail routes. + */ +function connOwnsAddress(conn: SidecarConnection, address: string): boolean { + return ( + conn.agentAddresses.has(address) || conn.workflowAddresses.has(address) + ); +} + +/** The deduped set of every address this connection owns (session + workflow). */ +function ownedAddresses(conn: SidecarConnection): Set { + return new Set([...conn.agentAddresses, ...conn.workflowAddresses]); +} + type PendingRequest = { requestId: string; ws: WsHandle; @@ -379,7 +405,13 @@ export function createSidecarRouter( switch (frame.type) { case "register": - handleRegister(ws, frame.sidecarId, frame.token, frame.agentAddresses); + handleRegister( + ws, + frame.sidecarId, + frame.token, + frame.agentAddresses, + frame.workflowAddresses ?? [], + ); break; case "reconnect": void handleReconnect( @@ -387,6 +419,7 @@ export function createSidecarRouter( frame.sidecarId, frame.token, frame.agentAddresses, + frame.workflowAddresses ?? [], frame.deployRefs ?? {}, ); break; @@ -499,6 +532,7 @@ export function createSidecarRouter( sidecarId: string, token: string, agentAddresses: string[], + workflowAddresses: string[] = [], ): void { if (validateToken !== undefined && !validateToken(sidecarId, token)) { logger.warn`Rejected registration from sidecar ${sidecarId}: invalid token`; @@ -519,9 +553,15 @@ export function createSidecarRouter( for (const addr of existing.agentAddresses) { addressIndex.delete(addr); } + for (const addr of existing.workflowAddresses) { + addressIndex.delete(addr); + } } const addrSet = new Set(agentAddresses); + // The frame carries the COMPLETE current live workflow-address set, so + // this replaces (not merges) the connection's workflow routing. + const workflowSet = new Set(workflowAddresses); // Clean up ghost entries from other connections that previously // owned addresses this sidecar is now claiming. @@ -550,9 +590,28 @@ export function createSidecarRouter( } } + // Same ghost cleanup for reclaimed workflow-substrate addresses: evict the + // address from the prior owner's set. Without this, when the superseded + // connection later closes (the abrupt-restart overlap window), its + // `handleClose` teardown would iterate the stale address and cancel THIS + // connection's live in-flight pack transfer / abandon its collector for + // the deployment it just took over. `cancelByAgent` is keyed by address, + // not by connection, so the stale close would hit the new owner. The + // session loop above evicts the reclaimed address the same way. + for (const addr of workflowSet) { + const prevWs = addressIndex.get(addr); + if (prevWs !== undefined && prevWs !== ws) { + const prevConn = connections.get(prevWs); + if (prevConn !== undefined) { + prevConn.workflowAddresses.delete(addr); + } + } + } + const conn: SidecarConnection = { sidecarId, agentAddresses: addrSet, + workflowAddresses: workflowSet, send(frame: HubFrame) { ws.send(JSON.stringify(frame)); }, @@ -573,8 +632,17 @@ export function createSidecarRouter( disconnectedAgents.delete(addr); } } + // Re-register workflow-substrate addresses for routing directly. They are + // hub-minted with no per-address key (no `agent_instance` row), so there + // is no challenge to run -- the same way they first entered `addressIndex` + // at deploy time via `sendAgentDeploy`. A later-connecting ws claiming the + // same address overwrites the pointer here; the prior owner's `handleClose` + // then no-ops on it via its ownership guard. + for (const addr of workflowSet) { + addressIndex.set(addr, ws); + } - logger.info`Sidecar ${sidecarId} registered with ${String(agentAddresses.length)} agents`; + logger.info`Sidecar ${sidecarId} registered with ${String(agentAddresses.length)} agents and ${String(workflowSet.size)} workflow deployments`; } async function handleReconnect( @@ -582,6 +650,7 @@ export function createSidecarRouter( sidecarId: string, token: string, agentAddresses: string[], + workflowAddresses: string[] = [], deployRefs: Record = {}, ): Promise { if (validateToken !== undefined && !validateToken(sidecarId, token)) { @@ -614,9 +683,10 @@ export function createSidecarRouter( // freshly-deployed agent from routing. const previouslyOwned = new Set(connections.get(ws)?.agentAddresses); - // Register the sidecar connection immediately (with no addresses) - // so it can receive frames while the challenge is pending. - handleRegister(ws, sidecarId, token, []); + // Register the sidecar connection immediately (with no session addresses, + // but WITH the workflow-substrate addresses -- those need no challenge) + // so it can receive frames while the session challenge is pending. + handleRegister(ws, sidecarId, token, [], workflowAddresses); const conn = connections.get(ws); if (conn === undefined) return; @@ -970,6 +1040,17 @@ export function createSidecarRouter( } } } + // Remove this connection's workflow-substrate routes. No disconnect queue + // is created: these addresses re-register (with the complete live set) + // when the sidecar reconnects, and their in-flight run state is + // reconstructed sidecar-locally, not from a hub-side queue. The ownership + // guard mirrors the session loop above so a takeover by a newer ws is not + // clobbered by the prior owner's close. + for (const addr of conn.workflowAddresses) { + if (addressIndex.get(addr) === ws) { + addressIndex.delete(addr); + } + } connections.delete(ws); // Cancel the liveness timer for this connection. @@ -1022,14 +1103,20 @@ export function createSidecarRouter( // across both receivers. The two receivers track their own in- // flight transferIds, so a pending workflow-run transfer for an // agent that just disconnected won't outlive the connection just - // because the agent-state receiver has nothing to cancel. - for (const addr of conn.agentAddresses) { + // because the agent-state receiver has nothing to cancel. Iterate the + // owned union so a reconnected workflow deployment's transfer is + // cancelled too; the deduped set avoids a double-cancel for an address + // that is in both sets. A reclaimed address is not present here -- the + // ghost cleanup in handleRegister evicts it from this (superseded) + // connection -- so a stale close does not cancel the new owner's work. + const owned = ownedAddresses(conn); + for (const addr of owned) { agentStatePackReceiver.cancelByAgent(addr); workflowRunPackReceiver.cancelByAgent(addr); } events.emit("sidecar.disconnect", { - agentAddresses: [...conn.agentAddresses], + ownedAddresses: [...owned], }); logger.info`Sidecar ${conn.sidecarId} disconnected`; @@ -1192,7 +1279,7 @@ export function createSidecarRouter( function handlePackPush(ws: WsHandle, frame: PackPushFrame): void { const conn = connections.get(ws); if (conn === undefined) return; - if (!conn.agentAddresses.has(frame.agentAddress)) { + if (!connOwnsAddress(conn, frame.agentAddress)) { logger.warn`Received repo.pack.push for unrouted agent ${frame.agentAddress}`; return; } @@ -1228,7 +1315,7 @@ export function createSidecarRouter( ): Promise { const conn = connections.get(ws); if (conn === undefined) return; - if (!conn.agentAddresses.has(frame.agentAddress)) { + if (!connOwnsAddress(conn, frame.agentAddress)) { logger.warn`Received repo.pack.done for unrouted agent ${frame.agentAddress}`; return; } diff --git a/packages/types/src/sidecar.ts b/packages/types/src/sidecar.ts index e019d568..2fce6612 100644 --- a/packages/types/src/sidecar.ts +++ b/packages/types/src/sidecar.ts @@ -31,6 +31,12 @@ export const RegisterFrame = type({ sidecarId: "string", token: "string", agentAddresses: "string[]", + // Live workflow-substrate deployment addresses (ins_dep_...) this sidecar + // currently hosts. Unlike `agentAddresses`, these are hub-minted and + // carry no per-address key, so the hub re-registers them for routing + // directly (no challenge) -- the same way they were first registered at + // deploy time. Absent on sidecars/paths that host none. + "workflowAddresses?": "string[]", }); export type RegisterFrame = typeof RegisterFrame.infer; @@ -44,6 +50,10 @@ export const ReconnectFrame = type({ sidecarId: "string", token: "string", agentAddresses: "string[]", + // See `RegisterFrame.workflowAddresses`. Carried on the reconnect frame too + // so a sidecar restoring both single-agent sessions and workflow + // deployments re-registers both in one connect. + "workflowAddresses?": "string[]", "deployRefs?": "Record", }); export type ReconnectFrame = typeof ReconnectFrame.infer; From 75f16d6ef12949cae105f348b03a3edde912f58f Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Thu, 2 Jul 2026 22:47:59 -0500 Subject: [PATCH 026/101] Ack the agent key for a single-step deployment The deploy ack returned the supervisor principal key for every workflow deployment. That is right for a genuine multi-step deployment, whose head is workflow-derived and has no agent identity, but wrong for a single-step head: that head IS an agent identity. It signs its own outbound mail and its reconnect challenges with the agent key, and the hub records the ack's key and later verifies the challenge signature against it. Acking the supervisor key there would make a reconnect challenge verify a supervisor-key signature that was never produced. The single-step branch already loads the agent keypair to register the head's outbound crypto; surface its public key as the ack instead. A multi-step deployment still acks the supervisor key its workflow-run events are signed with. This is inert for the single-step workflow deployments that exist today -- their head is workflow-derived, so the hub never records or verifies the acked key -- but it is the prerequisite for routing single-agent instance deploys through this path, where the head is a real instance whose reconnect must verify. --- apps/sidecar/src/workflow-host-wiring.test.ts | 40 ++++++++++++++++++- apps/sidecar/src/workflow-host-wiring.ts | 23 +++++++++-- 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/apps/sidecar/src/workflow-host-wiring.test.ts b/apps/sidecar/src/workflow-host-wiring.test.ts index 81b87b6c..f99814ad 100644 --- a/apps/sidecar/src/workflow-host-wiring.test.ts +++ b/apps/sidecar/src/workflow-host-wiring.test.ts @@ -1162,6 +1162,12 @@ describe("createSidecarDeployRouter multi-step branch", () => { * the threaded value echoed in the message. */ readyTimeoutMs?: number; + /** + * Fixed keypair the keyStore's `loadOrGenerateKey` returns for the head. + * The B-key test pins the single-step deploy ack to this agent key; when + * omitted a fresh keypair is minted per call as before. + */ + headKeyPair?: Awaited>; }) { const transport = opts.transport ?? createInMemoryTransport(); const keyPair = await generateKeyPair(); @@ -1200,7 +1206,7 @@ describe("createSidecarDeployRouter multi-step branch", () => { keyStore: { recordHubKey: () => undefined, loadOrGenerateKey: async () => ({ - keyPair: await generateKeyPair(), + keyPair: opts.headKeyPair ?? (await generateKeyPair()), isNew: false, }), } as unknown as Parameters< @@ -2406,4 +2412,36 @@ describe("createSidecarDeployRouter multi-step branch", () => { // wedged deploy leaves nothing for a later boot to re-spawn. expect(await recordExists(dataDir, deploymentId)).toBe(false); }); + + test("a single-step deploy acks the agent key, not the supervisor key", async () => { + // The single-step head IS an agent identity: it signs its own reconnect + // challenges with the agent key, and the hub records the ack's key into + // agent_instance.publicKey and verifies the challenge against it. So the + // ack must surface the AGENT key, not the supervisor principal key -- + // otherwise a rerouted instance's reconnect signature never matches. + const headKeyPair = await generateKeyPair(); + const spawner = makeReadyDrivingSpawner(10300); + const { router, keyPair: fixtureKeyPair } = await buildMultistepFixture({ + spawner: spawner.spawner, + headKeyPair, + multistepSubstrateEnv: { + SIDECAR_DATA_DIR: await createTempBaseDir("sidecar-bkey-data-"), + }, + }); + + const deployPromise = router.deploy( + singleStepFrame("ins_bkey@example.com", "wf-bkey"), + ); + await spawner.driveReadyFor(0); + const result = await deployPromise; + + // The supervisor principal key is derived from the fixture's signing seed + // (fixtureKeyPair); the head's agent key is the distinct headKeyPair. + expect(result.publicKey).toBe( + Buffer.from(headKeyPair.publicKey).toString("hex"), + ); + expect(result.publicKey).not.toBe( + Buffer.from(fixtureKeyPair.publicKey).toString("hex"), + ); + }); }); diff --git a/apps/sidecar/src/workflow-host-wiring.ts b/apps/sidecar/src/workflow-host-wiring.ts index d81ced80..b74de381 100644 --- a/apps/sidecar/src/workflow-host-wiring.ts +++ b/apps/sidecar/src/workflow-host-wiring.ts @@ -1189,10 +1189,23 @@ export function createSidecarDeployRouter(deps: { // agent identity whose keypair lives in the keyStore. Registration // happens before `spawn()` so the address is live the instant the // first reply routes outbound. + // The public key the deploy ack surfaces to the hub. For a single-step + // head it is the AGENT key, set inside the block below; a genuine + // multi-step deployment has no head agent identity and falls back to the + // supervisor principal key at the return. + let headAgentPublicKey: string | undefined; if (spec.definition.stepOrder.length === 1) { const { keyPair } = await deps.keyStore.loadOrGenerateKey( spec.agentAddress, ); + // A single-step head IS an agent identity: it signs its own outbound + // mail AND its reconnect challenges with this agent key (via the key + // store's signChallenge). The hub records the ack's key into + // `agent_instance.publicKey` (for a rerouted instance head, which has + // an instance row and is not workflow-derived) and verifies the + // reconnect challenge against it, so the ack MUST carry the agent key, + // not the supervisor principal key -- otherwise verification fails. + headAgentPublicKey = hexEncode(keyPair.publicKey); deps.transport.register( spec.agentAddress, deps.createAgentCrypto(keyPair), @@ -1287,12 +1300,16 @@ export function createSidecarDeployRouter(deps: { agentAddress: spec.agentAddress, }); - // Derive the ack public key BEFORE marking the spawn succeeded so an + // Resolve the ack public key BEFORE marking the spawn succeeded so an // (unreachable, deterministic) derivation failure unwinds the spawn // rather than leaving a live-but-unacked deployment whose slug the // caller then frees. Once `succeeded` is set the finally is a no-op - // and the deployment is retained. - const publicKey = await derivePrincipalPublicKeyHex(deps.signingKeySeed); + // and the deployment is retained. A single-step head acks its agent + // key (captured above); a multi-step deployment acks the supervisor + // principal key its workflow-run events are signed with. + const publicKey = + headAgentPublicKey ?? + (await derivePrincipalPublicKeyHex(deps.signingKeySeed)); succeeded = true; return { publicKey }; } finally { From a448ebb1b7a468f98bd5b8cb66d59f3e282b7c43 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Thu, 2 Jul 2026 23:36:48 -0500 Subject: [PATCH 027/101] Run single-agent instances as single-step workflows Deploying an agent instance took the legacy trivial in-process path: the sidecar built the harness in its own process. Route it through the deploy core instead, so an instance runs as a supervised workflow-process child on the same substrate as multi-step workflows -- the last thing keeping the trivial path alive. A new SessionService entry point wraps the instance's harness as a one-step workflow and deploys it at the head with the instance's real identity, then the production instance route calls it in place of launchSession. The real agentId is preserved (not collapsed to a derived deployment id) so the spawned child resolves the instance's skills and pinned tool packages; the head's ack carries the agent key, so reconnect verifies. No workflow_deployment row is written -- a plain instance has no workflow asset -- and the single step's inference source is pinned to the instance's default source, which the route already resolved and authorized. The child never runs the wrap's walk-only tool factories: it materializes tools from the deploy tree and calls the real agent builder reading only id, prompt, and capabilities. A tool-less instance gets an empty materialization slot and never hits the bare fallback that would touch those factories. An integration test proves it end to end -- a real sidecar spawns a real child that instantiates and runs the wrapped agent to a real inference reply -- the path unit tests leave mocked. launchSession is now unused in production and remains only as dead code alongside the rest of the trivial in-process machinery. --- Makefile | 2 +- packages/hub-api/src/app.test.ts | 5 + packages/hub-api/src/routes/agents.test.ts | 5 + packages/hub-api/src/routes/assets.test.ts | 5 + .../hub-api/src/routes/catalog-routes.test.ts | 5 + .../hub-api/src/routes/git-tokens.test.ts | 5 + packages/hub-api/src/routes/instances.test.ts | 18 +- packages/hub-api/src/routes/instances.ts | 8 +- packages/hub-api/src/routes/workflows.test.ts | 1 + packages/hub-sessions/src/session-service.ts | 85 ++++++++ tests/hub-api/asset-routes.test.ts | 1 + .../instance-reroute-real-agent.test.ts | 192 ++++++++++++++++++ 12 files changed, 328 insertions(+), 4 deletions(-) create mode 100644 tests/workflow-deploy/instance-reroute-real-agent.test.ts diff --git a/Makefile b/Makefile index 250ac8ca..1bf63a92 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ lint: FORCE test: FORCE bun test packages/ apps/ bin/ tests/agent/ tests/agent-audit-log/ tests/agent-blob-spill/ tests/agent-common/ tests/agent-multi-provider/ tests/agent-quickstart/ tests/agent-resume/ tests/agent-rewind/ tests/agent-rich-tool/ tests/agent-structured-payload/ tests/coding-agent/ tests/hub-agent/lib/ tests/inference-testing/ tests/tool-packaging/ tests/workflow/ - bun test --timeout 120000 tests/hub-agent/deploy-flow.test.ts tests/workflow-deploy/trivial-roundtrip.test.ts tests/workflow-deploy/multistep-signal.test.ts tests/workflow-deploy/single-step-real-agent.test.ts tests/workflow-deploy/single-step-message-input.test.ts tests/workflow-deploy/single-step-posix-tool.test.ts tests/workflow-deploy/single-step-grants-bridge.test.ts tests/workflow-deploy/single-step-event-threading.test.ts tests/workflow-deploy/single-step-conversation-durability.test.ts tests/workflow-deploy/single-step-full-lifecycle.test.ts tests/workflow-deploy/cross-process-custom-adapter.test.ts tests/workflow-deploy/conversation-state-wal.test.ts tests/workflow-deploy/drain-roundtrip.test.ts tests/workflow-deploy/child-workflow-roundtrip.test.ts tests/workflow-deploy/unresolvable-director.test.ts tests/workflow-deploy/fifo-mail.test.ts tests/workflow-deploy/mail-edge-cases.test.ts tests/inference/ tests/skill-attachment-flow.test.ts tests/hub-api/ tests/db/ + bun test --timeout 120000 tests/hub-agent/deploy-flow.test.ts tests/workflow-deploy/trivial-roundtrip.test.ts tests/workflow-deploy/multistep-signal.test.ts tests/workflow-deploy/single-step-real-agent.test.ts tests/workflow-deploy/instance-reroute-real-agent.test.ts tests/workflow-deploy/single-step-message-input.test.ts tests/workflow-deploy/single-step-posix-tool.test.ts tests/workflow-deploy/single-step-grants-bridge.test.ts tests/workflow-deploy/single-step-event-threading.test.ts tests/workflow-deploy/single-step-conversation-durability.test.ts tests/workflow-deploy/single-step-full-lifecycle.test.ts tests/workflow-deploy/cross-process-custom-adapter.test.ts tests/workflow-deploy/conversation-state-wal.test.ts tests/workflow-deploy/drain-roundtrip.test.ts tests/workflow-deploy/child-workflow-roundtrip.test.ts tests/workflow-deploy/unresolvable-director.test.ts tests/workflow-deploy/fifo-mail.test.ts tests/workflow-deploy/mail-edge-cases.test.ts tests/inference/ tests/skill-attachment-flow.test.ts tests/hub-api/ tests/db/ test-load: FORCE bun test --timeout 300000 tests/workflow-deploy/fifo-mail-load.test.ts diff --git a/packages/hub-api/src/app.test.ts b/packages/hub-api/src/app.test.ts index 282b4298..9b0937d5 100644 --- a/packages/hub-api/src/app.test.ts +++ b/packages/hub-api/src/app.test.ts @@ -23,6 +23,11 @@ const sessionService: SessionService = { launchSession(_params) { throw new Error("mock: sessionService.launchSession not implemented"); }, + deployInstanceAtHead(_params) { + throw new Error( + "mock: sessionService.deployInstanceAtHead not implemented", + ); + }, deployWorkflowDefinition(_params) { throw new Error( "mock: sessionService.deployWorkflowDefinition not implemented", diff --git a/packages/hub-api/src/routes/agents.test.ts b/packages/hub-api/src/routes/agents.test.ts index 69e56815..4f769eef 100644 --- a/packages/hub-api/src/routes/agents.test.ts +++ b/packages/hub-api/src/routes/agents.test.ts @@ -196,6 +196,11 @@ function createMockSessionService(): SessionService { launchSession: () => { throw new Error("mock: sessionService.launchSession not implemented"); }, + deployInstanceAtHead: () => { + throw new Error( + "mock: sessionService.deployInstanceAtHead not implemented", + ); + }, deployWorkflowDefinition: () => { throw new Error( "mock: sessionService.deployWorkflowDefinition not implemented", diff --git a/packages/hub-api/src/routes/assets.test.ts b/packages/hub-api/src/routes/assets.test.ts index 3d6b572f..e40cc3f7 100644 --- a/packages/hub-api/src/routes/assets.test.ts +++ b/packages/hub-api/src/routes/assets.test.ts @@ -312,6 +312,11 @@ function createMockSessionService(): SessionService { launchSession: () => { throw new Error("mock: sessionService.launchSession not implemented"); }, + deployInstanceAtHead: () => { + throw new Error( + "mock: sessionService.deployInstanceAtHead not implemented", + ); + }, deployWorkflowDefinition: () => { throw new Error( "mock: sessionService.deployWorkflowDefinition not implemented", diff --git a/packages/hub-api/src/routes/catalog-routes.test.ts b/packages/hub-api/src/routes/catalog-routes.test.ts index 7b2716af..4b173437 100644 --- a/packages/hub-api/src/routes/catalog-routes.test.ts +++ b/packages/hub-api/src/routes/catalog-routes.test.ts @@ -211,6 +211,11 @@ function createMockSessionService(): SessionService { launchSession: () => { throw new Error("mock: sessionService.launchSession not implemented"); }, + deployInstanceAtHead: () => { + throw new Error( + "mock: sessionService.deployInstanceAtHead not implemented", + ); + }, deployWorkflowDefinition: () => { throw new Error( "mock: sessionService.deployWorkflowDefinition not implemented", diff --git a/packages/hub-api/src/routes/git-tokens.test.ts b/packages/hub-api/src/routes/git-tokens.test.ts index 4c921403..3e0ced70 100644 --- a/packages/hub-api/src/routes/git-tokens.test.ts +++ b/packages/hub-api/src/routes/git-tokens.test.ts @@ -235,6 +235,11 @@ function createMockSessionService(): SessionService { launchSession: () => { throw new Error("mock: sessionService.launchSession not implemented"); }, + deployInstanceAtHead: () => { + throw new Error( + "mock: sessionService.deployInstanceAtHead not implemented", + ); + }, deployWorkflowDefinition: () => { throw new Error( "mock: sessionService.deployWorkflowDefinition not implemented", diff --git a/packages/hub-api/src/routes/instances.test.ts b/packages/hub-api/src/routes/instances.test.ts index 4cba4467..1b39240a 100644 --- a/packages/hub-api/src/routes/instances.test.ts +++ b/packages/hub-api/src/routes/instances.test.ts @@ -287,6 +287,9 @@ function createMockSessionService(): SessionService { launchSession(_params) { return notImpl("launchSession"); }, + deployInstanceAtHead(_params) { + return notImpl("deployInstanceAtHead"); + }, deployWorkflowDefinition(_params) { return notImpl("deployWorkflowDefinition"); }, @@ -660,6 +663,9 @@ describe("POST /agents/instances/:instanceId/mail", () => { launchSession() { throw new Error("not implemented"); }, + deployInstanceAtHead() { + throw new Error("not implemented"); + }, deployWorkflowDefinition() { throw new Error("not implemented"); }, @@ -811,6 +817,9 @@ describe("POST /agents/instances/:instanceId/mail attachments", () => { launchSession() { throw new Error("not implemented"); }, + deployInstanceAtHead() { + throw new Error("not implemented"); + }, deployWorkflowDefinition() { throw new Error("not implemented"); }, @@ -1244,6 +1253,7 @@ describe("POST /agents/instances seeds creator agent-state grant", () => { function createCapturingSessionService(): SessionService { return { launchSession: async () => undefined, + deployInstanceAtHead: async () => ({ publicKey: "pk-instance-mock" }), deployWorkflowDefinition: () => { throw new Error("mock: deployWorkflowDefinition not implemented"); }, @@ -1454,14 +1464,18 @@ describe("POST /agents/instances seeds creator agent-state grant", () => { expect(JSON.stringify(await res.json())).toContain("wallet_backed"); }); - test("passes catalog-ordered sources to launchSession and persists modelPreferences", async () => { + test("passes catalog-ordered sources to deployInstanceAtHead and persists modelPreferences", async () => { const inserts: TableInsert[] = []; let launchedSources: unknown; let launchedDefaultSource: unknown; const sessionService: SessionService = { - launchSession: async (params) => { + launchSession: () => { + throw new Error("mock: launchSession not implemented"); + }, + deployInstanceAtHead: async (params) => { launchedSources = params.config.sources; launchedDefaultSource = params.config.defaultSource; + return { publicKey: "pk-instance-mock" }; }, deployWorkflowDefinition: () => { throw new Error("mock: deployWorkflowDefinition not implemented"); diff --git a/packages/hub-api/src/routes/instances.ts b/packages/hub-api/src/routes/instances.ts index e9d0b2a5..1ed505d4 100644 --- a/packages/hub-api/src/routes/instances.ts +++ b/packages/hub-api/src/routes/instances.ts @@ -517,7 +517,13 @@ export function createInstanceRoutes({ eventCollectors.create(agentAddress, tenant.id, sessionId, instanceId); try { - await sessionService.launchSession({ + // Deploy the instance as a single-step workflow at the head: it runs + // as a supervised workflow-process child, not the legacy trivial + // in-process path. The real `agentId` (row.id) is passed so the child + // resolves the instance's skills and pinned tool packages. The + // returned head public key is surfaced separately via the sidecar's + // `agent.deploy.ack`, so the route discards it here. + await sessionService.deployInstanceAtHead({ agentAddress, agentId: row.id, instanceId, diff --git a/packages/hub-api/src/routes/workflows.test.ts b/packages/hub-api/src/routes/workflows.test.ts index abb0485c..f9d1632a 100644 --- a/packages/hub-api/src/routes/workflows.test.ts +++ b/packages/hub-api/src/routes/workflows.test.ts @@ -224,6 +224,7 @@ function createMockSessionService( } return { launchSession: () => notImpl("launchSession"), + deployInstanceAtHead: () => notImpl("deployInstanceAtHead"), deployWorkflowDefinition: (params) => { deployCalls.push(params); if (result === undefined) { diff --git a/packages/hub-sessions/src/session-service.ts b/packages/hub-sessions/src/session-service.ts index b4c8d690..1ef2927d 100644 --- a/packages/hub-sessions/src/session-service.ts +++ b/packages/hub-sessions/src/session-service.ts @@ -129,6 +129,24 @@ export type SessionService = { toolPackagePins?: readonly ToolPackagePin[]; }): Promise; + /** + * Deploy a single-agent instance through the single-step-at-head path, + * wrapping the harness as a one-step workflow and routing it through the + * deploy core with the instance's real identity. Replaces `launchSession` + * as the production instance-deploy entry point: the instance runs as a + * supervised workflow-process child rather than the legacy trivial + * in-process path. Writes no `workflow_deployment` row. Returns the head's + * agent-key ack (the key the head signs its reconnect challenges with). + */ + deployInstanceAtHead(params: { + agentAddress: string; + agentId: string; + instanceId: string; + config: HarnessConfig; + deployContent: DeployContent; + toolPackagePins?: readonly ToolPackagePin[]; + }): Promise<{ publicKey: string }>; + /** * Deploy a one-step workflow once at the head through the deploy core, * without the DB-backed `workflow_deployment` projection row. Stages the @@ -1013,6 +1031,72 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { }); } + /** + * Deploy a single-agent instance through the single-step-at-head path: wrap + * the harness as a one-step workflow (the same wrap `launchSession` uses) and + * route it through `deploySingleStepAtHead` with the instance's REAL identity + * -- so the head address IS the instance address and the deploy runs as a + * supervised workflow-process child, not the legacy trivial in-process path. + * + * Unlike the orchestrator's `runSingleStepAtHead`, this calls + * `deploySingleStepAtHead` directly with the route's real `agentId` + * (`row.id`), NOT a `deriveDeploymentAgentId(deploymentId)` -- the child + * resolves skills, deploy tree, and tool-package pins by `agentId`, so + * collapsing it to the deployment id would strip the instance's attachments. + * It writes no `workflow_deployment` row (a plain instance has no workflow + * asset) and carries no `trivialBindings`. Returns the head's agent-key ack. + */ + async function deployInstanceAtHead(params: { + agentAddress: string; + agentId: string; + instanceId: string; + config: HarnessConfig; + deployContent: DeployContent; + toolPackagePins?: readonly ToolPackagePin[]; + }): Promise<{ publicKey: string }> { + const { agentAddress, agentId, instanceId, config, deployContent } = params; + + const trivialAgent = wrapHarnessAsTrivialAgent({ config, deployContent }); + const workflow = defineWorkflow({ + id: `wf_${agentId}`, + agent: trivialAgent, + trigger: { type: "mail", to: agentAddress }, + }); + + // The sole step's id, read off the built definition. + const stepId = workflow.stepOrder[0]; + if (stepId === undefined) { + throw new Error( + `instance deploy for ${agentAddress}: the wrapped single-step workflow has an empty stepOrder`, + ); + } + + // Pin the step's inference source to the instance's default source. The + // instance route already resolved and authorized it against the tenant + // catalog, so the source is selected directly rather than re-run through + // the orchestrator's operator-approval gate. + const source = config.sources.find((s) => s.id === config.defaultSource); + if (source === undefined) { + throw new Error( + `instance deploy for ${agentAddress}: config.defaultSource ${JSON.stringify(config.defaultSource)} does not resolve to a HarnessConfig.sources entry`, + ); + } + + return deploySingleStepAtHead({ + agentAddress, + agentId, + instanceId, + config, + deployContent, + definition: workflow, + sources: { [stepId]: source }, + hubPublicKey: hexEncode(agentRepoStore.getSigningPublicKey()), + ...(params.toolPackagePins !== undefined + ? { toolPackagePins: params.toolPackagePins } + : {}), + }); + } + async function deployWorkflowDefinition( params: DeployWorkflowDefinitionParams, ): Promise { @@ -1495,6 +1579,7 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { return { launchSession, + deployInstanceAtHead, deploySingleStepAtHead, deployWorkflowDefinition, sendUserMessage, diff --git a/tests/hub-api/asset-routes.test.ts b/tests/hub-api/asset-routes.test.ts index c22ab3c1..b9d80850 100644 --- a/tests/hub-api/asset-routes.test.ts +++ b/tests/hub-api/asset-routes.test.ts @@ -162,6 +162,7 @@ function createMockSidecarRouter(): SidecarRouter { function createMockSessionService(): SessionService { return { launchSession: notImplemented("sessionService.launchSession"), + deployInstanceAtHead: notImplemented("sessionService.deployInstanceAtHead"), deployWorkflowDefinition: notImplemented( "sessionService.deployWorkflowDefinition", ), diff --git a/tests/workflow-deploy/instance-reroute-real-agent.test.ts b/tests/workflow-deploy/instance-reroute-real-agent.test.ts new file mode 100644 index 00000000..730baca1 --- /dev/null +++ b/tests/workflow-deploy/instance-reroute-real-agent.test.ts @@ -0,0 +1,192 @@ +// Instance-reroute real-agent round-trip integration test. +// +// The proof that a single-agent INSTANCE deploy, routed through +// `SessionService.deployInstanceAtHead` (which wraps the harness as a +// one-step workflow via `wrapHarnessAsTrivialAgent` and deploys it at the +// head), runs a REAL agent in the spawned workflow-process child -- against +// the real hub + real sidecar subprocess + mock inference fixture. This +// closes the gap that unit tests (mock spawner) leave open: that the +// walk-only trivial-wrap agent instantiates and runs in a real child. +// +// The child never invokes the wrap's walk-only tool factories: it +// materializes tools from the deploy tree and calls the real `createAgent` +// reading only id/systemPrompt/capabilities. A tool-less instance gets an +// empty materialization slot and never hits the bare-createAgent fallback, +// so the wrapped agent runs to a real inference reply. +// +// The mock inference server returns `I see these tools: `; with an +// empty tool set the reply is the stable prefix `I see these tools:`. + +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; + +import type { HarnessConfig } from "@intx/types/runtime"; +import { wrapHarnessAsTrivialAgent } from "@intx/workflow-deploy"; +import { defineWorkflow, type WorkflowDefinition } from "@intx/workflow"; +import { deriveTrivialDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; +import type { RepoId } from "@intx/hub-sessions"; + +import { + SESSION_ID, + SIDECAR_ID, + fireMailTrigger, + readWorkflowRunEvents, + startDeployFlowEnv, + waitForFirstRunId, + waitForWorkflowRunComplete, + type DeployFlowEnv, +} from "../hub-agent/lib/deploy-flow-env"; + +const DEPLOYMENT_DOMAIN = "integration.interchange"; +// A real instance identity: `ins_` + 32 hex, NOT a `dep_`-prefixed deployment +// id. The head address IS the instance address; the reroute keeps that +// identity rather than deriving a synthetic deployment agent id. +const INSTANCE_ID = `ins_${"b".repeat(32)}`; +const AGENT_ID = "agent-instance-reroute"; +const WORKFLOW_RUN_REF = "refs/heads/main"; + +// The mock inference server's reply for an empty tool set. +const EXPECTED_REPLY = "I see these tools:"; + +let env: DeployFlowEnv; + +beforeAll(async () => { + env = await startDeployFlowEnv(); +}); + +afterAll(async () => { + await env.teardown(); +}); + +describe("instance-reroute real-agent round-trip", () => { + test("sidecar registers with hub", () => { + expect(env.hub.router.getConnectedSidecars()).toContain(SIDECAR_ID); + }); + + test("deployInstanceAtHead runs the wrapped single agent in a real child", async () => { + const agentAddress = `${INSTANCE_ID}@${DEPLOYMENT_DOMAIN}`; + + const config: HarnessConfig = { + sessionId: SESSION_ID, + agentId: AGENT_ID, + tenantId: "tenant-1", + principalId: "prin_integration-reroute-1", + agentAddress, + systemPrompt: "You are the rerouted single-agent instance.", + tools: [], + grants: [], + sources: [ + { + id: "anthropic:mock-model", + provider: "anthropic", + baseURL: `http://localhost:${env.inference.server.port}`, + apiKey: "sk-mock", + model: "mock-model", + }, + ], + defaultSource: "anthropic:mock-model", + }; + const deployContent = { systemPrompt: config.systemPrompt }; + + // Reconstruct the definition `deployInstanceAtHead` builds internally, for + // the run-observation handle. Same deterministic inputs -> same wrap. + const workflow: WorkflowDefinition = defineWorkflow({ + id: `wf_${AGENT_ID}`, + agent: wrapHarnessAsTrivialAgent({ config, deployContent }), + trigger: { type: "mail", to: agentAddress }, + }); + + // Deploy the instance through the reroute entry point under test. + const { publicKey } = await env.hub.sessionService.deployInstanceAtHead({ + agentAddress, + agentId: AGENT_ID, + instanceId: INSTANCE_ID, + config, + deployContent, + }); + expect(publicKey).toMatch(/^[0-9a-f]{64}$/); + + const workflowRunRepoId: RepoId = { + kind: "workflow-run", + id: deriveTrivialDeploymentId(agentAddress), + }; + env.registerDeployment({ + deploymentId: INSTANCE_ID, + workflowDefinition: workflow, + workflowRunRepoId, + workflowRunRef: WORKFLOW_RUN_REF, + mailAddress: agentAddress, + }); + + // The instance's head address is routable from the hub after the deploy. + expect(env.hub.router.getRoutableAddresses()).toContain(agentAddress); + + const { messageId } = await fireMailTrigger(env, agentAddress, { + messageId: "", + }); + + const runId = await waitForFirstRunId(env, workflowRunRepoId, { + diagnostics: env.sidecarDiagnostics, + timeoutMs: 20_000, + }); + + const terminal = await waitForWorkflowRunComplete(env, INSTANCE_ID, runId, { + timeoutMs: 20_000, + diagnostics: env.sidecarDiagnostics, + }); + expect(terminal.type).toBe("RunCompleted"); + + const events = await readWorkflowRunEvents(env, INSTANCE_ID, runId); + const runStartedBody = events.find((e) => e.type === "RunStarted")?.body; + if (runStartedBody === undefined) throw new Error("missing RunStarted"); + expect(runStartedBody["consumedMessageId"]).toBe(messageId); + + const stepCompleted = events.find((e) => e.type === "StepCompleted"); + if (stepCompleted === undefined) { + throw new Error("missing StepCompleted for the wrapped single step"); + } + + // The proof: the step output is the REAL agent reply from `agent.send` + // (the mock provider's deterministic output) -- the wrapped trivial agent + // instantiated and ran in the child rather than throwing on its walk-only + // tool factories. + const reply = readStepReply(stepCompleted.body); + expect(reply).toBe(EXPECTED_REPLY); + + // The agent's inference call actually reached the mock provider. + expect(env.inference.requests.length).toBeGreaterThan(0); + }); +}); + +/** + * Extract the agent's reply string from a `StepCompleted` event body. A small + * `{ reply, turn }` output inlines as `inline:`. + */ +function readStepReply(body: Record): string { + const output = body["output"]; + if (typeof output !== "object" || output === null || !("ref" in output)) { + throw new Error( + `StepCompleted output is not a { ref } record: ${JSON.stringify(output)}`, + ); + } + const ref: unknown = output.ref; + if (typeof ref !== "string") { + throw new Error(`StepCompleted output ref is not a string: ${String(ref)}`); + } + const INLINE_PREFIX = "inline:"; + if (!ref.startsWith(INLINE_PREFIX)) { + throw new Error(`expected an inline output ref, got ${ref}`); + } + const parsed: unknown = JSON.parse(ref.slice(INLINE_PREFIX.length)); + if (typeof parsed !== "object" || parsed === null || !("reply" in parsed)) { + throw new Error( + `step output does not carry a reply field: ${JSON.stringify(parsed)}`, + ); + } + const reply: unknown = parsed.reply; + if (typeof reply !== "string") { + throw new Error( + `step output reply is not a string: ${JSON.stringify(parsed)}`, + ); + } + return reply; +} From 5624a6fdd8c4c5068094a178f89221d83679ee73 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Fri, 3 Jul 2026 01:07:06 -0500 Subject: [PATCH 028/101] Fail over inference sources per step in workflow children A single agent instance and every workflow step now run inside a supervised workflow-process child, but the deploy wire format pinned exactly one inference source per step. The child's reactor does runtime cross-provider failover by walking an ordered source list forward from its default, so pinning a single source silently disabled failover that the legacy in-process harness had. Carry each step's full ordered source chain end to end. The per-step sources map becomes a non-empty list keyed by step id across the wire frame, the supervisor deploy frame, the substrate-config env the child parses, and the on-disk restore record. The child builds the step env from the whole chain and pins the reactor's initial source to its head, so forward-only failover walks the tail. The single-agent-at-head path pins the instance's whole resolved source list rather than only its default. It asserts the list is non-empty and its head is the default source, because the reactor resolves the initial source by id and fails over forward with no wrap; a default that is not the head would leave part of the chain unreachable or disable failover entirely, so the deploy fails loudly instead. The source-admission gate and projection validator both walk every source in every step's chain, and the validator rejects an empty chain, so an unbuildable failover target or a missing initial source is caught at deploy and restore rather than deep in a running child. --- .../agent-key-registration-lifecycle.test.ts | 16 ++- .../src/workflow-deployment-record.test.ts | 48 ++++--- .../sidecar/src/workflow-deployment-record.ts | 7 +- ...orkflow-host-wiring-deploy-failure.test.ts | 16 ++- ...ow-host-wiring-undeploy-supervisor.test.ts | 16 ++- apps/sidecar/src/workflow-host-wiring.test.ts | 64 ++++++--- apps/sidecar/src/workflow-host-wiring.ts | 57 +++++--- ...low-substrate-factory-step-storage.test.ts | 28 +++- .../sidecar/src/workflow-substrate-factory.ts | 50 ++++--- .../hub-sessions/src/session-service.test.ts | 127 +++++++++++++++--- packages/hub-sessions/src/session-service.ts | 34 +++-- packages/types/src/sidecar.test.ts | 6 +- packages/types/src/sidecar.ts | 17 ++- .../src/orchestrator-fallback-source.test.ts | 34 +++-- .../src/orchestrator-nonagent-source.test.ts | 19 +-- .../workflow-deploy/src/orchestrator.test.ts | 6 +- packages/workflow-deploy/src/orchestrator.ts | 31 +++-- .../workflow-host/src/supervisor/types.ts | 4 +- 18 files changed, 411 insertions(+), 169 deletions(-) diff --git a/apps/sidecar/src/agent-key-registration-lifecycle.test.ts b/apps/sidecar/src/agent-key-registration-lifecycle.test.ts index fe680fba..668fddf6 100644 --- a/apps/sidecar/src/agent-key-registration-lifecycle.test.ts +++ b/apps/sidecar/src/agent-key-registration-lifecycle.test.ts @@ -277,13 +277,15 @@ describe("agent signing-key registration lifecycle on the host transport", () => steps: { "step-1": { kind: "step" } }, }, sources: { - "step-1": { - id: "step-1", - provider: "anthropic", - baseURL: "https://api.anthropic.com", - apiKey: "sk-step-1", - model: "claude-3-5", - }, + "step-1": [ + { + id: "step-1", + provider: "anthropic", + baseURL: "https://api.anthropic.com", + apiKey: "sk-step-1", + model: "claude-3-5", + }, + ], }, }, }; diff --git a/apps/sidecar/src/workflow-deployment-record.test.ts b/apps/sidecar/src/workflow-deployment-record.test.ts index 258a9282..db321dc1 100644 --- a/apps/sidecar/src/workflow-deployment-record.test.ts +++ b/apps/sidecar/src/workflow-deployment-record.test.ts @@ -34,13 +34,15 @@ const SINGLE_STEP: WorkflowDeploymentRecord = { agentAddress: "ins_abc123@tenant.example", definitionId: "wf_abc123", sources: { - "step-1": { - id: "anthropic:mock", - provider: "anthropic", - baseURL: "https://api.example/anthropic", - apiKey: "sk-x", - model: "claude-mock", - }, + "step-1": [ + { + id: "anthropic:mock", + provider: "anthropic", + baseURL: "https://api.example/anthropic", + apiKey: "sk-x", + model: "claude-mock", + }, + ], }, sessionId: "ses_1", hubPublicKey: "deadbeef", @@ -53,20 +55,24 @@ const MULTI_STEP: WorkflowDeploymentRecord = { agentAddress: "ins_dep_xyz@tenant.example", definitionId: "wf_xyz", sources: { - plan: { - id: "anthropic:mock", - provider: "anthropic", - baseURL: "https://api.example/anthropic", - apiKey: "sk-y", - model: "claude-mock", - }, - execute: { - id: "openai:mock", - provider: "openai", - baseURL: "https://api.example/openai", - apiKey: "sk-z", - model: "gpt-mock", - }, + plan: [ + { + id: "anthropic:mock", + provider: "anthropic", + baseURL: "https://api.example/anthropic", + apiKey: "sk-y", + model: "claude-mock", + }, + ], + execute: [ + { + id: "openai:mock", + provider: "openai", + baseURL: "https://api.example/openai", + apiKey: "sk-z", + model: "gpt-mock", + }, + ], }, }; diff --git a/apps/sidecar/src/workflow-deployment-record.ts b/apps/sidecar/src/workflow-deployment-record.ts index 9d1a5c4f..ed7da6b8 100644 --- a/apps/sidecar/src/workflow-deployment-record.ts +++ b/apps/sidecar/src/workflow-deployment-record.ts @@ -6,8 +6,9 @@ // deployments beside the run state they resume. // // It carries only the inputs that are otherwise frame/in-memory only: -// `sources` (pinned per-step inference sources, threaded to the child via -// the spawn env and durable nowhere else), `sessionId` (inference-event +// `sources` (each step's ordered inference-source failover chain, threaded to +// the child via the spawn env and durable nowhere else), `sessionId` +// (inference-event // correlation), and `hubPublicKey` (the head's deploy-pack / inbound // verification key, recorded only in memory today). The definition itself // lives in `assets/workflow//workflow.json`, referenced by @@ -49,7 +50,7 @@ export const WorkflowDeploymentRecord = type({ agentAddress: "string > 0", definitionId: "string > 0", sources: { - "[string]": InferenceSource, + "[string]": InferenceSource.array().atLeastLength(1), }, "sessionId?": "string > 0", "hubPublicKey?": "string > 0", diff --git a/apps/sidecar/src/workflow-host-wiring-deploy-failure.test.ts b/apps/sidecar/src/workflow-host-wiring-deploy-failure.test.ts index b4c36de4..1bb90c24 100644 --- a/apps/sidecar/src/workflow-host-wiring-deploy-failure.test.ts +++ b/apps/sidecar/src/workflow-host-wiring-deploy-failure.test.ts @@ -236,13 +236,15 @@ describe("deploy-failure registry leak", () => { steps: { s1: { kind: "step" } }, }, sources: { - s1: { - id: "primary", - provider: "anthropic", - baseURL: "https://api.anthropic.com", - apiKey: "sk-x", - model: "claude-3-5", - }, + s1: [ + { + id: "primary", + provider: "anthropic", + baseURL: "https://api.anthropic.com", + apiKey: "sk-x", + model: "claude-3-5", + }, + ], }, }, }; diff --git a/apps/sidecar/src/workflow-host-wiring-undeploy-supervisor.test.ts b/apps/sidecar/src/workflow-host-wiring-undeploy-supervisor.test.ts index fbd86af0..06ef2670 100644 --- a/apps/sidecar/src/workflow-host-wiring-undeploy-supervisor.test.ts +++ b/apps/sidecar/src/workflow-host-wiring-undeploy-supervisor.test.ts @@ -299,13 +299,15 @@ describe("createSidecarDeployRouter multi-step undeploy shuts the supervisor dow steps: { "step-1": { kind: "step" } }, }, sources: { - "step-1": { - id: "step-1", - provider: "anthropic", - baseURL: "https://api.anthropic.com", - apiKey: "sk-step-1", - model: "claude-3-5", - }, + "step-1": [ + { + id: "step-1", + provider: "anthropic", + baseURL: "https://api.anthropic.com", + apiKey: "sk-step-1", + model: "claude-3-5", + }, + ], }, }, }; diff --git a/apps/sidecar/src/workflow-host-wiring.test.ts b/apps/sidecar/src/workflow-host-wiring.test.ts index f99814ad..be32a839 100644 --- a/apps/sidecar/src/workflow-host-wiring.test.ts +++ b/apps/sidecar/src/workflow-host-wiring.test.ts @@ -591,10 +591,12 @@ function createSpawnTestRepoStore(tempBase: string): RepoStore { } type WorkflowProjection = NonNullable; -type InferenceSourceFixture = WorkflowProjection["sources"][string]; +// A single source: each step's `sources` value is an ordered failover chain, +// so the fixture element type is the chain's member. +type InferenceSourceFixture = WorkflowProjection["sources"][string][number]; type MultistepDeployArgs = { - sources: Record; + sources: WorkflowProjection["sources"]; definition: { id: string; triggers: unknown[]; @@ -642,10 +644,10 @@ function makeMultistepFrame(args: MultistepDeployArgs): AgentDeployFrame { }; } -function defaultMultistepSources(): Record { +function defaultMultistepSources(): WorkflowProjection["sources"] { return { - "step-1": makeInferenceSource("step-1"), - "step-2": makeInferenceSource("step-2"), + "step-1": [makeInferenceSource("step-1")], + "step-2": [makeInferenceSource("step-2")], }; } @@ -685,6 +687,32 @@ describe("validateWorkflowProjection", () => { ).toThrow(/sources is missing entry/); }); + test("rejects an empty sources chain for a stepOrder id", () => { + expect(() => + validateWorkflowProjection({ + definition: { + id: "w-1", + stepOrder: ["step-1"], + steps: { "step-1": {} }, + }, + sources: { "step-1": [] }, + }), + ).toThrow(/must be a non-empty array/); + }); + + test("rejects a non-array sources entry for a stepOrder id", () => { + expect(() => + validateWorkflowProjection({ + definition: { + id: "w-1", + stepOrder: ["step-1"], + steps: { "step-1": {} }, + }, + sources: { "step-1": {} }, + }), + ).toThrow(/must be a non-empty array/); + }); + test("accepts a well-formed projection", () => { expect(() => validateWorkflowProjection({ @@ -693,7 +721,7 @@ describe("validateWorkflowProjection", () => { stepOrder: ["step-1", "step-2"], steps: { "step-1": {}, "step-2": {} }, }, - sources: { "step-1": {}, "step-2": {} }, + sources: { "step-1": [{}], "step-2": [{}] }, }), ).not.toThrow(); }); @@ -1477,7 +1505,7 @@ describe("createSidecarDeployRouter multi-step branch", () => { stepOrder: ["step-1"], steps: { "step-1": { kind: "step" } }, }, - sources: { "step-1": makeInferenceSource("step-1") }, + sources: { "step-1": [makeInferenceSource("step-1")] }, }); await expect(router.deploy(frame)).rejects.toThrow( @@ -1505,7 +1533,7 @@ describe("createSidecarDeployRouter multi-step branch", () => { stepOrder: ["step-1"], steps: { "step-1": { kind: "step" } }, }, - sources: { "step-1": makeInferenceSource("step-1") }, + sources: { "step-1": [makeInferenceSource("step-1")] }, }); await expect(router.deploy(frame)).rejects.toThrow(/ENOENT/); @@ -1562,10 +1590,12 @@ describe("createSidecarDeployRouter multi-step branch", () => { steps: { "step-1": { kind: "step" } }, }, sources: { - "step-1": { - ...makeInferenceSource("step-1"), - provider: "ghost-provider", - }, + "step-1": [ + { + ...makeInferenceSource("step-1"), + provider: "ghost-provider", + }, + ], }, }); @@ -1594,7 +1624,7 @@ describe("createSidecarDeployRouter multi-step branch", () => { steps: { "step-1": { kind: "step" } }, }, sources: { - "step-1": makeInferenceSource("step-1"), + "step-1": [makeInferenceSource("step-1")], }, }); @@ -1621,7 +1651,7 @@ describe("createSidecarDeployRouter multi-step branch", () => { steps: { "step-1": { kind: "step" } }, }, sources: { - "step-1": makeInferenceSource("step-1"), + "step-1": [makeInferenceSource("step-1")], }, }); @@ -2118,7 +2148,7 @@ describe("createSidecarDeployRouter multi-step branch", () => { stepOrder: ["step-1"], steps: { "step-1": { kind: "step" } }, }, - sources: { "step-1": makeInferenceSource("step-1") }, + sources: { "step-1": [makeInferenceSource("step-1")] }, }); } @@ -2221,7 +2251,7 @@ describe("createSidecarDeployRouter multi-step branch", () => { version: 1, agentAddress: head, definitionId: "wf-missing-step", - sources: { "step-1": makeInferenceSource("step-1") }, + sources: { "step-1": [makeInferenceSource("step-1")] }, hubPublicKey: "hub-pk", }; await writeWorkflowDeploymentRecord(dataDir, deploymentId, record); @@ -2323,7 +2353,7 @@ describe("createSidecarDeployRouter multi-step branch", () => { version: 1, agentAddress: head, definitionId: "wf-mismatch", - sources: { "step-1": makeInferenceSource("step-1") }, + sources: { "step-1": [makeInferenceSource("step-1")] }, hubPublicKey: "hub-pk", }; await writeWorkflowDeploymentRecord(dataDir, wrongDir, record); diff --git a/apps/sidecar/src/workflow-host-wiring.ts b/apps/sidecar/src/workflow-host-wiring.ts index b74de381..880c5ead 100644 --- a/apps/sidecar/src/workflow-host-wiring.ts +++ b/apps/sidecar/src/workflow-host-wiring.ts @@ -516,11 +516,12 @@ export type SidecarWorkflowSupervisor = { const TRIVIAL_STEP_ID = "trivial"; /** - * Env key the multi-step branch uses to carry per-step inference source - * pins from `frame.workflow.sources` down to the workflow-process child. - * The substrate factory's `buildEnv` reads this and resolves a source - * per step at step invocation; the supervisor itself is opaque to the - * value (it is plumbed through `bindings.substrateEnv` verbatim). + * Env key the multi-step branch uses to carry each step's ordered + * inference-source failover chain from `frame.workflow.sources` down to + * the workflow-process child. The substrate factory's `buildEnv` reads + * this and resolves a step's chain at step invocation, feeding it to the + * reactor for forward-only failover; the supervisor itself is opaque to + * the value (it is plumbed through `bindings.substrateEnv` verbatim). * * Listed here so the router and the future substrate-factory consumer * spell the key the same way without a magic-string trip hazard. @@ -550,11 +551,14 @@ export const STEP_INFERENCE_SOURCES_ENV_KEY = "STEP_INFERENCE_SOURCES"; * entry be absent; presence is required so the workflow-process * child can resolve each step's primitive at run time. * - Every `stepOrder` entry has a corresponding `sources[id]` - * entry. The arktype narrow already enforces this; the re-check - * here surfaces a structured router-side error instead of an - * arktype validation failure at the wire boundary, which keeps + * entry, and that entry is a non-empty array (the step's ordered + * failover chain). The arktype narrow already enforces both; the + * re-check here surfaces a structured router-side error instead of + * an arktype validation failure at the wire boundary, which keeps * the failure shape consistent with the rest of the validations - * this function owns. + * this function owns. An empty chain would leave the reactor with + * no initial source, so it is rejected here rather than deferred to + * a deep-stack child failure. * * A rejection here surfaces as a thrown `Error` the link's deploy * frame caller converts into a structured failure reply. @@ -607,6 +611,13 @@ export function validateWorkflowProjection(projection: { `sidecar deploy router: workflow.sources is missing entry for stepId ${JSON.stringify(stepId)}`, ); } + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- sources is checked to be a non-null object above; this reads a value to re-check its array shape + const stepSources = (sources as Record)[stepId]; + if (!Array.isArray(stepSources) || stepSources.length === 0) { + throw new Error( + `sidecar deploy router: workflow.sources[${JSON.stringify(stepId)}] must be a non-empty array (the step's ordered inference-source failover chain)`, + ); + } } } @@ -1131,7 +1142,8 @@ export function createSidecarDeployRouter(deps: { // validator requires. The boot edge's `multistepSubstrateEnv` carries // the boot-edge constants; the four workflow-definition / workflow-run // identity keys are derived per-deploy here. `STEP_INFERENCE_SOURCES` - // threads the pinned per-step sources into the child. + // threads each step's ordered inference-source failover chain into the + // child. const substrateEnv: Record = { ...multistepSubstrateEnv, WORKFLOW_DEFINITION_REPO_ID: spec.definition.id, @@ -1350,17 +1362,21 @@ export function createSidecarDeployRouter(deps: { // supervisor. validateWorkflowProjection(projection); - // Source-admission gate: reject a deploy whose any step pins an + // Source-admission gate: reject a deploy where any step pins an // inference provider this sidecar cannot build, BEFORE any state is // claimed or the child is spawned. The throw propagates back through // the deploy frame so the hub's `deployWorkflow` rejects synchronously // at deploy time, rather than the child failing the run when the // step's inference first resolves. Covers single- and multi-step: the // projection's `narrow` guarantees every stepOrder entry has a - // `sources` entry. + // `sources` entry. Every source in a step's failover chain must be + // buildable -- a chain with an unbuildable tail would fail only after + // the reactor failed over onto it -- so this iterates the whole list. for (const stepId of projection.definition.stepOrder) { - const source = projection.sources[stepId]; - if (source !== undefined) deps.assertSourceBuildable(source); + const chain = projection.sources[stepId]; + if (chain !== undefined) { + for (const source of chain) deps.assertSourceBuildable(source); + } } // Reject a re-deploy of an address already live in this process BEFORE @@ -1729,12 +1745,15 @@ export function createSidecarDeployRouter(deps: { validateWorkflowProjection(projection); // Re-run the source-admission gate: refuse to restore a deployment - // whose pinned provider this sidecar can no longer build. The - // record is KEPT (not deleted) so a later boot with the provider - // restored retries it. + // whose pinned provider this sidecar can no longer build. Every + // source in a step's failover chain must be buildable, so this + // iterates the whole list. The record is KEPT (not deleted) so a + // later boot with the provider restored retries it. for (const stepId of projection.definition.stepOrder) { - const source = projection.sources[stepId]; - if (source !== undefined) deps.assertSourceBuildable(source); + const chain = projection.sources[stepId]; + if (chain !== undefined) { + for (const source of chain) deps.assertSourceBuildable(source); + } } const spec: WorkflowDeploySpec = { diff --git a/apps/sidecar/src/workflow-substrate-factory-step-storage.test.ts b/apps/sidecar/src/workflow-substrate-factory-step-storage.test.ts index 2f863d15..326437ab 100644 --- a/apps/sidecar/src/workflow-substrate-factory-step-storage.test.ts +++ b/apps/sidecar/src/workflow-substrate-factory-step-storage.test.ts @@ -86,9 +86,10 @@ function stubDurableConversationRegistry(): DurableConversationRegistry { function buildDeps(opts: { dataDir: string; durableConversation?: DurableConversationRegistry; + sourceChain?: InferenceSource[]; }): SidecarStepBuildEnvDeps { return { - table: { [STEP_ID]: SOURCE }, + table: { [STEP_ID]: opts.sourceChain ?? [SOURCE] }, dataDir: opts.dataDir, workflowRunRepoId: WORKFLOW_RUN_REPO_ID, signer: (payload: string) => Promise.resolve(`sig:${payload.length}`), @@ -173,4 +174,29 @@ describe("createSidecarStepBuildEnv per-step scratch keying", () => { // The cold path never parks scratch under the warm sub-root. expect(env1.workdir).not.toContain(path.join("warm", STEP_ID)); }); + + test("feeds the reactor the whole failover chain with the head pinned as default", async () => { + // The reactor resolves its initial source by `defaultSource` and fails + // over forward across `sources`; the child must hand it the full ordered + // chain, not just the active source, or cross-provider failover is lost. + const failoverSource: InferenceSource = { + id: "failover", + provider: "openai", + baseURL: "https://api.openai.com", + apiKey: "sk-failover", + model: "gpt-failover", + }; + const chain = [SOURCE, failoverSource]; + const dataDir = await makeTempDir(); + const buildEnv = createSidecarStepBuildEnv( + buildDeps({ dataDir, sourceChain: chain }), + ); + + const env: StepEnvBase = await buildEnv(requestForRun("run-1")); + + // The whole chain reaches the reactor, ordered, with element 0 as the + // initial source. + expect(env.sources).toEqual(chain); + expect(env.defaultSource).toBe(SOURCE.id); + }); }); diff --git a/apps/sidecar/src/workflow-substrate-factory.ts b/apps/sidecar/src/workflow-substrate-factory.ts index ecec5a94..a76cf71d 100644 --- a/apps/sidecar/src/workflow-substrate-factory.ts +++ b/apps/sidecar/src/workflow-substrate-factory.ts @@ -179,15 +179,17 @@ function parseByteCap(raw: string, name: string): number { } /** - * Per-step `InferenceSource` table parsed from the spawn-time + * Per-step inference-source table parsed from the spawn-time * `STEP_INFERENCE_SOURCES` env entry. The deploy router serializes - * `frame.workflow.sources` (a `Record`) as + * `frame.workflow.sources` (a `Record`) as * JSON and threads it through the supervisor's `substrateEnv`; the * factory parses and validates the table once at construction time - * and pins it for `buildEnv` lookups. + * and pins it for `buildEnv` lookups. Each value is the step's ordered + * failover chain -- element 0 is the active source, the tail are the + * reactor's forward-only failover targets -- so the list is non-empty. */ const StepInferenceSourceTable = type({ - "[string]": InferenceSource, + "[string]": InferenceSource.array().atLeastLength(1), }); type StepInferenceSourceTable = typeof StepInferenceSourceTable.infer; @@ -260,24 +262,26 @@ export function parseAdapterManifest(raw: string): AdapterManifest { } /** - * Resolve the per-step `InferenceSource` pinned at factory - * construction. The supervisor's multi-step branch only invokes a - * step whose `stepId` appears in `frame.workflow.sources`; a lookup + * Resolve the per-step inference-source failover chain pinned at + * factory construction. The supervisor's multi-step branch only invokes + * a step whose `stepId` appears in `frame.workflow.sources`; a lookup * miss here is a programmer error in the supervisor, not a wire-side * failure, and the resolver surfaces it with the missing `stepId` - * named. + * named. The returned list is the step's ordered chain (element 0 the + * active source, the tail the reactor's failover targets); the table's + * arktype guarantees it is non-empty. */ function createStepInferenceSourceResolver( table: StepInferenceSourceTable, -): (stepId: string) => InferenceSource { - return (stepId: string): InferenceSource => { - const source = table[stepId]; - if (source === undefined) { +): (stepId: string) => InferenceSource[] { + return (stepId: string): InferenceSource[] => { + const sources = table[stepId]; + if (sources === undefined) { throw new Error( `sidecar workflow-child step invoker buildEnv: no InferenceSource pinned for stepId ${JSON.stringify(stepId)}; the supervisor must populate frame.workflow.sources for every stepOrder entry`, ); } - return source; + return sources; }; } @@ -526,7 +530,16 @@ export function createSidecarStepBuildEnv( "sidecar workflow-child step invoker buildEnv: AuthorizeContext.attempt is required to root per-step storage per attempt; the workflow runtime must populate attempt on every step-originated invocation", ); } - const source = resolveStepInferenceSource(stepId); + const sources = resolveStepInferenceSource(stepId); + // The resolver's arktype guarantees a non-empty chain; assert it here so + // the reactor's initial-source pin (element 0) is a checked fact rather + // than an unchecked index. + const activeSource = sources[0]; + if (activeSource === undefined) { + throw new Error( + `sidecar workflow-child step invoker buildEnv: empty InferenceSource chain pinned for stepId ${JSON.stringify(stepId)}`, + ); + } // Root the per-step scratch (workspace + tool tarball-cache + // apply-state). The cold (multi-step) path keys it per @@ -612,8 +625,13 @@ export function createSidecarStepBuildEnv( // narrower type). const env: StepEnvBase & { transport: MessageTransport; address: string } = { - sources: [source], - defaultSource: source.id, + // Feed the reactor the step's full ordered failover chain and pin + // its initial source to element 0. The reactor resolves the initial + // source by id and fails over forward through `sources`, so this + // restores cross-source failover inside the workflow-child, matching + // the legacy in-process harness. + sources, + defaultSource: activeSource.id, storage, workdir, audit: storage, diff --git a/packages/hub-sessions/src/session-service.test.ts b/packages/hub-sessions/src/session-service.test.ts index 35592c83..cf05ab1e 100644 --- a/packages/hub-sessions/src/session-service.test.ts +++ b/packages/hub-sessions/src/session-service.test.ts @@ -1679,21 +1679,26 @@ describe("sendMultiStepDeployFrame", () => { execute: step({ agent: stubAgent, after: ["plan"] }), }, }); + // Each step's value is its ordered inference-source failover chain. const sources = { - plan: { - id: "src-plan", - provider: "anthropic", - baseURL: "https://api.example/anthropic", - apiKey: "secret-plan", - model: "mock-model", - }, - execute: { - id: "src-execute", - provider: "anthropic", - baseURL: "https://api.example/anthropic", - apiKey: "secret-execute", - model: "mock-model", - }, + plan: [ + { + id: "src-plan", + provider: "anthropic", + baseURL: "https://api.example/anthropic", + apiKey: "secret-plan", + model: "mock-model", + }, + ], + execute: [ + { + id: "src-execute", + provider: "anthropic", + baseURL: "https://api.example/anthropic", + apiKey: "secret-execute", + model: "mock-model", + }, + ], }; const config: HarnessConfig = { sessionId: "ses-multi", @@ -1704,7 +1709,7 @@ describe("sendMultiStepDeployFrame", () => { systemPrompt: "deployment-level", tools: [], grants: [], - sources: Object.values(sources), + sources: Object.values(sources).flat(), defaultSource: "src-plan", }; @@ -1731,3 +1736,95 @@ describe("sendMultiStepDeployFrame", () => { expect(sent.sources).toEqual(sources); }); }); + +describe("deployInstanceAtHead inference-source pinning", () => { + const MULTI_SOURCE_CONFIG: HarnessConfig = { + ...MOCK_CONFIG, + sources: [ + { + id: "primary", + provider: "anthropic", + baseURL: "https://api.example/anthropic", + apiKey: "sk-primary", + model: "claude-mock", + }, + { + id: "failover", + provider: "openai", + baseURL: "https://api.example/openai", + apiKey: "sk-failover", + model: "gpt-mock", + }, + ], + defaultSource: "primary", + }; + + test("pins the instance's full ordered source chain to the sole step", async () => { + const router = createMockRouter(); + // Capture the workflow frame off `sendAgentDeploy` with its typed + // signature rather than casting the tracker's `unknown[]` args. + const sentWorkflows: Parameters[2][] = []; + router.sendAgentDeploy = (( + _agentAddress: string, + _config: HarnessConfig, + workflow?: Parameters[2], + ) => { + sentWorkflows.push(workflow); + return Promise.resolve({ publicKey: "mock-public-key" }); + }) as SidecarRouter["sendAgentDeploy"]; + const repoStore = createMockRepoStore(); + const service = createSessionService({ + sidecarRouter: router, + agentRepoStore: repoStore, + }); + + await service.deployInstanceAtHead({ + agentAddress: AGENT_ADDRESS, + agentId: AGENT_ID, + instanceId: INSTANCE_ID, + config: MULTI_SOURCE_CONFIG, + deployContent: MOCK_CONTENT, + }); + + const workflow = sentWorkflows[0]; + if (workflow === undefined) { + throw new Error("sendAgentDeploy was not called with a workflow frame"); + } + const chains = Object.values(workflow.sources); + // A single-step-at-head deploy has exactly one step, so one chain. + expect(chains).toHaveLength(1); + // The reactor fails over forward across the whole chain, so the head + // pins `config.sources` verbatim rather than only the default source. + expect(chains[0]).toEqual(MULTI_SOURCE_CONFIG.sources); + }); + + test("rejects a config whose first source is not the default source", async () => { + const router = createMockRouter(); + const repoStore = createMockRepoStore(); + const service = createSessionService({ + sidecarRouter: router, + agentRepoStore: repoStore, + }); + + // The reactor resolves its initial source by `defaultSource` and fails + // over forward with no wrap; a default that is not element 0 would leave + // the head unreachable, so the deploy must fail loudly. + const misordered: HarnessConfig = { + ...MULTI_SOURCE_CONFIG, + defaultSource: "failover", + }; + await expect( + service.deployInstanceAtHead({ + agentAddress: AGENT_ADDRESS, + agentId: AGENT_ID, + instanceId: INSTANCE_ID, + config: misordered, + deployContent: MOCK_CONTENT, + }), + ).rejects.toThrow(/must be the default source/); + + expect(router.calls.some((c) => c.method === "sendAgentDeploy")).toBe( + false, + ); + }); +}); diff --git a/packages/hub-sessions/src/session-service.ts b/packages/hub-sessions/src/session-service.ts index 1ef2927d..ff1ad566 100644 --- a/packages/hub-sessions/src/session-service.ts +++ b/packages/hub-sessions/src/session-service.ts @@ -520,7 +520,7 @@ export async function sendMultiStepDeployFrame(args: { agentAddress: string; config: HarnessConfig; definition: WorkflowDefinition; - sources: Record; + sources: Record; }): Promise<{ publicKey: string }> { // The wire validator's projection types `stepOrder` and `triggers` // as mutable arrays while `WorkflowDefinition` declares them as @@ -638,7 +638,7 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { */ workflowFrame?: { definition: WorkflowDefinition; - sources: Record; + sources: Record; }; }): Promise<{ publicKey: string } | undefined> { const { agentAddress, agentId, instanceId, config, deployContent } = params; @@ -1071,14 +1071,28 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { ); } - // Pin the step's inference source to the instance's default source. The - // instance route already resolved and authorized it against the tenant - // catalog, so the source is selected directly rather than re-run through - // the orchestrator's operator-approval gate. - const source = config.sources.find((s) => s.id === config.defaultSource); - if (source === undefined) { + // Pin the step's inference sources to the instance's FULL ordered source + // chain so the workflow-process child's reactor fails over across it at + // runtime, matching the legacy in-process harness. The route already + // resolved and authorized `config.sources` against the tenant catalog, so + // it is pinned directly rather than re-run through the orchestrator's + // operator-approval gate. + // + // Fail loud on the invariant the reactor depends on: the reactor resolves + // its initial source by id (`defaultSource`) and fails over FORWARD-ONLY + // with no wrap, so the default must be element 0 or part of the chain is + // unreachable -- and if the default were last, failover would silently + // no-op. The route guarantees `config.sources[0].id === config.defaultSource` + // (head = active); assert it here so a future reordering fails loudly + // rather than silently disabling failover. + if (config.sources.length === 0) { + throw new Error( + `instance deploy for ${agentAddress}: config.sources is empty; at least the default source is required`, + ); + } + if (config.sources[0]?.id !== config.defaultSource) { throw new Error( - `instance deploy for ${agentAddress}: config.defaultSource ${JSON.stringify(config.defaultSource)} does not resolve to a HarnessConfig.sources entry`, + `instance deploy for ${agentAddress}: config.sources[0] (${JSON.stringify(config.sources[0]?.id)}) must be the default source ${JSON.stringify(config.defaultSource)}; the reactor fails over forward from the default and would otherwise skip the head`, ); } @@ -1089,7 +1103,7 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { config, deployContent, definition: workflow, - sources: { [stepId]: source }, + sources: { [stepId]: config.sources }, hubPublicKey: hexEncode(agentRepoStore.getSigningPublicKey()), ...(params.toolPackagePins !== undefined ? { toolPackagePins: params.toolPackagePins } diff --git a/packages/types/src/sidecar.test.ts b/packages/types/src/sidecar.test.ts index c33c7e41..fcd954ac 100644 --- a/packages/types/src/sidecar.test.ts +++ b/packages/types/src/sidecar.test.ts @@ -160,7 +160,7 @@ describe("AgentDeployFrame", () => { stepOrder: ["plan", "act"], steps: { plan: {}, act: {} }, }, - sources: { plan: stepSource, act: stepSource }, + sources: { plan: [stepSource], act: [stepSource] }, }, }); expect(result instanceof type.errors).toBe(false); @@ -191,7 +191,7 @@ describe("AgentDeployFrame", () => { stepOrder: ["plan", "act"], steps: { plan: {}, act: {} }, }, - sources: { plan: stepSource }, + sources: { plan: [stepSource] }, }, }); expect(result instanceof type.errors).toBe(true); @@ -210,7 +210,7 @@ describe("AgentDeployFrame", () => { stepOrder: ["plan"], steps: { plan: {} }, }, - sources: { plan: stepSource }, + sources: { plan: [stepSource] }, }, }); expect(result instanceof type.errors).toBe(true); diff --git a/packages/types/src/sidecar.ts b/packages/types/src/sidecar.ts index 2fce6612..ec1b1adc 100644 --- a/packages/types/src/sidecar.ts +++ b/packages/types/src/sidecar.ts @@ -280,11 +280,16 @@ export type DrainDeliverFrame = typeof DrainDeliverFrame.infer; * of authoring-time primitive shape lives on the workflow definition * surface in `@intx/workflow`, not on the wire. * - * `sources` pins one inference source per step in `definition.stepOrder` - * so the workflow-process child can resolve inference at step invocation - * without a round trip to the hub. Every `stepOrder` entry must have a - * matching `sources` entry; the validator rejects frames that violate - * this invariant at the boundary. + * `sources` pins an ordered, non-empty inference-source list per step in + * `definition.stepOrder` so the workflow-process child can resolve inference + * at step invocation without a round trip to the hub. The list is the step's + * failover chain: element 0 is the active source (its id is the step's + * `defaultSource`), and the reactor fails over forward through the tail on a + * transient inference error. A workflow step pins a single-element list (no + * per-step failover); a single-agent instance pins the instance's full + * ordered source chain. Every `stepOrder` entry must have a matching + * `sources` entry; the validator rejects frames that violate this at the + * boundary. */ export const AgentDeployWorkflow = type({ definition: type({ @@ -295,7 +300,7 @@ export const AgentDeployWorkflow = type({ "state?": "Record", "+": "delete", }), - sources: { "[string]": InferenceSource }, + sources: { "[string]": InferenceSource.array().atLeastLength(1) }, }).narrow((value, ctx) => { for (const stepId of value.definition.stepOrder) { if (!Object.prototype.hasOwnProperty.call(value.sources, stepId)) { diff --git a/packages/workflow-deploy/src/orchestrator-fallback-source.test.ts b/packages/workflow-deploy/src/orchestrator-fallback-source.test.ts index 1510a90e..257119cf 100644 --- a/packages/workflow-deploy/src/orchestrator-fallback-source.test.ts +++ b/packages/workflow-deploy/src/orchestrator-fallback-source.test.ts @@ -240,13 +240,16 @@ describe("pickStepInferenceSource (agent step)", () => { expect(deps.sources).toHaveLength(1); const sources = deps.sources[0]; if (sources === undefined) throw new Error("missing sources"); - expect(sources.only).toEqual({ - id: "src-default", - provider: "openai", - baseURL: "https://api.example/openai", - apiKey: "secret", - model: "default-model", - }); + // The step pins a single-element failover chain around the picked source. + expect(sources.only).toEqual([ + { + id: "src-default", + provider: "openai", + baseURL: "https://api.example/openai", + apiKey: "secret", + model: "default-model", + }, + ]); }); test("uses the agent's preferred source when it matches an approved HarnessConfig source", async () => { @@ -305,13 +308,16 @@ describe("pickStepInferenceSource (agent step)", () => { expect(deps.sources).toHaveLength(1); const sources = deps.sources[0]; if (sources === undefined) throw new Error("missing sources"); - expect(sources.only).toEqual({ - id: "src-preferred", - provider: "anthropic", - baseURL: "https://api.example/anthropic", - apiKey: "secret-a", - model: "preferred-model", - }); + // The step pins a single-element failover chain around the picked source. + expect(sources.only).toEqual([ + { + id: "src-preferred", + provider: "anthropic", + baseURL: "https://api.example/anthropic", + apiKey: "secret-a", + model: "preferred-model", + }, + ]); }); test("regression: unapproved fallback is also rejected at the approval gate when the walk surfaces it", async () => { diff --git a/packages/workflow-deploy/src/orchestrator-nonagent-source.test.ts b/packages/workflow-deploy/src/orchestrator-nonagent-source.test.ts index dd9435c8..012c29a1 100644 --- a/packages/workflow-deploy/src/orchestrator-nonagent-source.test.ts +++ b/packages/workflow-deploy/src/orchestrator-nonagent-source.test.ts @@ -312,14 +312,17 @@ describe("pickStepInferenceSource (non-agent step)", () => { expect(deps.sources).toHaveLength(1); const sources = deps.sources[0]; if (sources === undefined) throw new Error("missing sources"); - // Both steps land on the approved source. + // Both steps land on the approved source, each pinned as a single-element + // failover chain. expect(sources.worker).toBeDefined(); - expect(sources.cooldown).toEqual({ - id: "src-anthropic", - provider: "anthropic", - baseURL: "https://api.example/anthropic", - apiKey: "secret-a", - model: "worker-model", - }); + expect(sources.cooldown).toEqual([ + { + id: "src-anthropic", + provider: "anthropic", + baseURL: "https://api.example/anthropic", + apiKey: "secret-a", + model: "worker-model", + }, + ]); }); }); diff --git a/packages/workflow-deploy/src/orchestrator.test.ts b/packages/workflow-deploy/src/orchestrator.test.ts index f53dd67b..c1c3b5f2 100644 --- a/packages/workflow-deploy/src/orchestrator.test.ts +++ b/packages/workflow-deploy/src/orchestrator.test.ts @@ -409,8 +409,10 @@ describe("createWorkflowDeployOrchestrator", () => { expect(Object.keys(handoff.sources).sort()).toEqual(["execute", "plan"]); const baseSource = HARNESS_CONFIG_BASE.sources[0]; if (baseSource === undefined) throw new Error("missing base source"); - expect(handoff.sources.plan).toEqual(baseSource); - expect(handoff.sources.execute).toEqual(baseSource); + // Each step pins a single-element failover chain: the multi-step branch + // wraps its one picked source in a list. + expect(handoff.sources.plan).toEqual([baseSource]); + expect(handoff.sources.execute).toEqual([baseSource]); expect(result).toEqual({ kind: "multi-step", publicKey: "ff".repeat(32), diff --git a/packages/workflow-deploy/src/orchestrator.ts b/packages/workflow-deploy/src/orchestrator.ts index 74e3086e..1305efb9 100644 --- a/packages/workflow-deploy/src/orchestrator.ts +++ b/packages/workflow-deploy/src/orchestrator.ts @@ -115,7 +115,7 @@ export type SendMultiStepDeployFn = (params: { agentId: string; config: HarnessConfig; definition: WorkflowDefinition; - sources: Record; + sources: Record; hubPublicKey: string; }) => Promise; @@ -143,7 +143,7 @@ export type DeploySingleStepFn = (params: { config: HarnessConfig; deployContent: DeployContent; definition: WorkflowDefinition; - sources: Record; + sources: Record; hubPublicKey: string; toolPackagePins?: readonly ToolPackagePin[]; }) => Promise; @@ -563,7 +563,11 @@ async function runSingleStepAtHead(args: { config: headConfig, deployContent: headDeployContent, definition: deploy.workflow, - sources: { [stepId]: source }, + // A workflow step pins a single source (no per-step failover): wrap the + // one operator-approved source in a one-element list. The per-step + // failover chain is intentionally an instance-only concern; a workflow + // step preserves its prior single-source behavior. + sources: { [stepId]: [source] }, hubPublicKey: deploy.hubPublicKey, ...(deploy.toolPackagePins !== undefined ? { toolPackagePins: deploy.toolPackagePins } @@ -606,7 +610,7 @@ async function runMultiStepBranch(args: { config: HarnessConfig; deployContent: DeployContent; }; - const sources: Record = {}; + const sources: Record = {}; const prepared: PreparedStep[] = []; for (const stepId of deploy.workflow.stepOrder) { const primitive = deploy.workflow.steps[stepId]; @@ -617,13 +621,18 @@ async function runMultiStepBranch(args: { ); } const stepAgent = extractAgent(primitive); - sources[stepId] = pickStepInferenceSource({ - stepAgent, - stepId, - workflowId: deploy.workflow.id, - config: deploy.config, - operatorApprovals: deploy.operatorApprovals, - }); + // A workflow step pins a single source (no per-step failover), wrapped in + // a one-element list. Per-step failover chains are an instance-only + // concern; this preserves prior workflow-step behavior. + sources[stepId] = [ + pickStepInferenceSource({ + stepAgent, + stepId, + workflowId: deploy.workflow.id, + config: deploy.config, + operatorApprovals: deploy.operatorApprovals, + }), + ]; const agentAddress = deriveStepAddress({ deploymentId, stepId, diff --git a/packages/workflow-host/src/supervisor/types.ts b/packages/workflow-host/src/supervisor/types.ts index a3864310..c928159b 100644 --- a/packages/workflow-host/src/supervisor/types.ts +++ b/packages/workflow-host/src/supervisor/types.ts @@ -207,7 +207,7 @@ export type SubprocessSpawner = (args: { * to branch into `supervisor.spawn()` instead of `trivialLaunch`. The * field carries the workflow definition (so the supervisor can * construct the per-step substrate env without round-tripping the - * hub) and the per-step inference source pins keyed by + * hub) and each step's ordered inference-source failover chain keyed by * `definition.stepOrder` step ids. */ export interface SupervisorDeployFrame { @@ -217,7 +217,7 @@ export interface SupervisorDeployFrame { hubPublicKey: string; workflow?: { definition: WorkflowDefinition; - sources: Record; + sources: Record; }; } From 84602e397a51159c897432b3942dc799ea8a4dcc Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Fri, 3 Jul 2026 05:54:53 -0500 Subject: [PATCH 029/101] Prove inference-source failover in a real workflow child Unit coverage shows the per-step source chain is threaded to the workflow-process child, but not that the child's reactor honors it at runtime across the process boundary. Add an end-to-end test that deploys a single-agent instance with a two-element source chain whose head is a dead provider returning HTTP 500 and whose tail is the healthy mock inference server. The child exhausts its mechanical retries against the head, fails over forward to the tail, and returns the tail's live reply. Failover is proven by the reply equalling the healthy source's output: the dead head only ever returns 500s and cannot produce that string. Run completion alone proves nothing, because a total inference failure also completes the run with a synthesized provider-error reply; a sibling negative-control test pins that contract so the reply-equality proof is grounded rather than assumed. --- Makefile | 2 +- .../instance-failover-real-agent.test.ts | 310 ++++++++++++++++++ 2 files changed, 311 insertions(+), 1 deletion(-) create mode 100644 tests/workflow-deploy/instance-failover-real-agent.test.ts diff --git a/Makefile b/Makefile index 1bf63a92..e2ebc723 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ lint: FORCE test: FORCE bun test packages/ apps/ bin/ tests/agent/ tests/agent-audit-log/ tests/agent-blob-spill/ tests/agent-common/ tests/agent-multi-provider/ tests/agent-quickstart/ tests/agent-resume/ tests/agent-rewind/ tests/agent-rich-tool/ tests/agent-structured-payload/ tests/coding-agent/ tests/hub-agent/lib/ tests/inference-testing/ tests/tool-packaging/ tests/workflow/ - bun test --timeout 120000 tests/hub-agent/deploy-flow.test.ts tests/workflow-deploy/trivial-roundtrip.test.ts tests/workflow-deploy/multistep-signal.test.ts tests/workflow-deploy/single-step-real-agent.test.ts tests/workflow-deploy/instance-reroute-real-agent.test.ts tests/workflow-deploy/single-step-message-input.test.ts tests/workflow-deploy/single-step-posix-tool.test.ts tests/workflow-deploy/single-step-grants-bridge.test.ts tests/workflow-deploy/single-step-event-threading.test.ts tests/workflow-deploy/single-step-conversation-durability.test.ts tests/workflow-deploy/single-step-full-lifecycle.test.ts tests/workflow-deploy/cross-process-custom-adapter.test.ts tests/workflow-deploy/conversation-state-wal.test.ts tests/workflow-deploy/drain-roundtrip.test.ts tests/workflow-deploy/child-workflow-roundtrip.test.ts tests/workflow-deploy/unresolvable-director.test.ts tests/workflow-deploy/fifo-mail.test.ts tests/workflow-deploy/mail-edge-cases.test.ts tests/inference/ tests/skill-attachment-flow.test.ts tests/hub-api/ tests/db/ + bun test --timeout 120000 tests/hub-agent/deploy-flow.test.ts tests/workflow-deploy/trivial-roundtrip.test.ts tests/workflow-deploy/multistep-signal.test.ts tests/workflow-deploy/single-step-real-agent.test.ts tests/workflow-deploy/instance-reroute-real-agent.test.ts tests/workflow-deploy/instance-failover-real-agent.test.ts tests/workflow-deploy/single-step-message-input.test.ts tests/workflow-deploy/single-step-posix-tool.test.ts tests/workflow-deploy/single-step-grants-bridge.test.ts tests/workflow-deploy/single-step-event-threading.test.ts tests/workflow-deploy/single-step-conversation-durability.test.ts tests/workflow-deploy/single-step-full-lifecycle.test.ts tests/workflow-deploy/cross-process-custom-adapter.test.ts tests/workflow-deploy/conversation-state-wal.test.ts tests/workflow-deploy/drain-roundtrip.test.ts tests/workflow-deploy/child-workflow-roundtrip.test.ts tests/workflow-deploy/unresolvable-director.test.ts tests/workflow-deploy/fifo-mail.test.ts tests/workflow-deploy/mail-edge-cases.test.ts tests/inference/ tests/skill-attachment-flow.test.ts tests/hub-api/ tests/db/ test-load: FORCE bun test --timeout 300000 tests/workflow-deploy/fifo-mail-load.test.ts diff --git a/tests/workflow-deploy/instance-failover-real-agent.test.ts b/tests/workflow-deploy/instance-failover-real-agent.test.ts new file mode 100644 index 00000000..6fd12a2b --- /dev/null +++ b/tests/workflow-deploy/instance-failover-real-agent.test.ts @@ -0,0 +1,310 @@ +// Inference-source failover real-agent round-trip integration test. +// +// The proof that the per-step inference-source failover chain is HONORED at +// runtime by a REAL spawned workflow-process child -- not merely threaded +// through the wire. A single-agent instance is deployed with a two-element +// source chain whose head is a dead provider (HTTP 500) and whose tail is the +// healthy mock inference server. The child's reactor exhausts its mechanical +// retries against the head, fails over forward to the tail, and returns the +// tail's live reply. This closes the gap the unit tests leave open: they prove +// the child builds `env.sources` from the whole chain, but only a real child +// against a real provider pair proves the reactor actually advances across it. +// +// A dead head maps to `retryable` (HTTP 5xx), so the harness retries it a few +// times before the reactor fails over -- exercising the full +// harness-retry-then-reactor-failover path. +// +// The discriminating proof is that the step reply EQUALS the healthy tail's +// output. Run completion alone proves nothing: a total inference failure with +// no failover target ALSO completes -- the agent synthesizes a graceful +// provider-error reply and the run lands `RunCompleted`. The second test here +// pins that contract as a negative control, so the first test's reliance on +// reply-equality (rather than run completion) is grounded rather than assumed. + +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; + +import type { HarnessConfig } from "@intx/types/runtime"; +import { wrapHarnessAsTrivialAgent } from "@intx/workflow-deploy"; +import { defineWorkflow, type WorkflowDefinition } from "@intx/workflow"; +import { deriveTrivialDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; +import type { RepoId } from "@intx/hub-sessions"; + +import { + SESSION_ID, + SIDECAR_ID, + fireMailTrigger, + readWorkflowRunEvents, + startDeployFlowEnv, + waitForFirstRunId, + waitForWorkflowRunComplete, + type DeployFlowEnv, +} from "../hub-agent/lib/deploy-flow-env"; + +const DEPLOYMENT_DOMAIN = "integration.interchange"; +// A real instance identity, distinct from the reroute test's `b`-filled id so +// the two integration tests never collide on a substrate slug. +const INSTANCE_ID = `ins_${"c".repeat(32)}`; +const AGENT_ID = "agent-instance-failover"; +const WORKFLOW_RUN_REF = "refs/heads/main"; + +// The mock inference server's reply for an empty tool set. +const EXPECTED_REPLY = "I see these tools:"; + +let env: DeployFlowEnv; + +// The dead head source. Always returns HTTP 500 (a `retryable` category), so +// the child's reactor fails over off it. `startDeployFlowEnv().teardown()` does +// not own this server, so it is started and stopped symmetrically here. +let headRequests = 0; +let deadHead: ReturnType; + +beforeAll(async () => { + env = await startDeployFlowEnv(); + deadHead = Bun.serve({ + port: 0, + fetch() { + headRequests += 1; + return new Response("upstream boom", { status: 500 }); + }, + }); +}); + +afterAll(async () => { + await env.teardown(); + await deadHead.stop(true); +}); + +describe("instance inference-source failover real-agent round-trip", () => { + test("sidecar registers with hub", () => { + expect(env.hub.router.getConnectedSidecars()).toContain(SIDECAR_ID); + }); + + test("the child fails over from a dead head source to the healthy tail", async () => { + const agentAddress = `${INSTANCE_ID}@${DEPLOYMENT_DOMAIN}`; + + // A two-element failover chain: element 0 is the dead head (default), the + // tail is the healthy mock inference server. The head must be element 0 and + // equal `defaultSource` -- the reactor resolves its initial source by id and + // fails over forward, so the deploy asserts that invariant. + const config: HarnessConfig = { + sessionId: SESSION_ID, + agentId: AGENT_ID, + tenantId: "tenant-1", + principalId: "prin_integration-failover-1", + agentAddress, + systemPrompt: "You are the failover single-agent instance.", + tools: [], + grants: [], + sources: [ + { + id: "anthropic:dead-head", + provider: "anthropic", + baseURL: `http://localhost:${deadHead.port}`, + apiKey: "sk-dead", + model: "mock-model", + }, + { + id: "anthropic:mock-model", + provider: "anthropic", + baseURL: `http://localhost:${env.inference.server.port}`, + apiKey: "sk-mock", + model: "mock-model", + }, + ], + defaultSource: "anthropic:dead-head", + }; + const deployContent = { systemPrompt: config.systemPrompt }; + + // Reconstruct the definition `deployInstanceAtHead` builds internally, for + // the run-observation handle. Same deterministic inputs -> same wrap. + const workflow: WorkflowDefinition = defineWorkflow({ + id: `wf_${AGENT_ID}`, + agent: wrapHarnessAsTrivialAgent({ config, deployContent }), + trigger: { type: "mail", to: agentAddress }, + }); + + const { publicKey } = await env.hub.sessionService.deployInstanceAtHead({ + agentAddress, + agentId: AGENT_ID, + instanceId: INSTANCE_ID, + config, + deployContent, + }); + expect(publicKey).toMatch(/^[0-9a-f]{64}$/); + + const workflowRunRepoId: RepoId = { + kind: "workflow-run", + id: deriveTrivialDeploymentId(agentAddress), + }; + env.registerDeployment({ + deploymentId: INSTANCE_ID, + workflowDefinition: workflow, + workflowRunRepoId, + workflowRunRef: WORKFLOW_RUN_REF, + mailAddress: agentAddress, + }); + + expect(env.hub.router.getRoutableAddresses()).toContain(agentAddress); + + await fireMailTrigger(env, agentAddress, { + messageId: "", + }); + + const runId = await waitForFirstRunId(env, workflowRunRepoId, { + diagnostics: env.sidecarDiagnostics, + timeoutMs: 20_000, + }); + + const terminal = await waitForWorkflowRunComplete(env, INSTANCE_ID, runId, { + timeoutMs: 20_000, + diagnostics: env.sidecarDiagnostics, + }); + // A sanity check, not the proof: the run reaches a normal terminal. This + // does NOT discriminate working from broken failover on its own -- the + // negative-control test below shows a total inference failure also lands + // `RunCompleted`. The reply-equality assertion is what proves failover. + expect(terminal.type).toBe("RunCompleted"); + + const events = await readWorkflowRunEvents(env, INSTANCE_ID, runId); + const stepCompleted = events.find((e) => e.type === "StepCompleted"); + if (stepCompleted === undefined) { + throw new Error("missing StepCompleted for the wrapped single step"); + } + + // The proof of failover: the reply EQUALS the healthy tail's output. The + // dead head returns nothing but 500s, so it cannot produce this string; a + // broken failover would instead surface the agent's synthesized + // provider-error reply (the negative control below). + const reply = readStepReply(stepCompleted.body); + expect(reply).toBe(EXPECTED_REPLY); + + // The head was tried first (the default pins to element 0 every cycle), then + // the healthy tail actually served the reply. Both counts are `>= 1` rather + // than exact: the precise retry count is the harness retry policy's contract, + // not this cross-process test's. + expect(headRequests).toBeGreaterThanOrEqual(1); + expect(env.inference.requests.length).toBeGreaterThanOrEqual(1); + }); + + test("a chain with no healthy source completes with a synthesized error reply", async () => { + // Negative control grounding the test above: with the dead head as the SOLE + // source there is nothing to fail over to, yet the run still lands + // `RunCompleted` -- the agent synthesizes a graceful provider-error reply + // rather than failing the run. This is why the failover test cannot lean on + // the terminal type and instead proves failover through reply-equality. + const soleDeadInstanceId = `ins_${"d".repeat(32)}`; + const soleDeadAgentId = "agent-instance-failover-negctl"; + const agentAddress = `${soleDeadInstanceId}@${DEPLOYMENT_DOMAIN}`; + + const config: HarnessConfig = { + sessionId: SESSION_ID, + agentId: soleDeadAgentId, + tenantId: "tenant-1", + principalId: "prin_integration-failover-negctl-1", + agentAddress, + systemPrompt: "You are the failover negative-control instance.", + tools: [], + grants: [], + sources: [ + { + id: "anthropic:dead-head", + provider: "anthropic", + baseURL: `http://localhost:${deadHead.port}`, + apiKey: "sk-dead", + model: "mock-model", + }, + ], + defaultSource: "anthropic:dead-head", + }; + const deployContent = { systemPrompt: config.systemPrompt }; + + const workflow: WorkflowDefinition = defineWorkflow({ + id: `wf_${soleDeadAgentId}`, + agent: wrapHarnessAsTrivialAgent({ config, deployContent }), + trigger: { type: "mail", to: agentAddress }, + }); + + await env.hub.sessionService.deployInstanceAtHead({ + agentAddress, + agentId: soleDeadAgentId, + instanceId: soleDeadInstanceId, + config, + deployContent, + }); + + const workflowRunRepoId: RepoId = { + kind: "workflow-run", + id: deriveTrivialDeploymentId(agentAddress), + }; + env.registerDeployment({ + deploymentId: soleDeadInstanceId, + workflowDefinition: workflow, + workflowRunRepoId, + workflowRunRef: WORKFLOW_RUN_REF, + mailAddress: agentAddress, + }); + + await fireMailTrigger(env, agentAddress, { + messageId: "", + }); + + const runId = await waitForFirstRunId(env, workflowRunRepoId, { + diagnostics: env.sidecarDiagnostics, + timeoutMs: 20_000, + }); + + const terminal = await waitForWorkflowRunComplete( + env, + soleDeadInstanceId, + runId, + { timeoutMs: 20_000, diagnostics: env.sidecarDiagnostics }, + ); + // Total inference failure still completes the run. + expect(terminal.type).toBe("RunCompleted"); + + const events = await readWorkflowRunEvents(env, soleDeadInstanceId, runId); + const stepCompleted = events.find((e) => e.type === "StepCompleted"); + if (stepCompleted === undefined) { + throw new Error("missing StepCompleted for the sole-dead-source step"); + } + + // The reply is the synthesized error, NOT the healthy tail's output -- + // exactly the outcome the failover test's reply-equality assertion rules out. + const reply = readStepReply(stepCompleted.body); + expect(reply).not.toBe(EXPECTED_REPLY); + }); +}); + +/** + * Extract the agent's reply string from a `StepCompleted` event body. A small + * `{ reply, turn }` output inlines as `inline:`. + */ +function readStepReply(body: Record): string { + const output = body["output"]; + if (typeof output !== "object" || output === null || !("ref" in output)) { + throw new Error( + `StepCompleted output is not a { ref } record: ${JSON.stringify(output)}`, + ); + } + const ref: unknown = output.ref; + if (typeof ref !== "string") { + throw new Error(`StepCompleted output ref is not a string: ${String(ref)}`); + } + const INLINE_PREFIX = "inline:"; + if (!ref.startsWith(INLINE_PREFIX)) { + throw new Error(`expected an inline output ref, got ${ref}`); + } + const parsed: unknown = JSON.parse(ref.slice(INLINE_PREFIX.length)); + if (typeof parsed !== "object" || parsed === null || !("reply" in parsed)) { + throw new Error( + `step output does not carry a reply field: ${JSON.stringify(parsed)}`, + ); + } + const reply: unknown = parsed.reply; + if (typeof reply !== "string") { + throw new Error( + `step output reply is not a string: ${JSON.stringify(parsed)}`, + ); + } + return reply; +} From b51eb722c26b124d5545cf210d9a4f639fda9cb6 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Fri, 3 Jul 2026 06:34:32 -0500 Subject: [PATCH 030/101] Provision launched agents without the trivial workflow wrap The agent launcher built a throwaway single-step workflow, synthesized an approval set for it, and ran it through the workflow-deploy orchestrator -- which, for that wrap, did nothing but round-trip back into the same deploy-tree, pack, and session-start phases. None of the wrap reached the wire: the repo writer wrote nothing, the approval set gated an agent that always passed, and the orchestrator result was discarded. Collapse the launcher to call the provision phases directly, and have the multi-step branch's per-step launch callback share that one method rather than duplicating the same phase call. The approval-set builder and the no-op workflow repo writer that only the wrap used are removed. The launch behavior is unchanged, which the deploy integration suite and the launcher's own unit tests confirm. The wrap used to narrow the orchestrator's deploy content through the manifest-validating bridge before writing it. Export that bridge so the integration fixtures forward orchestrator-shaped deploy content the same validated way rather than casting `unknown`, keeping the validation the collapse would otherwise drop. --- packages/hub-sessions/src/index.ts | 1 + packages/hub-sessions/src/session-service.ts | 128 ++++-------------- tests/hub-agent/lib/deploy-flow-env.ts | 27 ++-- .../workflow-deploy/launch-session-bridge.ts | 24 ++-- 4 files changed, 50 insertions(+), 130 deletions(-) diff --git a/packages/hub-sessions/src/index.ts b/packages/hub-sessions/src/index.ts index 6b48c71e..a2273a0d 100644 --- a/packages/hub-sessions/src/index.ts +++ b/packages/hub-sessions/src/index.ts @@ -6,6 +6,7 @@ export { export { createSessionService, SessionLaunchError, + bridgeOrchestratorDeployContent, type SessionService, type DeployWorkflowDefinitionParams, type DeployWorkflowDefinitionResult, diff --git a/packages/hub-sessions/src/session-service.ts b/packages/hub-sessions/src/session-service.ts index ff1ad566..60baef81 100644 --- a/packages/hub-sessions/src/session-service.ts +++ b/packages/hub-sessions/src/session-service.ts @@ -3,7 +3,6 @@ import { and, eq } from "drizzle-orm"; import { createDefaultDirectorRegistry, - defaultDirectorFactory, type DirectorRegistry, } from "@intx/agent"; import { getLogger } from "@intx/log"; @@ -435,45 +434,6 @@ function resolveMountPath(row: AgentAssetWithAsset): string { } } -/** - * Compute the operator-approval set the workflow-deploy orchestrator - * needs for a trivial-wrap deploy. The wrapped agent carries no - * capability list and no director ref (see `wrapHarnessAsTrivialAgent`), - * but `HarnessConfig.tools` projects onto the synthesized agent's - * `toolFactories` so the walk emits `tool:` grants the gate - * checks. The approval set mirrors that projection: every tool the - * `HarnessConfig` already names gets the matching `tool:` approval, - * the per-source inference grants land alongside the default-director - * grant, and the trigger-derived mail grants close the set out. - * - * The legacy agent-deploy path has already authorized the deployment - * in toto -- the harness's `tools` array is the operator-supplied - * surface the hub ships to the sidecar -- and re-running deploy - * through the workflow surface must not synthesize a fresh approval - * prompt for grants the legacy path implicitly approved. The shape - * here keeps the gate honest (an unapproved tool fails the deploy) - * while the legacy passthrough remains bit-for-bit on the wire. - */ -function buildTrivialApprovalSet(args: { - agentAddress: string; - config: HarnessConfig; -}): ApprovalSet { - const approvals = new Set(); - for (const tool of args.config.tools) { - approvals.add(`tool:${tool.name}`); - } - for (const source of args.config.sources) { - approvals.add(`inference.source:${source.provider}:${source.model}`); - } - approvals.add(`director:${defaultDirectorFactory.id}`); - approvals.add(`mail.address:${args.agentAddress}`); - const at = args.agentAddress.lastIndexOf("@"); - if (at >= 0 && at < args.agentAddress.length - 1) { - approvals.add(`mail.send:${args.agentAddress.slice(at + 1)}`); - } - return approvals; -} - /** * Translate the orchestrator's structural `DeployContent` (which types * `toolPackageManifest` as `unknown`) back into the hub-sessions @@ -481,8 +441,12 @@ function buildTrivialApprovalSet(args: { * caller supplied, but the surface type widens `toolPackageManifest` to * `unknown`; the validator narrows it back to the canonical shape * `agentRepoStore.writeDeployTree` consumes. + * + * Exported so a test fixture that forwards orchestrator-shaped deploy + * content into `launchSession` narrows it the same validated way the + * production multi-step callback does, rather than casting `unknown`. */ -function bridgeOrchestratorDeployContent( +export function bridgeOrchestratorDeployContent( content: OrchestratorDeployContent, ): DeployContent { const bridged: DeployContent = { systemPrompt: content.systemPrompt }; @@ -548,22 +512,6 @@ export async function sendMultiStepDeployFrame(args: { }); } -/** - * No-op `WorkflowRepoWriter` used by the legacy `launchSession` - * delegate. The workflow-deploy orchestrator writes a workflow repo - * tree before per-step launch; the legacy agent-deploy path never - * materialized a workflow repo and the integration-test surface does - * not exercise one, so the trivial wrap skips the write rather than - * inventing a phantom repo. - */ -function createNoopWorkflowRepoWriter(): WorkflowRepoWriter { - return { - async writeWorkflowRepo(_args) { - return; - }, - }; -} - /** * `WorkflowRepoWriter` backed by the hub's repo substrate. Writes the * orchestrator-produced workflow tree (`workflow.json`, @@ -942,12 +890,11 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { directorRegistry: DirectorRegistry; deployArgs: DeployWorkflowArgs; }): Promise { - const launchSessionCallback: LaunchSessionFn = async ( - orchestratorParams, - ) => { - // The legacy launch path (no `workflowFrame`) provisions a warm - // agent and returns no deploy-ack key; the void return is discarded. - await executeLaunchPhases({ + // The per-step launcher: the same warm-harness provision the public + // `launchSession` runs, with the orchestrator's structural + // `DeployContent` narrowed back to the hub-sessions shape first. + const launchSessionCallback: LaunchSessionFn = (orchestratorParams) => + launchSession({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, @@ -959,7 +906,6 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { ? { toolPackagePins: orchestratorParams.toolPackagePins } : {}), }); - }; const sendMultiStepDeployCallback: SendMultiStepDeployFn = (deployParams) => sendMultiStepDeployFrame({ @@ -982,14 +928,14 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { } /** - * Legacy agent-deploy entry point preserved bit-for-bit at its wire - * shape. The body now constructs a single-step trivial workflow from - * the deploy's `HarnessConfig` + `DeployContent`, synthesizes the - * matching operator-approval set, and delegates to the workflow-deploy - * orchestrator. The orchestrator's trivial branch round-trips back - * into `executeLaunchPhases` with the original `trivialBindings`, - * which preserves every on-disk and wire-level surface the legacy - * agent-deploy path exposed. + * Provision one agent at its address through the deploy-tree + pack + + * session-start phases (`executeLaunchPhases` with no workflow frame -- + * the warm-harness provision path). This is the launcher the multi-step + * workflow branch drives per step via the orchestrator's `launchSession` + * callback, and the launcher the integration-test fixture drives + * directly. A single-agent instance deploy uses `deployInstanceAtHead` + * and a single-step workflow uses the head hand-off; neither routes + * here. The method carries no workflow projection of its own. */ async function launchSession(params: { agentAddress: string; @@ -999,35 +945,15 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { deployContent: DeployContent; toolPackagePins?: readonly ToolPackagePin[]; }): Promise { - const { agentAddress, agentId, instanceId, config, deployContent } = params; - - const trivialAgent = wrapHarnessAsTrivialAgent({ - config, - deployContent, - }); - const workflow = defineWorkflow({ - id: `wf_${agentId}`, - agent: trivialAgent, - trigger: { type: "mail", to: agentAddress }, - }); - const operatorApprovals = buildTrivialApprovalSet({ - agentAddress, - config, - }); - - await runWorkflowDeploy({ - workflowRepo: createNoopWorkflowRepoWriter(), - directorRegistry: createDefaultDirectorRegistry(), - deployArgs: { - workflow, - trivialBindings: { agentAddress, agentId, instanceId }, - config, - deployContent, - ...(params.toolPackagePins !== undefined - ? { toolPackagePins: params.toolPackagePins } - : {}), - operatorApprovals, - }, + await executeLaunchPhases({ + agentAddress: params.agentAddress, + agentId: params.agentId, + instanceId: params.instanceId, + config: params.config, + deployContent: params.deployContent, + ...(params.toolPackagePins !== undefined + ? { toolPackagePins: params.toolPackagePins } + : {}), }); } diff --git a/tests/hub-agent/lib/deploy-flow-env.ts b/tests/hub-agent/lib/deploy-flow-env.ts index 26aeb549..b899be67 100644 --- a/tests/hub-agent/lib/deploy-flow-env.ts +++ b/tests/hub-agent/lib/deploy-flow-env.ts @@ -45,6 +45,7 @@ import { type MessageHeaders, } from "@intx/mime"; import { + bridgeOrchestratorDeployContent, createAgentRepoStore, createSessionService, createSidecarRouter, @@ -1121,29 +1122,21 @@ export async function deployWorkflow( id: workflowRunRepoSlug, }; - // Route every per-step launch through the session service. The - // session service's `launchSession` itself routes through the - // orchestrator's trivial branch, so this preserves the bit-identical - // trivial round-trip the existing deploy-flow test asserts. - // - // `launchSession`'s `deployContent` parameter widens - // `toolPackageManifest` to `unknown` in the orchestrator's surface - // shape; the session-service's `bridgeOrchestratorDeployContent` - // narrows it back at the inner boundary, so the cast here only - // crosses the structural-shape gap between the orchestrator's - // `OrchestratorDeployContent` and the session-service's - // `DeployContent`. + // Route every per-step launch through the session service, mirroring + // the production multi-step branch, which drives the orchestrator's + // per-step `launchSession` callback against the same method. The + // orchestrator's `DeployContent` widens `toolPackageManifest` to + // `unknown`; `bridgeOrchestratorDeployContent` narrows and validates it + // back to the hub-sessions shape -- the same bridge production uses. const launchSession: LaunchSessionFn = async (orchestratorParams) => { - const deployContent = orchestratorParams.deployContent; await env.hub.sessionService.launchSession({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, config: orchestratorParams.config, - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the session-service's launchSession invokes the orchestrator internally and re-narrows `toolPackageManifest` via arktype; this fixture forwards the orchestrator-shaped deploy content as-is - deployContent: deployContent as Parameters< - SessionService["launchSession"] - >[0]["deployContent"], + deployContent: bridgeOrchestratorDeployContent( + orchestratorParams.deployContent, + ), ...(orchestratorParams.toolPackagePins !== undefined ? { toolPackagePins: orchestratorParams.toolPackagePins } : {}), diff --git a/tests/workflow-deploy/launch-session-bridge.ts b/tests/workflow-deploy/launch-session-bridge.ts index 55bb2277..68149566 100644 --- a/tests/workflow-deploy/launch-session-bridge.ts +++ b/tests/workflow-deploy/launch-session-bridge.ts @@ -1,23 +1,23 @@ -import type { DeployContent as LaunchDeployContent } from "@intx/hub-sessions"; +import { + bridgeOrchestratorDeployContent, + type DeployContent as LaunchDeployContent, +} from "@intx/hub-sessions"; import type { DeployContent as OrchestratorDeployContent } from "@intx/workflow-deploy"; /** * Bridge the orchestrator's `DeployContent` to hub-sessions' * `DeployContent` at the `launchSession` boundary. * - * `@intx/workflow-deploy` widens `toolPackageManifest` to `unknown` - * so the package does not need a runtime dep on `@intx/hub-sessions` - * (see the `DeployContent` docblock in - * `packages/workflow-deploy/src/orchestrator.ts`). That docblock - * declares the orchestrator type a structural mirror of hub-sessions' - * `DeployContent`, so the narrowing is sound -- but unprovable to - * the type checker. This helper centralizes the one allowed - * `eslint-disable` so each `LaunchSessionFn` callback in the test - * fixtures does not need its own. + * `@intx/workflow-deploy` widens `toolPackageManifest` to `unknown` so + * the package does not need a runtime dep on `@intx/hub-sessions` (see + * the `DeployContent` docblock in + * `packages/workflow-deploy/src/orchestrator.ts`). This delegates to the + * same `bridgeOrchestratorDeployContent` the production multi-step + * callback uses, which validates and narrows `toolPackageManifest` back + * to the canonical shape rather than casting `unknown`. */ export function toLaunchDeployContent( deployContent: OrchestratorDeployContent, ): LaunchDeployContent { - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- see file docblock; the orchestrator's DeployContent docblock declares this structural mirror - return deployContent as LaunchDeployContent; + return bridgeOrchestratorDeployContent(deployContent); } From 82778e4bc453a58fc01ddcbc12841a4ffa398ee0 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Fri, 3 Jul 2026 07:14:31 -0500 Subject: [PATCH 031/101] Delete the orchestrator's dead trivial deploy branch The trivial deploy branch took a single-step workflow that arrived with a caller-supplied address triple and reused it unchanged, reproducing the legacy agent-deploy shape. Nothing produces that triple anymore: the only caller went away when the agent launcher stopped wrapping itself as a one-step trivial workflow. What remains produces the triple only in tests. Remove the branch and everything reachable only through it: the trivial dispatch, the predicate that selected it, the address-triple argument, the trivial result variant, and the module header describing the trivial-versus-derived address dichotomy. The deploy-time guard that rejected a trivial result goes with them, since the result can no longer be trivial. Every surviving deploy routes by step count -- a one-step workflow deploys once at the head, more steps derive per-step addresses -- so the orchestrator tests that drove the branch through the address triple are retargeted onto those paths or removed, and the toolPackagePins and repo-before-launch coverage they carried is preserved against the multi-step branch. --- Makefile | 2 +- packages/hub-sessions/src/session-service.ts | 30 +- packages/workflow-deploy/README.md | 24 +- .../workflow-deploy/src/orchestrator.test.ts | 138 +++----- packages/workflow-deploy/src/orchestrator.ts | 177 +++-------- tests/hub-agent/lib/deploy-flow-env.test.ts | 90 ------ tests/hub-agent/lib/deploy-flow-env.ts | 84 ++--- tests/workflow-deploy/fifo-mail-load.test.ts | 11 +- .../workflow-deploy/trivial-roundtrip.test.ts | 297 ------------------ .../unresolvable-director.test.ts | 6 +- 10 files changed, 141 insertions(+), 718 deletions(-) delete mode 100644 tests/workflow-deploy/trivial-roundtrip.test.ts diff --git a/Makefile b/Makefile index e2ebc723..387dfaa6 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ lint: FORCE test: FORCE bun test packages/ apps/ bin/ tests/agent/ tests/agent-audit-log/ tests/agent-blob-spill/ tests/agent-common/ tests/agent-multi-provider/ tests/agent-quickstart/ tests/agent-resume/ tests/agent-rewind/ tests/agent-rich-tool/ tests/agent-structured-payload/ tests/coding-agent/ tests/hub-agent/lib/ tests/inference-testing/ tests/tool-packaging/ tests/workflow/ - bun test --timeout 120000 tests/hub-agent/deploy-flow.test.ts tests/workflow-deploy/trivial-roundtrip.test.ts tests/workflow-deploy/multistep-signal.test.ts tests/workflow-deploy/single-step-real-agent.test.ts tests/workflow-deploy/instance-reroute-real-agent.test.ts tests/workflow-deploy/instance-failover-real-agent.test.ts tests/workflow-deploy/single-step-message-input.test.ts tests/workflow-deploy/single-step-posix-tool.test.ts tests/workflow-deploy/single-step-grants-bridge.test.ts tests/workflow-deploy/single-step-event-threading.test.ts tests/workflow-deploy/single-step-conversation-durability.test.ts tests/workflow-deploy/single-step-full-lifecycle.test.ts tests/workflow-deploy/cross-process-custom-adapter.test.ts tests/workflow-deploy/conversation-state-wal.test.ts tests/workflow-deploy/drain-roundtrip.test.ts tests/workflow-deploy/child-workflow-roundtrip.test.ts tests/workflow-deploy/unresolvable-director.test.ts tests/workflow-deploy/fifo-mail.test.ts tests/workflow-deploy/mail-edge-cases.test.ts tests/inference/ tests/skill-attachment-flow.test.ts tests/hub-api/ tests/db/ + bun test --timeout 120000 tests/hub-agent/deploy-flow.test.ts tests/workflow-deploy/multistep-signal.test.ts tests/workflow-deploy/single-step-real-agent.test.ts tests/workflow-deploy/instance-reroute-real-agent.test.ts tests/workflow-deploy/instance-failover-real-agent.test.ts tests/workflow-deploy/single-step-message-input.test.ts tests/workflow-deploy/single-step-posix-tool.test.ts tests/workflow-deploy/single-step-grants-bridge.test.ts tests/workflow-deploy/single-step-event-threading.test.ts tests/workflow-deploy/single-step-conversation-durability.test.ts tests/workflow-deploy/single-step-full-lifecycle.test.ts tests/workflow-deploy/cross-process-custom-adapter.test.ts tests/workflow-deploy/conversation-state-wal.test.ts tests/workflow-deploy/drain-roundtrip.test.ts tests/workflow-deploy/child-workflow-roundtrip.test.ts tests/workflow-deploy/unresolvable-director.test.ts tests/workflow-deploy/fifo-mail.test.ts tests/workflow-deploy/mail-edge-cases.test.ts tests/inference/ tests/skill-attachment-flow.test.ts tests/hub-api/ tests/db/ test-load: FORCE bun test --timeout 300000 tests/workflow-deploy/fifo-mail-load.test.ts diff --git a/packages/hub-sessions/src/session-service.ts b/packages/hub-sessions/src/session-service.ts index 60baef81..2a15d25f 100644 --- a/packages/hub-sessions/src/session-service.ts +++ b/packages/hub-sessions/src/session-service.ts @@ -160,9 +160,9 @@ export type SessionService = { /** * Deploy a multi-step `WorkflowDefinition` through the workflow-deploy * orchestrator's multi-step branch. This is the general workflow - * deploy entry point: it carries no `trivialBindings` and is not - * coupled to a single agent's credential/session model the way - * `launchSession` is. The orchestrator derives every per-step address + * deploy entry point: it is not coupled to a single agent's + * credential/session model the way `launchSession` is. The + * orchestrator derives every per-step address * from `deploymentId` + `deploymentDomain`, provisions each step's * agent-state repo via the shared per-agent deploy phases, writes the * workflow repo, and fires the deployment-level `agent.deploy` frame. @@ -561,12 +561,12 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { } /** - * Drive the per-agent deploy + session-start phases. Factored out of - * `launchSession` so the workflow-deploy orchestrator's trivial branch - * can call back into the exact phases the legacy agent-deploy path - * owns. The body here is the legacy `launchSession` body verbatim; - * `launchSession` itself now wraps the call in a single-step workflow - * and routes through the orchestrator. + * Drive the per-agent deploy + session-start phases (deploy-tree write, + * asset-pack fan-out, provision frame, pack, session start). Shared by + * the public `launchSession` (warm-harness provision, no workflow frame) + * and the single-step head hand-off (`deploySingleStepAtHead`, which + * passes a `workflowFrame` so the sidecar spawns the workflow-process + * child instead of a warm harness). */ async function executeLaunchPhases(params: { agentAddress: string; @@ -970,7 +970,7 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { * resolves skills, deploy tree, and tool-package pins by `agentId`, so * collapsing it to the deployment id would strip the instance's attachments. * It writes no `workflow_deployment` row (a plain instance has no workflow - * asset) and carries no `trivialBindings`. Returns the head's agent-key ack. + * asset). Returns the head's agent-key ack. */ async function deployInstanceAtHead(params: { agentAddress: string; @@ -1079,16 +1079,6 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { : {}), }, }); - if (result.kind !== "multi-step") { - // The general deploy path always omits trivialBindings, so the - // orchestrator must take the multi-step branch. A trivial result - // here means the orchestrator's branch decision diverged from this - // method's contract; surface it loudly rather than recording a - // projection row for a deployment that never reached the sidecar. - throw new Error( - `deployWorkflowDefinition expected a multi-step deploy for ${deploymentId} but the orchestrator returned a ${result.kind} result`, - ); - } if (db === undefined) { throw new Error( diff --git a/packages/workflow-deploy/README.md b/packages/workflow-deploy/README.md index eef96c90..dc12be8b 100644 --- a/packages/workflow-deploy/README.md +++ b/packages/workflow-deploy/README.md @@ -6,26 +6,18 @@ and the workflow deploy orchestrator. This package is the deploy-side counterpart to `@intx/workflow`. It takes a `WorkflowDefinition`, computes the per-step grant declarations the workflow will require, gates them against an -operator-supplied `ApprovalSet`, and routes the deployment along -one of two paths. - -The orchestrator branches on the trivial-vs-multi-step dichotomy: - -- **Trivial workflow** (single step, caller supplies - `trivialBindings`): preserve the caller's existing agent address - and write the underlying agent's deploy tree onto the existing - `agent-state` repo via the legacy path. The on-disk and on-wire - surfaces stay bit-identical to what the pre-collapse - `SessionService.launchSession` produced. +operator-supplied `ApprovalSet`, and routes the deployment by step +count: + +- **Single-step workflow**: the lone step has no distinct address -- + it IS the deployment head. Deploy once at the head + (`ins_@`) through the single-step + hand-off, staging the head's deploy tree and firing the + `agent.deploy` frame in one call. - **Multi-step workflow**: derive per-step agent addresses as `ins_-@`, instantiate one `agent-state` repo per step, and write per-step deploy trees. -The asymmetry is intentional: it is the agent-deploy uniformity -claim's escape hatch. Without it, the existing -`tests/hub-agent/deploy-flow.test.ts` could not pass with zero -source changes after the collapse. - Public surface: - `createWorkflowDeployOrchestrator(opts)` — the orchestrator. diff --git a/packages/workflow-deploy/src/orchestrator.test.ts b/packages/workflow-deploy/src/orchestrator.test.ts index c1c3b5f2..54f01a19 100644 --- a/packages/workflow-deploy/src/orchestrator.test.ts +++ b/packages/workflow-deploy/src/orchestrator.test.ts @@ -220,10 +220,14 @@ function createRecordingSingleStepDeploy(publicKey = "ff".repeat(32)): { } describe("createWorkflowDeployOrchestrator", () => { - describe("trivial branch", () => { - test("preserves the existing agent address and launches once", async () => { - const agent = makeAgent("legacy-agent"); - const workflow = makeTrivialWorkflow(agent); + describe("deploy provisioning", () => { + test("passes through toolPackagePins to every step launch", async () => { + const workflow = makeMultiStepWorkflow(); + const planAgent = workflow.steps.plan; + const executeAgent = workflow.steps.execute; + if (planAgent?.kind !== "step" || executeAgent?.kind !== "step") { + throw new Error("expected both steps to be step primitives"); + } const directorRegistry = createDefaultDirectorRegistry(); const workflowRepo = createRecordingWorkflowRepoWriter(); const launch = createRecordingLaunch(); @@ -234,72 +238,37 @@ describe("createWorkflowDeployOrchestrator", () => { launchSession: launch.fn, sendMultiStepDeploy: multiStep.fn, }); - - const approvals = approvedGrantsForWorkflow(workflow, [agent]); - - const result = await orchestrator.deployWorkflow({ - workflow, - trivialBindings: { - agentAddress: "ins_legacy-agent@integration.interchange", - agentId: "legacy-agent", - instanceId: "instance-legacy", - }, - config: HARNESS_CONFIG_BASE, - deployContent: DEPLOY_CONTENT_BASE, - operatorApprovals: approvals, - }); - - expect(launch.launches).toHaveLength(1); - const launched = launch.launches[0]; - if (launched === undefined) throw new Error("missing launch"); - expect(launched.agentAddress).toBe( - "ins_legacy-agent@integration.interchange", - ); - expect(launched.agentId).toBe("legacy-agent"); - expect(launched.instanceId).toBe("instance-legacy"); - expect(launched.config).toEqual(HARNESS_CONFIG_BASE); - expect(launched.deployContent).toEqual(DEPLOY_CONTENT_BASE); - // Regression: the trivial branch does NOT invoke the multi-step - // deploy hand-off. - expect(multiStep.calls).toHaveLength(0); - expect(result).toEqual({ kind: "trivial" }); - }); - - test("passes through toolPackagePins to the launch", async () => { - const agent = makeAgent("legacy-agent"); - const workflow = makeTrivialWorkflow(agent); - const directorRegistry = createDefaultDirectorRegistry(); - const workflowRepo = createRecordingWorkflowRepoWriter(); - const launch = createRecordingLaunch(); - const orchestrator = createWorkflowDeployOrchestrator({ - directorRegistry, - workflowRepo, - launchSession: launch.fn, - }); - const approvals = approvedGrantsForWorkflow(workflow, [agent]); + const approvals = approvedGrantsForWorkflow(workflow, [ + planAgent.agent, + executeAgent.agent, + ]); const pins = [{ name: "@vendor/pkg", version: "1.0.0" }] as const; await orchestrator.deployWorkflow({ workflow, - trivialBindings: { - agentAddress: "ins_legacy-agent@integration.interchange", - agentId: "legacy-agent", - instanceId: "instance-legacy", - }, + deploymentId: "dep_pins", + deploymentDomain: "workflow.interchange", config: HARNESS_CONFIG_BASE, deployContent: DEPLOY_CONTENT_BASE, + hubPublicKey: "00".repeat(32), toolPackagePins: pins, operatorApprovals: approvals, }); - expect(launch.launches[0]?.toolPackagePins).toEqual(pins); + expect(launch.launches).toHaveLength(2); + for (const launched of launch.launches) { + expect(launched.toolPackagePins).toEqual(pins); + } }); - test("writes the workflow repo before launching", async () => { - const agent = makeAgent("legacy-agent"); - const workflow = makeTrivialWorkflow(agent); + test("writes the workflow repo before launching any step", async () => { + const workflow = makeMultiStepWorkflow(); + const planAgent = workflow.steps.plan; + const executeAgent = workflow.steps.execute; + if (planAgent?.kind !== "step" || executeAgent?.kind !== "step") { + throw new Error("expected both steps to be step primitives"); + } const directorRegistry = createDefaultDirectorRegistry(); - const workflowRepo = createRecordingWorkflowRepoWriter(); const order: string[] = []; const launch: LaunchSessionFn = async () => { order.push("launch"); @@ -307,39 +276,35 @@ describe("createWorkflowDeployOrchestrator", () => { const recordingRepo: WorkflowRepoWriter = { async writeWorkflowRepo(args) { order.push("repo"); - workflowRepo.writes.push({ - workflowRepoId: args.workflowRepoId, - files: new Map(args.files), - }); + expect(args.files.has("workflow.json")).toBe(true); + expect(args.files.has("capability-declarations.json")).toBe(true); + expect(args.files.has(".gitignore")).toBe(true); }, }; + const multiStep = createRecordingMultiStepDeploy(); const orchestrator = createWorkflowDeployOrchestrator({ directorRegistry, workflowRepo: recordingRepo, launchSession: launch, + sendMultiStepDeploy: multiStep.fn, }); - const approvals = approvedGrantsForWorkflow(workflow, [agent]); + const approvals = approvedGrantsForWorkflow(workflow, [ + planAgent.agent, + executeAgent.agent, + ]); await orchestrator.deployWorkflow({ workflow, - trivialBindings: { - agentAddress: "ins_legacy-agent@integration.interchange", - agentId: "legacy-agent", - instanceId: "instance-legacy", - }, + deploymentId: "dep_order", + deploymentDomain: "workflow.interchange", config: HARNESS_CONFIG_BASE, deployContent: DEPLOY_CONTENT_BASE, + hubPublicKey: "00".repeat(32), operatorApprovals: approvals, }); - expect(order).toEqual(["repo", "launch"]); - expect(workflowRepo.writes).toHaveLength(1); - const write = workflowRepo.writes[0]; - if (write === undefined) throw new Error("missing write"); - expect(write.workflowRepoId).toBe("wf_trivial"); - expect(write.files.has("workflow.json")).toBe(true); - expect(write.files.has("capability-declarations.json")).toBe(true); - expect(write.files.has(".gitignore")).toBe(true); + // The workflow repo lands before any per-step agent-state write. + expect(order).toEqual(["repo", "launch", "launch"]); }); }); @@ -595,7 +560,7 @@ describe("createWorkflowDeployOrchestrator", () => { ).rejects.toBeInstanceOf(MultiStepDeploymentArgsMissingError); }); - test("single-step workflow without trivialBindings deploys once at the head", async () => { + test("single-step workflow deploys once at the head", async () => { const agent = makeAgent("only"); const workflow = makeTrivialWorkflow(agent); const directorRegistry = createDefaultDirectorRegistry(); @@ -725,11 +690,8 @@ describe("createWorkflowDeployOrchestrator", () => { try { await orchestrator.deployWorkflow({ workflow, - trivialBindings: { - agentAddress: "ins_legacy-agent@integration.interchange", - agentId: "legacy-agent", - instanceId: "instance-legacy", - }, + deploymentId: "dep_legacy", + deploymentDomain: "workflow.interchange", config: HARNESS_CONFIG_BASE, deployContent: DEPLOY_CONTENT_BASE, operatorApprovals: incompleteApprovals, @@ -765,11 +727,8 @@ describe("createWorkflowDeployOrchestrator", () => { try { await orchestrator.deployWorkflow({ workflow, - trivialBindings: { - agentAddress: "ins_legacy-agent@integration.interchange", - agentId: "legacy-agent", - instanceId: "instance-legacy", - }, + deploymentId: "dep_legacy", + deploymentDomain: "workflow.interchange", config: HARNESS_CONFIG_BASE, deployContent: DEPLOY_CONTENT_BASE, operatorApprovals: new Set(), @@ -824,11 +783,8 @@ describe("createWorkflowDeployOrchestrator", () => { try { await orchestrator.deployWorkflow({ workflow, - trivialBindings: { - agentAddress: "ins_legacy-agent@integration.interchange", - agentId: "legacy-agent", - instanceId: "instance-legacy", - }, + deploymentId: "dep_legacy", + deploymentDomain: "workflow.interchange", config: HARNESS_CONFIG_BASE, deployContent: DEPLOY_CONTENT_BASE, operatorApprovals: broad, diff --git a/packages/workflow-deploy/src/orchestrator.ts b/packages/workflow-deploy/src/orchestrator.ts index 1305efb9..bb662be1 100644 --- a/packages/workflow-deploy/src/orchestrator.ts +++ b/packages/workflow-deploy/src/orchestrator.ts @@ -1,38 +1,26 @@ // Workflow-deploy orchestrator. // -// ========================================================================= -// THE LOAD-BEARING DICHOTOMY -// ========================================================================= +// A deploy validates the workflow, runs the capability walk, and gates on +// operator approval, then routes by step count. // -// Trivial-workflow path preserves the legacy address shape; multi-step -// uses the derived shape; this asymmetry is the agent-deploy uniformity -// claim's escape hatch. +// A one-step workflow has no distinct step address: the lone step IS the +// deployment head. It deploys once at the head (`deriveDeploymentAddress`) +// through the single-step hand-off -- the tree staging and the +// `agent.deploy` frame collapse onto one head deploy, with no per-step +// provisioning loop. // -// A workflow with exactly one step that arrives with `trivialBindings` -// reuses the deployment's existing `` -// triple unchanged. The deploy tree lands on the same per-agent -// `agent-state` repo the legacy agent-deploy path writes to, and the -// per-step content (system prompt, tool-package pins, asset attachments) -// flows through `launchSession` with bit-identical wire and on-disk -// shape. The legacy `tests/hub-agent/deploy-flow.test.ts` is the -// invariant the trivial branch must round-trip. -// -// A workflow with more than one step (or one without `trivialBindings`) -// derives per-step agent addresses of the form -// `ins_-@`, instantiates one -// agent-state repo per step keyed by the derived address, and writes +// A workflow with more than one step derives per-step agent addresses of +// the form `ins_-@`, instantiates +// one agent-state repo per step keyed by the derived address, and writes // each step's deploy tree onto its own repo. The derivation is a pure // function of `(deploymentId, stepId, deploymentDomain)`, so the // supervisor reconstructs the same addresses at spawn time without any // per-deploy state. // -// Both branches first validate the workflow, run the capability walk, -// and gate on operator approval. The workflow definition envelope plus -// the walk's per-step grant declarations land on a `workflow` repo -// before any agent-state write happens; if the workflow repo write -// fails, no agent-state repo is created. -// -// ========================================================================= +// The workflow definition envelope plus the walk's per-step grant +// declarations land on a `workflow` repo before any agent-state write +// happens; if the workflow repo write fails, no agent-state repo is +// created. import type { AgentDefinition, @@ -160,14 +148,14 @@ export interface MultiStepDeployResult { } /** - * Result returned by `deployWorkflow`. The trivial branch returns - * `{ kind: "trivial" }`; the multi-step branch surfaces the supervisor - * public key collected from the sidecar's `agent.deploy.ack` so the - * caller can stash it alongside the deployment record. + * Result returned by `deployWorkflow`. Surfaces the supervisor public key + * collected from the sidecar's `agent.deploy.ack` so the caller can stash + * it alongside the deployment record. */ -export type DeployWorkflowResult = - | { readonly kind: "trivial" } - | { readonly kind: "multi-step"; readonly publicKey: string }; +export type DeployWorkflowResult = { + readonly kind: "multi-step"; + readonly publicKey: string; +}; /** * Minimal interface for writing the workflow repo. The orchestrator @@ -191,28 +179,22 @@ export interface WorkflowDeployOrchestratorDeps { * orchestrator. */ readonly directorRegistry: DirectorRegistry; - /** - * Writes the workflow repo's deploy tree. The trivial-branch and the - * multi-step branch both call this once. - */ + /** Writes the workflow repo's deploy tree. Every deploy calls this once. */ readonly workflowRepo: WorkflowRepoWriter; /** - * Performs the per-agent deploy + session start. The orchestrator - * calls this once per step (once total in the trivial branch). In - * production this is `SessionService.launchSession`; tests pass a - * tracking stub. + * Performs the per-agent deploy + session start. The multi-step branch + * calls this once per step. In production this is + * `SessionService.launchSession`; tests pass a tracking stub. */ readonly launchSession: LaunchSessionFn; /** * Fires the deployment-level `agent.deploy` frame that carries the * workflow definition and per-step source pins to the sidecar. The * multi-step branch calls this exactly once, after every per-step - * `agent-state` repo has been provisioned via `launchSession`. The - * trivial branch never calls it. + * `agent-state` repo has been provisioned via `launchSession`. * - * Optional so existing callers that only exercise the trivial branch - * (e.g. the legacy agent-deploy passthrough) do not have to wire a - * stub. The multi-step branch fails fast with + * Optional so a caller that only exercises the single-step branch does + * not have to wire a stub. The multi-step branch fails fast with * `MultiStepDeployHandoffMissingError` if the dep is absent. */ readonly sendMultiStepDeploy?: SendMultiStepDeployFn; @@ -233,30 +215,15 @@ export interface DeployWorkflowArgs { /** The workflow definition the orchestrator validates and deploys. */ readonly workflow: WorkflowDefinition; /** - * Pre-existing per-agent address binding. Required for the trivial - * branch (workflow with exactly one step that wraps an existing - * agent-deploy). Absent for the multi-step branch -- the orchestrator - * derives per-step addresses from `deploymentId` and the workflow. - */ - readonly trivialBindings?: { - readonly agentAddress: string; - readonly agentId: string; - readonly instanceId: string; - }; - /** - * Stable identifier the multi-step branch concatenates into derived - * agent addresses. Required when `trivialBindings` is absent. + * Stable identifier the branch concatenates into derived agent + * addresses. Required. */ readonly deploymentId?: string; /** - * Mail-domain for the deployment. Required when `trivialBindings` is - * absent. The multi-step branch derives per-step addresses as - * `ins_-@`. - * - * The plan's `deployWorkflow` shape lists `deploymentId` but elides - * `deploymentDomain`; the derivation needs both, so the API surfaces - * the second parameter explicitly. Trivial deployments do not consume - * it because they reuse the existing `agentAddress`. + * Mail-domain for the deployment. Required. The multi-step branch + * derives per-step addresses as + * `ins_-@`; the single-step + * branch deploys the lone step at `ins_@`. */ readonly deploymentDomain?: string; /** @@ -281,11 +248,9 @@ export interface DeployWorkflowArgs { */ readonly operatorApprovals: ApprovalSet; /** - * Hex-encoded hub Ed25519 public key the multi-step branch threads - * onto the `agent.deploy` frame so the sidecar can verify the - * deploy-tree commit signatures. Required when the multi-step branch - * is reached; ignored by the trivial branch (whose `launchSession` - * delegate already carries the hub key through its own wire surface). + * Hex-encoded hub Ed25519 public key threaded onto the `agent.deploy` + * frame so the sidecar can verify the deploy-tree commit signatures. + * Required for both deploy paths (single-step head and multi-step). */ readonly hubPublicKey?: string; } @@ -317,7 +282,7 @@ export class WorkflowDefinitionInvalidError extends Error { export class MultiStepDeploymentArgsMissingError extends Error { constructor(missing: string) { super( - `multi-step deploy requires ${missing}; supply both deploymentId and deploymentDomain (or pass trivialBindings for a single-step workflow)`, + `deploy requires ${missing}; supply both deploymentId and deploymentDomain`, ); this.name = "MultiStepDeploymentArgsMissingError"; } @@ -325,7 +290,7 @@ export class MultiStepDeploymentArgsMissingError extends Error { /** * Error thrown when the multi-step branch is reached but the - * `sendMultiStepDeploy` dependency was not wired. The trivial branch + * `sendMultiStepDeploy` dependency was not wired. The single-step branch * does not consult this dep, so the dep is optional on the deps record; * callers that may take the multi-step branch must wire it. */ @@ -341,7 +306,7 @@ export class MultiStepDeployHandoffMissingError extends Error { /** * Error thrown when the single-step branch is reached but the * `deploySingleStepAtHead` dependency was not wired. Parallel to - * `MultiStepDeployHandoffMissingError`; the trivial branch does not + * `MultiStepDeployHandoffMissingError`; the multi-step branch does not * consult this dep, so it is optional on the deps record. */ export class SingleStepDeployHandoffMissingError extends Error { @@ -391,7 +356,8 @@ function formatApprovalDeniedMessage( /** * Build a `WorkflowDeployOrchestrator`. The orchestrator owns the - * trivial-vs-multi-step decision; its deps own everything else. + * step-count routing (single-step head vs multi-step derived); its deps + * own everything else. */ export function createWorkflowDeployOrchestrator( deps: WorkflowDeployOrchestratorDeps, @@ -425,21 +391,6 @@ export function createWorkflowDeployOrchestrator( workflowRepo, }); - const trivial = isTrivialDeploy(args); - if (trivial !== null) { - await launchSession({ - agentAddress: trivial.bindings.agentAddress, - agentId: trivial.bindings.agentId, - instanceId: trivial.bindings.instanceId, - config: args.config, - deployContent: args.deployContent, - ...(args.toolPackagePins !== undefined - ? { toolPackagePins: args.toolPackagePins } - : {}), - }); - return { kind: "trivial" }; - } - // A one-step workflow has no distinct steps: the lone step IS the // head. It deploys once at the head (no per-step provisioning loop), // so it routes through the dedicated single-step hand-off rather @@ -463,23 +414,6 @@ export function createWorkflowDeployOrchestrator( }; } -/** - * Decide whether the deploy takes the trivial branch. A trivial deploy - * requires a single-step workflow AND a `trivialBindings` triple from - * the caller. Either condition alone is not enough: multi-step - * workflows never reuse a single legacy address, and a single-step - * workflow without `trivialBindings` is the multi-step branch's - * degenerate case (one derived address). The asymmetry is the - * load-bearing escape hatch documented in this module's header. - */ -function isTrivialDeploy( - args: DeployWorkflowArgs, -): { bindings: NonNullable } | null { - if (args.workflow.stepOrder.length !== 1) return null; - if (args.trivialBindings === undefined) return null; - return { bindings: args.trivialBindings }; -} - /** * Deploy a one-step workflow once at the head. The lone step has no * distinct per-step address -- it IS the head (`deriveDeploymentAddress`) @@ -972,28 +906,21 @@ function serializeWalk(walk: CapabilityWalkResult): unknown { } /** - * Build a trivial `AgentDefinition` from a `HarnessConfig` and a - * `DeployContent`. The orchestrator's trivial branch hands the resulting - * shape off to consumers that want to inspect an agent definition for a - * legacy agent-deploy that never went through the workflow surface. - * - * The wrap is the load-bearing transformation behind the trivial - * round-trip claim: the legacy deploy-flow exposes `HarnessConfig` plus - * `DeployContent`, and `SessionService.launchSession` collapses onto - * `deployWorkflow` by synthesizing a single-step workflow from those - * two values via this function. The deploy tree itself - * (`deployContent.systemPrompt`, the harness's `tools` and `grants` - * arrays) is the source of truth for runtime behaviour; the wrap - * synthesizes only the surfaces the capability walk needs to gate the - * deploy against the operator-approval set. + * Build an `AgentDefinition` from a `HarnessConfig` and a + * `DeployContent`. `SessionService.deployInstanceAtHead` uses it to wrap + * a single-agent instance's harness as a one-step workflow and deploy it + * at the head. The deploy tree itself (`deployContent.systemPrompt`, the + * harness's `tools` and `grants` arrays) is the source of truth for + * runtime behaviour; the wrap synthesizes only the surfaces the + * capability walk needs to gate the deploy against the operator-approval + * set. * * The walk inspects `agent.toolFactories[i].id` to emit `tool:` * grants. The wrap projects each `HarnessConfig.tools[i].name` onto a * synthesized `AnnotatedToolFactory` whose `id` matches; the factory * function itself is never invoked on the walk path. Skipping this - * projection would let the gate admit every trivial deploy regardless - * of what `HarnessConfig.tools` named, breaking the uniformity claim's - * substance at the approval layer. + * projection would let the gate admit every deploy regardless of what + * `HarnessConfig.tools` named, weakening the approval gate. */ export function wrapHarnessAsTrivialAgent(args: { config: HarnessConfig; diff --git a/tests/hub-agent/lib/deploy-flow-env.test.ts b/tests/hub-agent/lib/deploy-flow-env.test.ts index 250e5a66..c62863a6 100644 --- a/tests/hub-agent/lib/deploy-flow-env.test.ts +++ b/tests/hub-agent/lib/deploy-flow-env.test.ts @@ -10,7 +10,6 @@ import { describe, test, expect, afterAll, beforeAll } from "bun:test"; import { WORKFLOW_RUN_TERMINAL_TYPES, - deployWorkflow, injectSignal, readWorkflowRunEvents, simulateProcessingCrash, @@ -20,13 +19,6 @@ import { type DeploymentHandle, type HubEnv, } from "./deploy-flow-env"; -import { deriveTrivialDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; -import { defaultDirectorFactory } from "@intx/agent"; -import { defineWorkflow } from "@intx/workflow/definition"; -import { wrapHarnessAsTrivialAgent } from "@intx/workflow-deploy"; -import type { ApprovalSet } from "@intx/workflow-deploy"; -import type { HarnessConfig } from "@intx/types/runtime"; -import type { SessionService } from "@intx/hub-sessions"; import fs from "node:fs"; @@ -147,88 +139,6 @@ describe("deploy-flow-env helpers smoke tests", () => { ).rejects.toThrow(/timed out/); }); - test("deployWorkflow returns a slug-derived workflowRunRepoId.id for trivial deploys", async () => { - // The supervisor's trivial branch projects the agent address - // through `deriveTrivialDeploymentId` before writing workflow-run - // events; downstream helpers (and the consumer in 13a) read the - // resulting slug off the handle. Verify the helper's reported - // id matches what the supervisor will actually commit to. - const agentAddress = "ins_slug-probe@integration.interchange"; - const agentId = "slug-probe"; - const expectedSlug = deriveTrivialDeploymentId(agentAddress); - expect(expectedSlug).not.toBe(agentAddress); - - const config: HarnessConfig = { - sessionId: "ses-slug-1", - agentId, - tenantId: "tenant-1", - principalId: "prin-1", - agentAddress, - systemPrompt: "slug-probe-prompt", - tools: [], - grants: [], - sources: [ - { - id: "src-slug-1", - provider: "anthropic", - baseURL: "https://api.example/anthropic", - apiKey: "secret-key", - model: "mock-model", - }, - ], - defaultSource: "src-slug-1", - }; - const deployContent = { systemPrompt: "slug-probe-prompt" }; - const trivialAgent = wrapHarnessAsTrivialAgent({ config, deployContent }); - const workflow = defineWorkflow({ - id: `wf_${agentId}`, - agent: trivialAgent, - trigger: { type: "mail", to: agentAddress }, - }); - const approvals = new Set(); - for (const source of config.sources) { - approvals.add(`inference.source:${source.provider}:${source.model}`); - } - approvals.add(`director:${defaultDirectorFactory.id}`); - approvals.add(`mail.address:${agentAddress}`); - const at = agentAddress.lastIndexOf("@"); - if (at < 0 || at >= agentAddress.length - 1) { - throw new Error("unreachable"); - } - approvals.add(`mail.send:${agentAddress.slice(at + 1)}`); - const operatorApprovals: ApprovalSet = approvals; - - // Replace `launchSession` with a no-op so deployWorkflow's - // trivial path completes without engaging sidecar provisioning. - // The slug derivation is the contract under test and happens - // independently of the launch payload. - const originalLaunch = env.hub.sessionService.launchSession.bind( - env.hub.sessionService, - ); - const stubLaunch: SessionService["launchSession"] = async () => { - // The trivial branch's launch is not the contract under test; - // a no-op completes the deploy without engaging the sidecar. - }; - env.hub.sessionService.launchSession = stubLaunch; - try { - const handle = await deployWorkflow(env, workflow, { - config, - deployContent, - operatorApprovals, - trivialBindings: { - agentAddress, - agentId, - instanceId: `inst_${agentId}`, - }, - }); - expect(handle.workflowRunRepoId.kind).toBe("workflow-run"); - expect(handle.workflowRunRepoId.id).toBe(expectedSlug); - expect(handle.workflowRunRepoId.id).not.toBe(agentAddress); - } finally { - env.hub.sessionService.launchSession = originalLaunch; - } - }); - test("simulateProcessingCrash composes enqueueInbox + dequeueToProcessing", async () => { const address = "ins_smoke-test@integration.interchange"; const messageId = ""; diff --git a/tests/hub-agent/lib/deploy-flow-env.ts b/tests/hub-agent/lib/deploy-flow-env.ts index b899be67..454a66cd 100644 --- a/tests/hub-agent/lib/deploy-flow-env.ts +++ b/tests/hub-agent/lib/deploy-flow-env.ts @@ -76,7 +76,6 @@ import type { HarnessConfig } from "@intx/types/runtime"; import type { ToolPackagePin } from "@intx/types/tool-packages"; import type { ApprovalSet } from "@intx/workflow-deploy"; import type { WorkflowDefinition } from "@intx/workflow"; -import { deriveTrivialDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; export const AGENT_ADDRESS = "ins_test-agent@integration.interchange"; export const AGENT_ID = "ins_test-agent"; @@ -1011,22 +1010,14 @@ export type DeployWorkflowOpts = { * branch from the step's agent definition. */ deployContent: { systemPrompt: string }; - /** Pre-existing per-agent address binding for the trivial branch. */ - trivialBindings?: { - agentAddress: string; - agentId: string; - instanceId: string; - }; /** - * Stable identifier the multi-step branch concatenates into derived - * agent addresses. Required when `trivialBindings` is absent. + * Stable identifier the branch concatenates into derived agent + * addresses. Required. */ - deploymentId?: string; + deploymentId: string; /** - * Mail-domain for the deployment. Required when `trivialBindings` is - * absent. Defaults to the integration test's canonical domain so - * callers that exercise the multi-step branch with the default - * fixture wiring do not have to thread the domain through. + * Mail-domain for the deployment. Defaults to the integration test's + * canonical domain so callers do not have to thread the domain through. */ deploymentDomain?: string; /** Tool-package pins to ship with every step's deploy. */ @@ -1044,8 +1035,7 @@ export type DeployWorkflowOpts = { workflowRunRef?: string; /** * Optional override for the deployment's mail address. Defaults to - * the trivial bindings' `agentAddress` (trivial branch) or - * `ins_@` (multi-step branch). + * `ins_@`. */ deploymentMailAddress?: string; }; @@ -1064,11 +1054,9 @@ export type DeployWorkflowHandle = { /** * Build a workflow-deploy orchestrator wired against the env's hub - * substrate and run it against the supplied workflow. Trivial - * single-step workflows route through `env.hub.sessionService.launchSession` - * (which itself invokes the orchestrator's trivial branch); multi-step - * workflows derive per-step addresses and route each launch through - * the same `launchSession` callback. + * substrate and run it against the supplied workflow. The orchestrator + * derives per-step addresses and routes each launch through the + * `launchSession` callback. * * Registers the resulting handle on `env.deployments` so the other * Phase I helpers (event reads, signal injection, drain initiation, @@ -1081,45 +1069,16 @@ export async function deployWorkflow( ): Promise { const workflowRunRef = opts.workflowRunRef ?? DEFAULT_WORKFLOW_RUN_REF; - let deploymentId: string; - let mailAddress: string; - if (opts.trivialBindings !== undefined) { - if (workflow.stepOrder.length !== 1) { - throw new Error( - `deployWorkflow: trivialBindings supplied for a ${String(workflow.stepOrder.length)}-step workflow; trivial deploy requires exactly one step`, - ); - } - deploymentId = opts.trivialBindings.agentAddress; - mailAddress = - opts.deploymentMailAddress ?? opts.trivialBindings.agentAddress; - } else { - const explicit = opts.deploymentId; - if (explicit === undefined) { - throw new Error( - "deployWorkflow: deploymentId is required when trivialBindings is absent", - ); - } - deploymentId = explicit; - const deploymentDomain = opts.deploymentDomain ?? DEFAULT_DEPLOYMENT_DOMAIN; - mailAddress = - opts.deploymentMailAddress ?? `ins_${deploymentId}@${deploymentDomain}`; - } + const deploymentId = opts.deploymentId; + const deploymentDomain = opts.deploymentDomain ?? DEFAULT_DEPLOYMENT_DOMAIN; + const mailAddress = + opts.deploymentMailAddress ?? `ins_${deploymentId}@${deploymentDomain}`; - // The supervisor's trivial branch projects the agent address into - // a substrate-safe slug via `deriveTrivialDeploymentId` before - // writing workflow-run events (see - // `apps/sidecar/src/workflow-host-wiring.ts`). The fixture must - // report the same slug so downstream helpers query the repo the - // supervisor actually committed to. Multi-step deployments supply - // their own substrate-safe `deploymentId` and pass through - // unchanged. - const workflowRunRepoSlug = - opts.trivialBindings !== undefined - ? deriveTrivialDeploymentId(opts.trivialBindings.agentAddress) - : deploymentId; + // The deployment supplies its own substrate-safe `deploymentId`, which + // is the workflow-run repo slug the supervisor commits under. const workflowRunRepoId: RepoId = { kind: "workflow-run", - id: workflowRunRepoSlug, + id: deploymentId, }; // Route every per-step launch through the session service, mirroring @@ -1171,18 +1130,11 @@ export async function deployWorkflow( await orchestrator.deployWorkflow({ workflow, + deploymentId, + deploymentDomain, config: opts.config, deployContent: opts.deployContent, operatorApprovals: opts.operatorApprovals, - ...(opts.trivialBindings !== undefined - ? { trivialBindings: opts.trivialBindings } - : {}), - ...(opts.trivialBindings === undefined - ? { - deploymentId, - deploymentDomain: opts.deploymentDomain ?? DEFAULT_DEPLOYMENT_DOMAIN, - } - : {}), ...(opts.toolPackagePins !== undefined ? { toolPackagePins: opts.toolPackagePins } : {}), diff --git a/tests/workflow-deploy/fifo-mail-load.test.ts b/tests/workflow-deploy/fifo-mail-load.test.ts index 4c1bf4c3..227d2a50 100644 --- a/tests/workflow-deploy/fifo-mail-load.test.ts +++ b/tests/workflow-deploy/fifo-mail-load.test.ts @@ -134,13 +134,10 @@ describe("FIFO mail-trigger serialization under load", () => { test(`${String(LOAD_MAIL_COUNT)} mails dispatch in arrival order under load`, async () => { // Coverage-gap follow-up to the 3-mail case in - // fifo-mail.test.ts. The orchestrator's `isTrivialDeploy` requires - // `stepOrder.length === 1` AND `trivialBindings` present; - // absent the trivial-bindings (this fixture does not pass - // them), a single-step workflow still routes through the - // supervisor's FIFO inbox dispatch loop. The load test - // therefore uses a single-step workflow to keep per-mail - // commit pressure tractable in CI: every mail run still + // fifo-mail.test.ts. A single-step workflow still routes through + // the supervisor's FIFO inbox dispatch loop, so the load test + // uses one to keep per-mail commit pressure tractable in CI: + // every mail run still // exercises inbox -> processing -> trigger.fire -> wait for // terminal -> markConsumed, but trims one StepStarted + // StepCompleted commit pair off the per-run pack-push diff --git a/tests/workflow-deploy/trivial-roundtrip.test.ts b/tests/workflow-deploy/trivial-roundtrip.test.ts deleted file mode 100644 index 1f3c2f7d..00000000 --- a/tests/workflow-deploy/trivial-roundtrip.test.ts +++ /dev/null @@ -1,297 +0,0 @@ -// Trivial-workflow round-trip integration test. -// -// The runtime-side uniformity gate. Deploys a single-step workflow via -// the workflow-deploy orchestrator's trivial branch against the real -// hub + real sidecar subprocess + mock inference fixture, fires a mail -// trigger at the deployment's address, and asserts the canonical -// workflow-run event chain (`RunStarted` -> `StepStarted` -> -// `StepCompleted` -> `RunCompleted`) materializes in the workflow-run -// repo with the expected `consumedMessageId` correlation against the -// mail's `Message-Id`. The test additionally asserts the legacy -// deploy-flow surface (inference receives tools, the `agent.event` -// capture sees `inference.start`) still holds. -// -// The pre-landed `deploy-flow-env` fixture supplies every helper; this -// file does not modify the fixture. - -import fs from "node:fs"; - -import { afterAll, beforeAll, describe, expect, test } from "bun:test"; -import git from "isomorphic-git"; - -import { defineAgent } from "@intx/agent"; -import type { HarnessConfig } from "@intx/types/runtime"; -import { defineWorkflow } from "@intx/workflow"; - -import { - AGENT_ADDRESS, - AGENT_ID, - SESSION_ID, - SIDECAR_ID, - deployWorkflow, - fireMailTrigger, - readWorkflowRunEvents, - startDeployFlowEnv, - waitFor, - WORKFLOW_RUN_TERMINAL_TYPES, - type DeployFlowEnv, - type WorkflowRunEvent, -} from "../hub-agent/lib/deploy-flow-env"; - -let env: DeployFlowEnv; - -beforeAll(async () => { - env = await startDeployFlowEnv(); -}); - -afterAll(async () => { - await env.teardown(); -}); - -describe("trivial workflow round-trip", () => { - test("sidecar registers with hub", () => { - expect(env.hub.router.getConnectedSidecars()).toContain(SIDECAR_ID); - }); - - test("canonical event chain materializes end-to-end", async () => { - const config: HarnessConfig = { - sessionId: SESSION_ID, - agentId: AGENT_ID, - tenantId: "tenant-1", - principalId: "prin_integration-1", - agentAddress: AGENT_ADDRESS, - systemPrompt: "Fallback prompt (overridden by deploy tree)", - tools: [], - grants: [], - sources: [ - { - id: "anthropic:mock-model", - provider: "anthropic", - baseURL: `http://localhost:${env.inference.server.port}`, - apiKey: "sk-mock", - model: "mock-model", - }, - ], - defaultSource: "anthropic:mock-model", - }; - - // Mirror the legacy deploy-flow fixture's agent shape so the - // trivial-vs-multi-step dichotomy fires the trivial branch: a - // single step, single inference source, no tool factories, no - // capabilities. The workflow-deploy orchestrator's `isTrivialDeploy` - // routes on `stepOrder.length === 1 && trivialBindings !== - // undefined`; supplying both forces the trivial branch. - const agent = defineAgent({ - id: AGENT_ID, - systemPrompt: "You are an integration test agent.", - tools: [], - capabilities: [], - inference: { - sources: [{ provider: "anthropic", model: "mock-model" }], - }, - }); - - const workflow = defineWorkflow({ - id: `wf_${AGENT_ID}`, - agent, - trigger: { type: "mail", to: AGENT_ADDRESS }, - }); - - // Approval shape matches what `buildTrivialApprovalSet` computes - // for a `wrapHarnessAsTrivialAgent` deploy: per-source inference - // grants, the default-director grant, and the mail-address + - // mail-send grants derived from the deployment's mailbox. - const mailDomain = AGENT_ADDRESS.slice(AGENT_ADDRESS.lastIndexOf("@") + 1); - const operatorApprovals = new Set([ - `inference.source:anthropic:mock-model`, - `director:@intx/agent/default`, - `mail.address:${AGENT_ADDRESS}`, - `mail.send:${mailDomain}`, - ]); - - const deployHandle = await deployWorkflow(env, workflow, { - config, - deployContent: { systemPrompt: "You are an integration test agent." }, - trivialBindings: { - agentAddress: AGENT_ADDRESS, - agentId: AGENT_ID, - instanceId: AGENT_ID, - }, - operatorApprovals, - toolPackagePins: [{ name: "@intx/tools-mail", version: "0.1.2" }], - }); - - // The deploy ack arrives once `executeLaunchPhases` finishes - // provisioning the sidecar. This is the trivial branch's "ready" - // signal: the supervisor's trivial-launch path round-trips through - // the same wire surface the legacy agent-deploy uses. - const publicKey = env.hub.deployAcks.get(AGENT_ADDRESS); - expect(publicKey).toBeDefined(); - if (publicKey === undefined) throw new Error("unreachable"); - expect(publicKey.length).toBeGreaterThan(0); - - expect(env.hub.router.getRoutableAddresses()).toContain(AGENT_ADDRESS); - - const { messageId } = await fireMailTrigger(env, AGENT_ADDRESS, { - messageId: "", - }); - - // Legacy assertion #1: inference saw the deploy-tree's tool - // surfaces. The trivial round-trip must preserve the wire-level - // surface the legacy deploy-flow test asserts. - await waitFor(() => env.inference.requests.length > 0, { - diagnostics: env.sidecarDiagnostics, - }); - const inferenceReq = env.inference.requests[0]; - if (inferenceReq === undefined) throw new Error("unreachable"); - const toolNames = (inferenceReq.tools ?? []).map((t) => t.name); - expect(toolNames).toContain("@intx/tools-mail/sidecar-bundle:mail_send"); - - // Legacy assertion #2: the `agent.event` capture saw an - // `inference.start` event for this session. - function hasEventType( - event: unknown, - type: string, - ): event is { type: string } { - return ( - typeof event === "object" && - event !== null && - "type" in event && - event.type === type - ); - } - await waitFor( - () => - env.hub.agentEvents.some((e) => - hasEventType(e.event, "inference.start"), - ), - { diagnostics: env.sidecarDiagnostics }, - ); - const inferenceStartEvent = env.hub.agentEvents.find((e) => - hasEventType(e.event, "inference.start"), - ); - if (inferenceStartEvent === undefined) throw new Error("unreachable"); - expect(inferenceStartEvent.addr).toBe(AGENT_ADDRESS); - expect(inferenceStartEvent.sid).toBe(SESSION_ID); - - // The runtime-side uniformity gate. The deployment's workflow-run - // repo must hold the canonical event chain for the run this mail - // fired. The supervisor mints the runId internally (sha256 of the - // raw RFC 2822 message bytes in the current wiring), so the test - // does not know it up front; the helper below polls `runs/` in - // the workflow-run repo for the first run that reaches a terminal - // event. - const terminal = await waitForTerminalEventForAnyRun( - env, - deployHandle.deploymentId, - { diagnostics: env.sidecarDiagnostics }, - ); - - expect(terminal.event.type).toBe("RunCompleted"); - - const events = await readWorkflowRunEvents( - env, - deployHandle.deploymentId, - terminal.runId, - ); - - const types = events.map((e) => e.type); - const runStartedIdx = types.indexOf("RunStarted"); - const stepStartedIdx = types.indexOf("StepStarted"); - const stepCompletedIdx = types.indexOf("StepCompleted"); - const runCompletedIdx = types.indexOf("RunCompleted"); - - expect(runStartedIdx).toBeGreaterThanOrEqual(0); - expect(stepStartedIdx).toBeGreaterThan(runStartedIdx); - expect(stepCompletedIdx).toBeGreaterThan(stepStartedIdx); - expect(runCompletedIdx).toBeGreaterThan(stepCompletedIdx); - - const runStartedBody = events[runStartedIdx]?.body; - if (runStartedBody === undefined) throw new Error("unreachable"); - expect(runStartedBody["consumedMessageId"]).toBe(messageId); - - const stepStartedBody = events[stepStartedIdx]?.body; - if (stepStartedBody === undefined) throw new Error("unreachable"); - expect(stepStartedBody["attempt"]).toBe(1); - expect(typeof stepStartedBody["stepId"]).toBe("string"); - }); -}); - -/** - * Poll the deployment's workflow-run repo for the first run that - * reaches a terminal event. The trivial-roundtrip test does not know - * the runId the runtime mints (the supervisor derives it from the - * inbound mail bytes), so the helper walks `runs/` in the repo tree - * and returns the first run whose log has reached terminal status. - */ -async function waitForTerminalEventForAnyRun( - env: DeployFlowEnv, - deploymentId: string, - opts: { timeoutMs?: number; diagnostics?: () => string } = {}, -): Promise<{ runId: string; event: WorkflowRunEvent }> { - const { timeoutMs = 10_000, diagnostics } = opts; - const start = Date.now(); - const handle = env.deployments.get(deploymentId); - if (handle === undefined) { - throw new Error( - `waitForTerminalEventForAnyRun: no deployment registered for ${deploymentId}`, - ); - } - let repoDir: string; - try { - repoDir = env.hub.agentRepoStore.repoStore.getRepoDir( - handle.workflowRunRepoId, - ); - } catch (cause) { - const message = cause instanceof Error ? cause.message : String(cause); - throw new Error( - `waitForTerminalEventForAnyRun: workflow-run repo identity for deployment ${deploymentId} is structurally invalid at the substrate boundary (${message}); the trivial-deploy path uses the agent address as the deploymentId, but the workflow-run repo id constraint rejects characters present in mail addresses. This is the runtime-side uniformity gate failure: the trivial branch cannot write workflow-run events because it cannot construct a valid workflow-run repo id.`, - ); - } - for (;;) { - let runIds: string[] = []; - try { - const oid = await git.resolveRef({ - fs, - dir: repoDir, - ref: handle.workflowRunRef, - }); - try { - const runsTree = await git.readTree({ - fs, - dir: repoDir, - oid, - filepath: "runs", - }); - runIds = runsTree.tree - .filter((entry) => entry.type === "tree") - .map((entry) => entry.path); - } catch { - runIds = []; - } - } catch { - runIds = []; - } - for (const runId of runIds) { - const events = await readWorkflowRunEvents(env, deploymentId, runId); - const terminal = events.find((e) => - WORKFLOW_RUN_TERMINAL_TYPES.has(e.type), - ); - if (terminal !== undefined) { - return { runId, event: terminal }; - } - } - if (Date.now() - start > timeoutMs) { - const diag = diagnostics?.(); - const ctx = diag ? `\n${diag}` : ""; - const inspected = - runIds.length === 0 - ? "" - : runIds.join(","); - throw new Error( - `waitForTerminalEventForAnyRun timed out after ${String(timeoutMs)}ms for ${deploymentId}; runs observed: ${inspected}; expected the canonical event chain (RunStarted -> StepStarted -> StepCompleted -> RunCompleted) under runs//events/.${ctx}`, - ); - } - await new Promise((r) => setTimeout(r, 50)); - } -} diff --git a/tests/workflow-deploy/unresolvable-director.test.ts b/tests/workflow-deploy/unresolvable-director.test.ts index 385bb2cf..a664f23e 100644 --- a/tests/workflow-deploy/unresolvable-director.test.ts +++ b/tests/workflow-deploy/unresolvable-director.test.ts @@ -130,11 +130,7 @@ describe("unresolvable-director deploy rejection", () => { await deployWorkflow(env, workflow, { config, deployContent: { systemPrompt: "You are an integration test agent." }, - trivialBindings: { - agentAddress: AGENT_ADDRESS, - agentId: AGENT_ID, - instanceId: AGENT_ID, - }, + deploymentId: AGENT_ID, operatorApprovals, }); } catch (err) { From cf1ffbd5e2b2163b50bc11be322d2a6dbb96d8cd Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Fri, 3 Jul 2026 12:32:34 -0500 Subject: [PATCH 032/101] Register the deployment signing identity for every workflow deploy Every step of a deployed workflow signs its outbound mail as the one deployment mail address, and the host transport must hold a signing identity for any address that emits outbound mail or the signed send throws "not registered". The sidecar registered that identity only for a single-step deployment, so a step of a genuine multi-step deployment that sent outbound mail had its send rejected inside the step. Register the deployment address on the transport for both the single- and multi-step spawn, keeping the head-only work -- the deploy-ack agent key, the head repo init, and the recorded hub key -- gated to the single-step head. A multi-step deployment address is workflow-derived, incurs no reconnect challenge, and records no instance public key, so its ack keeps returning the supervisor principal key; the registered keypair is used purely for outbound signing. A new integration test deploys a two-step workflow whose sending step performs a signed outbound send and asserts the sidecar forwards a delivered outbound frame signed as the deployment address -- which the send can only produce once that address is registered. The deploy-flow fixture gains an outbound-mail capture to observe the forwarded frame. --- Makefile | 2 +- apps/sidecar/src/workflow-host-wiring.ts | 68 ++-- tests/hub-agent/lib/deploy-flow-env.ts | 17 + .../multistep-signed-send.test.ts | 302 ++++++++++++++++++ 4 files changed, 358 insertions(+), 31 deletions(-) create mode 100644 tests/workflow-deploy/multistep-signed-send.test.ts diff --git a/Makefile b/Makefile index 387dfaa6..ca0d630a 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ lint: FORCE test: FORCE bun test packages/ apps/ bin/ tests/agent/ tests/agent-audit-log/ tests/agent-blob-spill/ tests/agent-common/ tests/agent-multi-provider/ tests/agent-quickstart/ tests/agent-resume/ tests/agent-rewind/ tests/agent-rich-tool/ tests/agent-structured-payload/ tests/coding-agent/ tests/hub-agent/lib/ tests/inference-testing/ tests/tool-packaging/ tests/workflow/ - bun test --timeout 120000 tests/hub-agent/deploy-flow.test.ts tests/workflow-deploy/multistep-signal.test.ts tests/workflow-deploy/single-step-real-agent.test.ts tests/workflow-deploy/instance-reroute-real-agent.test.ts tests/workflow-deploy/instance-failover-real-agent.test.ts tests/workflow-deploy/single-step-message-input.test.ts tests/workflow-deploy/single-step-posix-tool.test.ts tests/workflow-deploy/single-step-grants-bridge.test.ts tests/workflow-deploy/single-step-event-threading.test.ts tests/workflow-deploy/single-step-conversation-durability.test.ts tests/workflow-deploy/single-step-full-lifecycle.test.ts tests/workflow-deploy/cross-process-custom-adapter.test.ts tests/workflow-deploy/conversation-state-wal.test.ts tests/workflow-deploy/drain-roundtrip.test.ts tests/workflow-deploy/child-workflow-roundtrip.test.ts tests/workflow-deploy/unresolvable-director.test.ts tests/workflow-deploy/fifo-mail.test.ts tests/workflow-deploy/mail-edge-cases.test.ts tests/inference/ tests/skill-attachment-flow.test.ts tests/hub-api/ tests/db/ + bun test --timeout 120000 tests/hub-agent/deploy-flow.test.ts tests/workflow-deploy/multistep-signal.test.ts tests/workflow-deploy/single-step-real-agent.test.ts tests/workflow-deploy/instance-reroute-real-agent.test.ts tests/workflow-deploy/instance-failover-real-agent.test.ts tests/workflow-deploy/multistep-signed-send.test.ts tests/workflow-deploy/single-step-message-input.test.ts tests/workflow-deploy/single-step-posix-tool.test.ts tests/workflow-deploy/single-step-grants-bridge.test.ts tests/workflow-deploy/single-step-event-threading.test.ts tests/workflow-deploy/single-step-conversation-durability.test.ts tests/workflow-deploy/single-step-full-lifecycle.test.ts tests/workflow-deploy/cross-process-custom-adapter.test.ts tests/workflow-deploy/conversation-state-wal.test.ts tests/workflow-deploy/drain-roundtrip.test.ts tests/workflow-deploy/child-workflow-roundtrip.test.ts tests/workflow-deploy/unresolvable-director.test.ts tests/workflow-deploy/fifo-mail.test.ts tests/workflow-deploy/mail-edge-cases.test.ts tests/inference/ tests/skill-attachment-flow.test.ts tests/hub-api/ tests/db/ test-load: FORCE bun test --timeout 300000 tests/workflow-deploy/fifo-mail-load.test.ts diff --git a/apps/sidecar/src/workflow-host-wiring.ts b/apps/sidecar/src/workflow-host-wiring.ts index 880c5ead..65d583c4 100644 --- a/apps/sidecar/src/workflow-host-wiring.ts +++ b/apps/sidecar/src/workflow-host-wiring.ts @@ -1096,14 +1096,15 @@ export function createSidecarDeployRouter(deps: { async function spawnWorkflowDeployment( spec: WorkflowDeploySpec, ): Promise { - // Fail loud if this address already has a live supervisor. A single-step - // deploy would fault anyway (its `transport.register` throws on a - // duplicate), but a genuine multi-step head never registers on the - // transport and the `activeSupervisors.set` below would silently clobber - // the running deployment's handle. Both the deploy path and the boot - // restore path route through here, so this is the single transition guard - // against a double-spawn -- notably a boot restore racing a legacy - // restore for the same address (the B-reroute follow-up relies on it). + // Fail loud if this address already has a live supervisor. Both single- + // and multi-step now register on the transport, so both carry the + // `transport.register` duplicate-throw backstop; this `has()` check is the + // primary early guard that gives a clean error before that lower-level + // throw and before the `activeSupervisors.set` below could clobber the + // running deployment's handle. Both the deploy path and the boot restore + // path route through here, so this is the single transition guard against + // a double-spawn -- notably a boot restore racing a legacy restore for the + // same address (the B-reroute follow-up relies on it). if (activeSupervisors.has(spec.agentAddress)) { throw new Error( `sidecar deploy router: a supervisor is already active for ${spec.agentAddress}; refusing to spawn a second`, @@ -1194,22 +1195,34 @@ export function createSidecarDeployRouter(deps: { : {}), }); - // OUTBOUND half of mailbox ownership (§3a): register the spawned - // agent's signing key on the host transport so the supervisor signs - // the agent's replies as the AGENT's identity. Gated on the - // single-step deploy, where the deployment mail address IS the legacy - // agent identity whose keypair lives in the keyStore. Registration - // happens before `spawn()` so the address is live the instant the - // first reply routes outbound. + // OUTBOUND half of mailbox ownership (§3a): register a signing key for + // the deployment mail address on the host transport so the supervisor + // signs the deployment's outbound mail. Every step -- single- or + // multi-step -- signs its outbound sends as `spec.agentAddress` (the + // one deployment mail address; no per-step sender reaches the host + // transport), so the transport MUST hold a `CryptoProvider` for it or + // `getTransportFor(senderAddress).send` throws "not registered". + // Registration happens before `spawn()` so the address is live the + // instant the first reply routes outbound. + const { keyPair } = await deps.keyStore.loadOrGenerateKey( + spec.agentAddress, + ); + deps.transport.register( + spec.agentAddress, + deps.createAgentCrypto(keyPair), + ); + agentTransportRegistered = true; + // The public key the deploy ack surfaces to the hub. For a single-step // head it is the AGENT key, set inside the block below; a genuine // multi-step deployment has no head agent identity and falls back to the - // supervisor principal key at the return. + // supervisor principal key at the return. (The registered deployment + // keypair above is used purely for outbound signing; a multi-step + // deployment address is workflow-derived, incurs no reconnect challenge, + // and so records no `agent_instance.publicKey` -- carrying it on the ack + // would be data written nowhere and read nowhere.) let headAgentPublicKey: string | undefined; if (spec.definition.stepOrder.length === 1) { - const { keyPair } = await deps.keyStore.loadOrGenerateKey( - spec.agentAddress, - ); // A single-step head IS an agent identity: it signs its own outbound // mail AND its reconnect challenges with this agent key (via the key // store's signChallenge). The hub records the ack's key into @@ -1218,11 +1231,6 @@ export function createSidecarDeployRouter(deps: { // reconnect challenge against it, so the ack MUST carry the agent key, // not the supervisor principal key -- otherwise verification fails. headAgentPublicKey = hexEncode(keyPair.publicKey); - deps.transport.register( - spec.agentAddress, - deps.createAgentCrypto(keyPair), - ); - agentTransportRegistered = true; // A single-step workflow stages its deploy tree at the head (the // lone step IS the head). Initialize the head's on-disk deploy-tree @@ -1657,12 +1665,12 @@ export function createSidecarDeployRouter(deps: { if (wired !== undefined) { activeSupervisors.delete(frame.agentAddress); await wired.supervisor.shutdown(); - // Drop the agent's transport registration installed at spawn for - // the single-step launched-agent deploy (OUTBOUND half of - // mailbox ownership, §3a). `unregister` is a no-op when the - // address was never registered (a genuine multi-step deploy - // whose derived per-step addresses carry no host keypair), so it - // is safe to call unconditionally for any spawned deployment. + // Drop the deployment address's transport registration installed at + // spawn (OUTBOUND half of mailbox ownership, §3a). Both single- and + // multi-step register the deployment address for outbound signing, so + // this tears down a real registration for either; `unregister` is a + // no-op only if the spawn failed before registering, so it is safe to + // call unconditionally for any spawned deployment. deps.transport.unregister(frame.agentAddress); // Reclaim the deployment's per-step local-disk scratch now that // its supervisor + workflow-process child are torn down. The diff --git a/tests/hub-agent/lib/deploy-flow-env.ts b/tests/hub-agent/lib/deploy-flow-env.ts index 454a66cd..b8b7ad90 100644 --- a/tests/hub-agent/lib/deploy-flow-env.ts +++ b/tests/hub-agent/lib/deploy-flow-env.ts @@ -469,6 +469,13 @@ export type HubEnv = { deployAcks: Map; statePacks: { agentAddress: string; ref: string; commitSha: string }[]; statePackReceiveFailures: { agentAddress: string; error: string }[]; + /** + * Every delivered `mail.outbound` frame the sidecar forwarded to the hub + * for persistence, keyed by the signing sender. A frame reaches here only + * after the sidecar signed and delivered the send, so its presence proves + * the sender's identity was registered on the host transport. + */ + outboundMail: { senderAddress: string; recipients: string[] }[]; hubDataDir: string; }; @@ -486,6 +493,7 @@ export async function startHub( const deployAcks = new Map(); const statePacks: HubEnv["statePacks"] = []; const statePackReceiveFailures: HubEnv["statePackReceiveFailures"] = []; + const outboundMail: HubEnv["outboundMail"] = []; const hubDataDir = await fs.promises.mkdtemp( path.join(os.tmpdir(), "hub-data-"), @@ -562,6 +570,14 @@ export async function startHub( } return { accepted: true }; }, + // Capture delivered outbound mail the sidecar forwards for + // persistence. Recording the signing sender is enough for the + // integration assertions; no durable row is minted, so this returns + // an empty result set. + persistMail({ senderAddress, recipients }) { + outboundMail.push({ senderAddress, recipients }); + return Promise.resolve([]); + }, }, }); router.events.on("agent.event", ({ agentAddress, sessionId, event }) => { @@ -722,6 +738,7 @@ export async function startHub( deployAcks, statePacks, statePackReceiveFailures, + outboundMail, hubDataDir, }; } diff --git a/tests/workflow-deploy/multistep-signed-send.test.ts b/tests/workflow-deploy/multistep-signed-send.test.ts new file mode 100644 index 00000000..eab2ae51 --- /dev/null +++ b/tests/workflow-deploy/multistep-signed-send.test.ts @@ -0,0 +1,302 @@ +// Multi-step signed outbound send regression test. +// +// A genuine multi-step (2+ step) workflow deployment must register a +// signing identity for its deployment mail address on the host transport, +// exactly as a single-step deployment does. Every step of a multi-step +// deployment signs its outbound mail as the ONE deployment-wide address +// (`ins_@`), so if that address is not registered a +// step's `env.transport.send` rejects with "not registered", the step +// fails, and the run fails. +// +// This deploys a two-step workflow whose sending step calls the +// transport-backed `mail_send` tool (the mock inference drives the tool +// call on the first request that exposes it). The tool routes through the +// real outbound chain -- supervisor-backed transport -> outbound bridge -> +// `outbound.message` IPC -> supervisor `sendOutbound` -> host transport +// SIGNED send -- so the send reaches the host transport as the deployment +// address. The sidecar forwards the delivered `mail.outbound` frame to the +// hub for persistence, where the fixture captures its signing sender. A +// captured frame whose sender is the deployment address is a load-bearing +// proof that the address held a registered signing identity; a registration +// gap would reject the send inside the step and forward no frame. +// +// The sender is a step of a multi-step deployment (not a single-step head), +// so this covers the deployment-scoped registration the single-step path +// already had and the multi-step path lacked. + +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; + +import { defineAgent, createDefaultDirectorRegistry } from "@intx/agent"; +import type { HarnessConfig } from "@intx/types/runtime"; +import type { ToolPackagePin } from "@intx/types/tool-packages"; +import { WireGrantRule } from "@intx/types/grant-wire"; +import { defineWorkflow, step, type WorkflowDefinition } from "@intx/workflow"; +import { + createWorkflowDeployOrchestrator, + deriveDeploymentAddress, + type ApprovalSet, + type LaunchSessionFn, + type SendMultiStepDeployFn, + type WorkflowRepoWriter, +} from "@intx/workflow-deploy"; +import { deriveTrivialDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; +import type { RepoId, WorkflowRunHubPrincipal } from "@intx/hub-sessions"; +import { DEFAULT_ASSET_REF } from "@intx/hub-sessions"; + +import { + SESSION_ID, + SIDECAR_ID, + fireMailTrigger, + startDeployFlowEnv, + waitFor, + waitForFirstRunId, + waitForWorkflowRunComplete, + type DeployFlowEnv, +} from "../hub-agent/lib/deploy-flow-env"; +import { toLaunchDeployContent } from "./launch-session-bridge"; + +const DEPLOYMENT_DOMAIN = "integration.interchange"; +const DEPLOYMENT_ID = "multistep-signed-send-1"; +const WORKFLOW_RUN_REF = "refs/heads/main"; +// The first step in `stepOrder`; the mock drives its inference to call the +// mail tool, so it is the step that performs the signed send. +const SENDER_STEP_ID = "send"; + +const TOOL_NAME = "@intx/tools-mail/sidecar-bundle:mail_send"; +const GRANTED_RESOURCE = `tool:${TOOL_NAME}`; +const SENTINEL_FILENAME = "multistep-signed-send-receipt.txt"; + +const TOOL_PINS: readonly ToolPackagePin[] = [ + { name: "@intx/tools-mail", version: "0.1.2" }, +]; + +const GRANTED_RULE: WireGrantRule = { + id: "grant-tool-invoke", + resource: GRANTED_RESOURCE, + action: "invoke", + effect: "allow", + origin: "creator", + conditions: null, + expiresAt: null, + roleId: null, + principalId: null, +}; + +let env: DeployFlowEnv; +let deploymentMailAddress: string; + +beforeAll(async () => { + deploymentMailAddress = deriveDeploymentAddress({ + deploymentId: DEPLOYMENT_ID, + deploymentDomain: DEPLOYMENT_DOMAIN, + }); + // The transport-backed `mail_send` bundle sends through the real outbound + // chain and sentinels on receipt; `inferenceToolCall` drives the model to + // call it. The send targets the deployment's own address -- a local, + // registered recipient once the deployment identity is on the transport -- + // so the signed send delivers without a remote leg. + env = await startDeployFlowEnv({ + transportBackedMailTool: true, + inferenceToolCall: { + toolName: TOOL_NAME, + input: { to: deploymentMailAddress, body: SENTINEL_FILENAME }, + }, + }); +}); + +afterAll(async () => { + await env.teardown(); +}); + +describe("multi-step signed outbound send", () => { + test("sidecar registers with hub", () => { + expect(env.hub.router.getConnectedSidecars()).toContain(SIDECAR_ID); + }); + + test("a step of a multi-step deployment signs and sends outbound mail", async () => { + const sendAgent = defineAgent({ + id: "agent-multistep-send", + systemPrompt: "You are the sending step agent.", + tools: [], + capabilities: [], + inference: { + sources: [{ provider: "anthropic", model: "mock-model" }], + }, + }); + const tailAgent = defineAgent({ + id: "agent-multistep-tail", + systemPrompt: "You are the trailing step agent.", + tools: [], + capabilities: [], + inference: { + sources: [{ provider: "anthropic", model: "mock-model" }], + }, + }); + + const workflow: WorkflowDefinition = defineWorkflow({ + id: `wf_${DEPLOYMENT_ID}`, + trigger: { type: "mail", to: deploymentMailAddress }, + steps: { + [SENDER_STEP_ID]: step({ agent: sendAgent }), + tail: step({ agent: tailAgent, after: [SENDER_STEP_ID] }), + }, + }); + + const config: HarnessConfig = { + sessionId: SESSION_ID, + agentId: `ins_${DEPLOYMENT_ID}`, + tenantId: "tenant-1", + principalId: "prin_integration-1", + agentAddress: deploymentMailAddress, + systemPrompt: "Fallback prompt (overridden per step by orchestrator)", + tools: [], + grants: [GRANTED_RULE], + sources: [ + { + id: "anthropic:mock-model", + provider: "anthropic", + baseURL: `http://localhost:${env.inference.server.port}`, + apiKey: "sk-mock", + model: "mock-model", + }, + ], + defaultSource: "anthropic:mock-model", + }; + + const operatorApprovals: ApprovalSet = new Set([ + "inference.source:anthropic:mock-model", + "director:@intx/agent/default", + `mail.address:${deploymentMailAddress}`, + `mail.send:${DEPLOYMENT_DOMAIN}`, + ]); + + const launchSession: LaunchSessionFn = async (orchestratorParams) => { + await env.hub.sessionService.launchSession({ + agentAddress: orchestratorParams.agentAddress, + agentId: orchestratorParams.agentId, + instanceId: orchestratorParams.instanceId, + config: orchestratorParams.config, + deployContent: toLaunchDeployContent(orchestratorParams.deployContent), + ...(orchestratorParams.toolPackagePins !== undefined + ? { toolPackagePins: orchestratorParams.toolPackagePins } + : {}), + }); + }; + + const sendMultiStepDeploy: SendMultiStepDeployFn = async (params) => + env.hub.router.sendAgentDeploy(params.agentAddress, params.config, { + definition: { + id: params.definition.id, + triggers: [...params.definition.triggers], + stepOrder: [...params.definition.stepOrder], + steps: params.definition.steps as Record, + ...(params.definition.state !== undefined + ? { state: params.definition.state } + : {}), + }, + sources: params.sources, + }); + + const workflowRepo: WorkflowRepoWriter = { + async writeWorkflowRepo(args) { + const repoId: RepoId = { kind: "workflow", id: args.workflowRepoId }; + const principal: WorkflowRunHubPrincipal = { kind: "hub" }; + const files: Record = {}; + for (const [k, v] of args.files) { + files[k] = v; + } + await env.hub.agentRepoStore.repoStore.writeTree( + principal, + repoId, + DEFAULT_ASSET_REF, + { + files, + message: `multistep-signed-send test: write workflow repo ${args.workflowRepoId}`, + }, + ); + }, + }; + + const orchestrator = createWorkflowDeployOrchestrator({ + directorRegistry: createDefaultDirectorRegistry(), + workflowRepo, + launchSession, + sendMultiStepDeploy, + }); + + let result: Awaited>; + try { + result = await orchestrator.deployWorkflow({ + workflow, + config, + deployContent: { systemPrompt: config.systemPrompt }, + operatorApprovals, + deploymentId: DEPLOYMENT_ID, + deploymentDomain: DEPLOYMENT_DOMAIN, + hubPublicKey: "00".repeat(32), + toolPackagePins: TOOL_PINS, + }); + } catch (cause) { + const message = cause instanceof Error ? cause.message : String(cause); + const diag = env.sidecarDiagnostics(); + throw new Error( + `deployWorkflow failed: ${message}\n${diag.length > 0 ? diag : ""}`, + { cause }, + ); + } + expect(result.kind).toBe("multi-step"); + + const workflowRunRepoId: RepoId = { + kind: "workflow-run", + id: deriveTrivialDeploymentId(deploymentMailAddress), + }; + env.registerDeployment({ + deploymentId: DEPLOYMENT_ID, + workflowDefinition: workflow, + workflowRunRepoId, + workflowRunRef: WORKFLOW_RUN_REF, + mailAddress: deploymentMailAddress, + }); + + expect(env.hub.router.getRoutableAddresses()).toContain( + deploymentMailAddress, + ); + + await fireMailTrigger(env, deploymentMailAddress, { + messageId: "", + }); + + const runId = await waitForFirstRunId(env, workflowRunRepoId, { + diagnostics: env.sidecarDiagnostics, + timeoutMs: 20_000, + }); + + const terminal = await waitForWorkflowRunComplete( + env, + DEPLOYMENT_ID, + runId, + { timeoutMs: 20_000, diagnostics: env.sidecarDiagnostics }, + ); + expect(terminal.type).toBe("RunCompleted"); + + // The proof of the fix: the sidecar signed and delivered a `mail.outbound` + // frame whose SIGNING SENDER is the deployment mail address. A multi-step + // step signs its outbound sends as that one deployment address, so the + // frame reaches the hub only when the address holds a registered signing + // identity on the host transport. Without the registration the send throws + // "not registered" inside the step and no frame is ever forwarded, leaving + // `outboundMail` empty. + await waitFor( + () => + env.hub.outboundMail.some( + (m) => m.senderAddress === deploymentMailAddress, + ), + { timeoutMs: 20_000, diagnostics: env.sidecarDiagnostics }, + ); + const signedSend = env.hub.outboundMail.find( + (m) => m.senderAddress === deploymentMailAddress, + ); + expect(signedSend).toBeDefined(); + expect(signedSend?.recipients).toContain(deploymentMailAddress); + }); +}); From 50a0cfa157b3a3ec5b9ccdfa8cdc3552a8298d60 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Fri, 3 Jul 2026 14:06:08 -0500 Subject: [PATCH 033/101] Separate warm-harness start from deploy staging The launch machinery folded the warm hub-agent session start into the same routine that stages a deploy -- resolve assets, write the deploy tree, provision, and deliver the deploy and asset packs. The single-step head hand-off already skipped the start by threading a workflow-frame flag through that routine, an inversion that made the staging routine carry knowledge of who does and does not want a warm harness. Lift the session start into its own step. The staging routine now ends at pack delivery and never starts a harness; the legacy agent launcher calls the new start step after staging, and the single-step head hand-off simply does not. Behavior is unchanged for both callers, which the deploy integration suite confirms. --- packages/hub-sessions/src/session-service.ts | 75 +++++++++++--------- 1 file changed, 41 insertions(+), 34 deletions(-) diff --git a/packages/hub-sessions/src/session-service.ts b/packages/hub-sessions/src/session-service.ts index 2a15d25f..99e4b3e9 100644 --- a/packages/hub-sessions/src/session-service.ts +++ b/packages/hub-sessions/src/session-service.ts @@ -561,12 +561,14 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { } /** - * Drive the per-agent deploy + session-start phases (deploy-tree write, - * asset-pack fan-out, provision frame, pack, session start). Shared by - * the public `launchSession` (warm-harness provision, no workflow frame) - * and the single-step head hand-off (`deploySingleStepAtHead`, which - * passes a `workflowFrame` so the sidecar spawns the workflow-process - * child instead of a warm harness). + * Stage a deploy on the sidecar: resolve assets and tool packages, write + * the deploy tree, provision the agent, and deliver the deploy + asset + * packs (Phases 0-2b). Does NOT start the warm harness -- callers that + * want one (the legacy agent-deploy path) invoke `startWarmSession` + * afterward. Shared by the public `launchSession` (warm-harness provision, + * no workflow frame) and the single-step head hand-off + * (`deploySingleStepAtHead`, which passes a `workflowFrame` so the sidecar + * spawns the workflow-process child instead of a warm harness). */ async function executeLaunchPhases(params: { agentAddress: string; @@ -580,9 +582,8 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { * deployment `agent.deploy` frame carrying the workflow definition + * source pins (the sidecar initializes the head repo on receipt and * spawns the workflow-process child) instead of the plain provision - * frame, and Phase 3's warm-harness `sendSessionStart` is skipped (the - * supervised child drives the agent, not a warm hub-agent harness). - * The returned supervisor public key comes from that frame's ack. + * frame. The returned supervisor public key comes from that frame's + * ack; the caller skips `startWarmSession` for a workflow deploy. */ workflowFrame?: { definition: WorkflowDefinition; @@ -824,32 +825,37 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { } } - // Phase 3: Warm-harness session start. A single-step workflow deploy - // skips this: the supervised workflow-process child drives the agent - // per message through the supervisor, so there is no warm hub-agent - // harness to start. The legacy agent-deploy path starts its warm - // harness here. - if (workflowFrame === undefined) { - try { - await sidecarRouter.sendSessionStart(agentAddress); - } catch (err) { - await attemptCleanup(agentAddress, "start", err); - throw new SessionLaunchError("start", err, false); - } - } - return deployAckPublicKey === undefined ? undefined : { publicKey: deployAckPublicKey }; } + /** + * Start the warm hub-agent harness for a staged deploy. The legacy + * agent-deploy path calls this after `executeLaunchPhases` stages the + * deploy tree and packs; a single-step workflow deploy skips it entirely, + * because the supervised workflow-process child drives the agent per + * message through the supervisor and there is no warm harness to start. + * A failed start tears the sidecar deployment down so no zombie agent is + * left behind. + */ + async function startWarmSession(agentAddress: string): Promise { + try { + await sidecarRouter.sendSessionStart(agentAddress); + } catch (err) { + await attemptCleanup(agentAddress, "start", err); + throw new SessionLaunchError("start", err, false); + } + } + /** * Deploy a one-step workflow once at the head. Reuses the full * launch-phase machinery (deploy-tree write, pack, asset fan-out) via * `executeLaunchPhases`, swapping the Phase 1 provision frame for the - * workflow frame and skipping the Phase 3 warm-harness start. The - * workflow frame makes the sidecar initialize the head repo and spawn - * the workflow-process child; the follow-up pack lands the head's deploy + * workflow frame; it never calls `startWarmSession`, so no warm harness + * starts. The workflow frame makes the sidecar initialize the head repo + * and spawn the workflow-process child; the follow-up pack lands the + * head's deploy * tree. Returns the supervisor's principal public key from the frame's * ack. A workflow-frame launch always yields a deploy-ack key; its * absence is a wiring bug, not a tolerable case. @@ -928,14 +934,14 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { } /** - * Provision one agent at its address through the deploy-tree + pack + - * session-start phases (`executeLaunchPhases` with no workflow frame -- - * the warm-harness provision path). This is the launcher the multi-step - * workflow branch drives per step via the orchestrator's `launchSession` - * callback, and the launcher the integration-test fixture drives - * directly. A single-agent instance deploy uses `deployInstanceAtHead` - * and a single-step workflow uses the head hand-off; neither routes - * here. The method carries no workflow projection of its own. + * Provision one agent at its address: stage the deploy tree + packs + * (`executeLaunchPhases` with no workflow frame) and then start its warm + * harness. This is the launcher the multi-step workflow branch drives per + * step via the orchestrator's `launchSession` callback, and the launcher + * the integration-test fixture drives directly. A single-agent instance + * deploy uses `deployInstanceAtHead` and a single-step workflow uses the + * head hand-off; neither routes here. The method carries no workflow + * projection of its own. */ async function launchSession(params: { agentAddress: string; @@ -955,6 +961,7 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { ? { toolPackagePins: params.toolPackagePins } : {}), }); + await startWarmSession(params.agentAddress); } /** From b5572f19dc5cf9885f88aade6e94fa9d738f49ca Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Fri, 3 Jul 2026 17:36:32 -0500 Subject: [PATCH 034/101] Add transient per-step route binding to the sidecar router A multi-step workflow deploy will stage each step's deploy tree by delivering packs to the step's derived address before the deployment frame spawns the child. A pack routes through the hub's address index, which today is populated only by a full agent-deploy frame. Staging a step without that frame needs a way to make the step address routable for the pack window alone. Add bind and unbind methods to the sidecar router. Bind resolves the deployment's sidecar and registers the step address in the keyless workflow-address routing set, so a pack finds the connection; unbind removes it once the step's packs land. The step address is hub-minted and workflow-derived, so it carries no per-address key and never joins the challenged session-address set. Per-step addresses are not routed at runtime -- mail, signals, and drains use the deployment address -- so the binding is transient: it is never persisted into the reconnect set and a reconnect after unbind does not resurrect it. A sidecar drop mid-stage reclaims a still-bound route through the existing connection teardown. --- packages/hub-api/src/routes/agents.test.ts | 2 + packages/hub-api/src/routes/assets.test.ts | 2 + .../hub-api/src/routes/catalog-routes.test.ts | 2 + .../hub-api/src/routes/git-tokens.test.ts | 2 + packages/hub-api/src/routes/instances.test.ts | 6 ++ packages/hub-api/src/routes/workflows.test.ts | 2 + .../hub-sessions/src/session-service.test.ts | 2 + .../src/ws/sidecar-handler.test.ts | 95 +++++++++++++++++++ .../hub-sessions/src/ws/sidecar-handler.ts | 63 ++++++++++++ tests/hub-api/asset-routes.test.ts | 2 + 10 files changed, 178 insertions(+) diff --git a/packages/hub-api/src/routes/agents.test.ts b/packages/hub-api/src/routes/agents.test.ts index 4f769eef..d8704860 100644 --- a/packages/hub-api/src/routes/agents.test.ts +++ b/packages/hub-api/src/routes/agents.test.ts @@ -179,6 +179,8 @@ function createMockSidecarRouter(): SidecarRouter { sendGrantsUpdate: () => notImpl("sendGrantsUpdate"), sendSourcesUpdate: () => notImpl("sendSourcesUpdate"), sendPack: () => notImpl("sendPack"), + bindStepRoute: () => notImpl("bindStepRoute"), + unbindStepRoute: () => notImpl("unbindStepRoute"), sendSyncRequest: () => notImpl("sendSyncRequest"), sendSignalDeliver: () => notImpl("sendSignalDeliver"), sendDrain: () => notImpl("sendDrain"), diff --git a/packages/hub-api/src/routes/assets.test.ts b/packages/hub-api/src/routes/assets.test.ts index e40cc3f7..665a99d0 100644 --- a/packages/hub-api/src/routes/assets.test.ts +++ b/packages/hub-api/src/routes/assets.test.ts @@ -295,6 +295,8 @@ function createMockSidecarRouter(): SidecarRouter { sendGrantsUpdate: () => notImpl("sendGrantsUpdate"), sendSourcesUpdate: () => notImpl("sendSourcesUpdate"), sendPack: () => notImpl("sendPack"), + bindStepRoute: () => notImpl("bindStepRoute"), + unbindStepRoute: () => notImpl("unbindStepRoute"), sendSyncRequest: () => notImpl("sendSyncRequest"), sendSignalDeliver: () => notImpl("sendSignalDeliver"), sendDrain: () => notImpl("sendDrain"), diff --git a/packages/hub-api/src/routes/catalog-routes.test.ts b/packages/hub-api/src/routes/catalog-routes.test.ts index 4b173437..8d780c0f 100644 --- a/packages/hub-api/src/routes/catalog-routes.test.ts +++ b/packages/hub-api/src/routes/catalog-routes.test.ts @@ -194,6 +194,8 @@ function createMockSidecarRouter(): SidecarRouter { // Mutations fire a fire-and-forget source push; resolve so it is a no-op. sendSourcesUpdate: () => Promise.resolve(), sendPack: () => notImpl("sendPack"), + bindStepRoute: () => notImpl("bindStepRoute"), + unbindStepRoute: () => notImpl("unbindStepRoute"), sendSyncRequest: () => notImpl("sendSyncRequest"), sendSignalDeliver: () => notImpl("sendSignalDeliver"), sendDrain: () => notImpl("sendDrain"), diff --git a/packages/hub-api/src/routes/git-tokens.test.ts b/packages/hub-api/src/routes/git-tokens.test.ts index 3e0ced70..01d5c608 100644 --- a/packages/hub-api/src/routes/git-tokens.test.ts +++ b/packages/hub-api/src/routes/git-tokens.test.ts @@ -218,6 +218,8 @@ function createMockSidecarRouter(): SidecarRouter { sendGrantsUpdate: () => notImpl("sendGrantsUpdate"), sendSourcesUpdate: () => notImpl("sendSourcesUpdate"), sendPack: () => notImpl("sendPack"), + bindStepRoute: () => notImpl("bindStepRoute"), + unbindStepRoute: () => notImpl("unbindStepRoute"), sendSyncRequest: () => notImpl("sendSyncRequest"), sendSignalDeliver: () => notImpl("sendSignalDeliver"), sendDrain: () => notImpl("sendDrain"), diff --git a/packages/hub-api/src/routes/instances.test.ts b/packages/hub-api/src/routes/instances.test.ts index 1b39240a..0f2bbdfe 100644 --- a/packages/hub-api/src/routes/instances.test.ts +++ b/packages/hub-api/src/routes/instances.test.ts @@ -255,6 +255,12 @@ function createMockSidecarRouter( sendPack(_addr, _pack, _ref, _sha) { return notImpl("sendPack"); }, + bindStepRoute(_stepAddress) { + notImpl("bindStepRoute"); + }, + unbindStepRoute(_stepAddress) { + notImpl("unbindStepRoute"); + }, sendSyncRequest(_addr) { notImpl("sendSyncRequest"); }, diff --git a/packages/hub-api/src/routes/workflows.test.ts b/packages/hub-api/src/routes/workflows.test.ts index f9d1632a..f0b2bf84 100644 --- a/packages/hub-api/src/routes/workflows.test.ts +++ b/packages/hub-api/src/routes/workflows.test.ts @@ -201,6 +201,8 @@ function createMockSidecarRouter( sendGrantsUpdate: () => notImpl("sendGrantsUpdate"), sendSourcesUpdate: () => notImpl("sendSourcesUpdate"), sendPack: () => notImpl("sendPack"), + bindStepRoute: () => notImpl("bindStepRoute"), + unbindStepRoute: () => notImpl("unbindStepRoute"), sendSyncRequest: () => notImpl("sendSyncRequest"), sendSignalDeliver: (opts) => { signalCalls.push(opts); diff --git a/packages/hub-sessions/src/session-service.test.ts b/packages/hub-sessions/src/session-service.test.ts index cf05ab1e..167af434 100644 --- a/packages/hub-sessions/src/session-service.test.ts +++ b/packages/hub-sessions/src/session-service.test.ts @@ -95,6 +95,8 @@ function createMockRouter(): SidecarRouter & { }); return Promise.resolve(); }) as SidecarRouter["sendPack"], + bindStepRoute: () => undefined, + unbindStepRoute: () => undefined, sendSyncRequest: track( "sendSyncRequest", ) as SidecarRouter["sendSyncRequest"], diff --git a/packages/hub-sessions/src/ws/sidecar-handler.test.ts b/packages/hub-sessions/src/ws/sidecar-handler.test.ts index 998516c0..17cf73f1 100644 --- a/packages/hub-sessions/src/ws/sidecar-handler.test.ts +++ b/packages/hub-sessions/src/ws/sidecar-handler.test.ts @@ -289,6 +289,101 @@ describe("SidecarRouter", () => { }); }); + describe("per-step route bind/unbind", () => { + const DEPLOYMENT_ADDR = "ins_dep_multi@local"; + const STEP_ADDR = "ins_dep_multi-step1@local"; + const ZERO_SHA = "0".repeat(40); + + function registerDeploymentSidecar(): ReturnType { + const ws = createMockWs(); + router.handleOpen(ws); + router.handleMessage( + ws, + JSON.stringify({ + type: "register", + sidecarId: "sc-1", + token: "tok", + agentAddresses: [DEPLOYMENT_ADDR], + }), + ); + return ws; + } + + test("bind makes a step address routable; unbind removes it", async () => { + registerDeploymentSidecar(); + + // An unbound step address has no route: `sendPack` rejects before + // touching the wire. + await expect( + router.sendPack( + STEP_ADDR, + new Uint8Array([1]), + "refs/heads/main", + ZERO_SHA, + ), + ).rejects.toThrow(/No sidecar connected/); + + router.bindStepRoute(STEP_ADDR); + expect(router.getRoutableAddresses()).toContain(STEP_ADDR); + // Routable: `routeMail` (and `sendPack`) now resolve the sidecar. + expect(router.routeMail(STEP_ADDR, "dGVzdA==")).toBe(true); + + router.unbindStepRoute(STEP_ADDR); + expect(router.getRoutableAddresses()).not.toContain(STEP_ADDR); + await expect( + router.sendPack( + STEP_ADDR, + new Uint8Array([1]), + "refs/heads/main", + ZERO_SHA, + ), + ).rejects.toThrow(/No sidecar connected/); + }); + + test("a reconnect after unbind does not resurrect the transient route", () => { + const ws = registerDeploymentSidecar(); + router.bindStepRoute(STEP_ADDR); + router.unbindStepRoute(STEP_ADDR); + + // The sidecar reconnects announcing only the deployment's workflow + // address -- never the transient per-step address -- so the step route + // stays gone (the binding never entered the reconnect set). + router.handleMessage( + ws, + JSON.stringify({ + type: "reconnect", + sidecarId: "sc-1", + token: "tok", + agentAddresses: [], + workflowAddresses: [DEPLOYMENT_ADDR], + }), + ); + + expect(router.getRoutableAddresses()).not.toContain(STEP_ADDR); + }); + + test("bind throws when no sidecar is connected", () => { + expect(() => router.bindStepRoute(STEP_ADDR)).toThrow(/No sidecar/); + }); + + test("unbind is a no-op for an address that was never bound", () => { + registerDeploymentSidecar(); + expect(() => router.unbindStepRoute(STEP_ADDR)).not.toThrow(); + expect(router.getRoutableAddresses()).not.toContain(STEP_ADDR); + }); + + test("handleClose reclaims a still-bound step route", () => { + const ws = registerDeploymentSidecar(); + router.bindStepRoute(STEP_ADDR); + expect(router.getRoutableAddresses()).toContain(STEP_ADDR); + + // A sidecar drop mid-stage tears the transient binding down with the + // rest of the connection's addresses -- no stale route survives. + router.handleClose(ws); + expect(router.getRoutableAddresses()).not.toContain(STEP_ADDR); + }); + }); + describe("mail routing", () => { test("routes mail between two sidecars", () => { const ws1 = createMockWs(); diff --git a/packages/hub-sessions/src/ws/sidecar-handler.ts b/packages/hub-sessions/src/ws/sidecar-handler.ts index 0057f874..7a86b6f6 100644 --- a/packages/hub-sessions/src/ws/sidecar-handler.ts +++ b/packages/hub-sessions/src/ws/sidecar-handler.ts @@ -141,6 +141,20 @@ export type SidecarRouter = { commitSha: string, options?: SendPackOptions, ): Promise; + /** + * Bind a per-step workflow-substrate address to a sidecar for the staging + * window of a multi-step deploy, so `sendPack` can route the step's deploy + * and asset packs before the deployment-level frame spawns the child. The + * address enters the keyless `workflowAddresses` routing set; call + * `unbindStepRoute` once the step's packs land. Throws if no sidecar is + * available. + */ + bindStepRoute(stepAddress: string): void; + /** + * Remove a per-step route bound by `bindStepRoute`. Idempotent: an unbound + * address is a no-op. + */ + unbindStepRoute(stepAddress: string): void; sendSyncRequest(agentAddress: string): void; /** * Deliver a workflow-run signal to the sidecar that hosts the named @@ -1385,6 +1399,53 @@ export function createSidecarRouter( } } + /** + * Bind a per-step workflow-substrate address to a sidecar for the staging + * window of a multi-step deploy, so `sendPack` can route the step's deploy + * and asset packs before the deployment-level frame spawns the child. + * + * The address is hub-minted and workflow-derived (no per-address key), so + * it enters the keyless `workflowAddresses` set -- never the challenged + * `agentAddresses` set -- and is torn down by `unbindStepRoute` once the + * step's packs land. `handleClose` reclaims it if the sidecar drops + * mid-stage. Per-step addresses are not runtime-routed (mail, signals, and + * drains use the deployment address), so the binding is transient: it is + * never persisted into the reconnect set and never resurrected on + * reconnect. + */ + function bindStepRoute(stepAddress: string): void { + const ws = + addressIndex.get(stepAddress) ?? findSidecarForNewAgent(stepAddress); + if (ws === undefined) { + throw new Error( + `No sidecar available to stage workflow step "${stepAddress}"`, + ); + } + const conn = connections.get(ws); + if (conn === undefined) { + throw new Error( + `No sidecar connected to stage workflow step "${stepAddress}"`, + ); + } + conn.workflowAddresses.add(stepAddress); + addressIndex.set(stepAddress, ws); + } + + /** + * Remove a per-step route bound by `bindStepRoute` once the step's packs + * have landed. Idempotent: an address that was never bound (or already + * unbound, e.g. by a mid-stage `handleClose`) is a no-op. + */ + function unbindStepRoute(stepAddress: string): void { + const ws = addressIndex.get(stepAddress); + if (ws === undefined) return; + const conn = connections.get(ws); + if (conn !== undefined) { + conn.workflowAddresses.delete(stepAddress); + } + addressIndex.delete(stepAddress); + } + // Pack transfers may take longer than session requests due to data volume. const PACK_TIMEOUT_MS = requestTimeoutMs * 4; @@ -1892,6 +1953,8 @@ export function createSidecarRouter( sendGrantsUpdate, sendSourcesUpdate, sendPack, + bindStepRoute, + unbindStepRoute, sendSyncRequest, sendSignalDeliver, sendDrain, diff --git a/tests/hub-api/asset-routes.test.ts b/tests/hub-api/asset-routes.test.ts index b9d80850..4d34b469 100644 --- a/tests/hub-api/asset-routes.test.ts +++ b/tests/hub-api/asset-routes.test.ts @@ -147,6 +147,8 @@ function createMockSidecarRouter(): SidecarRouter { sendGrantsUpdate: () => notImpl("sendGrantsUpdate"), sendSourcesUpdate: () => notImpl("sendSourcesUpdate"), sendPack: () => notImpl("sendPack"), + bindStepRoute: () => notImpl("bindStepRoute"), + unbindStepRoute: () => notImpl("unbindStepRoute"), sendSyncRequest: () => notImpl("sendSyncRequest"), sendSignalDeliver: () => notImpl("sendSignalDeliver"), sendDrain: () => notImpl("sendDrain"), From 2ec622266d09ff5b166bd6ab7f78fecec3476c17 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Fri, 3 Jul 2026 17:55:36 -0500 Subject: [PATCH 035/101] Provision multi-step workflow steps without spawning Staging a multi-step workflow step delivers its deploy tree by pack to the step's derived address, but a full-closure deploy pack still needs an initialized agent-state repo to apply into and the hub key recorded to verify the pack commit. A full agent-deploy frame does both, but it also spawns a warm harness the step never uses. Add a no-spawn provision path. A new deploy frame flag marks a per-step provision; on it the sidecar initializes the step's agent-state repo and records the hub key -- the same harness-free seam the single-step head uses -- and constructs no supervisor or child. The deployment-level workflow frame, fired once after every step is provisioned, spawns the child that reads each staged step tree from disk. The router method that sends the frame waits for the ack so the caller can safely deliver the deploy pack afterward, and reuses the transient step route rather than registering a session address. The provision returns the sidecar principal key so the ack carries a key like the multi-step ack; a per-step address is workflow-derived and records no instance key, so the hub discards it. --- apps/sidecar/src/workflow-host-wiring.test.ts | 83 +++++++++++++++++++ apps/sidecar/src/workflow-host-wiring.ts | 29 +++++++ packages/hub-api/src/routes/agents.test.ts | 1 + packages/hub-api/src/routes/assets.test.ts | 1 + .../hub-api/src/routes/catalog-routes.test.ts | 1 + .../hub-api/src/routes/git-tokens.test.ts | 1 + packages/hub-api/src/routes/instances.test.ts | 3 + packages/hub-api/src/routes/workflows.test.ts | 1 + .../hub-sessions/src/session-service.test.ts | 3 + .../hub-sessions/src/ws/sidecar-handler.ts | 81 ++++++++++++++++++ packages/types/src/sidecar.ts | 16 +++- tests/hub-api/asset-routes.test.ts | 1 + 12 files changed, 217 insertions(+), 4 deletions(-) diff --git a/apps/sidecar/src/workflow-host-wiring.test.ts b/apps/sidecar/src/workflow-host-wiring.test.ts index be32a839..9f176c54 100644 --- a/apps/sidecar/src/workflow-host-wiring.test.ts +++ b/apps/sidecar/src/workflow-host-wiring.test.ts @@ -443,6 +443,89 @@ describe("createSidecarDeployRouter wires the InferenceEvent subscription to rec }); }); +describe("createSidecarDeployRouter provision-step (no-spawn) mode", () => { + test("a provisionStep frame inits the repo and records the hub key without spawning", async () => { + const transport = createInMemoryTransport(); + const keyPair = await generateKeyPair(); + + const initRepoCalls: string[] = []; + const recordHubKeyCalls: { address: string; hubKey: string }[] = []; + // A spawn registers one per-agent InferenceEvent listener; a no-spawn + // provision registers none. + const agentEventRegistrations: string[] = []; + + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- provisionStep touches no RepoStore method; the Proxy throws if it ever does + const repoStore = new Proxy({} as RepoStore, { + get(_target, prop) { + return () => { + throw new Error(`stub RepoStore: ${String(prop)} not implemented`); + }; + }, + }); + + const router = createSidecarDeployRouter({ + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- provisionStep exercises only initRepo + sessions: { + initRepo: async (a: string) => { + initRepoCalls.push(a); + }, + } as unknown as Parameters< + typeof createSidecarDeployRouter + >[0]["sessions"], + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- provisionStep exercises only recordHubKey + keyStore: { + recordHubKey: (a: string, h: string) => { + recordHubKeyCalls.push({ address: a, hubKey: h }); + }, + } as unknown as Parameters< + typeof createSidecarDeployRouter + >[0]["keyStore"], + onAgentEvent: (address: string) => { + agentEventRegistrations.push(address); + return () => undefined; + }, + transport, + repoStore, + signingKeySeed: keyPair.privateKey, + createAgentCrypto: createEd25519Crypto, + assertSourceBuildable: () => undefined, + registerDeployment: () => undefined, + unregisterDeployment: () => undefined, + }); + + const STEP_ADDR = "ins_dep_abc-step1@example.com"; + const HUB_KEY = "aa".repeat(32); + const result = await router.deploy({ + type: "agent.deploy", + agentAddress: STEP_ADDR, + agentId: "ins_dep_abc-step1", + hubPublicKey: HUB_KEY, + provisionStep: true, + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- provisionStep never inspects config + config: {} as unknown as Parameters< + ReturnType["deploy"] + >[0]["config"], + }); + + // The step's agent-state repo is initialized and the hub key recorded, + // so the follow-up full-closure deploy pack applies into a repo and + // verifies against the recorded key. + expect(initRepoCalls).toEqual([STEP_ADDR]); + expect(recordHubKeyCalls).toEqual([ + { address: STEP_ADDR, hubKey: HUB_KEY }, + ]); + + // The ack carries the sidecar principal key (the hub discards it for a + // workflow-derived per-step address). + expect(result.publicKey).toMatch(/^[0-9a-f]{64}$/); + + // Nothing spawned: no supervisor, so no active address and no per-agent + // event listener registered. + expect(router.activeAddresses()).toEqual([]); + expect(agentEventRegistrations).toEqual([]); + }); +}); + // -------------------------------------------------------------------- // Multi-step branch tests // -------------------------------------------------------------------- diff --git a/apps/sidecar/src/workflow-host-wiring.ts b/apps/sidecar/src/workflow-host-wiring.ts index 65d583c4..c24177e0 100644 --- a/apps/sidecar/src/workflow-host-wiring.ts +++ b/apps/sidecar/src/workflow-host-wiring.ts @@ -1360,6 +1360,32 @@ export function createSidecarDeployRouter(deps: { } } + /** + * Provision one step of a multi-step deploy WITHOUT spawning. The hub + * stages each step's deploy tree before firing the deployment-level + * workflow frame; a full-closure deploy pack still needs an initialized + * agent-state repo to apply into and the hub key recorded to verify the + * pack commit signature. This does exactly those two things -- the same + * harness-free `initRepo` + `recordHubKey` seam the single-step head uses + * -- and constructs no supervisor or child. The deployment-level workflow + * frame (fired once after every step is provisioned) spawns the child, + * which reads each step's staged deploy tree from disk. + * + * Returns the sidecar's principal public key so the link's + * `agent.deploy.ack` carries a key, matching the multi-step ack. A + * per-step address is workflow-derived and records no `agent_instance` + * key, so the hub discards this value. + */ + async function provisionStep( + frame: AgentDeployFrame, + ): Promise { + await deps.sessions.initRepo(frame.agentAddress); + deps.keyStore.recordHubKey(frame.agentAddress, frame.hubPublicKey); + return { + publicKey: await derivePrincipalPublicKeyHex(deps.signingKeySeed), + }; + } + async function deployMultiStep( frame: AgentDeployFrame, projection: NonNullable, @@ -1514,6 +1540,9 @@ export function createSidecarDeployRouter(deps: { return { async deploy(frame): Promise { + if (frame.provisionStep === true) { + return await provisionStep(frame); + } if (frame.workflow !== undefined) { return await deployMultiStep(frame, frame.workflow); } diff --git a/packages/hub-api/src/routes/agents.test.ts b/packages/hub-api/src/routes/agents.test.ts index d8704860..d848a9d4 100644 --- a/packages/hub-api/src/routes/agents.test.ts +++ b/packages/hub-api/src/routes/agents.test.ts @@ -179,6 +179,7 @@ function createMockSidecarRouter(): SidecarRouter { sendGrantsUpdate: () => notImpl("sendGrantsUpdate"), sendSourcesUpdate: () => notImpl("sendSourcesUpdate"), sendPack: () => notImpl("sendPack"), + sendProvisionStep: () => notImpl("sendProvisionStep"), bindStepRoute: () => notImpl("bindStepRoute"), unbindStepRoute: () => notImpl("unbindStepRoute"), sendSyncRequest: () => notImpl("sendSyncRequest"), diff --git a/packages/hub-api/src/routes/assets.test.ts b/packages/hub-api/src/routes/assets.test.ts index 665a99d0..5f962ba4 100644 --- a/packages/hub-api/src/routes/assets.test.ts +++ b/packages/hub-api/src/routes/assets.test.ts @@ -295,6 +295,7 @@ function createMockSidecarRouter(): SidecarRouter { sendGrantsUpdate: () => notImpl("sendGrantsUpdate"), sendSourcesUpdate: () => notImpl("sendSourcesUpdate"), sendPack: () => notImpl("sendPack"), + sendProvisionStep: () => notImpl("sendProvisionStep"), bindStepRoute: () => notImpl("bindStepRoute"), unbindStepRoute: () => notImpl("unbindStepRoute"), sendSyncRequest: () => notImpl("sendSyncRequest"), diff --git a/packages/hub-api/src/routes/catalog-routes.test.ts b/packages/hub-api/src/routes/catalog-routes.test.ts index 8d780c0f..961539e5 100644 --- a/packages/hub-api/src/routes/catalog-routes.test.ts +++ b/packages/hub-api/src/routes/catalog-routes.test.ts @@ -194,6 +194,7 @@ function createMockSidecarRouter(): SidecarRouter { // Mutations fire a fire-and-forget source push; resolve so it is a no-op. sendSourcesUpdate: () => Promise.resolve(), sendPack: () => notImpl("sendPack"), + sendProvisionStep: () => notImpl("sendProvisionStep"), bindStepRoute: () => notImpl("bindStepRoute"), unbindStepRoute: () => notImpl("unbindStepRoute"), sendSyncRequest: () => notImpl("sendSyncRequest"), diff --git a/packages/hub-api/src/routes/git-tokens.test.ts b/packages/hub-api/src/routes/git-tokens.test.ts index 01d5c608..8187d433 100644 --- a/packages/hub-api/src/routes/git-tokens.test.ts +++ b/packages/hub-api/src/routes/git-tokens.test.ts @@ -218,6 +218,7 @@ function createMockSidecarRouter(): SidecarRouter { sendGrantsUpdate: () => notImpl("sendGrantsUpdate"), sendSourcesUpdate: () => notImpl("sendSourcesUpdate"), sendPack: () => notImpl("sendPack"), + sendProvisionStep: () => notImpl("sendProvisionStep"), bindStepRoute: () => notImpl("bindStepRoute"), unbindStepRoute: () => notImpl("unbindStepRoute"), sendSyncRequest: () => notImpl("sendSyncRequest"), diff --git a/packages/hub-api/src/routes/instances.test.ts b/packages/hub-api/src/routes/instances.test.ts index 0f2bbdfe..18a9c326 100644 --- a/packages/hub-api/src/routes/instances.test.ts +++ b/packages/hub-api/src/routes/instances.test.ts @@ -255,6 +255,9 @@ function createMockSidecarRouter( sendPack(_addr, _pack, _ref, _sha) { return notImpl("sendPack"); }, + sendProvisionStep(_agentAddress, _config) { + return notImpl("sendProvisionStep"); + }, bindStepRoute(_stepAddress) { notImpl("bindStepRoute"); }, diff --git a/packages/hub-api/src/routes/workflows.test.ts b/packages/hub-api/src/routes/workflows.test.ts index f0b2bf84..79942c60 100644 --- a/packages/hub-api/src/routes/workflows.test.ts +++ b/packages/hub-api/src/routes/workflows.test.ts @@ -201,6 +201,7 @@ function createMockSidecarRouter( sendGrantsUpdate: () => notImpl("sendGrantsUpdate"), sendSourcesUpdate: () => notImpl("sendSourcesUpdate"), sendPack: () => notImpl("sendPack"), + sendProvisionStep: () => notImpl("sendProvisionStep"), bindStepRoute: () => notImpl("bindStepRoute"), unbindStepRoute: () => notImpl("unbindStepRoute"), sendSyncRequest: () => notImpl("sendSyncRequest"), diff --git a/packages/hub-sessions/src/session-service.test.ts b/packages/hub-sessions/src/session-service.test.ts index 167af434..ca743e7b 100644 --- a/packages/hub-sessions/src/session-service.test.ts +++ b/packages/hub-sessions/src/session-service.test.ts @@ -95,6 +95,9 @@ function createMockRouter(): SidecarRouter & { }); return Promise.resolve(); }) as SidecarRouter["sendPack"], + sendProvisionStep: track( + "sendProvisionStep", + ) as SidecarRouter["sendProvisionStep"], bindStepRoute: () => undefined, unbindStepRoute: () => undefined, sendSyncRequest: track( diff --git a/packages/hub-sessions/src/ws/sidecar-handler.ts b/packages/hub-sessions/src/ws/sidecar-handler.ts index 7a86b6f6..a0ae951b 100644 --- a/packages/hub-sessions/src/ws/sidecar-handler.ts +++ b/packages/hub-sessions/src/ws/sidecar-handler.ts @@ -155,6 +155,14 @@ export type SidecarRouter = { * address is a no-op. */ unbindStepRoute(stepAddress: string): void; + /** + * Provision one step of a multi-step deploy on the sidecar WITHOUT + * spawning: the sidecar initializes the step's agent-state repo and + * records the hub key so the follow-up deploy pack applies and verifies. + * The step address must already be bound via `bindStepRoute`. Resolves + * once the sidecar acks, so the caller can then deliver the deploy pack. + */ + sendProvisionStep(agentAddress: string, config: HarnessConfig): Promise; sendSyncRequest(agentAddress: string): void; /** * Deliver a workflow-run signal to the sidecar that hosts the named @@ -1681,6 +1689,78 @@ export function createSidecarRouter( }); } + /** + * Provision one step of a multi-step deploy on the sidecar WITHOUT + * spawning: the sidecar initializes the step's agent-state repo and + * records the hub key, so the follow-up deploy pack applies into a repo + * and verifies against the recorded key -- but no supervisor or child is + * constructed. The deployment-level workflow frame, sent once after every + * step is provisioned, spawns the child. + * + * The step address must already be bound via `bindStepRoute`, which + * resolves and records the sidecar; this reuses that route rather than + * touching `agentAddresses`. Waits for the sidecar's `agent.deploy.ack` + * so the caller can safely deliver the deploy pack afterward. On failure + * the caller owns tearing the route down via `unbindStepRoute`. + */ + function sendProvisionStep( + agentAddress: string, + harnessConfig: HarnessConfig, + ): Promise { + if (hubPublicKeyHex === undefined) { + throw new Error("Hub signing key is required for step provisioning"); + } + if (pendingDeploys.has(agentAddress)) { + throw new Error(`Deploy already in progress for agent "${agentAddress}"`); + } + const ws = addressIndex.get(agentAddress); + if (ws === undefined) { + throw new Error( + `Step route for "${agentAddress}" is not bound; call bindStepRoute before provisioning`, + ); + } + const conn = connections.get(ws); + if (conn === undefined) { + throw new Error(`No sidecar connected for agent "${agentAddress}"`); + } + + const hubKey = hubPublicKeyHex; + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + pendingDeploys.delete(agentAddress); + reject( + new Error( + `Step provision of "${agentAddress}" timed out after ${requestTimeoutMs}ms`, + ), + ); + }, requestTimeoutMs); + // The sidecar's `agent.deploy.ack` resolves this through + // `resolveDeployPending` -> `req.resolve()`. The per-step address is + // workflow-derived and records no hub-side key, so the ack's public + // key is not needed and this resolves void. + pendingDeploys.set(agentAddress, { + agentAddress, + ws, + resolve() { + resolve(); + }, + reject(error: string) { + reject(new Error(error)); + }, + timer, + }); + + conn.send({ + type: "agent.deploy", + agentAddress, + agentId: harnessConfig.agentId, + config: harnessConfig, + hubPublicKey: hubKey, + provisionStep: true, + }); + }); + } + function findSidecarForNewAgent(_agentAddress: string): WsHandle | undefined { const first = connections.entries().next(); if (first.done) return undefined; @@ -1955,6 +2035,7 @@ export function createSidecarRouter( sendPack, bindStepRoute, unbindStepRoute, + sendProvisionStep, sendSyncRequest, sendSignalDeliver, sendDrain, diff --git a/packages/types/src/sidecar.ts b/packages/types/src/sidecar.ts index ec1b1adc..9a4347a8 100644 --- a/packages/types/src/sidecar.ts +++ b/packages/types/src/sidecar.ts @@ -317,10 +317,17 @@ export type AgentDeployWorkflow = typeof AgentDeployWorkflow.infer; * Deploy an agent to this sidecar. The sidecar initializes a harness from * the config. * - * `workflow` is set only on multi-step deployments. Absent on every - * trivial-shape frame (every frame the system currently emits); the - * deploy router uses field presence to discriminate the two branches - * without consulting `config`. + * The deploy router discriminates three shapes by field presence without + * consulting `config`: + * - `workflow` set: a workflow deployment (single-step head or multi-step) + * that spawns the supervised workflow-process child. + * - `provisionStep` true: a no-spawn per-step provision of a multi-step + * deploy -- the sidecar initializes the step's agent-state repo and + * records the hub key so the follow-up deploy pack applies and verifies, + * but spawns nothing. The deployment-level `workflow` frame (sent once + * after every step is provisioned) spawns the child. + * - neither: the legacy trivial in-process harness deploy. + * `workflow` and `provisionStep` are mutually exclusive. */ export const AgentDeployFrame = type({ type: "'agent.deploy'", @@ -329,6 +336,7 @@ export const AgentDeployFrame = type({ config: HarnessConfig, hubPublicKey: "string", "workflow?": AgentDeployWorkflow, + "provisionStep?": "boolean", }); export type AgentDeployFrame = typeof AgentDeployFrame.infer; diff --git a/tests/hub-api/asset-routes.test.ts b/tests/hub-api/asset-routes.test.ts index 4d34b469..d0491454 100644 --- a/tests/hub-api/asset-routes.test.ts +++ b/tests/hub-api/asset-routes.test.ts @@ -147,6 +147,7 @@ function createMockSidecarRouter(): SidecarRouter { sendGrantsUpdate: () => notImpl("sendGrantsUpdate"), sendSourcesUpdate: () => notImpl("sendSourcesUpdate"), sendPack: () => notImpl("sendPack"), + sendProvisionStep: () => notImpl("sendProvisionStep"), bindStepRoute: () => notImpl("bindStepRoute"), unbindStepRoute: () => notImpl("unbindStepRoute"), sendSyncRequest: () => notImpl("sendSyncRequest"), From 8417bc0475bb6414f367095d47221ca80ae9bb90 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Fri, 3 Jul 2026 18:00:40 -0500 Subject: [PATCH 036/101] Exercise the reconnect path in the per-step route test The test asserting a reconnect does not resurrect an unbound per-step route ran against a router with no public-key lookup configured, so the reconnect handler bailed and closed the socket before re-registering anything. The assertion held only because the earlier unbind had already removed the address -- it never observed a real reconnect. Give the test a router with the lookup configured so the reconnect actually re-registers, register the deployment address as a workflow address, and assert the socket stays open, the deployment address is routable again, and the unbound step address stays gone. The router routes whatever the reconnect frame announces, so this pins the intended contract: a transient step route never enters the reconnect announcement and so is not resurrected. --- .../src/ws/sidecar-handler.test.ts | 44 +++++++++++++++---- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/packages/hub-sessions/src/ws/sidecar-handler.test.ts b/packages/hub-sessions/src/ws/sidecar-handler.test.ts index 17cf73f1..fb456b9a 100644 --- a/packages/hub-sessions/src/ws/sidecar-handler.test.ts +++ b/packages/hub-sessions/src/ws/sidecar-handler.test.ts @@ -340,15 +340,36 @@ describe("SidecarRouter", () => { ).rejects.toThrow(/No sidecar connected/); }); - test("a reconnect after unbind does not resurrect the transient route", () => { - const ws = registerDeploymentSidecar(); - router.bindStepRoute(STEP_ADDR); - router.unbindStepRoute(STEP_ADDR); + test("a reconnect after unbind does not resurrect the transient route", async () => { + // A real reconnect only runs when `lookupPublicKey` is configured + // (otherwise `handleReconnect` bails before re-registering anything). + // Use a dedicated router with the lookup so the reconnect actually + // executes and this exercises the re-registration path. + const reconnectRouter = createSidecarRouter({ + requestTimeoutMs: 500, + hubPublicKey: TEST_HUB_KEY, + lookups: { lookupPublicKey: async () => null }, + }); + const ws = createMockWs(); + reconnectRouter.handleOpen(ws); + // The deployment address is a workflow-substrate address (no challenge). + reconnectRouter.handleMessage( + ws, + JSON.stringify({ + type: "register", + sidecarId: "sc-1", + token: "tok", + agentAddresses: [], + workflowAddresses: [DEPLOYMENT_ADDR], + }), + ); + reconnectRouter.bindStepRoute(STEP_ADDR); + reconnectRouter.unbindStepRoute(STEP_ADDR); // The sidecar reconnects announcing only the deployment's workflow - // address -- never the transient per-step address -- so the step route - // stays gone (the binding never entered the reconnect set). - router.handleMessage( + // address -- never the transient per-step address, which was never + // persisted into the reconnect announcement. + reconnectRouter.handleMessage( ws, JSON.stringify({ type: "reconnect", @@ -358,8 +379,15 @@ describe("SidecarRouter", () => { workflowAddresses: [DEPLOYMENT_ADDR], }), ); + // Let the async reconnect re-registration settle. + await new Promise((r) => setTimeout(r, 0)); - expect(router.getRoutableAddresses()).not.toContain(STEP_ADDR); + // The reconnect actually ran (it did not bail and close the socket). + expect(ws.closed).toBe(false); + // The deployment address is routable again; the transient step route + // stays gone. + expect(reconnectRouter.getRoutableAddresses()).toContain(DEPLOYMENT_ADDR); + expect(reconnectRouter.getRoutableAddresses()).not.toContain(STEP_ADDR); }); test("bind throws when no sidecar is connected", () => { From ad16a80558de97086757dd864d53b088c4924986 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Fri, 3 Jul 2026 18:19:14 -0500 Subject: [PATCH 037/101] Stage multi-step workflow steps without a warm harness Each step of a multi-step workflow deploy was provisioned as its own warm in-process agent harness: the per-step launch sent a plain provision frame that ran the sidecar's trivial branch, then started a session. Nothing consumed those harnesses -- the supervised workflow-process child, spawned once by the deployment-level frame, materializes each step's agent from the deploy tree on disk and runs the step itself. Stage each step instead. A new staging entry point binds a transient route for the step address, fires the no-spawn provision frame so the sidecar inits the step's agent-state repo and records the hub key, delivers the deploy and asset packs, and unbinds the route once they land -- no provision frame for a warm harness and no session start. The route is dropped in a finally, so a failed stage leaks no per-step route. The deployment-level frame still writes the step grants and spawns the child, which reads the staged trees. The multi-step branch and the integration fixture both drive steps through the staging entry point, and the multi-step integration tests run against it end to end. A unit test pins the negative contract: a staged step binds, provisions, packs, and unbinds, and never sends a warm provision frame or a session start. --- packages/hub-api/src/app.test.ts | 3 + packages/hub-api/src/routes/agents.test.ts | 3 + packages/hub-api/src/routes/assets.test.ts | 3 + .../hub-api/src/routes/catalog-routes.test.ts | 3 + .../hub-api/src/routes/git-tokens.test.ts | 3 + packages/hub-api/src/routes/instances.test.ts | 13 + packages/hub-api/src/routes/workflows.test.ts | 1 + .../hub-sessions/src/session-service.test.ts | 69 ++++- packages/hub-sessions/src/session-service.ts | 283 ++++++++++++------ tests/hub-agent/lib/deploy-flow-env.ts | 11 +- tests/hub-api/asset-routes.test.ts | 1 + .../child-workflow-roundtrip.test.ts | 6 +- .../cross-process-custom-adapter.test.ts | 2 +- tests/workflow-deploy/drain-roundtrip.test.ts | 4 +- tests/workflow-deploy/fifo-mail-load.test.ts | 2 +- tests/workflow-deploy/fifo-mail.test.ts | 2 +- tests/workflow-deploy/latency-gate.bench.ts | 2 +- tests/workflow-deploy/mail-edge-cases.test.ts | 2 +- .../workflow-deploy/multistep-signal.test.ts | 2 +- .../multistep-signed-send.test.ts | 2 +- .../single-step-full-lifecycle.test.ts | 2 +- .../single-step-grants-bridge.test.ts | 2 +- .../single-step-message-input.test.ts | 2 +- .../single-step-posix-tool.test.ts | 2 +- .../single-step-real-agent.test.ts | 2 +- 25 files changed, 311 insertions(+), 116 deletions(-) diff --git a/packages/hub-api/src/app.test.ts b/packages/hub-api/src/app.test.ts index 9b0937d5..ba55f26a 100644 --- a/packages/hub-api/src/app.test.ts +++ b/packages/hub-api/src/app.test.ts @@ -23,6 +23,9 @@ const sessionService: SessionService = { launchSession(_params) { throw new Error("mock: sessionService.launchSession not implemented"); }, + stageWorkflowStep(_params) { + throw new Error("mock: sessionService.stageWorkflowStep not implemented"); + }, deployInstanceAtHead(_params) { throw new Error( "mock: sessionService.deployInstanceAtHead not implemented", diff --git a/packages/hub-api/src/routes/agents.test.ts b/packages/hub-api/src/routes/agents.test.ts index d848a9d4..487c768c 100644 --- a/packages/hub-api/src/routes/agents.test.ts +++ b/packages/hub-api/src/routes/agents.test.ts @@ -199,6 +199,9 @@ function createMockSessionService(): SessionService { launchSession: () => { throw new Error("mock: sessionService.launchSession not implemented"); }, + stageWorkflowStep: () => { + throw new Error("mock: sessionService.stageWorkflowStep not implemented"); + }, deployInstanceAtHead: () => { throw new Error( "mock: sessionService.deployInstanceAtHead not implemented", diff --git a/packages/hub-api/src/routes/assets.test.ts b/packages/hub-api/src/routes/assets.test.ts index 5f962ba4..c33d1e85 100644 --- a/packages/hub-api/src/routes/assets.test.ts +++ b/packages/hub-api/src/routes/assets.test.ts @@ -315,6 +315,9 @@ function createMockSessionService(): SessionService { launchSession: () => { throw new Error("mock: sessionService.launchSession not implemented"); }, + stageWorkflowStep: () => { + throw new Error("mock: sessionService.stageWorkflowStep not implemented"); + }, deployInstanceAtHead: () => { throw new Error( "mock: sessionService.deployInstanceAtHead not implemented", diff --git a/packages/hub-api/src/routes/catalog-routes.test.ts b/packages/hub-api/src/routes/catalog-routes.test.ts index 961539e5..5a3b0aa6 100644 --- a/packages/hub-api/src/routes/catalog-routes.test.ts +++ b/packages/hub-api/src/routes/catalog-routes.test.ts @@ -214,6 +214,9 @@ function createMockSessionService(): SessionService { launchSession: () => { throw new Error("mock: sessionService.launchSession not implemented"); }, + stageWorkflowStep: () => { + throw new Error("mock: sessionService.stageWorkflowStep not implemented"); + }, deployInstanceAtHead: () => { throw new Error( "mock: sessionService.deployInstanceAtHead not implemented", diff --git a/packages/hub-api/src/routes/git-tokens.test.ts b/packages/hub-api/src/routes/git-tokens.test.ts index 8187d433..fbed1673 100644 --- a/packages/hub-api/src/routes/git-tokens.test.ts +++ b/packages/hub-api/src/routes/git-tokens.test.ts @@ -238,6 +238,9 @@ function createMockSessionService(): SessionService { launchSession: () => { throw new Error("mock: sessionService.launchSession not implemented"); }, + stageWorkflowStep: () => { + throw new Error("mock: sessionService.stageWorkflowStep not implemented"); + }, deployInstanceAtHead: () => { throw new Error( "mock: sessionService.deployInstanceAtHead not implemented", diff --git a/packages/hub-api/src/routes/instances.test.ts b/packages/hub-api/src/routes/instances.test.ts index 18a9c326..264df3d4 100644 --- a/packages/hub-api/src/routes/instances.test.ts +++ b/packages/hub-api/src/routes/instances.test.ts @@ -296,6 +296,9 @@ function createMockSessionService(): SessionService { launchSession(_params) { return notImpl("launchSession"); }, + stageWorkflowStep(_params) { + return notImpl("stageWorkflowStep"); + }, deployInstanceAtHead(_params) { return notImpl("deployInstanceAtHead"); }, @@ -672,6 +675,9 @@ describe("POST /agents/instances/:instanceId/mail", () => { launchSession() { throw new Error("not implemented"); }, + stageWorkflowStep() { + throw new Error("not implemented"); + }, deployInstanceAtHead() { throw new Error("not implemented"); }, @@ -826,6 +832,9 @@ describe("POST /agents/instances/:instanceId/mail attachments", () => { launchSession() { throw new Error("not implemented"); }, + stageWorkflowStep() { + throw new Error("not implemented"); + }, deployInstanceAtHead() { throw new Error("not implemented"); }, @@ -1262,6 +1271,7 @@ describe("POST /agents/instances seeds creator agent-state grant", () => { function createCapturingSessionService(): SessionService { return { launchSession: async () => undefined, + stageWorkflowStep: async () => undefined, deployInstanceAtHead: async () => ({ publicKey: "pk-instance-mock" }), deployWorkflowDefinition: () => { throw new Error("mock: deployWorkflowDefinition not implemented"); @@ -1481,6 +1491,9 @@ describe("POST /agents/instances seeds creator agent-state grant", () => { launchSession: () => { throw new Error("mock: launchSession not implemented"); }, + stageWorkflowStep: () => { + throw new Error("mock: stageWorkflowStep not implemented"); + }, deployInstanceAtHead: async (params) => { launchedSources = params.config.sources; launchedDefaultSource = params.config.defaultSource; diff --git a/packages/hub-api/src/routes/workflows.test.ts b/packages/hub-api/src/routes/workflows.test.ts index 79942c60..9b3f88c0 100644 --- a/packages/hub-api/src/routes/workflows.test.ts +++ b/packages/hub-api/src/routes/workflows.test.ts @@ -227,6 +227,7 @@ function createMockSessionService( } return { launchSession: () => notImpl("launchSession"), + stageWorkflowStep: () => notImpl("stageWorkflowStep"), deployInstanceAtHead: () => notImpl("deployInstanceAtHead"), deployWorkflowDefinition: (params) => { deployCalls.push(params); diff --git a/packages/hub-sessions/src/session-service.test.ts b/packages/hub-sessions/src/session-service.test.ts index ca743e7b..f23a9203 100644 --- a/packages/hub-sessions/src/session-service.test.ts +++ b/packages/hub-sessions/src/session-service.test.ts @@ -98,8 +98,12 @@ function createMockRouter(): SidecarRouter & { sendProvisionStep: track( "sendProvisionStep", ) as SidecarRouter["sendProvisionStep"], - bindStepRoute: () => undefined, - unbindStepRoute: () => undefined, + bindStepRoute(stepAddress: string) { + calls.push({ method: "bindStepRoute", args: [stepAddress] }); + }, + unbindStepRoute(stepAddress: string) { + calls.push({ method: "unbindStepRoute", args: [stepAddress] }); + }, sendSyncRequest: track( "sendSyncRequest", ) as SidecarRouter["sendSyncRequest"], @@ -356,6 +360,67 @@ describe("SessionService", () => { ]); }); + test("stageWorkflowStep stages without a warm harness", async () => { + const service = createSessionService({ + sidecarRouter: router, + agentRepoStore: repoStore, + }); + + await service.stageWorkflowStep({ + agentAddress: AGENT_ADDRESS, + agentId: AGENT_ID, + instanceId: INSTANCE_ID, + config: MOCK_CONFIG, + deployContent: MOCK_CONTENT, + }); + + const methods = [ + ...repoStore.calls.map((c) => c.method), + ...router.calls.map((c) => c.method), + ]; + + // A stage-only per-step deploy binds a transient route, fires the + // no-spawn provision frame, delivers the deploy pack, and unbinds the + // route -- and NEVER provisions a warm harness (`sendAgentDeploy` with + // no workflow frame) or starts a session (`sendSessionStart`). + expect(methods).toEqual([ + "writeDeployTree", + "createDeployPack", + "bindStepRoute", + "sendProvisionStep", + "sendPack", + "unbindStepRoute", + ]); + expect(methods).not.toContain("sendAgentDeploy"); + expect(methods).not.toContain("sendSessionStart"); + }); + + test("stageWorkflowStep unbinds the route even when the pack fails", async () => { + router.sendPack = () => Promise.reject(new Error("pack failed")); + + const service = createSessionService({ + sidecarRouter: router, + agentRepoStore: repoStore, + }); + + await service + .stageWorkflowStep({ + agentAddress: AGENT_ADDRESS, + agentId: AGENT_ID, + instanceId: INSTANCE_ID, + config: MOCK_CONFIG, + deployContent: MOCK_CONTENT, + }) + .catch((e: unknown) => e); + + const routerMethods = router.calls.map((c) => c.method); + // The transient route is dropped in the `finally`, even on failure, so no + // stale per-step route leaks. A stage-only step has no warm harness to + // tear down, so it never undeploys. + expect(routerMethods).toContain("unbindStepRoute"); + expect(routerMethods).not.toContain("sendAgentUndeploy"); + }); + test("launchSession cleans up on pack failure", async () => { router.sendPack = () => Promise.reject(new Error("pack failed")); diff --git a/packages/hub-sessions/src/session-service.ts b/packages/hub-sessions/src/session-service.ts index 99e4b3e9..2c6773c6 100644 --- a/packages/hub-sessions/src/session-service.ts +++ b/packages/hub-sessions/src/session-service.ts @@ -128,6 +128,24 @@ export type SessionService = { toolPackagePins?: readonly ToolPackagePin[]; }): Promise; + /** + * Stage one step of a multi-step workflow deploy: bind a transient route + * for the step address, fire a no-spawn provision frame (init the step's + * agent-state repo and record the hub key), deliver the deploy + asset + * packs, and unbind the route -- no warm harness. The multi-step branch + * stages every step this way before firing the deployment-level workflow + * frame that spawns the supervised child; the child reads each staged step + * tree from disk and runs the step itself. + */ + stageWorkflowStep(params: { + agentAddress: string; + agentId: string; + instanceId: string; + config: HarnessConfig; + deployContent: DeployContent; + toolPackagePins?: readonly ToolPackagePin[]; + }): Promise; + /** * Deploy a single-agent instance through the single-step-at-head path, * wrapping the harness as a one-step workflow and routing it through the @@ -565,10 +583,15 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { * the deploy tree, provision the agent, and deliver the deploy + asset * packs (Phases 0-2b). Does NOT start the warm harness -- callers that * want one (the legacy agent-deploy path) invoke `startWarmSession` - * afterward. Shared by the public `launchSession` (warm-harness provision, - * no workflow frame) and the single-step head hand-off - * (`deploySingleStepAtHead`, which passes a `workflowFrame` so the sidecar - * spawns the workflow-process child instead of a warm harness). + * afterward. Phase 1's provision has three shapes: + * - `workflowFrame` set: the single-step head hand-off fires the + * deployment `agent.deploy` frame that spawns the workflow-process + * child. Returns the supervisor public key. + * - `stageOnly` set: a multi-step per-step stage binds a transient route + * for the step address, fires a no-spawn provision frame (init repo + + * record hub key), and unbinds the route once the packs land. No warm + * harness, no child. + * - neither: the legacy plain provision frame (warm harness). */ async function executeLaunchPhases(params: { agentAddress: string; @@ -584,14 +607,32 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { * spawns the workflow-process child) instead of the plain provision * frame. The returned supervisor public key comes from that frame's * ack; the caller skips `startWarmSession` for a workflow deploy. + * + * Mutually exclusive with `stageOnly`. */ workflowFrame?: { definition: WorkflowDefinition; sources: Record; }; + /** + * Multi-step per-step stage. When true, Phase 1 binds a transient route + * for the step address, fires a no-spawn provision frame (the sidecar + * inits the step's agent-state repo and records the hub key), delivers + * the deploy + asset packs, and unbinds the route -- no provision of a + * warm harness and no child. The deployment-level workflow frame, sent + * once after every step is staged, spawns the child. Returns no ack. + * Mutually exclusive with `workflowFrame`. + */ + stageOnly?: boolean; }): Promise<{ publicKey: string } | undefined> { const { agentAddress, agentId, instanceId, config, deployContent } = params; const toolPackagePins = params.toolPackagePins ?? []; + const stageOnly = params.stageOnly ?? false; + if (params.workflowFrame !== undefined && stageOnly) { + throw new Error( + "executeLaunchPhases: workflowFrame and stageOnly are mutually exclusive", + ); + } const workflowFrame = params.workflowFrame; // Phase 0: Resolve attached assets first so the skill index is in @@ -742,92 +783,118 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { throw new SessionLaunchError("write", err, false); } - // Phase 1: Provision on sidecar. A single-step workflow deploy sends - // the deployment `agent.deploy` frame carrying the workflow definition - // + source pins INSTEAD of the plain provision frame: the sidecar's - // deploy router initializes the head repo on receipt (so the Phase 2 - // pack has a repo to apply into) and spawns the workflow-process - // child. The plain-provision frame stays for the legacy agent-deploy - // passthrough. Firing the frame before the Phase 2 pack is the - // ordering barrier -- the head repo must exist before the pack - // applies. The ack's supervisor public key is surfaced to the caller. - let deployAckPublicKey: string | undefined; - try { - if (workflowFrame !== undefined) { - const ack = await sendMultiStepDeployFrame({ - sidecarRouter, - agentAddress, - config, - definition: workflowFrame.definition, - sources: workflowFrame.sources, - }); - deployAckPublicKey = ack.publicKey; - } else { - await sidecarRouter.sendAgentDeploy(agentAddress, config); + // A stage-only per-step deploy binds a transient route for the step + // address so the packs below route to the deployment's sidecar; the + // route is held only for the pack window and dropped in the `finally`. + if (stageOnly) { + try { + sidecarRouter.bindStepRoute(agentAddress); + } catch (err) { + throw new SessionLaunchError("provision", err, false); } - } catch (err) { - throw new SessionLaunchError("provision", err, false); } - - // Phases 2-3: Pack delivery and session start. If either fails, - // attempt cleanup so the sidecar doesn't retain a zombie agent. try { - await sidecarRouter.sendPack(agentAddress, pack, ref, commitSha); - } catch (err) { - await attemptCleanup(agentAddress, "pack", err); - throw new SessionLaunchError("pack", err, false); - } + // Phase 1: Provision on sidecar. A single-step workflow deploy sends + // the deployment `agent.deploy` frame carrying the workflow definition + // + source pins: the sidecar's deploy router initializes the head repo + // on receipt (so the Phase 2 pack has a repo to apply into) and spawns + // the workflow-process child. A stage-only per-step deploy sends a + // no-spawn provision frame: the sidecar inits the step's agent-state + // repo and records the hub key, but spawns nothing. The plain-provision + // frame stays for the legacy agent-deploy passthrough. Firing the frame + // before the Phase 2 pack is the ordering barrier -- the repo must + // exist before the pack applies. A workflow frame's ack surfaces the + // supervisor public key to the caller. + let deployAckPublicKey: string | undefined; + try { + if (workflowFrame !== undefined) { + const ack = await sendMultiStepDeployFrame({ + sidecarRouter, + agentAddress, + config, + definition: workflowFrame.definition, + sources: workflowFrame.sources, + }); + deployAckPublicKey = ack.publicKey; + } else if (stageOnly) { + await sidecarRouter.sendProvisionStep(agentAddress, config); + } else { + await sidecarRouter.sendAgentDeploy(agentAddress, config); + } + } catch (err) { + throw new SessionLaunchError("provision", err, false); + } - // Phase 2b: Asset-pack fan-out. For each attached asset, build a - // pack, insert the manifest row, then send the pack. The manifest - // insert MUST happen before the pack send: if the sidecar acks - // but the row is missing, the session has materialization without - // a recorded manifest. If the row insert fails, the pack send - // must not happen. - // - // The fan-out covers two sources: the agent's direct attachments - // (skills, today) and the package-registry assets the tool-package - // resolver picked from. The latter live behind tenant inheritance - // rather than a per-agent attachment row, so the session service - // synthesizes the attachment view in `manifestAssetAttachments`. - // - // Both sources can name the same `package-registry` asset — a - // direct attachment and a resolver pin would each compute - // `mountPath = "package-registries//"` and collide on - // the `(instanceId, mountPath)` PK in `session_asset`. Dedup by - // asset id BEFORE the inserts and let the direct attachment win: - // it is an explicit operator action and carries an `agentAssetId` - // the audit query joins against. The resolver-derived row would - // produce the same materialized contents, so dropping it is - // semantically lossless. - const fanOut: ResolvedAttachment[] = dedupAttachmentsByAssetId({ - direct: attachments, - resolved: manifestAssetAttachments, - }); - if (assetService !== undefined && fanOut.length > 0) { - // Track every successfully committed attachment so a later - // fan-out failure can roll back the earlier rows in lockstep - // with the sidecar undeploy. Without this, fan-out[0] succeeds, - // fan-out[1] fails, attemptCleanup tears down the sidecar — but - // fan-out[0]'s session_asset row survives and a future - // materialization query reads a manifest the sidecar no longer - // honors. - const committed: ResolvedAttachment[] = []; - for (const att of fanOut) { - try { - await sendAttachmentPack(instanceId, agentAddress, att); - committed.push(att); - } catch (err) { - await rollbackCommittedAttachments(instanceId, committed); - await attemptCleanup(agentAddress, "pack", err); - throw new SessionLaunchError("pack", err, false); + // Phase 2: Pack delivery. On failure, the warm/workflow paths tear the + // sidecar deployment down; a stage-only step has no supervisor to + // undeploy, so it only drops its transient route (in the `finally`). + // The step's inited agent-state repo is left on the sidecar: the + // orchestrator aborts the whole deploy before the deployment frame is + // sent, so there is nothing to undeploy, and a redeploy of the same + // deployment overwrites the orphaned repo. This is an acceptable minor + // leak on the exceptional staging-failure path, not a live-path cost. + try { + await sidecarRouter.sendPack(agentAddress, pack, ref, commitSha); + } catch (err) { + if (!stageOnly) await attemptCleanup(agentAddress, "pack", err); + throw new SessionLaunchError("pack", err, false); + } + + // Phase 2b: Asset-pack fan-out. For each attached asset, build a + // pack, insert the manifest row, then send the pack. The manifest + // insert MUST happen before the pack send: if the sidecar acks + // but the row is missing, the session has materialization without + // a recorded manifest. If the row insert fails, the pack send + // must not happen. + // + // The fan-out covers two sources: the agent's direct attachments + // (skills, today) and the package-registry assets the tool-package + // resolver picked from. The latter live behind tenant inheritance + // rather than a per-agent attachment row, so the session service + // synthesizes the attachment view in `manifestAssetAttachments`. + // + // Both sources can name the same `package-registry` asset — a + // direct attachment and a resolver pin would each compute + // `mountPath = "package-registries//"` and collide on + // the `(instanceId, mountPath)` PK in `session_asset`. Dedup by + // asset id BEFORE the inserts and let the direct attachment win: + // it is an explicit operator action and carries an `agentAssetId` + // the audit query joins against. The resolver-derived row would + // produce the same materialized contents, so dropping it is + // semantically lossless. + const fanOut: ResolvedAttachment[] = dedupAttachmentsByAssetId({ + direct: attachments, + resolved: manifestAssetAttachments, + }); + if (assetService !== undefined && fanOut.length > 0) { + // Track every successfully committed attachment so a later + // fan-out failure can roll back the earlier rows in lockstep + // with the sidecar undeploy. Without this, fan-out[0] succeeds, + // fan-out[1] fails, attemptCleanup tears down the sidecar — but + // fan-out[0]'s session_asset row survives and a future + // materialization query reads a manifest the sidecar no longer + // honors. + const committed: ResolvedAttachment[] = []; + for (const att of fanOut) { + try { + await sendAttachmentPack(instanceId, agentAddress, att); + committed.push(att); + } catch (err) { + await rollbackCommittedAttachments(instanceId, committed); + if (!stageOnly) await attemptCleanup(agentAddress, "pack", err); + throw new SessionLaunchError("pack", err, false); + } } } - } - return deployAckPublicKey === undefined - ? undefined - : { publicKey: deployAckPublicKey }; + return deployAckPublicKey === undefined + ? undefined + : { publicKey: deployAckPublicKey }; + } finally { + if (stageOnly) { + sidecarRouter.unbindStepRoute(agentAddress); + } + } } /** @@ -896,11 +963,12 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { directorRegistry: DirectorRegistry; deployArgs: DeployWorkflowArgs; }): Promise { - // The per-step launcher: the same warm-harness provision the public - // `launchSession` runs, with the orchestrator's structural - // `DeployContent` narrowed back to the hub-sessions shape first. + // The per-step launcher: stage each step's deploy tree WITHOUT a warm + // harness (the supervised child runs the step), with the orchestrator's + // structural `DeployContent` narrowed back to the hub-sessions shape + // first. const launchSessionCallback: LaunchSessionFn = (orchestratorParams) => - launchSession({ + stageWorkflowStep({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, @@ -936,11 +1004,10 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { /** * Provision one agent at its address: stage the deploy tree + packs * (`executeLaunchPhases` with no workflow frame) and then start its warm - * harness. This is the launcher the multi-step workflow branch drives per - * step via the orchestrator's `launchSession` callback, and the launcher - * the integration-test fixture drives directly. A single-agent instance - * deploy uses `deployInstanceAtHead` and a single-step workflow uses the - * head hand-off; neither routes here. The method carries no workflow + * harness. This is the legacy warm-harness deploy with no production + * caller: a multi-step step stages without a harness (`stageWorkflowStep`), + * a single-agent instance uses `deployInstanceAtHead`, and a single-step + * workflow uses the head hand-off. The method carries no workflow * projection of its own. */ async function launchSession(params: { @@ -964,6 +1031,37 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { await startWarmSession(params.agentAddress); } + /** + * Stage one step of a multi-step workflow deploy: bind a transient route + * for the step address, fire a no-spawn provision frame (the sidecar inits + * the step's agent-state repo and records the hub key), deliver the deploy + * and asset packs, and unbind the route -- no warm harness. The multi-step + * branch stages every step this way, then fires ONE deployment-level + * workflow frame that writes the step grants and spawns the supervised + * workflow-process child; the child reads each step's staged deploy tree + * from disk and runs the step itself. + */ + async function stageWorkflowStep(params: { + agentAddress: string; + agentId: string; + instanceId: string; + config: HarnessConfig; + deployContent: DeployContent; + toolPackagePins?: readonly ToolPackagePin[]; + }): Promise { + await executeLaunchPhases({ + agentAddress: params.agentAddress, + agentId: params.agentId, + instanceId: params.instanceId, + config: params.config, + deployContent: params.deployContent, + stageOnly: true, + ...(params.toolPackagePins !== undefined + ? { toolPackagePins: params.toolPackagePins } + : {}), + }); + } + /** * Deploy a single-agent instance through the single-step-at-head path: wrap * the harness as a one-step workflow (the same wrap `launchSession` uses) and @@ -1516,6 +1614,7 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { return { launchSession, + stageWorkflowStep, deployInstanceAtHead, deploySingleStepAtHead, deployWorkflowDefinition, diff --git a/tests/hub-agent/lib/deploy-flow-env.ts b/tests/hub-agent/lib/deploy-flow-env.ts index b8b7ad90..b017b896 100644 --- a/tests/hub-agent/lib/deploy-flow-env.ts +++ b/tests/hub-agent/lib/deploy-flow-env.ts @@ -1100,12 +1100,13 @@ export async function deployWorkflow( // Route every per-step launch through the session service, mirroring // the production multi-step branch, which drives the orchestrator's - // per-step `launchSession` callback against the same method. The - // orchestrator's `DeployContent` widens `toolPackageManifest` to - // `unknown`; `bridgeOrchestratorDeployContent` narrows and validates it - // back to the hub-sessions shape -- the same bridge production uses. + // per-step launch callback against `stageWorkflowStep` (stage-only, no + // warm harness). The orchestrator's `DeployContent` widens + // `toolPackageManifest` to `unknown`; `bridgeOrchestratorDeployContent` + // narrows and validates it back to the hub-sessions shape -- the same + // bridge production uses. const launchSession: LaunchSessionFn = async (orchestratorParams) => { - await env.hub.sessionService.launchSession({ + await env.hub.sessionService.stageWorkflowStep({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, diff --git a/tests/hub-api/asset-routes.test.ts b/tests/hub-api/asset-routes.test.ts index d0491454..644f8791 100644 --- a/tests/hub-api/asset-routes.test.ts +++ b/tests/hub-api/asset-routes.test.ts @@ -165,6 +165,7 @@ function createMockSidecarRouter(): SidecarRouter { function createMockSessionService(): SessionService { return { launchSession: notImplemented("sessionService.launchSession"), + stageWorkflowStep: notImplemented("sessionService.stageWorkflowStep"), deployInstanceAtHead: notImplemented("sessionService.deployInstanceAtHead"), deployWorkflowDefinition: notImplemented( "sessionService.deployWorkflowDefinition", diff --git a/tests/workflow-deploy/child-workflow-roundtrip.test.ts b/tests/workflow-deploy/child-workflow-roundtrip.test.ts index 5134b4c3..c6830ed8 100644 --- a/tests/workflow-deploy/child-workflow-roundtrip.test.ts +++ b/tests/workflow-deploy/child-workflow-roundtrip.test.ts @@ -194,7 +194,7 @@ describe("parent -> child workflow round-trip", () => { }); const launchSession: LaunchSessionFn = async (orchestratorParams) => { - await env.hub.sessionService.launchSession({ + await env.hub.sessionService.stageWorkflowStep({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, @@ -573,7 +573,7 @@ describe("parent -> child workflow round-trip", () => { }); const launchSession: LaunchSessionFn = async (orchestratorParams) => { - await env.hub.sessionService.launchSession({ + await env.hub.sessionService.stageWorkflowStep({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, @@ -932,7 +932,7 @@ describe("parent -> child workflow round-trip", () => { }); const launchSession: LaunchSessionFn = async (orchestratorParams) => { - await env.hub.sessionService.launchSession({ + await env.hub.sessionService.stageWorkflowStep({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, diff --git a/tests/workflow-deploy/cross-process-custom-adapter.test.ts b/tests/workflow-deploy/cross-process-custom-adapter.test.ts index ffd58a94..68557df0 100644 --- a/tests/workflow-deploy/cross-process-custom-adapter.test.ts +++ b/tests/workflow-deploy/cross-process-custom-adapter.test.ts @@ -158,7 +158,7 @@ async function deployCustomProviderWorkflow( ]); const launchSession: LaunchSessionFn = async (orchestratorParams) => { - await env.hub.sessionService.launchSession({ + await env.hub.sessionService.stageWorkflowStep({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, diff --git a/tests/workflow-deploy/drain-roundtrip.test.ts b/tests/workflow-deploy/drain-roundtrip.test.ts index 451a5759..f564be15 100644 --- a/tests/workflow-deploy/drain-roundtrip.test.ts +++ b/tests/workflow-deploy/drain-roundtrip.test.ts @@ -168,7 +168,7 @@ describe("drain round-trip", () => { const launchSession: LaunchSessionFn = async (orchestratorParams) => { const deployContent = orchestratorParams.deployContent; - await env.hub.sessionService.launchSession({ + await env.hub.sessionService.stageWorkflowStep({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, @@ -412,7 +412,7 @@ describe("drain round-trip", () => { ]); const launchSession: LaunchSessionFn = async (orchestratorParams) => { const deployContent = orchestratorParams.deployContent; - await env.hub.sessionService.launchSession({ + await env.hub.sessionService.stageWorkflowStep({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, diff --git a/tests/workflow-deploy/fifo-mail-load.test.ts b/tests/workflow-deploy/fifo-mail-load.test.ts index 227d2a50..3c44d712 100644 --- a/tests/workflow-deploy/fifo-mail-load.test.ts +++ b/tests/workflow-deploy/fifo-mail-load.test.ts @@ -196,7 +196,7 @@ describe("FIFO mail-trigger serialization under load", () => { const launchSession: LaunchSessionFn = async (orchestratorParams) => { const deployContent = orchestratorParams.deployContent; - await env.hub.sessionService.launchSession({ + await env.hub.sessionService.stageWorkflowStep({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, diff --git a/tests/workflow-deploy/fifo-mail.test.ts b/tests/workflow-deploy/fifo-mail.test.ts index e60abbae..1d9c39c2 100644 --- a/tests/workflow-deploy/fifo-mail.test.ts +++ b/tests/workflow-deploy/fifo-mail.test.ts @@ -184,7 +184,7 @@ describe("FIFO mail-trigger serialization", () => { const launchSession: LaunchSessionFn = async (orchestratorParams) => { const deployContent = orchestratorParams.deployContent; - await env.hub.sessionService.launchSession({ + await env.hub.sessionService.stageWorkflowStep({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, diff --git a/tests/workflow-deploy/latency-gate.bench.ts b/tests/workflow-deploy/latency-gate.bench.ts index 637a9afe..7a511b82 100644 --- a/tests/workflow-deploy/latency-gate.bench.ts +++ b/tests/workflow-deploy/latency-gate.bench.ts @@ -366,7 +366,7 @@ async function runUnified(opts: { ]); const launchSession: LaunchSessionFn = async (orchestratorParams) => { - await env.hub.sessionService.launchSession({ + await env.hub.sessionService.stageWorkflowStep({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, diff --git a/tests/workflow-deploy/mail-edge-cases.test.ts b/tests/workflow-deploy/mail-edge-cases.test.ts index 6ded567a..be3c6f37 100644 --- a/tests/workflow-deploy/mail-edge-cases.test.ts +++ b/tests/workflow-deploy/mail-edge-cases.test.ts @@ -405,7 +405,7 @@ async function deployEdgeWorkflow( const launchSession: LaunchSessionFn = async (orchestratorParams) => { const deployContent = orchestratorParams.deployContent; - await env.hub.sessionService.launchSession({ + await env.hub.sessionService.stageWorkflowStep({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, diff --git a/tests/workflow-deploy/multistep-signal.test.ts b/tests/workflow-deploy/multistep-signal.test.ts index 04aff63d..270ef386 100644 --- a/tests/workflow-deploy/multistep-signal.test.ts +++ b/tests/workflow-deploy/multistep-signal.test.ts @@ -160,7 +160,7 @@ describe("multi-step workflow round-trip with signal-await", () => { // `agent-state` repo is provisioned end-to-end. const launchSession: LaunchSessionFn = async (orchestratorParams) => { const deployContent = orchestratorParams.deployContent; - await env.hub.sessionService.launchSession({ + await env.hub.sessionService.stageWorkflowStep({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, diff --git a/tests/workflow-deploy/multistep-signed-send.test.ts b/tests/workflow-deploy/multistep-signed-send.test.ts index eab2ae51..e56b7bad 100644 --- a/tests/workflow-deploy/multistep-signed-send.test.ts +++ b/tests/workflow-deploy/multistep-signed-send.test.ts @@ -171,7 +171,7 @@ describe("multi-step signed outbound send", () => { ]); const launchSession: LaunchSessionFn = async (orchestratorParams) => { - await env.hub.sessionService.launchSession({ + await env.hub.sessionService.stageWorkflowStep({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, diff --git a/tests/workflow-deploy/single-step-full-lifecycle.test.ts b/tests/workflow-deploy/single-step-full-lifecycle.test.ts index 56a5b43d..02f9a180 100644 --- a/tests/workflow-deploy/single-step-full-lifecycle.test.ts +++ b/tests/workflow-deploy/single-step-full-lifecycle.test.ts @@ -238,7 +238,7 @@ describe("single-step full lifecycle on the unified child path (Phase 4.6)", () ]); const launchSession: LaunchSessionFn = async (orchestratorParams) => { - await env.hub.sessionService.launchSession({ + await env.hub.sessionService.stageWorkflowStep({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, diff --git a/tests/workflow-deploy/single-step-grants-bridge.test.ts b/tests/workflow-deploy/single-step-grants-bridge.test.ts index 2bd36bf4..617d0cd1 100644 --- a/tests/workflow-deploy/single-step-grants-bridge.test.ts +++ b/tests/workflow-deploy/single-step-grants-bridge.test.ts @@ -200,7 +200,7 @@ describe("single-step launched-agent grants bridge via spawned child", () => { ]); const launchSession: LaunchSessionFn = async (orchestratorParams) => { - await env.hub.sessionService.launchSession({ + await env.hub.sessionService.stageWorkflowStep({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, diff --git a/tests/workflow-deploy/single-step-message-input.test.ts b/tests/workflow-deploy/single-step-message-input.test.ts index 14769124..1f90779f 100644 --- a/tests/workflow-deploy/single-step-message-input.test.ts +++ b/tests/workflow-deploy/single-step-message-input.test.ts @@ -127,7 +127,7 @@ describe("single-step message-input round-trip", () => { const launchSession: LaunchSessionFn = async (orchestratorParams) => { const deployContent = orchestratorParams.deployContent; - await env.hub.sessionService.launchSession({ + await env.hub.sessionService.stageWorkflowStep({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, diff --git a/tests/workflow-deploy/single-step-posix-tool.test.ts b/tests/workflow-deploy/single-step-posix-tool.test.ts index d7c60866..0f6c4a06 100644 --- a/tests/workflow-deploy/single-step-posix-tool.test.ts +++ b/tests/workflow-deploy/single-step-posix-tool.test.ts @@ -158,7 +158,7 @@ describe("single-step posix-tool in-child execution", () => { ]); const launchSession: LaunchSessionFn = async (orchestratorParams) => { - await env.hub.sessionService.launchSession({ + await env.hub.sessionService.stageWorkflowStep({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, diff --git a/tests/workflow-deploy/single-step-real-agent.test.ts b/tests/workflow-deploy/single-step-real-agent.test.ts index 2699bd7d..96b3e379 100644 --- a/tests/workflow-deploy/single-step-real-agent.test.ts +++ b/tests/workflow-deploy/single-step-real-agent.test.ts @@ -135,7 +135,7 @@ describe("single-step real-agent round-trip", () => { const launchSession: LaunchSessionFn = async (orchestratorParams) => { const deployContent = orchestratorParams.deployContent; - await env.hub.sessionService.launchSession({ + await env.hub.sessionService.stageWorkflowStep({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, From 9827958733479f3435bdc5bdd765d95f0c19c4e4 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Fri, 3 Jul 2026 18:48:10 -0500 Subject: [PATCH 038/101] Cover shared deploy staging through the staged entry point The unit tests for the shared deploy-staging logic -- asset fan-out, the available-skills stanza, the tool-package resolver rows, fan-out rollback, and the write/provision phase aborts -- drove that logic through the warm launcher, even though the same logic is now live through the staging path. Point them at the staged entry point instead, so the coverage survives when the warm launcher is removed. This is a test-only change: the staged and warm paths run the same staging phases, so the assertions on rows, packs, and phase aborts hold unchanged, aside from provisioning through the no-spawn frame rather than the warm provision frame. --- .../hub-sessions/src/session-service.test.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/hub-sessions/src/session-service.test.ts b/packages/hub-sessions/src/session-service.test.ts index f23a9203..aa344219 100644 --- a/packages/hub-sessions/src/session-service.test.ts +++ b/packages/hub-sessions/src/session-service.test.ts @@ -516,7 +516,7 @@ describe("SessionService", () => { }); const err = await service - .launchSession({ + .stageWorkflowStep({ agentAddress: AGENT_ADDRESS, agentId: AGENT_ID, instanceId: INSTANCE_ID, @@ -533,7 +533,7 @@ describe("SessionService", () => { }); test("launchSession does not send pack on provision failure", async () => { - router.sendAgentDeploy = () => + router.sendProvisionStep = () => Promise.reject(new Error("provision failed")); const service = createSessionService({ @@ -542,7 +542,7 @@ describe("SessionService", () => { }); const err = await service - .launchSession({ + .stageWorkflowStep({ agentAddress: AGENT_ADDRESS, agentId: AGENT_ID, instanceId: INSTANCE_ID, @@ -804,7 +804,7 @@ describe("SessionService", () => { >, }); - await service.launchSession({ + await service.stageWorkflowStep({ agentAddress: AGENT_ADDRESS, agentId: AGENT_ID, instanceId: INSTANCE_ID, @@ -961,7 +961,7 @@ describe("SessionService", () => { >, }); - await service.launchSession({ + await service.stageWorkflowStep({ agentAddress: AGENT_ADDRESS, agentId: AGENT_ID, instanceId: INSTANCE_ID, @@ -1012,7 +1012,7 @@ describe("SessionService", () => { agentRepoStore: repoStore, }); - await service.launchSession({ + await service.stageWorkflowStep({ agentAddress: AGENT_ADDRESS, agentId: AGENT_ID, instanceId: INSTANCE_ID, @@ -1198,7 +1198,7 @@ describe("SessionService", () => { }, }); - await service.launchSession({ + await service.stageWorkflowStep({ agentAddress: AGENT_ADDRESS, agentId: AGENT_ID, instanceId: INSTANCE_ID, @@ -1313,7 +1313,7 @@ describe("SessionService", () => { let err: unknown; try { - await service.launchSession({ + await service.stageWorkflowStep({ agentAddress: AGENT_ADDRESS, agentId: AGENT_ID, instanceId: INSTANCE_ID, @@ -1484,7 +1484,7 @@ describe("SessionService", () => { let caught: unknown; try { - await service.launchSession({ + await service.stageWorkflowStep({ agentAddress: AGENT_ADDRESS, agentId: AGENT_ID, instanceId: INSTANCE_ID, From 6bbca878d4bfc0f6f5932b1d00e3907fa0622da1 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Fri, 3 Jul 2026 19:03:01 -0500 Subject: [PATCH 039/101] Remove the warm-harness launch entry point Nothing in production launches a warm in-process agent harness anymore: single-agent instances deploy at the head, and multi-step workflow steps stage without a harness. The public launch method that drove the warm path was reached only by tests. Delete the launch method, the warm session-start helper, and the provision arm of the staging routine that fired the plain provision frame. The staging routine now requires a workflow frame or a stage-only flag and fails loud on neither. The warm-lifecycle integration test and the warm-specific unit tests go with it; the shared staging logic keeps its unit coverage through the staged entry point. --- Makefile | 2 +- packages/hub-api/src/app.test.ts | 3 - packages/hub-api/src/routes/agents.test.ts | 3 - packages/hub-api/src/routes/assets.test.ts | 3 - .../hub-api/src/routes/catalog-routes.test.ts | 3 - .../hub-api/src/routes/git-tokens.test.ts | 3 - packages/hub-api/src/routes/instances.test.ts | 13 - packages/hub-api/src/routes/workflows.test.ts | 1 - .../hub-sessions/src/session-service.test.ts | 146 ------- packages/hub-sessions/src/session-service.ts | 91 +---- tests/hub-agent/deploy-flow.test.ts | 363 ------------------ tests/hub-agent/lib/deploy-flow-env.ts | 7 +- tests/hub-api/asset-routes.test.ts | 1 - tests/workflow-deploy/drain-roundtrip.test.ts | 7 +- tests/workflow-deploy/fifo-mail.test.ts | 11 +- .../latency-d2-attribution.bench.ts | 2 +- .../workflow-deploy/multistep-signal.test.ts | 12 +- 17 files changed, 27 insertions(+), 644 deletions(-) delete mode 100644 tests/hub-agent/deploy-flow.test.ts diff --git a/Makefile b/Makefile index ca0d630a..885c4a61 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ lint: FORCE test: FORCE bun test packages/ apps/ bin/ tests/agent/ tests/agent-audit-log/ tests/agent-blob-spill/ tests/agent-common/ tests/agent-multi-provider/ tests/agent-quickstart/ tests/agent-resume/ tests/agent-rewind/ tests/agent-rich-tool/ tests/agent-structured-payload/ tests/coding-agent/ tests/hub-agent/lib/ tests/inference-testing/ tests/tool-packaging/ tests/workflow/ - bun test --timeout 120000 tests/hub-agent/deploy-flow.test.ts tests/workflow-deploy/multistep-signal.test.ts tests/workflow-deploy/single-step-real-agent.test.ts tests/workflow-deploy/instance-reroute-real-agent.test.ts tests/workflow-deploy/instance-failover-real-agent.test.ts tests/workflow-deploy/multistep-signed-send.test.ts tests/workflow-deploy/single-step-message-input.test.ts tests/workflow-deploy/single-step-posix-tool.test.ts tests/workflow-deploy/single-step-grants-bridge.test.ts tests/workflow-deploy/single-step-event-threading.test.ts tests/workflow-deploy/single-step-conversation-durability.test.ts tests/workflow-deploy/single-step-full-lifecycle.test.ts tests/workflow-deploy/cross-process-custom-adapter.test.ts tests/workflow-deploy/conversation-state-wal.test.ts tests/workflow-deploy/drain-roundtrip.test.ts tests/workflow-deploy/child-workflow-roundtrip.test.ts tests/workflow-deploy/unresolvable-director.test.ts tests/workflow-deploy/fifo-mail.test.ts tests/workflow-deploy/mail-edge-cases.test.ts tests/inference/ tests/skill-attachment-flow.test.ts tests/hub-api/ tests/db/ + bun test --timeout 120000 tests/workflow-deploy/multistep-signal.test.ts tests/workflow-deploy/single-step-real-agent.test.ts tests/workflow-deploy/instance-reroute-real-agent.test.ts tests/workflow-deploy/instance-failover-real-agent.test.ts tests/workflow-deploy/multistep-signed-send.test.ts tests/workflow-deploy/single-step-message-input.test.ts tests/workflow-deploy/single-step-posix-tool.test.ts tests/workflow-deploy/single-step-grants-bridge.test.ts tests/workflow-deploy/single-step-event-threading.test.ts tests/workflow-deploy/single-step-conversation-durability.test.ts tests/workflow-deploy/single-step-full-lifecycle.test.ts tests/workflow-deploy/cross-process-custom-adapter.test.ts tests/workflow-deploy/conversation-state-wal.test.ts tests/workflow-deploy/drain-roundtrip.test.ts tests/workflow-deploy/child-workflow-roundtrip.test.ts tests/workflow-deploy/unresolvable-director.test.ts tests/workflow-deploy/fifo-mail.test.ts tests/workflow-deploy/mail-edge-cases.test.ts tests/inference/ tests/skill-attachment-flow.test.ts tests/hub-api/ tests/db/ test-load: FORCE bun test --timeout 300000 tests/workflow-deploy/fifo-mail-load.test.ts diff --git a/packages/hub-api/src/app.test.ts b/packages/hub-api/src/app.test.ts index ba55f26a..b7a6ac71 100644 --- a/packages/hub-api/src/app.test.ts +++ b/packages/hub-api/src/app.test.ts @@ -20,9 +20,6 @@ const OpenAPISpec = type({ const mockDb = {} as unknown as DB["db"]; const sidecarRouter = createSidecarRouter({}); const sessionService: SessionService = { - launchSession(_params) { - throw new Error("mock: sessionService.launchSession not implemented"); - }, stageWorkflowStep(_params) { throw new Error("mock: sessionService.stageWorkflowStep not implemented"); }, diff --git a/packages/hub-api/src/routes/agents.test.ts b/packages/hub-api/src/routes/agents.test.ts index 487c768c..59ae012e 100644 --- a/packages/hub-api/src/routes/agents.test.ts +++ b/packages/hub-api/src/routes/agents.test.ts @@ -196,9 +196,6 @@ function createMockSidecarRouter(): SidecarRouter { function createMockSessionService(): SessionService { return { - launchSession: () => { - throw new Error("mock: sessionService.launchSession not implemented"); - }, stageWorkflowStep: () => { throw new Error("mock: sessionService.stageWorkflowStep not implemented"); }, diff --git a/packages/hub-api/src/routes/assets.test.ts b/packages/hub-api/src/routes/assets.test.ts index c33d1e85..8ae07c73 100644 --- a/packages/hub-api/src/routes/assets.test.ts +++ b/packages/hub-api/src/routes/assets.test.ts @@ -312,9 +312,6 @@ function createMockSidecarRouter(): SidecarRouter { function createMockSessionService(): SessionService { return { - launchSession: () => { - throw new Error("mock: sessionService.launchSession not implemented"); - }, stageWorkflowStep: () => { throw new Error("mock: sessionService.stageWorkflowStep not implemented"); }, diff --git a/packages/hub-api/src/routes/catalog-routes.test.ts b/packages/hub-api/src/routes/catalog-routes.test.ts index 5a3b0aa6..f4745c72 100644 --- a/packages/hub-api/src/routes/catalog-routes.test.ts +++ b/packages/hub-api/src/routes/catalog-routes.test.ts @@ -211,9 +211,6 @@ function createMockSidecarRouter(): SidecarRouter { function createMockSessionService(): SessionService { return { - launchSession: () => { - throw new Error("mock: sessionService.launchSession not implemented"); - }, stageWorkflowStep: () => { throw new Error("mock: sessionService.stageWorkflowStep not implemented"); }, diff --git a/packages/hub-api/src/routes/git-tokens.test.ts b/packages/hub-api/src/routes/git-tokens.test.ts index fbed1673..3338c4ab 100644 --- a/packages/hub-api/src/routes/git-tokens.test.ts +++ b/packages/hub-api/src/routes/git-tokens.test.ts @@ -235,9 +235,6 @@ function createMockSidecarRouter(): SidecarRouter { function createMockSessionService(): SessionService { return { - launchSession: () => { - throw new Error("mock: sessionService.launchSession not implemented"); - }, stageWorkflowStep: () => { throw new Error("mock: sessionService.stageWorkflowStep not implemented"); }, diff --git a/packages/hub-api/src/routes/instances.test.ts b/packages/hub-api/src/routes/instances.test.ts index 264df3d4..12310984 100644 --- a/packages/hub-api/src/routes/instances.test.ts +++ b/packages/hub-api/src/routes/instances.test.ts @@ -293,9 +293,6 @@ function createMockSessionService(): SessionService { throw new Error(`mock: sessionService.${name} not implemented`); } return { - launchSession(_params) { - return notImpl("launchSession"); - }, stageWorkflowStep(_params) { return notImpl("stageWorkflowStep"); }, @@ -672,9 +669,6 @@ describe("POST /agents/instances/:instanceId/mail", () => { } { const captured: CapturedSendArgs[] = []; const service: SessionService = { - launchSession() { - throw new Error("not implemented"); - }, stageWorkflowStep() { throw new Error("not implemented"); }, @@ -829,9 +823,6 @@ describe("POST /agents/instances/:instanceId/mail attachments", () => { } { const captured: (MessageAttachment[] | undefined)[] = []; const service: SessionService = { - launchSession() { - throw new Error("not implemented"); - }, stageWorkflowStep() { throw new Error("not implemented"); }, @@ -1270,7 +1261,6 @@ describe("POST /agents/instances seeds creator agent-state grant", () => { function createCapturingSessionService(): SessionService { return { - launchSession: async () => undefined, stageWorkflowStep: async () => undefined, deployInstanceAtHead: async () => ({ publicKey: "pk-instance-mock" }), deployWorkflowDefinition: () => { @@ -1488,9 +1478,6 @@ describe("POST /agents/instances seeds creator agent-state grant", () => { let launchedSources: unknown; let launchedDefaultSource: unknown; const sessionService: SessionService = { - launchSession: () => { - throw new Error("mock: launchSession not implemented"); - }, stageWorkflowStep: () => { throw new Error("mock: stageWorkflowStep not implemented"); }, diff --git a/packages/hub-api/src/routes/workflows.test.ts b/packages/hub-api/src/routes/workflows.test.ts index 9b3f88c0..679ef44e 100644 --- a/packages/hub-api/src/routes/workflows.test.ts +++ b/packages/hub-api/src/routes/workflows.test.ts @@ -226,7 +226,6 @@ function createMockSessionService( throw new Error(`mock: sessionService.${name} not implemented`); } return { - launchSession: () => notImpl("launchSession"), stageWorkflowStep: () => notImpl("stageWorkflowStep"), deployInstanceAtHead: () => notImpl("deployInstanceAtHead"), deployWorkflowDefinition: (params) => { diff --git a/packages/hub-sessions/src/session-service.test.ts b/packages/hub-sessions/src/session-service.test.ts index aa344219..85afcd24 100644 --- a/packages/hub-sessions/src/session-service.test.ts +++ b/packages/hub-sessions/src/session-service.test.ts @@ -332,34 +332,6 @@ describe("SessionService", () => { repoStore = createMockRepoStore(); }); - test("launchSession calls steps in order", async () => { - const service = createSessionService({ - sidecarRouter: router, - agentRepoStore: repoStore, - }); - - await service.launchSession({ - agentAddress: AGENT_ADDRESS, - agentId: AGENT_ID, - instanceId: INSTANCE_ID, - config: MOCK_CONFIG, - deployContent: MOCK_CONTENT, - }); - - const methods = [ - ...repoStore.calls.map((c) => c.method), - ...router.calls.map((c) => c.method), - ]; - - expect(methods).toEqual([ - "writeDeployTree", - "createDeployPack", - "sendAgentDeploy", - "sendPack", - "sendSessionStart", - ]); - }); - test("stageWorkflowStep stages without a warm harness", async () => { const service = createSessionService({ sidecarRouter: router, @@ -421,92 +393,6 @@ describe("SessionService", () => { expect(routerMethods).not.toContain("sendAgentUndeploy"); }); - test("launchSession cleans up on pack failure", async () => { - router.sendPack = () => Promise.reject(new Error("pack failed")); - - const service = createSessionService({ - sidecarRouter: router, - agentRepoStore: repoStore, - }); - - const err = await service - .launchSession({ - agentAddress: AGENT_ADDRESS, - agentId: AGENT_ID, - instanceId: INSTANCE_ID, - config: MOCK_CONFIG, - deployContent: MOCK_CONTENT, - }) - .catch((e: unknown) => e); - - expect(err).toBeInstanceOf(SessionLaunchError); - if (!(err instanceof SessionLaunchError)) throw new Error("unreachable"); - expect(err.phase).toBe("pack"); - expect(err.leakedAgent).toBe(false); - - const routerMethods = router.calls.map((c) => c.method); - expect(routerMethods).toContain("sendAgentUndeploy"); - }); - - test("launchSession cleans up on session start failure", async () => { - router.sendSessionStart = () => Promise.reject(new Error("start failed")); - - const service = createSessionService({ - sidecarRouter: router, - agentRepoStore: repoStore, - }); - - const err = await service - .launchSession({ - agentAddress: AGENT_ADDRESS, - agentId: AGENT_ID, - instanceId: INSTANCE_ID, - config: MOCK_CONFIG, - deployContent: MOCK_CONTENT, - }) - .catch((e: unknown) => e); - - expect(err).toBeInstanceOf(SessionLaunchError); - if (!(err instanceof SessionLaunchError)) throw new Error("unreachable"); - expect(err.phase).toBe("start"); - expect(err.leakedAgent).toBe(false); - - const routerMethods = router.calls.map((c) => c.method); - expect(routerMethods).toContain("sendAgentUndeploy"); - }); - - test("launchSession reports leaked agent when cleanup fails", async () => { - router.sendPack = () => Promise.reject(new Error("pack failed")); - router.sendAgentUndeploy = () => - Promise.reject(new Error("cleanup failed")); - - const service = createSessionService({ - sidecarRouter: router, - agentRepoStore: repoStore, - }); - - const err = await service - .launchSession({ - agentAddress: AGENT_ADDRESS, - agentId: AGENT_ID, - instanceId: INSTANCE_ID, - config: MOCK_CONFIG, - deployContent: MOCK_CONTENT, - }) - .catch((e: unknown) => e); - - expect(err).toBeInstanceOf(SessionLaunchError); - if (!(err instanceof SessionLaunchError)) throw new Error("unreachable"); - expect(err.leakedAgent).toBe(true); - expect(err.phase).toBe("pack"); - - // The original error (pack failure) must be preserved as the cause, - // not the cleanup failure. - expect(err.cause).toBeInstanceOf(Error); - if (!(err.cause instanceof Error)) throw new Error("unreachable"); - expect(err.cause.message).toBe("pack failed"); - }); - test("launchSession does not provision on write failure", async () => { repoStore.writeDeployTree = () => Promise.reject(new Error("write failed")); @@ -1037,38 +923,6 @@ describe("SessionService", () => { expect(content.systemPrompt).not.toContain(""); }); - test("launchSession without assetService leaves the deploy-only flow unchanged", async () => { - const service = createSessionService({ - sidecarRouter: router, - agentRepoStore: repoStore, - }); - - await service.launchSession({ - agentAddress: AGENT_ADDRESS, - agentId: AGENT_ID, - instanceId: INSTANCE_ID, - config: MOCK_CONFIG, - deployContent: MOCK_CONTENT, - }); - - const methods = [ - ...repoStore.calls.map((c) => c.method), - ...router.calls.map((c) => c.method), - ]; - expect(methods).toEqual([ - "writeDeployTree", - "createDeployPack", - "sendAgentDeploy", - "sendPack", - "sendSessionStart", - ]); - - // No mountPath options on the single sendPack call. - const packCall = router.calls.find((c) => c.method === "sendPack"); - if (packCall === undefined) throw new Error("sendPack not called"); - expect(packCall.args[4]).toBeUndefined(); - }); - test("launchSession writes a resolved-source session_asset row for resolver-derived packs", async () => { // Build a single-tarball asset registry, fake the DB query path // the session service walks (`listAssetsForTenant` walks diff --git a/packages/hub-sessions/src/session-service.ts b/packages/hub-sessions/src/session-service.ts index 2c6773c6..5b863b76 100644 --- a/packages/hub-sessions/src/session-service.ts +++ b/packages/hub-sessions/src/session-service.ts @@ -95,39 +95,6 @@ export class SessionLaunchError extends Error { } export type SessionService = { - /** - * Orchestrate the full deploy lifecycle: - * 1. Write deploy tree to hub repo and produce packfile - * 2. Provision agent on sidecar (sendAgentDeploy) - * 3. Deliver deploy packfile (sendPack) - * 4. Fan-out attached asset packs: for each row returned by - * `assetService.listAgentAssets(agentId)`, resolve the - * mountPath, resolve the ref to a source commit SHA, build a - * pack, insert a `session_asset` row, and send the pack to the - * sidecar with `mountPath` set on the `repo.pack.done` frame. - * The manifest insert MUST precede the pack send. - * 5. Start session (sendSessionStart) - * - * On partial failure after provision, attempts cleanup via - * sendAgentUndeploy before re-throwing. - */ - launchSession(params: { - agentAddress: string; - agentId: string; - instanceId: string; - config: HarnessConfig; - deployContent: DeployContent; - /** - * The agent definition's pinned tool packages. When non-empty, the - * service uses its injected `toolPackageResolver` to compute the - * full pinned closure and ships the resulting manifest as part of - * the deploy tree. An empty array (or absent value) means no - * tool-package manifest is materialized; the sidecar's loader is - * a no-op for this deploy. - */ - toolPackagePins?: readonly ToolPackagePin[]; - }): Promise; - /** * Stage one step of a multi-step workflow deploy: bind a transient route * for the step address, fire a no-spawn provision frame (init the step's @@ -819,7 +786,14 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { } else if (stageOnly) { await sidecarRouter.sendProvisionStep(agentAddress, config); } else { - await sidecarRouter.sendAgentDeploy(agentAddress, config); + // Every caller supplies `workflowFrame` (single-step head) or + // `stageOnly` (multi-step per-step). A deploy with neither has no + // provisioning shape -- the legacy warm-harness path is gone -- so + // fail loud rather than ship a deploy pack the sidecar never + // provisioned a repo for. + throw new Error( + "executeLaunchPhases: a deploy requires either workflowFrame or stageOnly", + ); } } catch (err) { throw new SessionLaunchError("provision", err, false); @@ -897,24 +871,6 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { } } - /** - * Start the warm hub-agent harness for a staged deploy. The legacy - * agent-deploy path calls this after `executeLaunchPhases` stages the - * deploy tree and packs; a single-step workflow deploy skips it entirely, - * because the supervised workflow-process child drives the agent per - * message through the supervisor and there is no warm harness to start. - * A failed start tears the sidecar deployment down so no zombie agent is - * left behind. - */ - async function startWarmSession(agentAddress: string): Promise { - try { - await sidecarRouter.sendSessionStart(agentAddress); - } catch (err) { - await attemptCleanup(agentAddress, "start", err); - throw new SessionLaunchError("start", err, false); - } - } - /** * Deploy a one-step workflow once at the head. Reuses the full * launch-phase machinery (deploy-tree write, pack, asset fan-out) via @@ -1001,36 +957,6 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { return orchestrator.deployWorkflow(args.deployArgs); } - /** - * Provision one agent at its address: stage the deploy tree + packs - * (`executeLaunchPhases` with no workflow frame) and then start its warm - * harness. This is the legacy warm-harness deploy with no production - * caller: a multi-step step stages without a harness (`stageWorkflowStep`), - * a single-agent instance uses `deployInstanceAtHead`, and a single-step - * workflow uses the head hand-off. The method carries no workflow - * projection of its own. - */ - async function launchSession(params: { - agentAddress: string; - agentId: string; - instanceId: string; - config: HarnessConfig; - deployContent: DeployContent; - toolPackagePins?: readonly ToolPackagePin[]; - }): Promise { - await executeLaunchPhases({ - agentAddress: params.agentAddress, - agentId: params.agentId, - instanceId: params.instanceId, - config: params.config, - deployContent: params.deployContent, - ...(params.toolPackagePins !== undefined - ? { toolPackagePins: params.toolPackagePins } - : {}), - }); - await startWarmSession(params.agentAddress); - } - /** * Stage one step of a multi-step workflow deploy: bind a transient route * for the step address, fire a no-spawn provision frame (the sidecar inits @@ -1613,7 +1539,6 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { } return { - launchSession, stageWorkflowStep, deployInstanceAtHead, deploySingleStepAtHead, diff --git a/tests/hub-agent/deploy-flow.test.ts b/tests/hub-agent/deploy-flow.test.ts deleted file mode 100644 index a7238c35..00000000 --- a/tests/hub-agent/deploy-flow.test.ts +++ /dev/null @@ -1,363 +0,0 @@ -// Integration test: full deploy lifecycle through SessionService. -// -// Spins up a real WS server (hub side), spawns a real sidecar subprocess, -// and exercises the complete agent lifecycle orchestrated by SessionService: -// -// launchSession (write → pack → provision → deliver → start) -// → message → sync → endSession -// -// The gap this test fills: nobody else tests that AgentRepoStore.writeDeployTree -// → createDeployPack → sendPack produces a packfile the sidecar actually -// accepts and materializes correctly. -// -// Inference is mocked by a tiny HTTP server that echoes the tools it receives, -// so we can assert the model saw the correct tool definitions without calling -// a real LLM. - -import { describe, test, expect, afterAll, beforeAll } from "bun:test"; -import fs from "node:fs"; -import path from "node:path"; -import { - assembleSignedContent, - assembleMessage, - createDetachedSignatureFromProvider, - type MessageHeaders, -} from "@intx/mime"; -import { generateKeyPair, createEd25519Crypto } from "@intx/crypto"; -import { parseAgentId } from "@intx/hub-sessions"; -import { createAgentRepoStore as createSidecarRepoStore } from "@intx/hub-agent"; -import { base64Encode } from "@intx/types"; -import type { HarnessConfig } from "@intx/types/runtime"; -import git from "isomorphic-git"; - -import { - AGENT_ADDRESS, - AGENT_ID, - SESSION_ID, - SIDECAR_ID, - startDeployFlowEnv, - waitFor, - type DeployFlowEnv, -} from "./lib/deploy-flow-env"; - -let env: DeployFlowEnv; -let setupInferenceRequestIndex: number; -let setupAgentEventsBeforeMessage: number; - -// The downstream tests (send-message side-effect assertions, -// sync-request, endSession) each depend on prior lifecycle stages -// having run. Bun runs tests within a `describe` in source order, so -// the chain works when the file runs end-to-end, but `bun test -t -// ` would skip the prerequisite stages and a filtered run of -// `sync request` would fail with `No sidecar connected for agent` or -// `tree must include at least one state-bearing top-level entry`. -// Running the prerequisite stages here -- launchSession AND the first -// message routing, both of which populate the state the sync test -// then asserts against -- codifies the preconditions once so each -// individual test can run on its own under a `-t` filter. The tests -// below assert against the state these setup steps produced. -beforeAll(async () => { - env = await startDeployFlowEnv(); - - const config: HarnessConfig = { - sessionId: SESSION_ID, - agentId: AGENT_ID, - tenantId: "tenant-1", - principalId: "prin_integration-1", - agentAddress: AGENT_ADDRESS, - systemPrompt: "Fallback prompt (should be overridden by deploy tree)", - tools: [], - grants: [], - sources: [ - { - id: "anthropic:mock-model", - provider: "anthropic", - baseURL: `http://localhost:${env.inference.server.port}`, - apiKey: "sk-mock", - model: "mock-model", - }, - ], - defaultSource: "anthropic:mock-model", - }; - - await env.hub.sessionService.launchSession({ - agentAddress: AGENT_ADDRESS, - agentId: AGENT_ID, - instanceId: AGENT_ID, - config, - deployContent: { - systemPrompt: "You are an integration test agent.", - }, - // The hub is wired with a `workspace-builtins` package-registry - // asset containing a synthetic `@intx/tools-mail@0.1.2` tarball; - // pinning here exercises the resolver → asset-pack fan-out → - // sidecar loader path end-to-end. The model-facing tool surfaces - // in the message test below as - // `@intx/tools-mail/sidecar-bundle:mail_send`. - toolPackagePins: [{ name: "@intx/tools-mail", version: "0.1.2" }], - }); - - // Record the indices before the setup message lands so the - // message-side-effect test can locate the resulting inference - // request and the inference.start event by absolute position - // rather than by delta-from-an-earlier-mutation. - setupInferenceRequestIndex = env.inference.requests.length; - setupAgentEventsBeforeMessage = env.hub.agentEvents.length; - - const messageKeyPair = await generateKeyPair(); - const messageCrypto = createEd25519Crypto(messageKeyPair); - const messageHeaders: MessageHeaders = { - from: "user@integration.interchange", - to: [AGENT_ADDRESS], - cc: undefined, - date: new Date(), - messageId: "", - subject: undefined, - inReplyTo: undefined, - references: undefined, - mimeVersion: "1.0", - interchangeType: "conversation.message", - interchangeCorrelationId: undefined, - interchangeTenantId: undefined, - interchangeAgentId: undefined, - interchangeSessionId: SESSION_ID, - interchangeOfferingId: undefined, - interchangeSchemaVersion: undefined, - traceparent: undefined, - tracestate: undefined, - }; - const messageSignedContent = assembleSignedContent({ - kind: "conversation", - text: "Hello.", - }); - const messageSignature = await createDetachedSignatureFromProvider( - messageSignedContent, - messageCrypto, - ); - const messageRaw = assembleMessage( - messageHeaders, - messageSignedContent, - messageSignature, - ); - env.hub.router.routeMail(AGENT_ADDRESS, base64Encode(messageRaw)); - - // Wait for the FULL message lifecycle to land so the downstream - // sync test sees populated agent state on disk. The inference - // request arriving is not enough -- it only signals the reactor - // dispatched -- so we also wait for the `message.run.ended` - // event, which fires after the reactor's context-store write - // completes. Without this, the sync test races the state file - // appearing on disk and the supervisor pushes an empty pack - // (rejected by the kind handler with `tree must include at - // least one state-bearing top-level entry`). - function setupHasEventType(event: unknown, type: string): boolean { - return ( - typeof event === "object" && - event !== null && - "type" in event && - event.type === type - ); - } - await waitFor( - () => env.inference.requests.length > setupInferenceRequestIndex, - { timeoutMs: 30_000, diagnostics: env.sidecarDiagnostics }, - ); - await waitFor( - () => - env.hub.agentEvents - .slice(setupAgentEventsBeforeMessage) - .some((e) => setupHasEventType(e.event, "message.run.ended")), - { timeoutMs: 30_000, diagnostics: env.sidecarDiagnostics }, - ); -}); - -afterAll(async () => { - await env.teardown(); -}); - -describe("deploy flow integration", () => { - test("sidecar registers with hub", () => { - expect(env.hub.router.getConnectedSidecars()).toContain(SIDECAR_ID); - }); - - test("launchSession writes, packs, provisions, delivers, and starts", async () => { - // The launchSession call lives in `beforeAll` so the downstream - // lifecycle tests can run under a `-t` filter; this test asserts - // the side effects the call produced. - - // The deploy ack should have arrived (provision phase completed). - const publicKey = env.hub.deployAcks.get(AGENT_ADDRESS); - expect(publicKey).toBeDefined(); - if (publicKey === undefined) throw new Error("unreachable"); - expect(publicKey.length).toBeGreaterThan(0); - - // The agent should now be routable (session start completed). - expect(env.hub.router.getRoutableAddresses()).toContain(AGENT_ADDRESS); - - // The deploy tree should have landed on the sidecar's disk. - const agentDir = createSidecarRepoStore({ - dataDir: env.sidecar.dataDir, - }).getAgentDir(AGENT_ADDRESS); - - await waitFor( - async () => { - try { - await fs.promises.access(path.join(agentDir, "deploy", "prompt.md")); - return true; - } catch { - return false; - } - }, - { diagnostics: env.sidecarDiagnostics }, - ); - - const prompt = await fs.promises.readFile( - path.join(agentDir, "deploy", "prompt.md"), - "utf-8", - ); - expect(prompt).toContain("integration test agent"); - }); - - test("send message and inference receives the asset-backed mail tool", async () => { - // The setup message landed in `beforeAll`; this test asserts the - // resulting inference request and event chain. Reading by the - // recorded indices keeps the assertions correct regardless of - // whether prior tests in the suite also bumped the counters. - const req = env.inference.requests[setupInferenceRequestIndex]; - if (req === undefined) throw new Error("unreachable"); - const tools = req.tools ?? []; - const toolNames = tools.map((t) => t.name); - // The synthetic tarball seeded into the workspace-builtins - // package-registry asset publishes a single `mail_send` - // definition under the `@intx/tools-mail/sidecar-bundle` factory - // id; the loader prefixes the definition name with the factory id - // to yield the qualified tool name the model sees. - expect(toolNames).toContain("@intx/tools-mail/sidecar-bundle:mail_send"); - - // reactor.start may or may not have arrived before - // `setupAgentEventsBeforeMessage` was captured (it depends on how - // fast contextStore.load() resolves), so wait until we see an - // inference.start event rather than assuming it is the very first - // new event. - function hasEventType( - event: unknown, - type: string, - ): event is { type: string } { - return ( - typeof event === "object" && - event !== null && - "type" in event && - event.type === type - ); - } - - await waitFor( - () => - env.hub.agentEvents - .slice(setupAgentEventsBeforeMessage) - .some((e) => hasEventType(e.event, "inference.start")), - { diagnostics: env.sidecarDiagnostics }, - ); - - const inferenceStartEvent = env.hub.agentEvents - .slice(setupAgentEventsBeforeMessage) - .find((e) => hasEventType(e.event, "inference.start")); - if (inferenceStartEvent === undefined) throw new Error("unreachable"); - expect(inferenceStartEvent.addr).toBe(AGENT_ADDRESS); - expect(inferenceStartEvent.sid).toBe(SESSION_ID); - }); - - test("sync request triggers state push to hub repo", async () => { - const packCountBefore = env.hub.statePacks.length; - env.hub.router.sendSyncRequest(AGENT_ADDRESS); - - // 30s headroom: the state-pack push travels the - // sidecar -> hub pack-push pipeline and the default - // `waitFor` 10s budget flaked ~10% of runs on busier - // machines. - await waitFor(() => env.hub.statePacks.length > packCountBefore, { - timeoutMs: 30_000, - diagnostics: env.sidecarDiagnostics, - }); - - const last = env.hub.statePacks[env.hub.statePacks.length - 1]; - if (last === undefined) throw new Error("unreachable"); - expect(last.agentAddress).toBe(AGENT_ADDRESS); - expect(last.ref).toMatch(/^refs\//); - expect(last.commitSha).toMatch(/^[0-9a-f]{40}$/); - expect(env.hub.router.getConnectedSidecars()).toContain(SIDECAR_ID); - - // Verify the pack was actually persisted in the hub's git repo. - const hubAgentDir = path.join( - env.hub.hubDataDir, - "agents", - parseAgentId(AGENT_ADDRESS), - ); - const resolvedSha = await git.resolveRef({ - fs, - dir: hubAgentDir, - ref: last.ref, - }); - expect(resolvedSha).toBe(last.commitSha); - - // Verify the commit object is readable (pack was properly indexed). - const { commit } = await git.readCommit({ - fs, - dir: hubAgentDir, - oid: last.commitSha, - }); - expect(commit.tree).toMatch(/^[0-9a-f]{40}$/); - }); - - test("endSession drains an in-flight state read before tearing down the sidecar agent dir", async () => { - const failuresBefore = env.hub.statePackReceiveFailures.filter( - (f) => f.agentAddress === AGENT_ADDRESS, - ).length; - const packCountBefore = env.hub.statePacks.length; - const sidecarLogBefore = env.sidecar.stderr.length; - - // Race the teardown: the sync request makes the sidecar read the agent - // repo (createStatePack) at the same moment endSession deletes that - // repo's directory. The drain serializes the read ahead of the rm, so - // the read runs against a live directory instead of a vanishing one. - env.hub.router.sendSyncRequest(AGENT_ADDRESS); - await env.hub.sessionService.endSession(AGENT_ADDRESS, "test_complete"); - - // Agent should no longer be routable after ack. - expect(env.hub.router.getRoutableAddresses()).not.toContain(AGENT_ADDRESS); - - // The ack is sent after deleteAgentDir completes, so the directory - // is already gone by the time the promise resolves. - const agentDir = createSidecarRepoStore({ - dataDir: env.sidecar.dataDir, - }).getAgentDir(AGENT_ADDRESS); - const dirExists = await fs.promises - .access(agentDir) - .then(() => true) - .catch(() => false); - expect(dirExists).toBe(false); - - // Teardown still pushes agent state to the hub; wait for it to land so - // any sidecar-side read failure has had time to drain into the log. - await waitFor(() => env.hub.statePacks.length > packCountBefore, { - timeoutMs: 30_000, - diagnostics: env.sidecarDiagnostics, - }); - - // Teardown must log no state-push read failure. Both the undeploy push - // and the sync push log "State push failed for
" when their read - // hits a deleted directory; the drain keeps the read ahead of the rm, so - // with the fix this log never appears. This is a one-sided guard: on a - // fast machine the sync read can finish before the rm regardless, so a - // pass does not prove the read raced -- but a build that drops the drain - // and loses the race fails here. The deterministic ordering check is the - // session-manager unit test. The hub-side counter stays flat too, since a - // sidecar-side read throw never reaches the receive boundary. - const newSidecarLog = env.sidecar.stderr.slice(sidecarLogBefore).join(""); - expect(newSidecarLog).not.toContain("State push failed"); - const failuresAfter = env.hub.statePackReceiveFailures.filter( - (f) => f.agentAddress === AGENT_ADDRESS, - ).length; - expect(failuresAfter).toBe(failuresBefore); - }); -}); diff --git a/tests/hub-agent/lib/deploy-flow-env.ts b/tests/hub-agent/lib/deploy-flow-env.ts index b017b896..c1044e5c 100644 --- a/tests/hub-agent/lib/deploy-flow-env.ts +++ b/tests/hub-agent/lib/deploy-flow-env.ts @@ -1009,10 +1009,9 @@ const DEFAULT_WORKFLOW_RUN_REF = "refs/heads/main"; /** * Options accepted by `deployWorkflow`. The helper composes the - * workflow-deploy orchestrator against the env's hub substrate and - * routes per-step launches through `env.hub.sessionService.launchSession` - * so the trivial-branch round-trip and the multi-step branch both - * exercise the production code paths. + * workflow-deploy orchestrator against the env's hub substrate and routes + * per-step launches through `env.hub.sessionService.stageWorkflowStep`, the + * stage-only path production's multi-step branch runs. */ export type DeployWorkflowOpts = { /** diff --git a/tests/hub-api/asset-routes.test.ts b/tests/hub-api/asset-routes.test.ts index 644f8791..04c01a14 100644 --- a/tests/hub-api/asset-routes.test.ts +++ b/tests/hub-api/asset-routes.test.ts @@ -164,7 +164,6 @@ function createMockSidecarRouter(): SidecarRouter { function createMockSessionService(): SessionService { return { - launchSession: notImplemented("sessionService.launchSession"), stageWorkflowStep: notImplemented("sessionService.stageWorkflowStep"), deployInstanceAtHead: notImplemented("sessionService.deployInstanceAtHead"), deployWorkflowDefinition: notImplemented( diff --git a/tests/workflow-deploy/drain-roundtrip.test.ts b/tests/workflow-deploy/drain-roundtrip.test.ts index f564be15..5d0c0ca4 100644 --- a/tests/workflow-deploy/drain-roundtrip.test.ts +++ b/tests/workflow-deploy/drain-roundtrip.test.ts @@ -32,10 +32,9 @@ // at this test. // // The orchestrator's multi-step branch is composed in-test (matching -// the multi-step signal round-trip) because the pre-landed -// `deploy-flow-env` fixture wires only the trivial `launchSession` -// callback against `env.hub.sessionService.launchSession`; the -// multi-step `sendMultiStepDeploy` hand-off is supplied here against +// the multi-step signal round-trip): the per-step launch callback drives +// `env.hub.sessionService.stageWorkflowStep` (the stage-only path, no warm +// harness) and the `sendMultiStepDeploy` hand-off is supplied against // `env.hub.router.sendAgentDeploy` so the sidecar's deploy router // takes the workflow-process spawn path. diff --git a/tests/workflow-deploy/fifo-mail.test.ts b/tests/workflow-deploy/fifo-mail.test.ts index 1d9c39c2..bced36ab 100644 --- a/tests/workflow-deploy/fifo-mail.test.ts +++ b/tests/workflow-deploy/fifo-mail.test.ts @@ -42,12 +42,11 @@ // end) or a pre-spawn seeding hook on `deployWorkflow`. Adding either // is a separate landing. // -// The orchestrator's multi-step branch is composed in-test because -// the pre-landed `deploy-flow-env` fixture wires only the trivial -// `launchSession` callback against `env.hub.sessionService.launchSession`; -// the multi-step `sendMultiStepDeploy` hand-off is supplied here -// against `env.hub.router.sendAgentDeploy` so the sidecar's deploy -// router takes the workflow-process spawn path. This mirrors the +// The orchestrator's multi-step branch is composed in-test: the per-step +// launch callback drives `env.hub.sessionService.stageWorkflowStep` (the +// stage-only path, no warm harness) and the `sendMultiStepDeploy` hand-off +// is supplied against `env.hub.router.sendAgentDeploy` so the sidecar's +// deploy router takes the workflow-process spawn path. This mirrors the // multistep-signal and drain-roundtrip tests' shape so a regression // in any of the seven hops surfaces uniformly across the three. // diff --git a/tests/workflow-deploy/latency-d2-attribution.bench.ts b/tests/workflow-deploy/latency-d2-attribution.bench.ts index 63991d09..ea7472a3 100644 --- a/tests/workflow-deploy/latency-d2-attribution.bench.ts +++ b/tests/workflow-deploy/latency-d2-attribution.bench.ts @@ -403,7 +403,7 @@ async function runUnifiedD2(opts: { ]); const launchSession: LaunchSessionFn = async (orchestratorParams) => { - await env.hub.sessionService.launchSession({ + await env.hub.sessionService.stageWorkflowStep({ agentAddress: orchestratorParams.agentAddress, agentId: orchestratorParams.agentId, instanceId: orchestratorParams.instanceId, diff --git a/tests/workflow-deploy/multistep-signal.test.ts b/tests/workflow-deploy/multistep-signal.test.ts index 270ef386..77cd6cd8 100644 --- a/tests/workflow-deploy/multistep-signal.test.ts +++ b/tests/workflow-deploy/multistep-signal.test.ts @@ -6,12 +6,12 @@ // `SignalAwaited`, injects a signal via the host-side signal channel, // and asserts the runtime resumes through `step2` to `RunCompleted`. // -// The orchestrator's multi-step branch is composed in-test because the -// pre-landed `deploy-flow-env` fixture wires only the trivial -// `launchSession` callback against `env.hub.sessionService.launchSession`; -// the multi-step `sendMultiStepDeploy` hand-off is supplied here against -// `env.hub.router.sendAgentDeploy` so the sidecar's deploy router takes -// the workflow-process spawn path. The deployment handle is registered +// The orchestrator's multi-step branch is composed in-test: the per-step +// launch callback drives `env.hub.sessionService.stageWorkflowStep` (the +// stage-only path, no warm harness) and the `sendMultiStepDeploy` hand-off +// is supplied against `env.hub.router.sendAgentDeploy` so the sidecar's +// deploy router takes the workflow-process spawn path. The deployment +// handle is registered // on the env via `registerDeployment` so the fixture's `injectSignal`, // `readWorkflowRunEvents`, and `waitForWorkflowRunComplete` helpers can // resolve it by id. From 0ecc8befb900d65a819ac98b15224f857f0a661b Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Fri, 3 Jul 2026 19:22:19 -0500 Subject: [PATCH 040/101] Retire the session-start wire path The warm-harness deploy split provision from an explicit session-start frame that told the sidecar to launch inference. Nothing sends that frame now that every deploy stages through the workflow-run substrate, so the router method, the sidecar-side handler, and the session.start and session.start.ack frame types no longer have a caller. Delete sendSessionStart and its pending-request bookkeeping from the sidecar router, drop the session.start handler from the hub-agent link, and remove the two now-dead frame types from the wire schema. Tests that stubbed the router method lose the stub. --- packages/hub-agent/src/ws/hub-link.test.ts | 77 --------------- packages/hub-agent/src/ws/hub-link.ts | 22 ----- packages/hub-api/src/routes/agents.test.ts | 1 - packages/hub-api/src/routes/assets.test.ts | 1 - .../hub-api/src/routes/catalog-routes.test.ts | 1 - .../hub-api/src/routes/git-tokens.test.ts | 1 - packages/hub-api/src/routes/instances.test.ts | 3 - packages/hub-api/src/routes/workflows.test.ts | 1 - .../hub-sessions/src/session-service.test.ts | 6 +- .../hub-sessions/src/ws/sidecar-handler.ts | 98 +------------------ packages/types/src/sidecar.ts | 24 ----- tests/hub-api/asset-routes.test.ts | 1 - 12 files changed, 4 insertions(+), 232 deletions(-) diff --git a/packages/hub-agent/src/ws/hub-link.test.ts b/packages/hub-agent/src/ws/hub-link.test.ts index f08222ac..674f4372 100644 --- a/packages/hub-agent/src/ws/hub-link.test.ts +++ b/packages/hub-agent/src/ws/hub-link.test.ts @@ -444,83 +444,6 @@ describe("sidecar↔hub integration", () => { } }); - test("session.start starts the harness", async () => { - const transport = createInMemoryTransport(); - const sessions = createMockSessionManager(); - const client = createHubLink({ - hubURL: `ws://localhost:${env.server.port}/ws`, - sidecarId: "sc-start", - token: "test-token", - transport, - sessions, - ...withTestDeployBindings(sessions), - }); - - client.connect(); - try { - await waitFor(() => - env.router.getConnectedSidecars().includes("sc-start"), - ); - - await env.router.sendAgentDeploy( - "start-agent@test.interchange", - TEST_CONFIG, - ); - expect(sessions.started).toHaveLength(0); - - await env.router.sendSessionStart("start-agent@test.interchange"); - expect(sessions.started).toHaveLength(1); - expect(sessions.started[0]).toBe("start-agent@test.interchange"); - } finally { - client.close(); - await waitFor( - () => !env.router.getConnectedSidecars().includes("sc-start"), - ); - } - }); - - test("session.start failure removes agent from routing table", async () => { - const transport = createInMemoryTransport(); - const sessions = createMockSessionManager(); - const client = createHubLink({ - hubURL: `ws://localhost:${env.server.port}/ws`, - sidecarId: "sc-start-fail", - token: "test-token", - transport, - sessions, - ...withTestDeployBindings(sessions), - }); - - client.connect(); - try { - await waitFor(() => - env.router.getConnectedSidecars().includes("sc-start-fail"), - ); - - await env.router.sendAgentDeploy( - "start-fail@test.interchange", - TEST_CONFIG, - ); - - // Make startSession throw on the next call. - sessions.shouldThrow = "deploy tree missing"; - - await expect( - env.router.sendSessionStart("start-fail@test.interchange"), - ).rejects.toThrow("deploy tree missing"); - - // Failed session start should remove the agent from routing. - expect(env.router.getRoutableAddresses()).not.toContain( - "start-fail@test.interchange", - ); - } finally { - client.close(); - await waitFor( - () => !env.router.getConnectedSidecars().includes("sc-start-fail"), - ); - } - }); - test("sidecar forwards agent events to hub", async () => { const transport = createInMemoryTransport(); const sessions = createMockSessionManager(); diff --git a/packages/hub-agent/src/ws/hub-link.ts b/packages/hub-agent/src/ws/hub-link.ts index 9ab66595..f38b52b0 100644 --- a/packages/hub-agent/src/ws/hub-link.ts +++ b/packages/hub-agent/src/ws/hub-link.ts @@ -18,7 +18,6 @@ import { type ChallengeFrame, type ChallengeFailedFrame, type SessionAbortFrame, - type SessionStartFrame, type GrantsUpdateFrame, type SourcesUpdateFrame, type PackPushFrame, @@ -397,24 +396,6 @@ export function createHubLink(config: HubLinkConfig): HubLink { } } - async function handleSessionStart(frame: SessionStartFrame): Promise { - try { - await sessions.startSession(frame.agentAddress); - send({ - type: "session.start.ack", - agentAddress: frame.agentAddress, - }); - logger.info`Started session for ${frame.agentAddress}`; - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - send({ - type: "agent.error", - agentAddress: frame.agentAddress, - error: message, - }); - } - } - async function handleAgentUndeploy(frame: AgentUndeployFrame): Promise { let statePushed = false; @@ -939,9 +920,6 @@ export function createHubLink(config: HubLinkConfig): HubLink { case "agent.deploy": await handleAgentDeploy(frame); break; - case "session.start": - await handleSessionStart(frame); - break; case "agent.undeploy": await handleAgentUndeploy(frame); break; diff --git a/packages/hub-api/src/routes/agents.test.ts b/packages/hub-api/src/routes/agents.test.ts index 59ae012e..e074941d 100644 --- a/packages/hub-api/src/routes/agents.test.ts +++ b/packages/hub-api/src/routes/agents.test.ts @@ -174,7 +174,6 @@ function createMockSidecarRouter(): SidecarRouter { routeMail: () => notImpl("routeMail"), sendAgentDeploy: () => notImpl("sendAgentDeploy"), sendAgentUndeploy: () => notImpl("sendAgentUndeploy"), - sendSessionStart: () => notImpl("sendSessionStart"), sendSessionAbort: () => notImpl("sendSessionAbort"), sendGrantsUpdate: () => notImpl("sendGrantsUpdate"), sendSourcesUpdate: () => notImpl("sendSourcesUpdate"), diff --git a/packages/hub-api/src/routes/assets.test.ts b/packages/hub-api/src/routes/assets.test.ts index 8ae07c73..8874c448 100644 --- a/packages/hub-api/src/routes/assets.test.ts +++ b/packages/hub-api/src/routes/assets.test.ts @@ -290,7 +290,6 @@ function createMockSidecarRouter(): SidecarRouter { routeMail: () => notImpl("routeMail"), sendAgentDeploy: () => notImpl("sendAgentDeploy"), sendAgentUndeploy: () => notImpl("sendAgentUndeploy"), - sendSessionStart: () => notImpl("sendSessionStart"), sendSessionAbort: () => notImpl("sendSessionAbort"), sendGrantsUpdate: () => notImpl("sendGrantsUpdate"), sendSourcesUpdate: () => notImpl("sendSourcesUpdate"), diff --git a/packages/hub-api/src/routes/catalog-routes.test.ts b/packages/hub-api/src/routes/catalog-routes.test.ts index f4745c72..2424cb6b 100644 --- a/packages/hub-api/src/routes/catalog-routes.test.ts +++ b/packages/hub-api/src/routes/catalog-routes.test.ts @@ -188,7 +188,6 @@ function createMockSidecarRouter(): SidecarRouter { routeMail: () => notImpl("routeMail"), sendAgentDeploy: () => notImpl("sendAgentDeploy"), sendAgentUndeploy: () => notImpl("sendAgentUndeploy"), - sendSessionStart: () => notImpl("sendSessionStart"), sendSessionAbort: () => notImpl("sendSessionAbort"), sendGrantsUpdate: () => notImpl("sendGrantsUpdate"), // Mutations fire a fire-and-forget source push; resolve so it is a no-op. diff --git a/packages/hub-api/src/routes/git-tokens.test.ts b/packages/hub-api/src/routes/git-tokens.test.ts index 3338c4ab..8bc04a12 100644 --- a/packages/hub-api/src/routes/git-tokens.test.ts +++ b/packages/hub-api/src/routes/git-tokens.test.ts @@ -213,7 +213,6 @@ function createMockSidecarRouter(): SidecarRouter { routeMail: () => notImpl("routeMail"), sendAgentDeploy: () => notImpl("sendAgentDeploy"), sendAgentUndeploy: () => notImpl("sendAgentUndeploy"), - sendSessionStart: () => notImpl("sendSessionStart"), sendSessionAbort: () => notImpl("sendSessionAbort"), sendGrantsUpdate: () => notImpl("sendGrantsUpdate"), sendSourcesUpdate: () => notImpl("sendSourcesUpdate"), diff --git a/packages/hub-api/src/routes/instances.test.ts b/packages/hub-api/src/routes/instances.test.ts index 12310984..1b32717f 100644 --- a/packages/hub-api/src/routes/instances.test.ts +++ b/packages/hub-api/src/routes/instances.test.ts @@ -240,9 +240,6 @@ function createMockSidecarRouter( sendAgentUndeploy(_addr, _reason) { return notImpl("sendAgentUndeploy"); }, - sendSessionStart(_addr) { - return notImpl("sendSessionStart"); - }, sendSessionAbort(_addr, _reason) { return notImpl("sendSessionAbort"); }, diff --git a/packages/hub-api/src/routes/workflows.test.ts b/packages/hub-api/src/routes/workflows.test.ts index 679ef44e..5d2ecf00 100644 --- a/packages/hub-api/src/routes/workflows.test.ts +++ b/packages/hub-api/src/routes/workflows.test.ts @@ -196,7 +196,6 @@ function createMockSidecarRouter( }, sendAgentDeploy: () => notImpl("sendAgentDeploy"), sendAgentUndeploy: () => notImpl("sendAgentUndeploy"), - sendSessionStart: () => notImpl("sendSessionStart"), sendSessionAbort: () => notImpl("sendSessionAbort"), sendGrantsUpdate: () => notImpl("sendGrantsUpdate"), sendSourcesUpdate: () => notImpl("sendSourcesUpdate"), diff --git a/packages/hub-sessions/src/session-service.test.ts b/packages/hub-sessions/src/session-service.test.ts index 85afcd24..53e75660 100644 --- a/packages/hub-sessions/src/session-service.test.ts +++ b/packages/hub-sessions/src/session-service.test.ts @@ -70,9 +70,6 @@ function createMockRouter(): SidecarRouter & { sendAgentUndeploy: track( "sendAgentUndeploy", ) as SidecarRouter["sendAgentUndeploy"], - sendSessionStart: track( - "sendSessionStart", - ) as SidecarRouter["sendSessionStart"], sendSessionAbort: track( "sendSessionAbort", ) as SidecarRouter["sendSessionAbort"], @@ -354,7 +351,7 @@ describe("SessionService", () => { // A stage-only per-step deploy binds a transient route, fires the // no-spawn provision frame, delivers the deploy pack, and unbinds the // route -- and NEVER provisions a warm harness (`sendAgentDeploy` with - // no workflow frame) or starts a session (`sendSessionStart`). + // no workflow frame). expect(methods).toEqual([ "writeDeployTree", "createDeployPack", @@ -364,7 +361,6 @@ describe("SessionService", () => { "unbindStepRoute", ]); expect(methods).not.toContain("sendAgentDeploy"); - expect(methods).not.toContain("sendSessionStart"); }); test("stageWorkflowStep unbinds the route even when the pack fails", async () => { diff --git a/packages/hub-sessions/src/ws/sidecar-handler.ts b/packages/hub-sessions/src/ws/sidecar-handler.ts index a0ae951b..6349c434 100644 --- a/packages/hub-sessions/src/ws/sidecar-handler.ts +++ b/packages/hub-sessions/src/ws/sidecar-handler.ts @@ -126,7 +126,6 @@ export type SidecarRouter = { workflow?: AgentDeployFrame["workflow"], ): Promise<{ publicKey: string }>; sendAgentUndeploy(agentAddress: string, reason: string): Promise; - sendSessionStart(agentAddress: string): Promise; sendSessionAbort(agentAddress: string, reason: AbortReason): Promise; sendGrantsUpdate(agentAddress: string, grants: GrantRule[]): Promise; sendSourcesUpdate( @@ -319,16 +318,6 @@ export function createSidecarRouter( const pendingPacks = new Map(); let packCounter = 0; - // agentAddress → pending session start (resolved by session.start.ack) - type PendingSessionStart = { - agentAddress: string; - ws: WsHandle; - resolve(): void; - reject(error: string): void; - timer: ReturnType; - }; - const pendingSessionStarts = new Map(); - // agentAddress → pending undeploy (resolved by agent.undeploy.ack) type PendingUndeploy = { agentAddress: string; @@ -453,12 +442,8 @@ export function createSidecarRouter( break; case "agent.error": rejectDeployPending(frame.agentAddress, frame.error); - rejectSessionStartPending(frame.agentAddress, frame.error); rejectUndeployPending(frame.agentAddress, frame.error); break; - case "session.start.ack": - resolveSessionStartPending(frame.agentAddress); - break; case "agent.undeploy.ack": resolveUndeployPending(frame.agentAddress); break; @@ -1051,10 +1036,9 @@ export function createSidecarRouter( } // Create a queue entry so messages can accumulate while the - // sidecar is disconnected. Skip if the agent is being undeployed - // or has a pending session start — there is no point queuing - // messages for an agent being torn down or one that never started. - if (!pendingUndeploys.has(addr) && !pendingSessionStarts.has(addr)) { + // sidecar is disconnected. Skip if the agent is being undeployed -- + // there is no point queuing messages for an agent being torn down. + if (!pendingUndeploys.has(addr)) { const timer = setTimeout(() => { disconnectedAgents.delete(addr); }, disconnectQueueTTLMs); @@ -1105,14 +1089,6 @@ export function createSidecarRouter( pack.reject(`Sidecar ${conn.sidecarId} disconnected`); } - // Reject any in-flight session starts for this sidecar. - for (const [addr, req] of pendingSessionStarts) { - if (req.ws !== ws) continue; - clearTimeout(req.timer); - pendingSessionStarts.delete(addr); - req.reject(`Sidecar ${conn.sidecarId} disconnected`); - } - // Reject any in-flight undeploys for this sidecar. for (const [addr, req] of pendingUndeploys) { if (req.ws !== ws) continue; @@ -1224,28 +1200,6 @@ export function createSidecarRouter( entry.reject(reason); } - function resolveSessionStartPending(agentAddress: string): void { - const req = pendingSessionStarts.get(agentAddress); - if (req === undefined) { - logger.warn`Received session.start.ack for "${agentAddress}" with no pending start`; - return; - } - clearTimeout(req.timer); - pendingSessionStarts.delete(agentAddress); - req.resolve(); - } - - function rejectSessionStartPending( - agentAddress: string, - error: string, - ): void { - const req = pendingSessionStarts.get(agentAddress); - if (req === undefined) return; - clearTimeout(req.timer); - pendingSessionStarts.delete(agentAddress); - req.reject(error); - } - function resolveUndeployPending(agentAddress: string): void { const req = pendingUndeploys.get(agentAddress); if (req === undefined) { @@ -1817,51 +1771,6 @@ export function createSidecarRouter( }); } - function sendSessionStart(agentAddress: string): Promise { - const ws = addressIndex.get(agentAddress); - if (ws === undefined) { - return Promise.reject( - new Error(`No sidecar connected for agent "${agentAddress}"`), - ); - } - const conn = connections.get(ws); - if (conn === undefined) { - return Promise.reject( - new Error(`No sidecar connected for agent "${agentAddress}"`), - ); - } - - return new Promise((resolve, reject) => { - const timer = setTimeout(() => { - pendingSessionStarts.delete(agentAddress); - removeAgentAddress(ws, agentAddress); - reject( - new Error( - `Session start for "${agentAddress}" timed out after ${requestTimeoutMs}ms`, - ), - ); - }, requestTimeoutMs); - - pendingSessionStarts.set(agentAddress, { - agentAddress, - ws, - resolve() { - resolve(); - }, - reject(error: string) { - removeAgentAddress(ws, agentAddress); - reject(new Error(error)); - }, - timer, - }); - - conn.send({ - type: "session.start", - agentAddress, - }); - }); - } - async function sendSessionAbort( agentAddress: string, reason: AbortReason, @@ -2028,7 +1937,6 @@ export function createSidecarRouter( routeMail, sendAgentDeploy, sendAgentUndeploy, - sendSessionStart, sendSessionAbort, sendGrantsUpdate, sendSourcesUpdate, diff --git a/packages/types/src/sidecar.ts b/packages/types/src/sidecar.ts index 9a4347a8..302264b0 100644 --- a/packages/types/src/sidecar.ts +++ b/packages/types/src/sidecar.ts @@ -169,17 +169,6 @@ export const SessionErrorFrame = type({ }); export type SessionErrorFrame = typeof SessionErrorFrame.infer; -/** - * Acknowledges that the inference session has started for a provisioned - * agent. Sent in response to a session.start frame after the harness is - * running. - */ -export const SessionStartAckFrame = type({ - type: "'session.start.ack'", - agentAddress: "string", -}); -export type SessionStartAckFrame = typeof SessionStartAckFrame.infer; - /** * Acknowledges that an agent has been fully undeployed: harness stopped, * state pushed (best-effort), and directory deleted. @@ -352,17 +341,6 @@ export const AgentUndeployFrame = type({ }); export type AgentUndeployFrame = typeof AgentUndeployFrame.infer; -/** - * Start the inference session for a provisioned agent. Sent after the - * deploy pack has been applied so the harness can read deploy-tree tools - * and prompt from disk. - */ -export const SessionStartFrame = type({ - type: "'session.start'", - agentAddress: "string", -}); -export type SessionStartFrame = typeof SessionStartFrame.infer; - /** * Per-address cryptographic challenge. The sidecar must sign * `nonce || utf8(address)` with each agent's private key and respond @@ -739,7 +717,6 @@ export const SidecarFrame = RegisterFrame.or(ReconnectFrame) .or(PingFrame) .or(SessionAckFrame) .or(SessionErrorFrame) - .or(SessionStartAckFrame) .or(AgentUndeployAckFrame) .or(PackPushFrame) .or(PackDoneFrame) @@ -751,7 +728,6 @@ export type SidecarFrame = typeof SidecarFrame.infer; /** All frame types the hub sends to the sidecar. */ export const HubFrame = MailInboundFrame.or(AgentDeployFrame) .or(AgentUndeployFrame) - .or(SessionStartFrame) .or(ChallengeFrame) .or(ChallengeFailedFrame) .or(PongFrame) diff --git a/tests/hub-api/asset-routes.test.ts b/tests/hub-api/asset-routes.test.ts index 04c01a14..3a750e25 100644 --- a/tests/hub-api/asset-routes.test.ts +++ b/tests/hub-api/asset-routes.test.ts @@ -142,7 +142,6 @@ function createMockSidecarRouter(): SidecarRouter { routeMail: () => notImpl("routeMail"), sendAgentDeploy: () => notImpl("sendAgentDeploy"), sendAgentUndeploy: () => notImpl("sendAgentUndeploy"), - sendSessionStart: () => notImpl("sendSessionStart"), sendSessionAbort: () => notImpl("sendSessionAbort"), sendGrantsUpdate: () => notImpl("sendGrantsUpdate"), sendSourcesUpdate: () => notImpl("sendSourcesUpdate"), From 5e782e806422e6f68faabc258c699959a2eec08f Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Fri, 3 Jul 2026 20:06:35 -0500 Subject: [PATCH 041/101] Remove the in-process trivial deploy path Every agent deploy now stages through the workflow-run substrate: a provision-step frame primes a per-step repo, and a workflow frame spawns the supervised workflow-process child. The legacy trivial branch that provisioned a single agent in the host process, with the supervisor committing its run-event chain inline, no longer has a caller. Delete the sidecar router's trivial arm and the run-event projector it drove, and replace the fall-through with an explicit throw for a frame that carries neither shape. Drop the workflow-host supervisor's deploy entry point together with the TrivialLaunch binding and the inline run-event write cluster it fed, leaving that module to seal terminated runs only. Remove the router's onAgentEvent dependency, whose sole reader was the trivial branch. Relocate the slug-collision coverage onto a multi-step frame and delete the trivial-branch tests whose behavior is gone or already covered by the multi-step suite. --- .../agent-key-registration-lifecycle.test.ts | 1 - apps/sidecar/src/index.ts | 2 - ...orkflow-host-wiring-deploy-failure.test.ts | 82 +- ...ow-host-wiring-undeploy-supervisor.test.ts | 3 - apps/sidecar/src/workflow-host-wiring.test.ts | 765 +----------------- apps/sidecar/src/workflow-host-wiring.ts | 370 ++------- packages/hub-agent/src/ws/hub-link.ts | 9 +- packages/workflow-host/README.md | 58 +- packages/workflow-host/src/index.ts | 8 - .../workflow-host/src/supervisor/index.ts | 11 - .../src/supervisor/lifecycle-races.test.ts | 3 - .../supervisor/outbound-signed-send.test.ts | 3 - .../src/supervisor/recycle.test.ts | 3 - .../src/supervisor/run-event-signing.ts | 232 +----- .../src/supervisor/spawn-replay-fifo.test.ts | 3 - .../supervisor/stale-cohort-routing.test.ts | 3 - .../src/supervisor/substrate-write.test.ts | 3 - .../src/supervisor/supervisor.test.ts | 280 ------- .../src/supervisor/supervisor.ts | 73 +- .../workflow-host/src/supervisor/types.ts | 120 +-- 20 files changed, 148 insertions(+), 1884 deletions(-) diff --git a/apps/sidecar/src/agent-key-registration-lifecycle.test.ts b/apps/sidecar/src/agent-key-registration-lifecycle.test.ts index 668fddf6..1af4dd6f 100644 --- a/apps/sidecar/src/agent-key-registration-lifecycle.test.ts +++ b/apps/sidecar/src/agent-key-registration-lifecycle.test.ts @@ -247,7 +247,6 @@ describe("agent signing-key registration lifecycle on the host transport", () => } as unknown as Parameters< typeof createSidecarDeployRouter >[0]["keyStore"], - onAgentEvent: () => () => undefined, transport, repoStore, signingKeySeed: keyPair.privateKey, diff --git a/apps/sidecar/src/index.ts b/apps/sidecar/src/index.ts index 1725de09..e52a75a2 100644 --- a/apps/sidecar/src/index.ts +++ b/apps/sidecar/src/index.ts @@ -387,13 +387,11 @@ const orchestrator = createSidecarOrchestrator({ createDeployRouter: ({ sessions, keyStore, - onAgentEvent, publishWorkflowInferenceEvent, }) => { const router = createSidecarDeployRouter({ sessions, keyStore, - onAgentEvent, transport, repoStore: wrappedRepoStore, signingKeySeed: sidecarSigningKey.privateKey, diff --git a/apps/sidecar/src/workflow-host-wiring-deploy-failure.test.ts b/apps/sidecar/src/workflow-host-wiring-deploy-failure.test.ts index 1bb90c24..7853a977 100644 --- a/apps/sidecar/src/workflow-host-wiring-deploy-failure.test.ts +++ b/apps/sidecar/src/workflow-host-wiring-deploy-failure.test.ts @@ -1,19 +1,16 @@ // Pins H-A1: a deploy-router rejection must not leave the -// `DeploymentAddressRegistry` populated. The deploy router used to -// call `registerDeployment` BEFORE every step that can throw (the -// trivial branch's `provisionAgent`, the multi-step branch's asset -// materialization and `supervisor.spawn`). The link's -// `handleAgentDeploy` catches the rejection and sends `agent.error` -// without invoking `deployRouter.undeploy(frame)`, so the registry -// retained the `(deploymentId -> agentAddress)` mapping for a -// deployment that does not exist. The fix defers registration until -// every throwy step has succeeded; both branches must leave the -// registry clean on deploy failure. +// `DeploymentAddressRegistry` populated. The multi-step branch defers +// `registerDeployment` until every step that can throw (asset +// materialization, `supervisor.spawn`) has succeeded. The link's +// `handleAgentDeploy` catches a rejection and sends `agent.error` +// without invoking `deployRouter.undeploy(frame)`, so a premature +// registration would retain a `(deploymentId -> agentAddress)` mapping +// for a deployment that does not exist. The multi-step test drives that +// failure through a subprocess spawner that throws synchronously. // -// Both branches drive their failure path through this test: -// - trivial: `sessions.provisionAgent` throws. -// - multi-step: the subprocess spawner throws synchronously so -// `supervisor.spawn` rejects. +// The first test pins the router's frame-shape guard: a frame carrying +// neither `provisionStep` nor a workflow definition is rejected before +// any deploy work, so a malformed frame cannot leak registry state. import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; @@ -86,19 +83,19 @@ function makeRouterDeps() { } describe("deploy-failure registry leak", () => { - test("trivial deploy: provisionAgent throws and registry stays clean", async () => { + test("a frame carrying neither provisionStep nor a workflow is rejected before any deploy work", async () => { const { registry, mailRouter, signalRouter, drainRouter, transport } = makeRouterDeps(); - const sessions = stubFailingSessions(); - const keyStore = stubKeyStore(); - + let spawnerInvoked = false; const router = createSidecarDeployRouter({ - sessions, - keyStore, - onAgentEvent: () => () => undefined, + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- an unsupported frame throws before sessions is touched + sessions: {} as Parameters< + typeof createSidecarDeployRouter + >[0]["sessions"], + keyStore: stubKeyStore(), transport, - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- trivial branch reaches provisionAgent before any repoStore usage + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- an unsupported frame throws before any repoStore usage repoStore: {} as Parameters< typeof createSidecarDeployRouter >[0]["repoStore"], @@ -114,35 +111,31 @@ describe("deploy-failure registry leak", () => { multistepMailRouter: mailRouter, multistepSignalRouter: signalRouter, multistepDrainRouter: drainRouter, + multistepSubprocessSpawner: () => { + spawnerInvoked = true; + throw new Error("spawner must not run for an unsupported frame"); + }, }); + // Neither `provisionStep` nor a workflow definition: the router has + // no path to stage this frame through the substrate, so it rejects on + // shape before reaching any deploy work. const frame: AgentDeployFrame = { type: "agent.deploy", - agentAddress: "agent-fail@x.example", - agentId: "agent-fail", + agentAddress: "agent-unsupported@x.example", + agentId: "agent-unsupported", hubPublicKey: "00".repeat(32), - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- trivialLaunch surfaces config to the failing provisionAgent stub - config: { - agentAddress: "agent-fail@x.example", - agentId: "agent-fail", - sessionId: "session-fail", - sources: [], - defaultSource: "primary", - grants: [], - } as unknown as AgentDeployFrame["config"], + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- config is irrelevant; the frame is rejected on shape before config is read + config: {} as unknown as AgentDeployFrame["config"], }; - let threw = false; - try { - await router.deploy(frame); - } catch { - threw = true; - } - expect(threw).toBe(true); - - // `deriveTrivialDeploymentId` replaces every char outside - // `[a-zA-Z0-9_-]` with `-`, so `@` and `.` both become `-`. - const slug = "agent-fail-x-example"; + await expect(router.deploy(frame)).rejects.toThrow( + /unsupported deploy frame/, + ); + // The throw fires before any deploy work: no spawn, and the registry + // is never touched. + expect(spawnerInvoked).toBe(false); + const slug = "agent-unsupported-x-example"; expect(registry.resolve(slug)).toBeNull(); }); @@ -183,7 +176,6 @@ describe("deploy-failure registry leak", () => { const router = createSidecarDeployRouter({ sessions, keyStore, - onAgentEvent: () => () => undefined, transport, // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub: only getRepoDir + writeTree are exercised before the spawn-time failure repoStore: repoStoreStub as RepoStore, diff --git a/apps/sidecar/src/workflow-host-wiring-undeploy-supervisor.test.ts b/apps/sidecar/src/workflow-host-wiring-undeploy-supervisor.test.ts index 06ef2670..ebff0abf 100644 --- a/apps/sidecar/src/workflow-host-wiring-undeploy-supervisor.test.ts +++ b/apps/sidecar/src/workflow-host-wiring-undeploy-supervisor.test.ts @@ -258,9 +258,6 @@ describe("createSidecarDeployRouter multi-step undeploy shuts the supervisor dow } as unknown as Parameters< typeof createSidecarDeployRouter >[0]["keyStore"], - onAgentEvent: () => () => { - /* unused */ - }, transport, repoStore, signingKeySeed: keyPair.privateKey, diff --git a/apps/sidecar/src/workflow-host-wiring.test.ts b/apps/sidecar/src/workflow-host-wiring.test.ts index 9f176c54..b180525f 100644 --- a/apps/sidecar/src/workflow-host-wiring.test.ts +++ b/apps/sidecar/src/workflow-host-wiring.test.ts @@ -9,16 +9,13 @@ import { createInMemoryTransport } from "@intx/mail-memory"; import type { RepoId, RepoStore } from "@intx/hub-sessions"; import { createControlChannelSender, - type CommitRunEventResult, type EventPayload, type FrameReader, type NdjsonReader, type NdjsonWriter, type SubprocessHandle, type SubprocessSpawner, - type SupervisorRunEvent, } from "@intx/workflow-host"; -import type { InferenceEvent } from "@intx/types/runtime"; import type { AgentDeployFrame } from "@intx/types/sidecar"; import { @@ -26,10 +23,8 @@ import { createSidecarDeployRouter, createSidecarWorkflowSupervisor, deriveTrivialDeploymentId, - driveTrivialRunChain, STEP_INFERENCE_SOURCES_ENV_KEY, validateWorkflowProjection, - type TrivialRunCell, } from "./workflow-host-wiring"; import { createMultistepMailRouter, @@ -90,7 +85,6 @@ describe("createSidecarWorkflowSupervisor", () => { `${deploymentId}-${stepId}@example.com`, substrateEnv: { DATA_DIR: "/tmp/wire" }, subprocessSpawner: spawner, - trivialLaunch: () => Promise.resolve(), }); expect(typeof wired.supervisor.spawn).toBe("function"); @@ -128,7 +122,6 @@ describe("createSidecarWorkflowSupervisor", () => { subprocessSpawner: () => { throw new Error("spawner not invoked in this test"); }, - trivialLaunch: () => Promise.resolve(), }); // Without a subscriber, routeInbound is a no-op rather than a // throw -- the wiring's mail bus map is a per-address Set that @@ -139,310 +132,6 @@ describe("createSidecarWorkflowSupervisor", () => { }); }); -describe("driveTrivialRunChain projects reactor events onto the workflow-run chain", () => { - function makeRecorder(): { - calls: SupervisorRunEvent[]; - record: (e: SupervisorRunEvent) => Promise; - } { - const calls: SupervisorRunEvent[] = []; - return { - calls, - record: async (e) => { - calls.push(e); - return { - commitSha: "stub", - seq: calls.length - 1, - signature: { sig: new Uint8Array(64), principalKind: "supervisor" }, - }; - }, - }; - } - - test("completed run drives RunStarted, StepStarted, StepCompleted, RunCompleted", async () => { - const cell: TrivialRunCell = { runId: null, stepStarted: false }; - const { calls, record } = makeRecorder(); - - const started: InferenceEvent = { - type: "message.run.started", - seq: 0, - data: { messageId: "m-1", messageRunId: "r-1", receivedAt: 1 }, - }; - const inferStart: InferenceEvent = { - type: "inference.start", - seq: 1, - data: { model: "test" }, - }; - const ended: InferenceEvent = { - type: "message.run.ended", - seq: 2, - data: { messageRunId: "r-1", messageId: "m-1", status: "completed" }, - }; - - await driveTrivialRunChain(started, record, cell); - await driveTrivialRunChain(inferStart, record, cell); - // A second inference.start within the same bracket does NOT mint a - // duplicate StepStarted. - await driveTrivialRunChain(inferStart, record, cell); - await driveTrivialRunChain(ended, record, cell); - - const kinds = calls.map((c) => c.kind); - expect(kinds).toEqual([ - "RunStarted", - "StepStarted", - "StepCompleted", - "RunCompleted", - ]); - for (const call of calls) { - expect(call.runId).toBe("r-1"); - } - const runStarted = calls[0]; - if (runStarted?.kind !== "RunStarted") { - throw new Error("unreachable"); - } - expect(runStarted.consumedMessageId).toBe("m-1"); - expect(cell.runId).toBeNull(); - expect(cell.stepStarted).toBe(false); - }); - - test("failed run emits StepCompleted but not RunCompleted", async () => { - const cell: TrivialRunCell = { runId: null, stepStarted: false }; - const { calls, record } = makeRecorder(); - - await driveTrivialRunChain( - { - type: "message.run.started", - seq: 0, - data: { messageId: "m-2", messageRunId: "r-2", receivedAt: 1 }, - }, - record, - cell, - ); - await driveTrivialRunChain( - { type: "inference.start", seq: 1, data: { model: "test" } }, - record, - cell, - ); - await driveTrivialRunChain( - { - type: "message.run.ended", - seq: 2, - data: { - messageRunId: "r-2", - messageId: "m-2", - status: "failed", - error: { message: "boom" }, - }, - }, - record, - cell, - ); - - expect(calls.map((c) => c.kind)).toEqual([ - "RunStarted", - "StepStarted", - "StepCompleted", - ]); - }); - - test("inference.start without a live bracket is ignored", async () => { - const cell: TrivialRunCell = { runId: null, stepStarted: false }; - const { calls, record } = makeRecorder(); - - await driveTrivialRunChain( - { type: "inference.start", seq: 0, data: { model: "test" } }, - record, - cell, - ); - - expect(calls).toEqual([]); - expect(cell.runId).toBeNull(); - }); -}); - -describe("createSidecarDeployRouter wires the InferenceEvent subscription to recordRunEvent", () => { - test("the trivial-launch closure brackets a real mail trigger via onAgentEvent", async () => { - const transport = createInMemoryTransport(); - const keyPair = await generateKeyPair(); - - // Capture every `merge` payload the supervisor commits through the - // substrate. Each successful `recordRunEvent` call drives one - // writeTreePreservingPrefix invocation that runs `merge` against - // an empty existing-tree map; we read the files the merge wrote - // back out and snapshot their `type` discriminator. - const writtenEvents: string[] = []; - const repoStore: RepoStore = ((): RepoStore => { - // Mirror the real store's per-repo serialization (withRepoLock) so - // concurrent recordRunEvent writes land in invocation order even - // though the per-event signature is async. - let writeTail: Promise = Promise.resolve(); - const stub: Partial = { - getRepoDir(_repoId: RepoId): string { - return "/tmp/unused"; - }, - writeTreePreservingPrefix(_p, _id, _ref, args) { - const previous = writeTail; - let release: () => void = () => undefined; - writeTail = new Promise((resolve) => { - release = resolve; - }); - return (async () => { - await previous; - try { - const files = await args.merge(new Map()); - for (const value of Object.values(files)) { - const text = - value instanceof Uint8Array - ? new TextDecoder().decode(value) - : value; - const parsed: unknown = JSON.parse(text); - if ( - typeof parsed === "object" && - parsed !== null && - "type" in parsed && - typeof parsed.type === "string" - ) { - writtenEvents.push(parsed.type); - } - } - return { - commitSha: `c-${String(writtenEvents.length)}`, - newlyTerminalRuns: [], - }; - } finally { - release(); - } - })(); - }, - }; - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- in-test stub; the unused RepoStore methods are guarded by the Proxy below - return new Proxy(stub as RepoStore, { - get(target, prop, receiver) { - const value = Reflect.get(target, prop, receiver); - if (value !== undefined) return value; - return () => { - throw new Error(`stub RepoStore: ${String(prop)} not implemented`); - }; - }, - }); - })(); - - // Capture the per-agent InferenceEvent listener the trivial-launch - // closure registers so the test can fire events at it directly. - type CapturedListener = { - address: string; - listener: (e: InferenceEvent) => void; - }; - const captured: CapturedListener[] = []; - const onAgentEvent = ( - address: string, - listener: (e: InferenceEvent) => void, - ): (() => void) => { - captured.push({ address, listener }); - return () => { - const idx = captured.findIndex((c) => c.listener === listener); - if (idx >= 0) captured.splice(idx, 1); - }; - }; - - const router = createSidecarDeployRouter({ - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the single-step branch exercises initRepo; provisionAgent/persistHubPublicKey remain stubbed for the trivial-branch cases in this file - sessions: { - provisionAgent: async (_config: unknown) => ({ - publicKey: "pk-trivial", - keyPair: { - publicKey: new Uint8Array(32), - privateKey: new Uint8Array(32), - }, - }), - persistHubPublicKey: async (_a: string, _h: string) => { - /* no-op */ - }, - initRepo: async (_a: string) => { - /* no-op */ - }, - } as unknown as Parameters< - typeof createSidecarDeployRouter - >[0]["sessions"], - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the deploy-router test exercises only recordHubKey - keyStore: { - recordHubKey: (_a: string, _h: string) => { - /* no-op */ - }, - loadOrGenerateKey: async () => ({ - keyPair: await generateKeyPair(), - isNew: false, - }), - } as unknown as Parameters< - typeof createSidecarDeployRouter - >[0]["keyStore"], - onAgentEvent, - transport, - repoStore, - signingKeySeed: keyPair.privateKey, - createAgentCrypto: createEd25519Crypto, - assertSourceBuildable: () => undefined, - registerDeployment: () => { - /* the in-test repoStore is a stub; the pack-push facade is exercised separately */ - }, - unregisterDeployment: () => { - /* no-op for parity with registerDeployment in this stub */ - }, - }); - - const result = await router.deploy({ - type: "agent.deploy", - agentAddress: "trivial@example.com", - agentId: "trivial-agent", - hubPublicKey: "hub-pk", - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the trivial-launch closure passes config to the SessionManager mock above without inspecting it - config: {} as unknown as Parameters< - ReturnType["deploy"] - >[0]["config"], - }); - expect(result.publicKey).toBe("pk-trivial"); - - if (captured.length !== 1) { - throw new Error( - `expected exactly one onAgentEvent registration, got ${String(captured.length)}`, - ); - } - const entry = captured[0]; - if (entry === undefined) throw new Error("unreachable"); - // The slug derivation strips the `@example.com` suffix; the - // listener is registered against the original frame address. - expect(entry.address).toBe("trivial@example.com"); - - entry.listener({ - type: "message.run.started", - seq: 0, - data: { messageId: "m-1", messageRunId: "r-1", receivedAt: 1 }, - }); - entry.listener({ - type: "inference.start", - seq: 1, - data: { model: "test" }, - }); - entry.listener({ - type: "message.run.ended", - seq: 2, - data: { messageRunId: "r-1", messageId: "m-1", status: "completed" }, - }); - - // recordRunEvent fires are sequenced through Promises and the - // per-event signature is async; poll until all four events land. - for (let i = 0; i < 200 && writtenEvents.length < 4; i++) { - await new Promise((r) => setTimeout(r, 1)); - } - - expect(writtenEvents).toEqual([ - "RunStarted", - "StepStarted", - "StepCompleted", - "RunCompleted", - ]); - }); -}); - describe("createSidecarDeployRouter provision-step (no-spawn) mode", () => { test("a provisionStep frame inits the repo and records the hub key without spawning", async () => { const transport = createInMemoryTransport(); @@ -450,9 +139,6 @@ describe("createSidecarDeployRouter provision-step (no-spawn) mode", () => { const initRepoCalls: string[] = []; const recordHubKeyCalls: { address: string; hubKey: string }[] = []; - // A spawn registers one per-agent InferenceEvent listener; a no-spawn - // provision registers none. - const agentEventRegistrations: string[] = []; // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- provisionStep touches no RepoStore method; the Proxy throws if it ever does const repoStore = new Proxy({} as RepoStore, { @@ -480,10 +166,6 @@ describe("createSidecarDeployRouter provision-step (no-spawn) mode", () => { } as unknown as Parameters< typeof createSidecarDeployRouter >[0]["keyStore"], - onAgentEvent: (address: string) => { - agentEventRegistrations.push(address); - return () => undefined; - }, transport, repoStore, signingKeySeed: keyPair.privateKey, @@ -519,10 +201,8 @@ describe("createSidecarDeployRouter provision-step (no-spawn) mode", () => { // workflow-derived per-step address). expect(result.publicKey).toMatch(/^[0-9a-f]{64}$/); - // Nothing spawned: no supervisor, so no active address and no per-agent - // event listener registered. + // Nothing spawned: no supervisor, so no active address. expect(router.activeAddresses()).toEqual([]); - expect(agentEventRegistrations).toEqual([]); }); }); @@ -828,418 +508,6 @@ describe("computeWireDefinitionHash", () => { }); }); -describe("createSidecarDeployRouter trivial-frame regression", () => { - test("a frame without `workflow` still drives the trivial provisioning path", async () => { - const transport = createInMemoryTransport(); - const keyPair = await generateKeyPair(); - const repoStore = createMinimalStubRepoStore(); - - let provisionAgentCalled = false; - let spawnerInvoked = false; - - const router = createSidecarDeployRouter({ - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub; trivial branch exercises only provisionAgent + persistHubPublicKey - sessions: { - provisionAgent: async (_config: unknown) => { - provisionAgentCalled = true; - return { - publicKey: "pk-trivial-regression", - keyPair: { - publicKey: new Uint8Array(32), - privateKey: new Uint8Array(32), - }, - }; - }, - persistHubPublicKey: async (_a: string, _h: string) => { - /* no-op */ - }, - } as unknown as Parameters< - typeof createSidecarDeployRouter - >[0]["sessions"], - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub - keyStore: { - recordHubKey: (_a: string, _h: string) => { - /* no-op */ - }, - loadOrGenerateKey: async () => ({ - keyPair: await generateKeyPair(), - isNew: false, - }), - } as unknown as Parameters< - typeof createSidecarDeployRouter - >[0]["keyStore"], - onAgentEvent: () => () => { - /* unused in this test */ - }, - transport, - repoStore, - signingKeySeed: keyPair.privateKey, - createAgentCrypto: createEd25519Crypto, - assertSourceBuildable: () => undefined, - registerDeployment: () => { - /* no-op */ - }, - unregisterDeployment: () => { - /* no-op */ - }, - multistepSubprocessSpawner: () => { - spawnerInvoked = true; - throw new Error("the trivial branch must not invoke the spawner"); - }, - }); - - const result = await router.deploy({ - type: "agent.deploy", - agentAddress: "trivial-regression@example.com", - agentId: "trivial-agent", - hubPublicKey: "hub-pk", - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the trivial-launch closure passes config to the SessionManager mock above without inspecting it - config: {} as unknown as Parameters< - ReturnType["deploy"] - >[0]["config"], - }); - - expect(provisionAgentCalled).toBe(true); - expect(spawnerInvoked).toBe(false); - expect(result.publicKey).toBe("pk-trivial-regression"); - }); - - test("two distinct agent addresses whose deriveTrivialDeploymentId slugs collide are rejected at the second deploy", async () => { - // `deriveTrivialDeploymentId` substitutes every disallowed - // character with `-`, so two agent addresses that differ only in - // disallowed characters collapse to the same slug. The slug IS - // the workflow-run repoId, so a silent collision would let the - // second deploy overwrite the first deploy's repo state. The - // slug-claims map rejects the second deploy at the router edge. - const transport = createInMemoryTransport(); - const keyPair = await generateKeyPair(); - const repoStore = createMinimalStubRepoStore(); - const router = createSidecarDeployRouter({ - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub; trivial branch exercises only provisionAgent + persistHubPublicKey - sessions: { - provisionAgent: async (_config: unknown) => ({ - publicKey: "pk-slug-collision", - keyPair: { - publicKey: new Uint8Array(32), - privateKey: new Uint8Array(32), - }, - }), - persistHubPublicKey: async (_a: string, _h: string) => { - /* no-op */ - }, - } as unknown as Parameters< - typeof createSidecarDeployRouter - >[0]["sessions"], - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub - keyStore: { - recordHubKey: (_a: string, _h: string) => { - /* no-op */ - }, - loadOrGenerateKey: async () => ({ - keyPair: await generateKeyPair(), - isNew: false, - }), - } as unknown as Parameters< - typeof createSidecarDeployRouter - >[0]["keyStore"], - onAgentEvent: () => () => { - /* unused in this test */ - }, - transport, - repoStore, - signingKeySeed: keyPair.privateKey, - createAgentCrypto: createEd25519Crypto, - assertSourceBuildable: () => undefined, - registerDeployment: () => { - /* no-op */ - }, - unregisterDeployment: () => { - /* no-op */ - }, - multistepSubprocessSpawner: () => { - throw new Error("the trivial branch must not invoke the spawner"); - }, - }); - - // `agent@a.b.com` and `agent!a!b!com` both project to - // `agent-a-b-com` under the slug derivation. - const first = await router.deploy({ - type: "agent.deploy", - agentAddress: "agent@a.b.com", - agentId: "agent-id-1", - hubPublicKey: "hub-pk", - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the trivial-launch closure forwards config opaquely - config: {} as unknown as Parameters< - ReturnType["deploy"] - >[0]["config"], - }); - expect(first.publicKey).toBe("pk-slug-collision"); - - let caught: unknown; - try { - await router.deploy({ - type: "agent.deploy", - agentAddress: "agent!a!b!com", - agentId: "agent-id-2", - hubPublicKey: "hub-pk", - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- forwarded opaquely - config: {} as unknown as Parameters< - ReturnType["deploy"] - >[0]["config"], - }); - } catch (err) { - caught = err; - } - expect(caught).toBeInstanceOf(Error); - expect(caught instanceof Error && caught.message).toMatch( - /deriveTrivialDeploymentId collision/, - ); - expect(caught instanceof Error && caught.message).toMatch(/agent-a-b-com/); - }); - - test("re-deploying the same address is a no-op claim and succeeds", async () => { - // The slug-claims map records the FIRST claimer's agent - // address; a second claim from the SAME address must not - // throw. Otherwise idempotent re-deploys would be rejected as - // self-collisions. - const transport = createInMemoryTransport(); - const keyPair = await generateKeyPair(); - const repoStore = createMinimalStubRepoStore(); - let provisionCount = 0; - const router = createSidecarDeployRouter({ - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub - sessions: { - provisionAgent: async (_config: unknown) => { - provisionCount += 1; - return { - publicKey: `pk-redeploy-${String(provisionCount)}`, - keyPair: { - publicKey: new Uint8Array(32), - privateKey: new Uint8Array(32), - }, - }; - }, - persistHubPublicKey: async (_a: string, _h: string) => undefined, - } as unknown as Parameters< - typeof createSidecarDeployRouter - >[0]["sessions"], - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub - keyStore: { - recordHubKey: (_a: string, _h: string) => undefined, - loadOrGenerateKey: async () => ({ - keyPair: await generateKeyPair(), - isNew: false, - }), - } as unknown as Parameters< - typeof createSidecarDeployRouter - >[0]["keyStore"], - onAgentEvent: () => () => undefined, - transport, - repoStore, - signingKeySeed: keyPair.privateKey, - createAgentCrypto: createEd25519Crypto, - assertSourceBuildable: () => undefined, - registerDeployment: () => undefined, - unregisterDeployment: () => undefined, - multistepSubprocessSpawner: () => { - throw new Error("trivial branch must not invoke the spawner"); - }, - }); - - const first = await router.deploy({ - type: "agent.deploy", - agentAddress: "redeploy@example.com", - agentId: "redeploy-1", - hubPublicKey: "hub-pk", - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- forwarded opaquely - config: {} as unknown as Parameters< - ReturnType["deploy"] - >[0]["config"], - }); - const second = await router.deploy({ - type: "agent.deploy", - agentAddress: "redeploy@example.com", - agentId: "redeploy-2", - hubPublicKey: "hub-pk", - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- forwarded opaquely - config: {} as unknown as Parameters< - ReturnType["deploy"] - >[0]["config"], - }); - expect(first.publicKey).toBe("pk-redeploy-1"); - expect(second.publicKey).toBe("pk-redeploy-2"); - }); - - test("a failed deploy releases the slug so a subsequent deploy on the same address succeeds", async () => { - // Without the release-on-failure guard, the first deploy's - // `claimSlug` would leak after `provisionAgent` throws, and - // every subsequent retry on the same address would be rejected - // as a phantom collision. - const transport = createInMemoryTransport(); - const keyPair = await generateKeyPair(); - const repoStore = createMinimalStubRepoStore(); - let provisionCount = 0; - const router = createSidecarDeployRouter({ - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub - sessions: { - provisionAgent: async (_config: unknown) => { - provisionCount += 1; - if (provisionCount === 1) { - throw new Error("provision failed (synthetic)"); - } - return { - publicKey: "pk-after-retry", - keyPair: { - publicKey: new Uint8Array(32), - privateKey: new Uint8Array(32), - }, - }; - }, - persistHubPublicKey: async (_a: string, _h: string) => undefined, - } as unknown as Parameters< - typeof createSidecarDeployRouter - >[0]["sessions"], - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub - keyStore: { - recordHubKey: (_a: string, _h: string) => undefined, - loadOrGenerateKey: async () => ({ - keyPair: await generateKeyPair(), - isNew: false, - }), - } as unknown as Parameters< - typeof createSidecarDeployRouter - >[0]["keyStore"], - onAgentEvent: () => () => undefined, - transport, - repoStore, - signingKeySeed: keyPair.privateKey, - createAgentCrypto: createEd25519Crypto, - assertSourceBuildable: () => undefined, - registerDeployment: () => undefined, - unregisterDeployment: () => undefined, - multistepSubprocessSpawner: () => { - throw new Error("trivial branch must not invoke the spawner"); - }, - }); - - let firstCaught: unknown; - try { - await router.deploy({ - type: "agent.deploy", - agentAddress: "retry@example.com", - agentId: "retry-1", - hubPublicKey: "hub-pk", - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- forwarded opaquely - config: {} as unknown as Parameters< - ReturnType["deploy"] - >[0]["config"], - }); - } catch (err) { - firstCaught = err; - } - expect(firstCaught).toBeInstanceOf(Error); - expect(firstCaught instanceof Error && firstCaught.message).toMatch( - /provision failed \(synthetic\)/, - ); - - // Retry on the SAME address must succeed -- if the slug were - // leaked, this would throw `deriveTrivialDeploymentId collision`. - const retry = await router.deploy({ - type: "agent.deploy", - agentAddress: "retry@example.com", - agentId: "retry-2", - hubPublicKey: "hub-pk", - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- forwarded opaquely - config: {} as unknown as Parameters< - ReturnType["deploy"] - >[0]["config"], - }); - expect(retry.publicKey).toBe("pk-after-retry"); - }); - - test("undeploy releases the slug so a different-address deploy on the same slug succeeds", async () => { - // After deploy -> undeploy on `release@a.b.com` (slug - // `release-a-b-com`), a fresh deploy on `release!a!b!com` - // (same slug) must be accepted. Without `releaseSlug` running - // on undeploy, the slug would stay claimed and the second - // deploy would surface a phantom collision. - const transport = createInMemoryTransport(); - const keyPair = await generateKeyPair(); - const repoStore = createMinimalStubRepoStore(); - let provisionCount = 0; - const router = createSidecarDeployRouter({ - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub - sessions: { - provisionAgent: async (_config: unknown) => { - provisionCount += 1; - return { - publicKey: `pk-release-${String(provisionCount)}`, - keyPair: { - publicKey: new Uint8Array(32), - privateKey: new Uint8Array(32), - }, - }; - }, - persistHubPublicKey: async (_a: string, _h: string) => undefined, - destroySession: async (_a: string, _r: string) => undefined, - } as unknown as Parameters< - typeof createSidecarDeployRouter - >[0]["sessions"], - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub - keyStore: { - recordHubKey: (_a: string, _h: string) => undefined, - loadOrGenerateKey: async () => ({ - keyPair: await generateKeyPair(), - isNew: false, - }), - } as unknown as Parameters< - typeof createSidecarDeployRouter - >[0]["keyStore"], - onAgentEvent: () => () => undefined, - transport, - repoStore, - signingKeySeed: keyPair.privateKey, - createAgentCrypto: createEd25519Crypto, - assertSourceBuildable: () => undefined, - registerDeployment: () => undefined, - unregisterDeployment: () => undefined, - multistepSubprocessSpawner: () => { - throw new Error("trivial branch must not invoke the spawner"); - }, - }); - - await router.deploy({ - type: "agent.deploy", - agentAddress: "release@a.b.com", - agentId: "release-1", - hubPublicKey: "hub-pk", - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- forwarded opaquely - config: {} as unknown as Parameters< - ReturnType["deploy"] - >[0]["config"], - }); - if (router.undeploy === undefined) { - throw new Error("router.undeploy is required for this test"); - } - await router.undeploy({ - type: "agent.undeploy", - agentAddress: "release@a.b.com", - reason: "test", - }); - const reclaimed = await router.deploy({ - type: "agent.deploy", - agentAddress: "release!a!b!com", - agentId: "release-2", - hubPublicKey: "hub-pk", - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- forwarded opaquely - config: {} as unknown as Parameters< - ReturnType["deploy"] - >[0]["config"], - }); - expect(reclaimed.publicKey).toBe("pk-release-2"); - }); -}); - describe("createSidecarDeployRouter multi-step branch", () => { async function buildMultistepFixture(opts: { spawner: SubprocessSpawner; @@ -1323,9 +591,6 @@ describe("createSidecarDeployRouter multi-step branch", () => { } as unknown as Parameters< typeof createSidecarDeployRouter >[0]["keyStore"], - onAgentEvent: () => () => { - /* unused in multi-step branch */ - }, transport, repoStore, signingKeySeed: keyPair.privateKey, @@ -2557,4 +1822,32 @@ describe("createSidecarDeployRouter multi-step branch", () => { Buffer.from(fixtureKeyPair.publicKey).toString("hex"), ); }); + + test("two addresses whose deriveTrivialDeploymentId slugs collide are rejected at the second deploy", async () => { + // deriveTrivialDeploymentId substitutes every disallowed character with + // `-`, so two distinct addresses can collapse to the same slug. The slug + // IS the workflow-run repoId, so a silent collision would let the second + // deploy overwrite the first deploy's repo state. claimSlug rejects the + // second deploy at the router edge, before any spawn or repo write. + const spawner = makeReadyDrivingSpawner(10400); + const { router } = await buildMultistepFixture({ + spawner: spawner.spawner, + multistepSubstrateEnv: { + SIDECAR_DATA_DIR: await createTempBaseDir("sidecar-collision-data-"), + }, + }); + + // `ins_col.a@example.com` and `ins_col-a@example.com` both project to + // `ins_col-a-example-com` under the slug derivation. + const deployPromise = router.deploy( + singleStepFrame("ins_col.a@example.com", "wf-collide"), + ); + await spawner.driveReadyFor(0); + await deployPromise; + + await expect( + router.deploy(singleStepFrame("ins_col-a@example.com", "wf-collide")), + ).rejects.toThrow(/deriveTrivialDeploymentId collision/); + expect(spawner.spawnCount()).toBe(1); + }); }); diff --git a/apps/sidecar/src/workflow-host-wiring.ts b/apps/sidecar/src/workflow-host-wiring.ts index c24177e0..a1b50eb1 100644 --- a/apps/sidecar/src/workflow-host-wiring.ts +++ b/apps/sidecar/src/workflow-host-wiring.ts @@ -1,10 +1,8 @@ // Thin wiring module that constructs `createWorkflowSupervisor` with // this sidecar's host-specific bindings: the existing mail-bus // instance, the sidecar's Ed25519 signing keypair, the substrate -// RepoStore handle, `Bun.spawn` as the subprocess spawner, and a -// host-injected `trivialLaunch` callback that drives the legacy -// single-agent provisioning surface for trivial (1-step) deploys. -// Any logic that would benefit a future alternative-sidecar +// RepoStore handle, and `Bun.spawn` as the subprocess spawner. Any +// logic that would benefit a future alternative-sidecar // implementation lives inside `@intx/workflow-host`, not here. import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; @@ -42,12 +40,9 @@ import { type HubTransportMailBusAdapter, type NdjsonReader, type NdjsonWriter, - type RecordRunEvent, type SpawnOpts, type SubprocessHandle, type SubprocessSpawner, - type SupervisorRunEvent, - type TrivialLaunch, type WorkflowSupervisor, } from "@intx/workflow-host"; import { hexEncode } from "@intx/types"; @@ -434,13 +429,6 @@ export type CreateSidecarWorkflowSupervisorOpts = { deriveStepRepoId?: DeriveStepRepoId; /** Substrate-config keys propagated to the child via spawn-time env. */ substrateEnv: Record; - /** - * Host-injected callback the supervisor's trivial branch invokes. - * For the production sidecar this closes over the - * `SessionManager.provisionAgent` flow plus the hub-pairing-key - * recording the legacy `agent.deploy` handler performed inline. - */ - trivialLaunch: TrivialLaunch; /** * Override the subprocess spawner. Tests inject a deterministic * mock; production defaults to the `Bun.spawn`-backed @@ -488,33 +476,6 @@ export type SidecarWorkflowSupervisor = { getCredentialsSnapshot(): CredentialsSnapshot | null; }; -/** - * Construct the sidecar's `DeployRouter`. The router holds the - * `sessions` + `keyStore` closures the workflow-host supervisor's - * `trivialLaunch` needs and constructs a fresh per-deployment - * supervisor on every inbound `agent.deploy` frame. The trivial - * branch then calls back into `sessions.provisionAgent` plus the - * pre-existing hub-pairing-key recording so the bytes flowing - * through the deploy-flow gate test path stay bit-identical to the - * pre-supervisor path. - * - * Multi-step deploys (frames carrying a workflow definition with - * `steps.length >= 2`) route through `supervisor.deploy`'s - * multi-step branch -- the IPC channel, child spawn, and - * `credentialsSnapshot` assembly. The routing seam exists here so - * the frame-format extension that carries the workflow definition is - * a pure data-shape change. - */ -/** - * Stable step identifier the trivial workflow uses for its single - * step. The on-disk `StepStarted` / `StepCompleted` envelopes carry - * this value verbatim; it is opaque to the supervisor and the - * substrate, but downstream audit-log consumers join run events on - * it. The trivial workflow has exactly one step per run, so the id - * is a constant rather than a per-deploy mint. - */ -const TRIVIAL_STEP_ID = "trivial"; - /** * Env key the multi-step branch uses to carry each step's ordered * inference-source failover chain from `frame.workflow.sources` down to @@ -715,7 +676,6 @@ export interface SidecarDeployRouter extends DeployRouter { export function createSidecarDeployRouter(deps: { sessions: SessionManager; keyStore: AgentKeyStore; - onAgentEvent: SessionManager["onAgentEvent"]; transport: HubTransport; repoStore: RepoStore; signingKeySeed: Uint8Array; @@ -752,9 +712,9 @@ export function createSidecarDeployRouter(deps: { * Record a `(deploymentId -> agentAddress)` mapping the boot edge's * workflow-run pack push facade consults when it must address an * outbound pack frame. Fires once per inbound `agent.deploy` frame - * before the supervisor's `deploy()` call so the first `recordRunEvent` - * commit (which triggers the push hook) sees the mapping. Tests that - * do not exercise the pack push path may pass a no-op. + * before the deployment's supervisor spawns, so the first pack push + * the child triggers sees the mapping. Tests that do not exercise + * the pack push path may pass a no-op. */ registerDeployment: (entry: { deploymentId: string; @@ -779,16 +739,14 @@ export function createSidecarDeployRouter(deps: { * the workflow-process child's spawn-time env (see * `SIDECAR_SUBSTRATE_CONFIG_KEYS` in `workflow-substrate-factory.ts`). * The router merges `STEP_INFERENCE_SOURCES` on top per multi-step - * frame. Defaults to an empty record so trivial-only deployments do - * not require the boot edge to thread substrate config through the - * router. + * frame. Defaults to an empty record so a router built without + * substrate config (e.g. a test) needs no boot-edge threading. */ multistepSubstrateEnv?: Record; /** * Subprocess spawner the multi-step branch hands to the supervisor. * Defaults to the production `Bun.spawn`-backed * `defaultSubprocessSpawner`; tests inject a deterministic mock. - * The trivial branch never invokes the spawner. */ multistepSubprocessSpawner?: SubprocessSpawner; /** @@ -830,13 +788,12 @@ export function createSidecarDeployRouter(deps: { * `wired.routeInbound` against the deployment's mail address once * `supervisor.spawn` succeeds so inbound mail aimed at the * deployment address flows into the supervisor's mail-bus - * subscription. The trivial branch never touches this registry -- - * its mail path is the legacy session surface. + * subscription. * - * Optional so tests that exercise the trivial branch (or the - * multi-step branch without an end-to-end mail loop) can omit the - * binding; an absent registry simply means multi-step inbound mail - * cannot route through the hub-link until the wiring is plumbed. + * Optional so tests that exercise the multi-step branch without an + * end-to-end mail loop can omit the binding; an absent registry + * simply means multi-step inbound mail cannot route through the + * hub-link until the wiring is plumbed. */ multistepMailRouter?: MultistepMailRouter; /** @@ -850,10 +807,10 @@ export function createSidecarDeployRouter(deps: { * preserving the workflow-run repo's single-writer invariant on the * sidecar side. * - * Optional so tests that exercise the trivial branch (or the - * multi-step branch without an end-to-end signal loop) can omit the - * binding; an absent registry means hub-side signals cannot route - * through the hub-link until the wiring is plumbed. + * Optional so tests that exercise the multi-step branch without an + * end-to-end signal loop can omit the binding; an absent registry + * means hub-side signals cannot route through the hub-link until the + * wiring is plumbed. */ multistepSignalRouter?: MultistepSignalRouter; /** @@ -868,10 +825,10 @@ export function createSidecarDeployRouter(deps: { * signed `CancelRequested{origin: "supervisor-drain"}` against the * workflow-run repo when the deadline expires. * - * Optional so tests that exercise the trivial branch (or the - * multi-step branch without an end-to-end drain loop) can omit the - * binding; an absent registry means hub-side drain frames cannot - * route through the hub-link until the wiring is plumbed. + * Optional so tests that exercise the multi-step branch without an + * end-to-end drain loop can omit the binding; an absent registry + * means hub-side drain frames cannot route through the hub-link until + * the wiring is plumbed. */ multistepDrainRouter?: MultistepDrainRouter; /** @@ -932,9 +889,9 @@ export function createSidecarDeployRouter(deps: { // (`/workflow-step-state//...`). Resolved once // from the boot-edge substrate env so the undeploy hook can reclaim // the whole subtree. Absent only when the router is wired without - // substrate config (trivial-only paths / tests that never spawn a - // child), in which case no child ever rooted scratch and the - // undeploy reclaim is correctly skipped. + // substrate config (a test that never spawns a child), in which case + // no child ever rooted scratch and the undeploy reclaim is correctly + // skipped. const stepStateDataDir = multistepSubstrateEnv.SIDECAR_DATA_DIR; const multistepSpawner = deps.multistepSubprocessSpawner ?? defaultSubprocessSpawner; @@ -947,10 +904,7 @@ export function createSidecarDeployRouter(deps: { // frame; the supervisor owns the workflow-process child, its IPC // pipes, and its event-channel fd. The undeploy hook consults this // map to call `supervisor.shutdown()` so the child's lifetime ends - // with the deployment. The trivial branch never spawns a child -- - // its supervisor is constructed and immediately dropped after the - // deploy callback runs -- so no entry is recorded for trivial - // deploys and the undeploy hook's lookup is a no-op for them. + // with the deployment. const activeSupervisors = new Map(); // Slug-collision tracking. `deriveTrivialDeploymentId` substitutes @@ -970,6 +924,10 @@ export function createSidecarDeployRouter(deps: { `deriveTrivialDeploymentId collision: agent addresses ${JSON.stringify(existing)} and ${JSON.stringify(agentAddress)} both project to deploymentId ${JSON.stringify(deploymentId)}`, ); } + // A same-address re-claim is a defensive no-op: the `activeSupervisors` + // guard rejects a live re-deploy before claimSlug is re-invoked, and a + // failed or undeployed deploy releases the slug first, so in practice + // `existing` is only ever undefined or a different address here. slugClaims.set(deploymentId, agentAddress); } @@ -1173,14 +1131,6 @@ export function createSidecarDeployRouter(deps: { ...(deps.multistepBinaryPath !== undefined ? { binaryPath: deps.multistepBinaryPath } : {}), - // The multi-step branch never invokes trivialLaunch, but the - // supervisor's constructor requires the binding. Wire a sentinel - // that throws so a stray invocation surfaces loudly. - trivialLaunch: () => { - throw new Error( - "sidecar deploy router: trivialLaunch invoked on the multi-step branch; this is a programming bug", - ); - }, ...(deps.onDispatchTiming !== undefined ? { onDispatchTiming: deps.onDispatchTiming } : {}), @@ -1424,10 +1374,7 @@ export function createSidecarDeployRouter(deps: { // deployment's record (and the catch below would then delete it). A // re-deploy after `undeploy` passes: `undeploy` drops the // `activeSupervisors` entry; a failed deploy that unwound is not in the - // map either. Known boundary: the trivial branch (`frame.workflow` - // undefined) never enters `activeSupervisors`, so a trivial re-deploy - // against a live multi-step address is not caught here; that path is - // slated for removal, taking the boundary with it. + // map either. if (activeSupervisors.has(frame.agentAddress)) { throw new Error( `sidecar deploy router: ${frame.agentAddress} is already deployed; undeploy it before redeploying`, @@ -1546,121 +1493,13 @@ export function createSidecarDeployRouter(deps: { if (frame.workflow !== undefined) { return await deployMultiStep(frame, frame.workflow); } - let publicKey: string | undefined; - const deploymentId = deriveTrivialDeploymentId(frame.agentAddress); - claimSlug(deploymentId, frame.agentAddress); - // Same slug-release discipline as deployMultiStep's claim/try/catch: - // without it, any failure between `claimSlug` and the - // `registerDeployment` call below leaves the slug claimed forever - // (the undeploy hook never fires for failed deploys). - let trivialClaimedSlugSucceeded = false; - try { - const wired = createSidecarWorkflowSupervisor({ - transport: deps.transport, - repoStore: deps.repoStore, - signingKeySeed: deps.signingKeySeed, - workflowRunRepoId: { - kind: "workflow-run", - id: deploymentId, - }, - workflowRunRef: "refs/heads/main", - deploymentId, - // A trivial deploy is a single agent: one step, so the child's - // deploy-tree read collapses onto the head. - stepCount: 1, - deploymentMailAddress: frame.agentAddress, - deriveStepAddress: ({ deploymentId: dep, stepId }) => - `${dep}-${stepId}`, - substrateEnv: {}, - trivialLaunch: async (bindings) => { - const result = await deps.sessions.provisionAgent( - // The trivialLaunch contract treats `config` as opaque - // bytes the host minted; for the sidecar wiring the - // bytes are a `HarnessConfig` the frame carried - // verbatim, and `SessionManager.provisionAgent` - // expects exactly that. - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- frame.config is the validated HarnessConfig the link surfaced - bindings.config as Parameters< - SessionManager["provisionAgent"] - >[0], - ); - deps.keyStore.recordHubKey( - bindings.agentAddress, - bindings.hubPublicKey, - ); - await deps.sessions.persistHubPublicKey( - bindings.agentAddress, - bindings.hubPublicKey, - ); - publicKey = result.publicKey; - // Subscribe to per-agent InferenceEvents and project the - // reactor's run-bracket vocabulary onto the workflow-run - // event chain. The seam is a no-op until the harness - // dispatches the first `message.run.started`; the - // disposer is not held here because the trivial branch - // shares the agent's lifetime with the deployment and - // SessionManager prunes the listener set on - // destroySession through the closure's natural unbind. - // The reactor brackets one workflow-run per inbound - // mail; each `message.run.started` mints a fresh runId - // and brackets the chain. - const cell: TrivialRunCell = { - runId: null, - stepStarted: false, - }; - deps.onAgentEvent(bindings.agentAddress, (event) => { - driveTrivialRunChain(event, bindings.recordRunEvent, cell).catch( - (err: unknown) => { - // Capture rejections inside the listener so a substrate - // failure (e.g. the hub rejecting the workflow-run pack - // push) does not surface as an unhandled rejection on - // the host process. The trivial branch's audit chain - // is best-effort against the deploy path; persistent - // substrate or transport misconfigurations log loudly - // here without killing the agent's reactor. - const msg = err instanceof Error ? err.message : String(err); - logger.warn`trivial run-event recording failed for ${bindings.agentAddress}: ${msg}`; - }, - ); - }); - }, - ...(deps.consumedRetentionMs !== undefined - ? { consumedRetentionMs: deps.consumedRetentionMs } - : {}), - ...(deps.readyTimeoutMs !== undefined - ? { readyTimeoutMs: deps.readyTimeoutMs } - : {}), - }); - await wired.supervisor.deploy({ - agentAddress: frame.agentAddress, - agentId: frame.agentId, - config: frame.config, - hubPublicKey: frame.hubPublicKey, - }); - if (publicKey === undefined) { - throw new Error( - "sidecar deploy router: trivialLaunch did not surface a public key", - ); - } - // Register the deployment-address mapping last so a failure in - // `supervisor.deploy` (e.g. the host-supplied `trivialLaunch` - // callback's `provisionAgent` throwing) leaves the boot-edge - // `DeploymentAddressRegistry` untouched. The link's - // `handleAgentDeploy` catches a rejection here and surfaces - // `agent.error` without invoking the undeploy hook; a partial - // registration would persist a `(deploymentId -> agentAddress)` - // entry for a deployment that never finished standing up. - deps.registerDeployment({ - deploymentId, - agentAddress: frame.agentAddress, - }); - trivialClaimedSlugSucceeded = true; - return { publicKey }; - } finally { - if (!trivialClaimedSlugSucceeded) { - releaseSlug(deploymentId, frame.agentAddress); - } - } + // Every deploy stages through the workflow-run substrate: a + // provision-step frame primes the per-step repo, and a workflow + // frame spawns the supervised child. A frame carrying neither is + // an unsupported shape -- there is no in-process fall-through. + throw new Error( + `sidecar deploy router: unsupported deploy frame for ${frame.agentAddress}; a deploy must carry provisionStep or a workflow definition`, + ); }, async undeploy(frame): Promise { // Symmetric teardown for `deploy`: release the per-deployment @@ -1668,9 +1507,7 @@ export function createSidecarDeployRouter(deps: { // / `drain.deliver` / `mail.inbound` aimed at the dead deployment // address is rejected by the router rather than dispatched into // an orphan supervisor handler. The unregister calls are - // idempotent and safe to invoke for both branches even though - // the trivial branch never registers against the multi-step - // routers; those calls are no-ops when no handler is registered. + // idempotent -- they are no-ops when no handler is registered. // // Routers come down BEFORE the supervisor's `shutdown()` so any // hub-side frame racing the undeploy is dropped at the router @@ -1688,8 +1525,6 @@ export function createSidecarDeployRouter(deps: { // kill + `exited` await internally. The map entry is removed // before the await so a subsequent re-deploy on the same address // cannot observe a stale handle even if `shutdown()` rejects. - // Trivial deploys never enter the map, so this lookup is a no-op - // for the trivial branch. const wired = activeSupervisors.get(frame.agentAddress); if (wired !== undefined) { activeSupervisors.delete(frame.agentAddress); @@ -1735,9 +1570,9 @@ export function createSidecarDeployRouter(deps: { async restoreWorkflowDeployments(): Promise { const dataDir = stepStateDataDir; if (dataDir === undefined) { - // No substrate config was wired (a trivial-only or test router that - // never spawns a child): nothing was ever persisted under this data - // dir, so there is nothing to restore. + // No substrate config was wired (a test router that never spawns a + // child): nothing was ever persisted under this data dir, so there + // is nothing to restore. return; } @@ -1844,123 +1679,14 @@ export function createSidecarDeployRouter(deps: { }; } -/** - * Per-deployment cell tracking the active workflow-run bracket. The - * trivial branch holds one cell per agent: each `message.run.started` - * mints a fresh `runId`, the first `inference.start` after that flips - * `stepStarted` true, and the matching `message.run.ended` reads both - * back out before clearing the cell. Two outstanding brackets at once - * are not possible for the trivial path because the reactor - * serializes inbound messages. - */ -export interface TrivialRunCell { - runId: string | null; - stepStarted: boolean; -} - -/** - * Placeholder definition hash baked into the trivial workflow's - * `RunStarted` envelopes. The trivial workflow's content-addressed - * definition lands with the trivial-deploy capability walk; until - * then the on-disk envelope carries a stable sentinel so audit-log - * consumers see a consistent value across deployments. - */ -const TRIVIAL_DEFINITION_HASH = "trivial:v1"; - -/** - * Map one InferenceEvent into `recordRunEvent` calls when the - * reactor's run-bracket vocabulary lines up with a workflow-run - * lifecycle moment. The mapping is: - * - * - `message.run.started` -> RunStarted (mints a new workflow runId) - * - first `inference.start` after RunStarted -> StepStarted (attempt 1) - * - `message.run.ended` status=completed -> StepCompleted + RunCompleted - * - `message.run.ended` status=failed -> StepCompleted - * - * The runId is the `messageRunId` the reactor minted. `StepStarted` - * is suppressed if a second `inference.start` arrives within the - * same bracket (the trivial workflow's single step does not retry - * inside one run; the next attempt would be a separate workflow-run - * instance). - * - * The supervisor's `SupervisorRunEvent` union covers `RunCompleted` - * but not `RunFailed`; the trivial branch surfaces a failed run by - * recording `StepCompleted` against the step that failed (so the - * per-step audit trail closes) and letting the absence of a - * subsequent `RunCompleted` mark the run as unsuccessful for - * downstream consumers. Widening the supervisor's - * `SupervisorRunEvent` union to carry `RunFailed` lands when the - * substrate kind handler grows the matching signature-verification - * path. - */ -export async function driveTrivialRunChain( - event: InferenceEvent, - recordRunEvent: RecordRunEvent, - cell: TrivialRunCell, -): Promise { - if (event.type === "message.run.started") { - cell.runId = event.data.messageRunId; - cell.stepStarted = false; - const runStarted: SupervisorRunEvent = { - kind: "RunStarted", - runId: event.data.messageRunId, - at: new Date().toISOString(), - definitionHash: TRIVIAL_DEFINITION_HASH, - trigger: { - type: "mail", - payload: { messageId: event.data.messageId }, - }, - consumedMessageId: event.data.messageId, - }; - await recordRunEvent(runStarted); - return; - } - if (event.type === "inference.start") { - if (cell.runId === null) return; - if (cell.stepStarted) return; - cell.stepStarted = true; - await recordRunEvent({ - kind: "StepStarted", - runId: cell.runId, - at: new Date().toISOString(), - stepId: TRIVIAL_STEP_ID, - attempt: 1, - input: { ref: "refs/heads/main" }, - }); - return; - } - if (event.type === "message.run.ended") { - const runId = cell.runId; - if (runId === null) return; - cell.runId = null; - cell.stepStarted = false; - const at = new Date().toISOString(); - await recordRunEvent({ - kind: "StepCompleted", - runId, - at, - stepId: TRIVIAL_STEP_ID, - attempt: 1, - output: { ref: "refs/heads/main" }, - }); - if (event.data.status === "completed") { - await recordRunEvent({ - kind: "RunCompleted", - runId, - at, - }); - } - } -} - /** * Logical mail-audit reference the supervisor stamps onto every * inbox/processing/consumed envelope for sidecar-hosted deployments. * The substrate does not dereference the value; it is a host-side - * pointer the audit consumer joins on. The trivial-branch single-agent - * mail audit is keyed by the deployment id plus the parsed messageId, - * which is unique per inbound message and stable across the FIFO - * pipeline's enqueue/dequeue/markConsumed transitions. + * pointer the audit consumer joins on. The mail audit is keyed by the + * deployment id plus the parsed messageId, which is unique per inbound + * message and stable across the FIFO pipeline's + * enqueue/dequeue/markConsumed transitions. */ export function deriveSidecarMailAuditRef(deploymentId: string): ( messageId: string, @@ -1977,10 +1703,9 @@ export function deriveSidecarMailAuditRef(deploymentId: string): ( /** * Construct a per-deployment supervisor with the sidecar's bindings - * pre-wired. The host calls this once per `agent.deploy` frame; the - * supervisor's trivial branch routes the deploy through the - * host-supplied `trivialLaunch` callback so the on-wire and - * on-disk surfaces stay bit-identical to the pre-supervisor path. + * pre-wired. The router calls this once per multi-step `agent.deploy` + * frame to stand up the workflow-process child that hosts the + * deployment. */ export function createSidecarWorkflowSupervisor( opts: CreateSidecarWorkflowSupervisorOpts, @@ -2012,7 +1737,6 @@ export function createSidecarWorkflowSupervisor( ...(opts.deriveStepRepoId !== undefined ? { deriveStepRepoId: opts.deriveStepRepoId } : {}), - trivialLaunch: opts.trivialLaunch, deriveMailAuditRef: deriveSidecarMailAuditRef(opts.deploymentId), ...(opts.onDispatchTiming !== undefined ? { onDispatchTiming: opts.onDispatchTiming } diff --git a/packages/hub-agent/src/ws/hub-link.ts b/packages/hub-agent/src/ws/hub-link.ts index f38b52b0..65470f64 100644 --- a/packages/hub-agent/src/ws/hub-link.ts +++ b/packages/hub-agent/src/ws/hub-link.ts @@ -186,11 +186,10 @@ export type HubLinkConfig = { keyStore: AgentKeyStore; /** * Routes every inbound `agent.deploy` frame. Production wiring - * supplies a router that calls `supervisor.deploy(frame)` on a - * freshly-constructed workflow-host supervisor whose - * `trivialLaunch` closes over `SessionManager.provisionAgent`. - * The supervisor owns the trivial vs multi-step decision; the - * link does not re-decide. + * supplies a router that stages each deploy through the workflow-run + * substrate: a provision-step frame primes a per-step repo, and a + * workflow frame spawns the supervised workflow-process child. The + * router owns the routing decision; the link does not re-decide. */ deployRouter: DeployRouter; /** diff --git a/packages/workflow-host/README.md b/packages/workflow-host/README.md index cbbba62f..91d211ad 100644 --- a/packages/workflow-host/README.md +++ b/packages/workflow-host/README.md @@ -99,46 +99,30 @@ The constructor argument shape: `deriveStepAddress`, `deriveStepRepoId?`, `ipcKeyPairFactory?` — per-deployment configuration the supervisor needs in its closure state. -- `trivialLaunch: (bindings) => Promise` — host-injected - callback the supervisor invokes on the trivial branch of - `deploy(frame)`. See "Deploy routing" below for the trivial- - branch invariants the callback runs under. ### Deploy routing -`deploy(frame)` is the single ingress for inbound `agent.deploy` -frames. The supervisor decides between two branches and the host -does not re-decide: - -- **Trivial branch (1-step workflows).** The supervisor calls - `bindings.trivialLaunch(frame)` directly. No IPC channel opens. - No workflow-process child spawns. No mail-bus subscription - registers. No workflow-run event is emitted; `signAsPrincipal` - is not invoked. `getCredentialsSnapshot()` continues to return - `null`. The trivial branch is a true passthrough -- the host - callback owns the entire deploy, and the supervisor's other - bindings stay inert. -- **Multi-step branch (`steps.length >= 2`).** The supervisor - routes through the same lifecycle `spawn(opts)` runs: per-step - `agent-state` repo provisioning, key minting, child spawn via - `subprocessSpawner`, mail-bus registration, IPC handshake, and - `credentialsSnapshot` assembly. The multi-step branch's body - lands as `agent.deploy` frames are extended to carry a - `WorkflowDefinition`; the routing seam exists today. - -The `agent.deploy` wire frame currently carries only a -`HarnessConfig` (no workflow definition), so every frame today is -trivial. The supervisor codifies the seam now so the frame-format -extension lands as a pure data-shape change. - -The sidecar's production wiring lives at -`apps/sidecar/src/workflow-host-wiring.ts`: -`createSidecarDeployRouter` constructs a fresh per-deployment -supervisor on every inbound frame whose `trivialLaunch` closes -over `SessionManager.provisionAgent` plus the hub-pairing-key -recording the legacy handler performed inline. The bytes flowing -through the deploy-flow integration test path stay bit-identical -to the pre-supervisor surface. +The sidecar's deploy router is the single ingress for inbound +`agent.deploy` frames; its production wiring lives at +`apps/sidecar/src/workflow-host-wiring.ts` in +`createSidecarDeployRouter`. Every deploy stages through the +workflow-run substrate, and the router decides between two frame +shapes: + +- **Provision-step frame (`provisionStep: true`).** The router + primes the frame's per-step `agent-state` repo and records the + hub key, without constructing a supervisor or spawning a child. + The follow-up full-closure deploy pack then applies into the + primed repo and verifies against the recorded key. +- **Workflow frame (carries a `WorkflowDefinition`).** The router + constructs a fresh per-deployment supervisor and drives its + `spawn(opts)` lifecycle: per-step `agent-state` repo + provisioning, key minting, child spawn via `subprocessSpawner`, + mail-bus registration, IPC handshake, and `credentialsSnapshot` + assembly. + +A frame carrying neither shape is rejected -- there is no +in-process deploy path. ### Lifecycle diff --git a/packages/workflow-host/src/index.ts b/packages/workflow-host/src/index.ts index 3684e30f..339d0c1e 100644 --- a/packages/workflow-host/src/index.ts +++ b/packages/workflow-host/src/index.ts @@ -21,7 +21,6 @@ export { createWorkflowSupervisor, assembleCredentialsSnapshot, commitCancelRequested, - commitRunEvent, createDrainTimeoutAccumulator, createRecyclePolicy, defaultStepRepoId, @@ -40,8 +39,6 @@ export { type CancelRequestOpts, type CommitCancelRequestedOpts, type CommitCancelRequestedResult, - type CommitRunEventOpts, - type CommitRunEventResult, type CredentialsSnapshot, type CredentialsSnapshotStep, type DeliverSignalOpts, @@ -56,7 +53,6 @@ export { type MailAuditRef, type MailBusBindings, type PrincipalSigner, - type RecordRunEvent, type RecycleAttempt, type RecycleContext, type RecycleOpts, @@ -65,17 +61,13 @@ export { type RecyclePolicyBounds, type RecyclePolicyOpts, type SignedPayload, - type SupervisorRunEvent, type SpawnOpts, type SpawnResult, type SubprocessHandle, type SubprocessSpawner, - type SupervisorDeployFrame, type TerminalEventSource, type TerminalRunEvent, type TriggerRecycleOpts, - type TrivialLaunch, - type TrivialLaunchBindings, type DispatchTimingMark, type DispatchStructuralCounters, type DispatchSubstrateLeg, diff --git a/packages/workflow-host/src/supervisor/index.ts b/packages/workflow-host/src/supervisor/index.ts index 7b42e0cc..ae20d0bf 100644 --- a/packages/workflow-host/src/supervisor/index.ts +++ b/packages/workflow-host/src/supervisor/index.ts @@ -31,13 +31,6 @@ export { type CommitCancelRequestedResult, } from "./cancel-signing"; -export { - commitRunEvent, - type CommitRunEventOpts, - type CommitRunEventResult, - type SupervisorRunEvent, -} from "./run-event-signing"; - export { createDrainTimeoutAccumulator, DEFAULT_DRAIN_TIMEOUT_MS, @@ -72,15 +65,11 @@ export type { MailAuditRef, MailBusBindings, PrincipalSigner, - RecordRunEvent, SignedPayload, SubprocessHandle, SubprocessSpawner, - SupervisorDeployFrame, TerminalEventSource, TerminalRunEvent, - TrivialLaunch, - TrivialLaunchBindings, WorkflowSupervisorBindings, WorkflowSupervisorPrincipalKind, } from "./types"; diff --git a/packages/workflow-host/src/supervisor/lifecycle-races.test.ts b/packages/workflow-host/src/supervisor/lifecycle-races.test.ts index 0593480d..7689461a 100644 --- a/packages/workflow-host/src/supervisor/lifecycle-races.test.ts +++ b/packages/workflow-host/src/supervisor/lifecycle-races.test.ts @@ -351,9 +351,6 @@ async function buildBindings(opts: { readPrincipal: { kind: "supervisor" }, deriveStepAddress: ({ deploymentId, stepId }) => `${deploymentId}-${stepId}@example.com`, - trivialLaunch: () => { - throw new Error("trivialLaunch must not run in lifecycle-races tests"); - }, ipcKeyPairFactory: () => Promise.resolve(opts.ipcKeypair), inboxPrimitives: createMemoryInboxPrimitives(), }; diff --git a/packages/workflow-host/src/supervisor/outbound-signed-send.test.ts b/packages/workflow-host/src/supervisor/outbound-signed-send.test.ts index 8dff7d56..df1e75e5 100644 --- a/packages/workflow-host/src/supervisor/outbound-signed-send.test.ts +++ b/packages/workflow-host/src/supervisor/outbound-signed-send.test.ts @@ -273,9 +273,6 @@ describe("supervisor-backed outbound signed send (Phase 4.3)", () => { // getRepoDir resolves the dir; no grants file is needed because the // outbound path never reads grants. deriveStepRepoId: () => ({ kind: "agent-state", id: "outbound-dep" }), - trivialLaunch: () => { - throw new Error("trivialLaunch not used in this test"); - }, inboxPrimitives: createNoopInboxPrimitives(), ipcKeyPairFactory: () => Promise.resolve(supervisorIpcKeyPair), }); diff --git a/packages/workflow-host/src/supervisor/recycle.test.ts b/packages/workflow-host/src/supervisor/recycle.test.ts index b830a062..68bbce9a 100644 --- a/packages/workflow-host/src/supervisor/recycle.test.ts +++ b/packages/workflow-host/src/supervisor/recycle.test.ts @@ -476,9 +476,6 @@ async function buildBindings(opts: { readPrincipal: { kind: "supervisor" }, deriveStepAddress: ({ deploymentId, stepId }) => `${deploymentId}-${stepId}@example.com`, - trivialLaunch: () => { - throw new Error("trivialLaunch must not run in recycle tests"); - }, ipcKeyPairFactory: () => Promise.resolve(opts.ipcKeypair), inboxPrimitives: opts.inboxPrimitives ?? createMemoryInboxPrimitives(), ...(opts.recyclePolicy !== undefined diff --git a/packages/workflow-host/src/supervisor/run-event-signing.ts b/packages/workflow-host/src/supervisor/run-event-signing.ts index cbfeb62f..3662c2b0 100644 --- a/packages/workflow-host/src/supervisor/run-event-signing.ts +++ b/packages/workflow-host/src/supervisor/run-event-signing.ts @@ -1,18 +1,12 @@ -// Run-event signing path for the per-deployment supervisor. +// Run-event compaction path for the per-deployment supervisor. // -// The supervisor commits the canonical run-lifecycle event chain -// (`RunStarted`, `StepStarted`, `StepCompleted`, `RunCompleted`) to -// the workflow-run repo on behalf of the deployment. Both the trivial -// branch (no IPC, no child) and the multi-step branch's -// supervisor-side bookkeeping route through this module so the on- -// disk shape is identical regardless of process topology. -// -// The substrate-side workflow-run kind handler validates that the -// signing principal is `supervisor` and that the principal's -// `deploymentId` equals `repoId.id`; the surface here mirrors the -// `commitCancelRequested` shape so an audit-log walker sees the same -// envelope conventions across every event kind the supervisor -// authors. +// Once a workflow run terminates, the supervisor folds that run's +// per-event `events/.json` blobs into one combined `events.jsonl` +// and drops the per-event files. This shrinks the workflow-run repo's +// file count -- and every per-commit cost that scales with it -- +// without losing any event. The fold writes under the substrate's +// per-repo lock as the `supervisor` principal, whose `deploymentId` +// the workflow-run kind handler checks against `repoId.id`. import type { RepoId, @@ -23,10 +17,8 @@ import { WORKFLOW_RUN_EVENTS_FILE, encodeCombinedEventLog, } from "@intx/hub-sessions/substrate"; -import { hexEncode } from "@intx/types"; import { SUPERVISOR_PRINCIPAL_KIND } from "./cancel-signing"; -import type { PrincipalSigner, SignedPayload } from "./types"; const RUNS_PREFIX = "runs"; const EVENTS_DIR = "events"; @@ -37,204 +29,6 @@ const TERMINAL_EVENT_TYPES = new Set([ "RunCancelled", ]); -/** - * Shape of the run-lifecycle events the supervisor commits inline. - * The discriminator mirrors the workflow state-machine's - * `WorkflowEvent` union without pulling `@intx/workflow` into the - * supervisor module's dependency closure: the supervisor only needs - * to know the on-disk envelope shape, not the full transition - * semantics. The transition function in `@intx/workflow` is the - * authoritative validator for the chain. - */ -export type SupervisorRunEvent = - | { - readonly kind: "RunStarted"; - readonly runId: string; - readonly at: string; - readonly definitionHash: string; - readonly trigger: { readonly type: string; readonly payload: unknown }; - readonly consumedMessageId?: string; - } - | { - readonly kind: "StepStarted"; - readonly runId: string; - readonly at: string; - readonly stepId: string; - readonly attempt: number; - readonly input: { readonly ref: string }; - } - | { - readonly kind: "StepCompleted"; - readonly runId: string; - readonly at: string; - readonly stepId: string; - readonly attempt: number; - readonly output: { readonly ref: string }; - } - | { - readonly kind: "RunCompleted"; - readonly runId: string; - readonly at: string; - }; - -export type CommitRunEventOpts = { - /** Substrate handle the supervisor writes through. */ - substrate: SubstrateRepoStore; - /** Workflow-run repo for this deployment. */ - repoId: RepoId; - /** Events ref the workflow-run repo writes to. */ - ref: string; - /** Deployment id used to construct the supervisor principal. */ - deploymentId: string; - /** Event to commit. */ - event: SupervisorRunEvent; - /** Host-supplied per-principal signing callback. */ - signAsPrincipal: PrincipalSigner; -}; - -export type CommitRunEventResult = { - /** Substrate-assigned commit SHA the append produced. */ - commitSha: string; - /** Per-run sequence number the append landed at. */ - seq: number; - /** Signature the supervisor attached to the event. */ - signature: SignedPayload; -}; - -/** - * Build the canonical bytes the supervisor signs for a run-lifecycle - * event. The on-disk envelope adds a `signature` field after the - * signature is computed; signing the payload without that field keeps - * the verifier's reconstruction trivial (strip `signature`, - * canonicalize, verify). - * - * The substrate-level discriminator field on disk is `type` rather - * than `kind` so the workflow-run kind handler's `EventEnvelope` - * validator matches every supervisor-authored blob without a - * per-event-kind translation. - */ -function buildPayloadBytes(args: { - seq: number; - event: SupervisorRunEvent; -}): Uint8Array { - const canonical = canonicalize({ seq: args.seq, event: args.event }); - return new TextEncoder().encode(JSON.stringify(canonical)); -} - -function canonicalize(args: { - seq: number; - event: SupervisorRunEvent; -}): Record { - const { seq, event } = args; - switch (event.kind) { - case "RunStarted": { - const base: Record = { - type: "RunStarted", - seq, - runId: event.runId, - at: event.at, - definitionHash: event.definitionHash, - trigger: event.trigger, - }; - if (event.consumedMessageId !== undefined) { - base["consumedMessageId"] = event.consumedMessageId; - } - return base; - } - case "StepStarted": - return { - type: "StepStarted", - seq, - runId: event.runId, - at: event.at, - stepId: event.stepId, - attempt: event.attempt, - input: event.input, - }; - case "StepCompleted": - return { - type: "StepCompleted", - seq, - runId: event.runId, - at: event.at, - stepId: event.stepId, - attempt: event.attempt, - output: event.output, - }; - case "RunCompleted": - return { - type: "RunCompleted", - seq, - runId: event.runId, - at: event.at, - }; - } -} - -/** - * Commit a run-lifecycle event signed by the supervisor. The append - * is atomic against concurrent supervisor commits via the substrate's - * `writeTreePreservingPrefix` lock; the merge callback computes the - * next per-run seq by scanning the existing files under the run's - * `events/` subtree. - */ -export async function commitRunEvent( - opts: CommitRunEventOpts, -): Promise { - const prefix = `${RUNS_PREFIX}/${opts.event.runId}/${EVENTS_DIR}/`; - const principal: WorkflowRunSupervisorPrincipal = { - kind: SUPERVISOR_PRINCIPAL_KIND, - deploymentId: opts.deploymentId, - }; - let resolved: { seq: number; signature: SignedPayload } | null = null; - const { commitSha } = await opts.substrate.writeTreePreservingPrefix( - principal, - opts.repoId, - opts.ref, - { - preservePrefix: prefix, - merge: async (existing) => { - let maxSeq = -1; - for (const filepath of existing.keys()) { - const name = filepath.slice(prefix.length); - const match = EVENT_FILENAME_RE.exec(name); - if (match === null) continue; - const seqStr = match[1]; - if (seqStr === undefined) continue; - const seq = Number.parseInt(seqStr, 10); - if (seq > maxSeq) maxSeq = seq; - } - const nextSeq = maxSeq + 1; - const payloadBytes = buildPayloadBytes({ - seq: nextSeq, - event: opts.event, - }); - const signature = await opts.signAsPrincipal( - SUPERVISOR_PRINCIPAL_KIND, - payloadBytes, - ); - resolved = { seq: nextSeq, signature }; - const onDisk: Record = { - ...canonicalize({ seq: nextSeq, event: opts.event }), - signature: serializeSignedPayload(signature), - }; - const files: Record = {}; - for (const [k, v] of existing) files[k] = v; - files[`${prefix}${String(nextSeq)}.json`] = JSON.stringify(onDisk); - return files; - }, - message: `append ${opts.event.kind} for run ${opts.event.runId}`, - }, - ); - if (resolved === null) { - throw new Error( - `supervisor run-event-signing: merge callback did not assign a sequence number for run ${opts.event.runId}`, - ); - } - const final: { seq: number; signature: SignedPayload } = resolved; - return { commitSha, seq: final.seq, signature: final.signature }; -} - export type CompactRunEventsOpts = { /** Substrate handle the supervisor writes through. */ substrate: SubstrateRepoStore; @@ -352,13 +146,3 @@ function isErrnoNotFound(cause: unknown): boolean { if (cause === null || typeof cause !== "object") return false; return (cause as { code?: unknown }).code === "ENOENT"; } - -function serializeSignedPayload(signed: SignedPayload): { - principalKind: string; - sig: string; -} { - return { - principalKind: signed.principalKind, - sig: hexEncode(signed.sig), - }; -} diff --git a/packages/workflow-host/src/supervisor/spawn-replay-fifo.test.ts b/packages/workflow-host/src/supervisor/spawn-replay-fifo.test.ts index 653df60f..fcc185f6 100644 --- a/packages/workflow-host/src/supervisor/spawn-replay-fifo.test.ts +++ b/packages/workflow-host/src/supervisor/spawn-replay-fifo.test.ts @@ -553,9 +553,6 @@ async function boot(opts: { prefix: string }): Promise< readPrincipal: { kind: "supervisor" }, deriveStepAddress: ({ deploymentId, stepId }) => `${deploymentId}-${stepId}@example.com`, - trivialLaunch: () => { - throw new Error("trivialLaunch must not run in this test"); - }, ipcKeyPairFactory: () => Promise.resolve(supervisorIpcKeyPair), inboxPrimitives: gatedInbox.primitives, }; diff --git a/packages/workflow-host/src/supervisor/stale-cohort-routing.test.ts b/packages/workflow-host/src/supervisor/stale-cohort-routing.test.ts index 1c895571..e6e4132d 100644 --- a/packages/workflow-host/src/supervisor/stale-cohort-routing.test.ts +++ b/packages/workflow-host/src/supervisor/stale-cohort-routing.test.ts @@ -450,9 +450,6 @@ describe("H-S2 stale-cohort routing pinch-point", () => { readPrincipal: { kind: "supervisor" }, deriveStepAddress: ({ deploymentId, stepId }) => `${deploymentId}-${stepId}@example.com`, - trivialLaunch: () => { - throw new Error("not used"); - }, ipcKeyPairFactory: () => Promise.resolve(ipcKp), inboxPrimitives: wrappedInbox, }; diff --git a/packages/workflow-host/src/supervisor/substrate-write.test.ts b/packages/workflow-host/src/supervisor/substrate-write.test.ts index 87f29d71..bb2555b1 100644 --- a/packages/workflow-host/src/supervisor/substrate-write.test.ts +++ b/packages/workflow-host/src/supervisor/substrate-write.test.ts @@ -525,9 +525,6 @@ async function bootSupervisor(opts: { readPrincipal: { kind: "supervisor" }, deriveStepAddress: ({ deploymentId, stepId }) => `${deploymentId}-${stepId}@example.com`, - trivialLaunch: () => { - throw new Error("trivialLaunch must not run in this test"); - }, ipcKeyPairFactory: () => Promise.resolve(supervisorIpcKeyPair), inboxPrimitives, ...(opts.terminalWriteWatchdogMs !== undefined diff --git a/packages/workflow-host/src/supervisor/supervisor.test.ts b/packages/workflow-host/src/supervisor/supervisor.test.ts index 378abdd6..15db9552 100644 --- a/packages/workflow-host/src/supervisor/supervisor.test.ts +++ b/packages/workflow-host/src/supervisor/supervisor.test.ts @@ -19,7 +19,6 @@ import { type SubprocessSpawner, type SubprocessHandle, type SignedPayload, - type SupervisorRunEvent, type WorkflowSupervisorBindings, } from "./index"; import { @@ -86,27 +85,6 @@ function readCancelRequestedBlob( return validated; } -const RunEventBlob = type({ - type: "string", - seq: "number", - runId: "string", - at: "string", - signature: { - principalKind: "string", - sig: "string", - }, - "+": "ignore", -}); - -function readRunEventBlob(raw: string): typeof RunEventBlob.infer { - const parsed: unknown = JSON.parse(raw); - const validated = RunEventBlob(parsed); - if (validated instanceof type.errors) { - throw new Error(`unexpected run-event blob shape: ${validated.summary}`); - } - return validated; -} - async function makeTempDir(prefix: string): Promise { const dir = await fs.mkdtemp(path.join(os.tmpdir(), prefix)); return dir; @@ -484,7 +462,6 @@ async function buildBindings(opts: { spawner: SubprocessSpawner; signSpy: (kind: string, payload: Uint8Array) => SignedPayload; mailBus: MailBusBindings; - trivialLaunch?: WorkflowSupervisorBindings["trivialLaunch"]; onWrite?: (args: { principal: { kind: string }; repoId: RepoId; @@ -514,11 +491,6 @@ async function buildBindings(opts: { readPrincipal: { kind: "supervisor" }, deriveStepAddress: ({ deploymentId, stepId }) => `${deploymentId}-${stepId}@example.com`, - trivialLaunch: - opts.trivialLaunch ?? - (() => { - throw new Error("trivialLaunch not provided to this test binding"); - }), inboxPrimitives: opts.inboxPrimitives ?? createMemoryInboxPrimitives(), }; } @@ -551,7 +523,6 @@ describe("createWorkflowSupervisor", () => { mailBus: createMockMailBus(), }); const supervisor = createWorkflowSupervisor(bindings); - expect(typeof supervisor.deploy).toBe("function"); expect(typeof supervisor.spawn).toBe("function"); expect(typeof supervisor.requestCancel).toBe("function"); expect(typeof supervisor.shutdown).toBe("function"); @@ -1362,231 +1333,6 @@ describe("createWorkflowSupervisor", () => { expect(onDisk.signature.sig).toMatch(/^01[0-9a-f]+$/); }); - test("deploy routes the trivial branch through the host-injected trivialLaunch callback", async () => { - const baseDir = await makeTempDir("supervisor-deploy-trivial-"); - const trivialCalls: { - agentAddress: string; - agentId: string; - hubPublicKey: string; - config: unknown; - }[] = []; - const signSpyCalls: { kind: string }[] = []; - const bindings = await buildBindings({ - baseDir, - spawner: () => { - throw new Error( - "subprocessSpawner must not be invoked on the trivial branch", - ); - }, - signSpy: (kind) => { - signSpyCalls.push({ kind }); - return { sig: new Uint8Array(64), principalKind: "supervisor" }; - }, - mailBus: createMockMailBus(), - trivialLaunch: async (b) => { - trivialCalls.push({ - agentAddress: b.agentAddress, - agentId: b.agentId, - hubPublicKey: b.hubPublicKey, - config: b.config, - }); - }, - }); - const supervisor = createWorkflowSupervisor(bindings); - const frame = { - agentAddress: "agent-1@example.com", - agentId: "agent-1", - config: { sentinel: "config-bytes" }, - hubPublicKey: "deadbeef", - }; - await supervisor.deploy(frame); - expect(trivialCalls).toHaveLength(1); - expect(trivialCalls[0]).toEqual({ - agentAddress: "agent-1@example.com", - agentId: "agent-1", - hubPublicKey: "deadbeef", - config: { sentinel: "config-bytes" }, - }); - // No signAsPrincipal calls on the trivial branch -- the - // workflow-process-cancel path never engages. - expect(signSpyCalls).toEqual([]); - // credentialsSnapshot is multi-step-only; the trivial deploy - // does not assemble one. - expect(supervisor.getCredentialsSnapshot()).toBeNull(); - }); - - test("deploy does not register a mailbox or open IPC on the trivial branch", async () => { - const baseDir = await makeTempDir("supervisor-deploy-no-mailbus-"); - const mailBus = createMockMailBus(); - const bindings = await buildBindings({ - baseDir, - spawner: () => { - throw new Error("spawner must not be invoked on the trivial branch"); - }, - signSpy: () => ({ - sig: new Uint8Array(64), - principalKind: "supervisor", - }), - mailBus, - trivialLaunch: () => Promise.resolve(), - }); - const supervisor = createWorkflowSupervisor(bindings); - await supervisor.deploy({ - agentAddress: "agent-2@example.com", - agentId: "agent-2", - config: {}, - hubPublicKey: "cafef00d", - }); - // The mail bus is the multi-step branch's seam; the trivial - // branch must not touch it. - expect(mailBus.registered()).not.toContain("deployment-x@example.com"); - }); - - test("deploy hands recordRunEvent into trivialLaunch and commits the canonical four-event chain", async () => { - const baseDir = await makeTempDir("supervisor-deploy-run-events-"); - const signSpyCalls: { kind: string; payload: Uint8Array }[] = []; - const observedWrites: { - principal: { kind: string }; - repoId: RepoId; - ref: string; - files: Record; - }[] = []; - const bindings = await buildBindings({ - baseDir, - spawner: () => { - throw new Error("spawner must not be invoked on the trivial branch"); - }, - signSpy: (kind, payload) => { - signSpyCalls.push({ kind, payload }); - const sig = new Uint8Array(64); - sig[0] = signSpyCalls.length; - return { sig, principalKind: "supervisor" }; - }, - mailBus: createMockMailBus(), - onWrite: (args) => observedWrites.push(args), - statefulWrites: true, - trivialLaunch: async (b) => { - const runId = "run-trivial-1"; - const messageId = "msg-1"; - const stepId = "step-1"; - const chain: readonly SupervisorRunEvent[] = [ - { - kind: "RunStarted", - runId, - at: "2026-01-01T00:00:00.000Z", - definitionHash: "def-hash-trivial", - trigger: { type: "mail", payload: { to: b.agentAddress } }, - consumedMessageId: messageId, - }, - { - kind: "StepStarted", - runId, - at: "2026-01-01T00:00:00.001Z", - stepId, - attempt: 1, - input: { ref: "refs/heads/main" }, - }, - { - kind: "StepCompleted", - runId, - at: "2026-01-01T00:00:00.002Z", - stepId, - attempt: 1, - output: { ref: "refs/heads/main" }, - }, - { - kind: "RunCompleted", - runId, - at: "2026-01-01T00:00:00.003Z", - }, - ]; - for (const event of chain) { - await b.recordRunEvent(event); - } - }, - }); - const supervisor = createWorkflowSupervisor(bindings); - await supervisor.deploy({ - agentAddress: "agent-4@example.com", - agentId: "agent-4", - config: {}, - hubPublicKey: "abc", - }); - - // Every event in the chain flowed through signAsPrincipal with - // kind `"supervisor"` and against payload bytes containing the - // expected discriminator. - expect(signSpyCalls.map((c) => c.kind)).toEqual([ - "supervisor", - "supervisor", - "supervisor", - "supervisor", - ]); - const decodedPayloads = signSpyCalls.map((c) => - new TextDecoder().decode(c.payload), - ); - expect(decodedPayloads[0]).toContain("RunStarted"); - expect(decodedPayloads[0]).toContain("def-hash-trivial"); - expect(decodedPayloads[0]).toContain("msg-1"); - expect(decodedPayloads[1]).toContain("StepStarted"); - expect(decodedPayloads[2]).toContain("StepCompleted"); - expect(decodedPayloads[3]).toContain("RunCompleted"); - - // Every commit went through the supervisor principal against the - // deployment's workflow-run repo. - expect(observedWrites.length).toBe(4); - for (const write of observedWrites) { - expect(write.principal.kind).toBe("supervisor"); - expect(write.repoId).toEqual({ - kind: "workflow-run", - id: "deployment-x", - }); - expect(write.ref).toBe("refs/heads/main"); - } - - // The on-disk envelopes are filed under runs//events/.json - // with monotonically increasing seq from 0. - const expectedTypes = [ - "RunStarted", - "StepStarted", - "StepCompleted", - "RunCompleted", - ]; - for (const [index, write] of observedWrites.entries()) { - const expectedSeq = index; - const expectedPath = `runs/run-trivial-1/events/${String(expectedSeq)}.json`; - const bytes = write.files[expectedPath]; - if (bytes === undefined) { - throw new Error( - `commit ${String(index)} did not contain the expected event blob at ${expectedPath}`, - ); - } - const expectedType = expectedTypes[index]; - if (expectedType === undefined) { - throw new Error( - `unreachable: expectedTypes[${String(index)}] is undefined`, - ); - } - // Every prior commit's blob is also carried through the prefix- - // preserving merge so the substrate's append-only invariant - // holds; assert the count here so a regression that drops a - // prior blob surfaces at the test seam. - expect(Object.keys(write.files)).toHaveLength(expectedSeq + 1); - const blobJson = - typeof bytes === "string" ? bytes : new TextDecoder().decode(bytes); - const blob = readRunEventBlob(blobJson); - expect(blob.type).toBe(expectedType); - expect(blob.seq).toBe(expectedSeq); - expect(blob.runId).toBe("run-trivial-1"); - expect(blob.signature.principalKind).toBe("supervisor"); - expect(blob.signature.sig.length).toBe(128); - } - // The trivial branch does not assemble a credentials snapshot - // even though the event chain landed; observability and - // process topology are independent surfaces. - expect(supervisor.getCredentialsSnapshot()).toBeNull(); - }); - test("drain() threads the per-cohort terminal broadcaster into each accumulator's opts", async () => { const baseDir = await makeTempDir("supervisor-drain-terminal-source-"); await seedStepGrants( @@ -1894,32 +1640,6 @@ describe("createWorkflowSupervisor", () => { await supervisor.shutdown(); }); - test("deploy surfaces a trivialLaunch failure to the caller", async () => { - const baseDir = await makeTempDir("supervisor-deploy-error-"); - const bindings = await buildBindings({ - baseDir, - spawner: () => { - throw new Error("spawner must not be invoked on the trivial branch"); - }, - signSpy: () => ({ - sig: new Uint8Array(64), - principalKind: "supervisor", - }), - mailBus: createMockMailBus(), - trivialLaunch: () => - Promise.reject(new Error("provisionAgent failed in test")), - }); - const supervisor = createWorkflowSupervisor(bindings); - await expect( - supervisor.deploy({ - agentAddress: "agent-3@example.com", - agentId: "agent-3", - config: {}, - hubPublicKey: "abc", - }), - ).rejects.toThrow(/provisionAgent failed in test/); - }); - test("drain() is a no-op when the supervisor is idle (no spawn has run)", async () => { // Pins the defensive contract for an inbound drain.deliver frame // that lands while the supervisor has no in-flight runs to escalate diff --git a/packages/workflow-host/src/supervisor/supervisor.ts b/packages/workflow-host/src/supervisor/supervisor.ts index 7bfa8624..9d27a407 100644 --- a/packages/workflow-host/src/supervisor/supervisor.ts +++ b/packages/workflow-host/src/supervisor/supervisor.ts @@ -84,7 +84,7 @@ import { type CredentialsSnapshot, } from "./credentials"; import { commitCancelRequested } from "./cancel-signing"; -import { commitRunEvent, compactRunEvents } from "./run-event-signing"; +import { compactRunEvents } from "./run-event-signing"; import { createDrainTimeoutAccumulator, DEFAULT_DRAIN_TIMEOUT_MS, @@ -104,9 +104,7 @@ import type { DispatchSubstrateLeg, InboxPrimitives, MailAuditRef, - RecordRunEvent, SubprocessHandle, - SupervisorDeployFrame, TerminalEventSource, TerminalRunEvent, WorkflowSupervisorBindings, @@ -159,37 +157,6 @@ export const DEFAULT_TERMINAL_WRITE_WATCHDOG_MS = 30_000; * supervisor's internal state is encapsulated. */ export interface WorkflowSupervisor { - /** - * Single ingress for `agent.deploy` frames. The supervisor owns - * the routing decision between trivial (1-step) and multi-step - * workflows -- the host hands the frame off and never re-decides. - * - * Trivial branch (Option Z, locked): - * - Calls `bindings.trivialLaunch(frame)` directly. - * - Does NOT open an IPC channel. - * - Does NOT spawn a workflow-process child. - * - Does NOT emit any workflow-run event. `signAsPrincipal` - * is not invoked; for a trivial deploy there is no - * workflow-process child to cancel, so cancellation stays - * session-destroy at the host's layer. - * - Does NOT assemble a `credentialsSnapshot`; the trivial - * branch leaves `getCredentialsSnapshot()` returning `null`. - * - * Multi-step branch (`steps.length >= 2`): - * - Provisions per-step `agent-state` repos, mints keys, - * spawns the workflow-process child via `subprocessSpawner`, - * registers the deployment's mail address, waits for the - * child's `ready` frame, and assembles the - * `credentialsSnapshot`. This is the body of `spawn(opts)`, - * which is the multi-step branch's worker today. - * - * The `agent.deploy` wire frame today carries only a - * `HarnessConfig` (no workflow definition); every frame is - * therefore trivial. The supervisor codifies the seam so a frame- - * format extension that carries a `WorkflowDefinition` is a pure - * data-shape change. - */ - deploy(frame: SupervisorDeployFrame): Promise; /** * Spawn the workflow-process child, complete the IPC handshake, * assemble the credentialsSnapshot, register the deployment's mail @@ -617,43 +584,6 @@ export function createWorkflowSupervisor( let recyclePolicy: RecyclePolicy | null = null; let recycleInProgress = false; - async function deploy(frame: SupervisorDeployFrame): Promise { - // The `agent.deploy` wire frame currently carries only a - // `HarnessConfig`, which is the trivial-workflow shape (a single - // step derived from the agent's harness). The branching seam - // exists for the multi-step extension; for every frame today - // the supervisor calls trivialLaunch unchanged. - // - // Trivial-branch invariants: - // - No IPC opens, no child spawn, no mail-bus registration. - // - credentialsSnapshot is not assembled; the multi-step - // branch owns it. `getCredentialsSnapshot()` continues to - // return null on the trivial path. - // - Run-lifecycle events are committed inline from the - // supervisor process via `signAsPrincipal`. The supervisor - // hands `recordRunEvent` to the host's `trivialLaunch`; the - // host calls it from its per-message reactor / harness - // lifecycle moments (`message.run.started` / - // `message.run.ended`) so the on-disk event chain matches - // the multi-step branch byte-for-byte. - const recordRunEvent: RecordRunEvent = (event) => - commitRunEvent({ - substrate: bindings.repoStore, - repoId: bindings.workflowRunRepoId, - ref: bindings.workflowRunRef, - deploymentId: bindings.deploymentId, - event, - signAsPrincipal: bindings.signAsPrincipal, - }); - await bindings.trivialLaunch({ - agentAddress: frame.agentAddress, - agentId: frame.agentId, - config: frame.config, - hubPublicKey: frame.hubPublicKey, - recordRunEvent, - }); - } - function onChildCrash(reason: string): void { logger.error`workflow-process control channel crash: {reason}`; void shutdownInternal({ reason }); @@ -2450,7 +2380,6 @@ export function createWorkflowSupervisor( } return { - deploy, spawn, requestCancel, shutdown, diff --git a/packages/workflow-host/src/supervisor/types.ts b/packages/workflow-host/src/supervisor/types.ts index c928159b..164783fb 100644 --- a/packages/workflow-host/src/supervisor/types.ts +++ b/packages/workflow-host/src/supervisor/types.ts @@ -19,19 +19,10 @@ import type { RepoStore as SubstrateRepoStore, ReplayProcessingToInboxResult, } from "@intx/hub-sessions/substrate"; -import type { - InferenceSource, - OutboundMessage, - SendReceipt, -} from "@intx/types/runtime"; +import type { OutboundMessage, SendReceipt } from "@intx/types/runtime"; import type { RunCancelled, RunCompleted, RunFailed } from "@intx/workflow"; -import type { WorkflowDefinition } from "@intx/workflow/definition"; import type { FrameReader, NdjsonReader, NdjsonWriter } from "../ipc/index"; -import type { - CommitRunEventResult, - SupervisorRunEvent, -} from "./run-event-signing"; /** * Terminal workflow-run event the supervisor's drain accumulators @@ -193,108 +184,6 @@ export type SubprocessSpawner = (args: { env: Record; }) => SubprocessHandle; -/** - * Frame the supervisor receives at its deploy ingress. The shape is - * a structural projection of the sidecar's `agent.deploy` frame -- - * the supervisor does not depend on `@intx/types` for the wire - * type. `config` is opaque to the supervisor; the host owns its - * interpretation and passes it through to the trivial-launch - * callback (multi-step routing carries it into spawn-time env in - * later commits). - * - * `workflow` is the multi-step projection. Absent on every trivial- - * launch frame; presence is the discriminator the deploy router uses - * to branch into `supervisor.spawn()` instead of `trivialLaunch`. The - * field carries the workflow definition (so the supervisor can - * construct the per-step substrate env without round-tripping the - * hub) and each step's ordered inference-source failover chain keyed by - * `definition.stepOrder` step ids. - */ -export interface SupervisorDeployFrame { - agentAddress: string; - agentId: string; - config: unknown; - hubPublicKey: string; - workflow?: { - definition: WorkflowDefinition; - sources: Record; - }; -} - -/** - * Callback the supervisor hands to the host so the host's per-message - * reactor / harness lifecycle can drive the canonical run-event chain - * (`RunStarted` -> `StepStarted` -> `StepCompleted` -> `RunCompleted`) - * for the trivial deploy. The supervisor's closure resolves the - * workflow-run repo identity, mints the `signAsPrincipal` signature - * for each event, and commits the on-disk blob; the host calls this - * with the event payload at the appropriate reactor moment. - * - * The on-disk envelope is identical to the one the multi-step branch - * commits via its workflow-process child, which makes a trivial - * deployment's audit trail indistinguishable from a multi-step one - * from a downstream consumer's perspective. The split between the - * two branches is process topology (in-process commit vs IPC-forwarded - * commit), not observability. - */ -export type RecordRunEvent = ( - event: SupervisorRunEvent, -) => Promise; - -/** - * Arguments handed to `trivialLaunch`. Mirrors the deploy frame - * unchanged today (the trivial branch is a true passthrough); kept - * as its own type so future trivial-only context (e.g. a - * supervisor-derived deployment id) can attach without widening - * the deploy frame itself. - * - * `recordRunEvent` is the seam the host wires into its existing - * per-message reactor moments (`message.run.started` / - * `message.run.ended`) to drive the canonical workflow-run event - * chain inline from the supervisor's address space. Hosts that have - * not yet wired the reactor seam supply a `trivialLaunch` body that - * does not invoke the callback; the supervisor commits no events in - * that case but the capability is available without further wiring - * surgery. - */ -export interface TrivialLaunchBindings { - agentAddress: string; - agentId: string; - config: unknown; - hubPublicKey: string; - recordRunEvent: RecordRunEvent; -} - -/** - * Host-injected callback the supervisor invokes on the trivial - * branch. The supervisor's deploy() routes here for every single- - * step deployment; the callback owns the entire trivial deploy. - * - * Invariants preserved by the trivial branch: - * - * - The supervisor does not open an IPC channel. - * - The supervisor does not spawn a workflow-process child. - * - `credentialsSnapshot` is multi-step-only; the trivial branch - * leaves `getCredentialsSnapshot()` returning `null`. - * - * The supervisor DOES emit the canonical workflow-run event chain - * (`RunStarted` / `StepStarted` / `StepCompleted` / `RunCompleted`) - * for the trivial deploy inline from the supervisor process via - * `signAsPrincipal` against the workflow-run repo. The chain fires - * per inbound mail trigger (one run per fire) and is driven by the - * host calling `bindings.recordRunEvent(...)` from its reactor / - * harness lifecycle moments. The trivial-branch observability is - * therefore identical to the multi-step branch's; the two branches - * differ in process topology, not event surface. - * - * The host wires `trivialLaunch` against the legacy single-agent - * provisioning surface so the on-wire bytes and on-disk surfaces - * stay bit-identical to the pre-supervisor path; the `recordRunEvent` - * hook is additive (the host's reactor calls it from existing - * lifecycle brackets without changing the deploy-tree contents). - */ -export type TrivialLaunch = (bindings: TrivialLaunchBindings) => Promise; - /** * Logical pointer to the raw mail bytes the inbox claim-check * envelope stamps. The substrate stores this as the `mailAuditRef` @@ -448,13 +337,6 @@ export interface WorkflowSupervisorBindings { privateKey: Uint8Array; publicKey: Uint8Array; }>; - /** - * Host-injected callback the supervisor invokes on the trivial - * branch of `deploy(frame)`. Required for hosts that route deploy - * frames through `supervisor.deploy`; the multi-step branch does - * not consult it. See `TrivialLaunch` for the invariants. - */ - trivialLaunch: TrivialLaunch; /** * Operator-overridable per-deployment `drainTimeout` in * milliseconds. The supervisor's `drain()` path threads this value From a9b0c83a63244fd9704abd044d8eec18e3e8e901 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Sat, 4 Jul 2026 01:33:33 -0500 Subject: [PATCH 042/101] Retire the in-process session runtime Every agent now runs as a supervised workflow-process child on the workflow-run substrate, so the SessionManager no longer needs to build harnesses, provision agents, restore sessions from disk, or audit per-agent mail. What survives is the thin per-agent repo-op serializer the deploy path and the hub-link still call: deploy and asset-pack applies, state-pack and deploy-ref reads, and directory teardown. Strip SessionManager down to that surface, collapsing its config to the repo store alone. The hub-link open handler now ships a single register frame carrying the sidecar's workflow-deployment addresses instead of the empty-register-then-restore-then-reconnect dance, and its grants, sources, session-abort, and legacy mail-fallback handlers go with the methods they drove. The orchestrator stops constructing the harness builder and the connector-state and deploy-apply-error sinks for the session manager, and HarnessBuilder collapses to the source-admission check the deploy router still consults. The default sidecar harness builder drops its harness construction, keeping only that check. Rewrite the affected tests: the session-manager suite covers the surviving repo-op serialization, the hub-link register test asserts the single frame carries workflow addresses, and the verify-commit coverage moves onto the deploy path that now records the hub key. --- apps/sidecar/src/config.ts | 53 -- apps/sidecar/src/default-harness.ts | 340 +------- apps/sidecar/src/index.ts | 53 +- apps/sidecar/src/workflow-host-wiring.ts | 6 +- packages/hub-agent/src/harness-builder.ts | 100 +-- packages/hub-agent/src/index.ts | 14 +- .../hub-agent/src/session-manager.test.ts | 725 +++--------------- packages/hub-agent/src/session-manager.ts | 712 +---------------- .../hub-agent/src/sidecar-orchestrator.ts | 84 +- .../src/ws/hub-link-bootstrap-prune.test.ts | 71 +- .../src/ws/hub-link-mail-router.test.ts | 86 +-- packages/hub-agent/src/ws/hub-link.test.ts | 558 ++------------ packages/hub-agent/src/ws/hub-link.ts | 384 +++------- packages/tool-packaging/src/loader.ts | 10 +- .../src/mail-bus/hub-transport-adapter.ts | 22 +- 15 files changed, 390 insertions(+), 2828 deletions(-) diff --git a/apps/sidecar/src/config.ts b/apps/sidecar/src/config.ts index a9223a6f..21425e95 100644 --- a/apps/sidecar/src/config.ts +++ b/apps/sidecar/src/config.ts @@ -9,7 +9,6 @@ // re-reading env, so the boundary stays at the boot edge. import { AdapterManifest } from "@intx/inference"; -import type { GCPolicy, RetentionPolicy } from "@intx/storage-isogit"; const DEFAULT_CACHE_MAX_BYTES = 10 * 1024 * 1024 * 1024; @@ -69,55 +68,3 @@ export function readAdapterManifest(): AdapterManifest { } return AdapterManifest.assert(parsed); } - -// Write-path GC for the sidecar's deployed-agent repos. The reactor -// commits loose objects every cycle, so these repos accumulate loose -// objects far faster than packs; the loose threshold is the dominant -// trigger here. Retention defaults to tip-only: the sidecar treats the -// repo as the agent's current state, not a long-term archive, so dropping -// commit history keeps the repo small for reactor read/commit latency. An -// operator that needs the history preserved sets -// SIDECAR_AGENT_GC_RETENTION=keep-history. -const DEFAULT_SIDECAR_AGENT_GC_PACK_THRESHOLD = 16; -const DEFAULT_SIDECAR_AGENT_GC_LOOSE_THRESHOLD = 512; -const DEFAULT_SIDECAR_AGENT_GC_WARN_BYTES = 128 * 1024 * 1024; - -function readPositiveIntEnv(name: string, fallback: number): number { - const raw = process.env[name]; - if (raw === undefined || raw.trim() === "") return fallback; - const n = Number(raw); - if (!Number.isInteger(n) || n <= 0) { - throw new Error(`${name} must be a positive integer; got ${raw}`); - } - return n; -} - -function readRetentionEnv( - name: string, - fallback: RetentionPolicy, -): RetentionPolicy { - const raw = process.env[name]; - if (raw === undefined || raw.trim() === "") return fallback; - if (raw === "tip-only" || raw === "keep-history") return raw; - throw new Error( - `${name} must be "tip-only" or "keep-history"; got ${JSON.stringify(raw)}`, - ); -} - -export function readAgentGCPolicy(): GCPolicy { - return { - packThreshold: readPositiveIntEnv( - "SIDECAR_AGENT_GC_PACK_THRESHOLD", - DEFAULT_SIDECAR_AGENT_GC_PACK_THRESHOLD, - ), - looseThreshold: readPositiveIntEnv( - "SIDECAR_AGENT_GC_LOOSE_THRESHOLD", - DEFAULT_SIDECAR_AGENT_GC_LOOSE_THRESHOLD, - ), - warnBytes: readPositiveIntEnv( - "SIDECAR_AGENT_GC_WARN_BYTES", - DEFAULT_SIDECAR_AGENT_GC_WARN_BYTES, - ), - retention: readRetentionEnv("SIDECAR_AGENT_GC_RETENTION", "tip-only"), - }; -} diff --git a/apps/sidecar/src/default-harness.ts b/apps/sidecar/src/default-harness.ts index 3679c308..dac66522 100644 --- a/apps/sidecar/src/default-harness.ts +++ b/apps/sidecar/src/default-harness.ts @@ -1,73 +1,21 @@ // Default HarnessBuilder for the sidecar app. // -// Implements the HarnessBuilder seam declared by @intx/hub-agent using -// the concrete plugins the sidecar app ships with: posix + LSP tools, -// the mail tools package, the isogit-backed context and mail audit -// stores, the authz engine, and the inference runtime. Any host that -// wants a different mix of plugins ships its own builder; the package -// never sees these concrete dependencies. +// Implements the HarnessBuilder source-admission seam declared by +// @intx/hub-agent using the sidecar app's adapter registry. The package +// never sees the concrete inference dependencies the check consults. -import fs from "node:fs"; -import path from "node:path"; -import { evaluateGrants } from "@intx/authz"; -import { - type BaseEnv, - createDefaultDirectorRegistry, - defineAgent, -} from "@intx/agent"; -import { createHarness, type MailEnv } from "@intx/harness"; -import { - readDeployTree, - type HarnessBuilder, - type HarnessBundle, -} from "@intx/hub-agent"; -import { createDependencies, type AdapterRegistry } from "@intx/inference"; -import { getLogger } from "@intx/log"; -import { - createIsogitStore, - createMailAuditStore, - type GCPolicy, -} from "@intx/storage-isogit"; +import { type AdapterRegistry } from "@intx/inference"; +import type { HarnessBuilder } from "@intx/hub-agent"; import type { InferenceSource } from "@intx/types/runtime"; -import { materializeToolPackages } from "./tool-materialization"; - -const logger = getLogger(["sidecar", "harness-builder"]); - export interface DefaultHarnessBuilderConfig { - /** - * Root directory of the content-addressable tarball cache. The - * loader instantiated on each apply opens a fresh `TarballCache` - * against this path. - */ - readonly cacheRoot: string; - /** - * Cache size cap, resolved at the boot edge so the per-apply - * `TarballCache` receives a concrete value rather than re-reading - * env at a non-boundary call site. - */ - readonly cacheMaxBytes: number; - /** - * Per-tarball cap on the loader's HTTP-registry fetcher. Resolved - * at the boot edge from `SIDECAR_REGISTRY_MAX_TARBALL_BYTES` so the - * per-apply loader receives a concrete value. - */ - readonly registryMaxTarballBytes: number; /** * Adapter registry resolved once at the boot edge (built-ins merged - * with any operator-configured custom adapters). It backs both the - * `canBuildSource` membership check and the per-agent - * `env.deps.adapters` used to resolve inference adapters at run time, - * so the single-agent in-process path resolves the same provider set - * the operator configured. + * with any operator-configured custom adapters). It backs the + * `canBuildSource` membership check the deploy router calls to admit a + * step's pinned inference source before spawning. */ readonly adapters: AdapterRegistry; - /** - * Write-path GC policy for the per-agent context repo. Resolved at the - * boot edge and handed to `createIsogitStore` so the reactor's commits - * reclaim the repo once it crosses the policy's thresholds. - */ - readonly gcPolicy: GCPolicy; } export function createDefaultHarnessBuilder( @@ -81,277 +29,5 @@ export function createDefaultHarnessBuilder( ); } }, - - async build({ - agentAddress, - agentConfig, - sources, - defaultSource, - storeDir, - agentTransport, - crypto, - onEvent, - onConnectorStateChanged, - emitDeployApplyError, - }): Promise { - const signer = (payload: string) => crypto.signSSH(payload); - - const storage = await createIsogitStore( - storeDir, - signer, - config.gcPolicy, - ); - const mailStore = await createMailAuditStore(storeDir, signer); - - const deployTree = await readDeployTree(storeDir); - const systemPrompt = deployTree.systemPrompt ?? agentConfig.systemPrompt; - - // Materialize tool-package manifest (if present) via the loader. - // Failures emit a deploy.apply.error frame back to the hub and - // abort harness construction; the prior deploy remains active. - const { factories: loadedToolFactories, pluginFactories: loadedPlugins } = - await materializeToolPackages({ - rawManifestBytes: deployTree.toolPackageManifestRaw, - assetMounts: deployTree.assetMounts, - storeDir, - agentAddress, - cacheRoot: config.cacheRoot, - cacheMaxBytes: config.cacheMaxBytes, - registryMaxTarballBytes: config.registryMaxTarballBytes, - emitDeployApplyError, - }); - - const grantsRef = { current: agentConfig.grants }; - const { principalId, tenantId } = agentConfig; - const authorize = async ( - resource: string, - action: string, - _context: unknown, - ) => - evaluateGrants(grantsRef.current, resource, action, { - principalId, - tenantId, - }); - - const workDir = path.join(storeDir, "workspace"); - await fs.promises.mkdir(workDir, { recursive: true }); - - // Per the @intx/agent ToolBundle contract, the agent does not - // invoke `bundle.dispose` on normal shutdown — only on - // construction rollback. The caller of `createHarness` is - // responsible for disposing the bundle resources at session - // end. Wrap each loaded factory so its bundle's `dispose` - // (when present) lands in `capturedDisposers`; the harness's - // `disposers` array below replays them on session teardown. - // Dedupe by disposer-function identity: a factory whose bundle - // returns the same `dispose` closure on every invocation would - // otherwise re-append it on agent rebuild / hot-reapply paths - // and the teardown loop would call it once per push. A `Set` - // keyed on the closure reference collapses identical pushes - // while still capturing distinct disposers from genuinely - // distinct bundles. - const capturedDisposers = new Set<() => unknown>(); - const factoriesWithCapture = loadedToolFactories.map((f) => { - const wrapped = (env: BaseEnv) => { - const bundle = f(env); - if (bundle.dispose !== undefined) - capturedDisposers.add(bundle.dispose); - return bundle; - }; - // Freeze the harness wrapper so downstream consumers cannot - // mutate `wrapped.id` / `wrapped.requires`. The loader already - // froze the source factory's wrapper, but `Object.assign` here - // produces a fresh outer object whose own `id`/`requires` - // would be writable without this freeze — defeating the - // invariant the loader's wrapper documents. Object.freeze on - // the outer reference is enough; the inner `requires` array - // is already frozen at the loader layer. - return Object.freeze( - Object.assign(wrapped, { id: f.id, requires: f.requires }), - ); - }); - - const def = defineAgent({ - id: agentAddress, - systemPrompt, - // Mail, posix, and LSP travel through the loader path as - // ordinary tool packages — the agent definition pins them and - // the materialization above resolved their factories. The - // sidecar harness no longer constructs them directly. - tools: factoriesWithCapture, - capabilities: [], - inference: { - sources: sources.map((s) => ({ - provider: s.provider, - model: s.model, - })), - }, - }); - - const baseEnv: MailEnv = { - sources, - defaultSource, - storage, - workdir: workDir, - audit: storage, - authorize, - directors: createDefaultDirectorRegistry(), - // Resolve inference adapters through the boot-edge registry - // (built-ins + operator custom adapters). Without this the agent - // would fall back to the built-ins-only default and a - // custom-provider source would fail to resolve at run time even - // though `canBuildSource` admitted it. - deps: createDependencies(config.adapters), - transport: agentTransport, - address: agentAddress, - onConnectorStateChanged, - }; - - // Instantiate plugin factories one at a time so each successive - // factory sees the prior plugins' instances on `env.plugins`. - // The chain is rebuilt incrementally rather than handing every - // factory the same baseEnv (which would force a plugin that - // wanted to consume other plugins to receive an undefined slot). - // Posix's bundle reads `env.plugins` and threads ToolPlugin- - // shaped values into `createPosixTools`; LSP's plugin factory - // is what populates them. - // - // If a midway factory throws, every plugin instance already - // constructed has to release whatever it acquired (LSP starts a - // subprocess) before the construction error propagates. - // Otherwise a partial-success chain leaks resources the - // harness never owned. - const pluginInstances: unknown[] = []; - let chainEnv: MailEnv = baseEnv; - try { - for (const factory of loadedPlugins) { - const instance = factory(chainEnv); - pluginInstances.push(instance); - // Each iteration constructs a fresh `chainEnv` object so a - // factory observing `env.plugins` sees the prior chain - // entries. The non-`plugins` fields (storage, transport, - // address, etc.) are shared by reference across every - // chainEnv produced here, so factories must not compare - // envs by identity — equality checks across iterations - // would always return false even though the underlying - // substrate is the same object graph. - chainEnv = { - ...baseEnv, - plugins: [...pluginInstances], - }; - } - } catch (err) { - for (const instance of pluginInstances) { - if (instance === null || typeof instance !== "object") continue; - if (!("dispose" in instance)) continue; - const dispose: unknown = (instance as { dispose: unknown }).dispose; - if (typeof dispose !== "function") continue; - try { - // `await` accepts non-promise values verbatim, so this - // works whether the plugin's disposer is sync or async - // without the Promise.resolve indirection that would - // silently flatten a thrown-then-rejected pair. - const result: unknown = dispose.call(instance); - await result; - } catch (disposeErr) { - // A disposer that throws during harness construction - // rollback leaks whatever the plugin acquired (LSP starts - // a subprocess; a leak survives the harness teardown). At - // this point the tool-package apply itself already - // succeeded — materializeToolPackages returned — so the - // failure is not attributable to any specific apply - // attempt. Emitting a deploy.apply.error frame with a - // freshly-minted attemptId and a `none` previousDeployId - // would land in the hub's audit trail uncorrelated with - // any real apply record, so log the leak and rely on the - // sidecar log line; the hub's session-launch failure - // (raised by the outer throw below) already informs the - // operator that the harness did not come up. - logger.error`plugin dispose failed during harness construction rollback: ${disposeErr instanceof Error ? disposeErr.message : String(disposeErr)}`; - } - } - throw err; - } - const env: MailEnv = chainEnv; - - const harness = await createHarness(def, env); - - // The sidecar's event channel expects onEvent calls. The - // harness exposes the underlying agent's event stream; forward - // every event onto the legacy callback EXCEPT `message.received`, - // which is the single intentional exclusion. - // - // Why `message.received` is filtered: - // - It is an assembly-internal signal. The reactor emits it - // when an inbound mail dequeues, but the hub-facing audit - // chain expresses per-message work as the - // `message.run.started` / `message.run.ended` bracket pair, - // minted around the same dequeue. Forwarding both would - // double-count per-message work in downstream consumers - // (SessionManager, workflow-runtime translation) and would - // leak the dequeue-vs-bracket distinction into a layer that - // has no use for it. - // - The raw inbound mail bytes are already authoritative in - // the mail-audit store; the bracket events carry messageId - // plus the reactor-minted messageRunId, which is what the - // audit chain needs to correlate. - // - // The filter is an allowlist-of-everything-except, so new - // InferenceEvent members flow through by default. In particular, - // `message.run.started` and `message.run.ended` are forwarded - // unchanged; widening this filter to exclude other event types - // would silently drop hub-facing audit data and must be done - // deliberately. - // - // `harness.stream()` is invoked synchronously here so the - // underlying StreamConsumer is registered before any microtask - // runs. The IIFE below iterates the returned AsyncIterable; - // events emitted in the window between createHarness() - // resolving and the for-await loop starting are buffered by the - // consumer and delivered on the first iteration. - const events = harness.stream(); - let stopForward = false; - const forwardDone = (async () => { - try { - for await (const event of events) { - if (stopForward) break; - if (event.type === "message.received") continue; - onEvent(event); - } - } catch (cause) { - logger.warn`Event forwarder terminated: ${cause}`; - } - })(); - - return { - harness, - mailStore, - updateGrants(grants) { - grantsRef.current = grants; - }, - // The event-forwarder shutdown runs first so no more events - // reach a downstream that is about to release resources. Then - // every captured `ToolBundle.dispose` runs in registration - // order — the agent built the bundles in factory order, so - // disposing in the same order matches that build sequence. - // Each dispose is wrapped in its own try so one runner's - // failure does not stop the others from running. - disposers: [ - async () => { - stopForward = true; - await forwardDone; - }, - async () => { - for (const dispose of capturedDisposers) { - try { - await dispose(); - } catch (disposeErr) { - logger.warn`tool bundle dispose failed during session teardown: ${disposeErr instanceof Error ? disposeErr.message : String(disposeErr)}`; - } - } - }, - ], - }; - }, }; } diff --git a/apps/sidecar/src/index.ts b/apps/sidecar/src/index.ts index e52a75a2..99a7a105 100644 --- a/apps/sidecar/src/index.ts +++ b/apps/sidecar/src/index.ts @@ -17,7 +17,6 @@ import { loadAdapterRegistry } from "@intx/inference/providers"; import { readAdapterManifest, - readAgentGCPolicy, readCacheMaxBytes, readRegistryMaxTarballBytes, } from "./config"; @@ -26,9 +25,8 @@ import { createDefaultHarnessBuilder } from "./default-harness"; // graph so the supervisor surface is reachable from this binary. // `createSidecarDeployRouter` is the production routing the // orchestrator hands to the link's `agent.deploy` handler; every -// inbound frame flows through a freshly-constructed workflow-host -// supervisor whose trivial branch calls back into the sidecar's -// existing single-agent provisioning surface. +// inbound frame stages through the workflow-run substrate, spawning a +// supervised workflow-process child for a workflow deploy. import type { DispatchTimingMark } from "@intx/workflow-host"; import { @@ -58,9 +56,9 @@ function requireEnv(name: string): string { const dataDir = requireEnv("SIDECAR_DATA_DIR"); -// Resolve cache configuration at the boot edge so the per-apply loader -// inside the harness builder receives a concrete path and cap rather -// than re-reading env at non-boundary call sites. +// Resolve cache configuration at the boot edge so the workflow-child's +// per-apply loader receives a concrete path and cap through its spawn +// env rather than re-reading env at non-boundary call sites. const sidecarCacheDir = process.env["SIDECAR_CACHE_DIR"]; const cacheRoot = sidecarCacheDir !== undefined && sidecarCacheDir.trim() !== "" @@ -68,15 +66,14 @@ const cacheRoot = : path.join(dataDir, "cache", "tarballs"); const cacheMaxBytes = readCacheMaxBytes(); const registryMaxTarballBytes = readRegistryMaxTarballBytes(); -const agentGCPolicy = readAgentGCPolicy(); // Operator-configured custom inference adapters, resolved once at the // boot edge. `loadAdapterRegistry` merges the statically-linked // built-ins with any custom adapters the manifest names, importing each // custom module eagerly here so a bad specifier fails the sidecar at -// boot rather than at first inference. The SAME registry is threaded -// into the in-process single-agent harness builder below; the workflow -// child cannot receive this object across the fork, so the validated +// boot rather than at first inference. The SAME registry backs the +// deploy router's source-admission check below; the workflow child +// cannot receive this object across the fork, so the validated // manifest is serialized into the child's spawn env (see // `multistepSubstrateEnv`) and the child rebuilds an equivalent // registry from it. Specifiers are operator-config-only — the agent @@ -220,15 +217,13 @@ const agentRepoStore = createAgentRepoStore({ // agentAddress for hub-side routing. const deploymentAddressRegistry = createDeploymentAddressRegistry(); -// Per-deployment-address mail handler registry the hub-link consults -// before falling back to the legacy `transport.deliver` / -// `sessions.commitInboundMail` path. The deploy router's multi-step +// Per-deployment-address mail handler registry the hub-link consults on +// every inbound `mail.inbound` frame. The deploy router's multi-step // branch registers `wired.routeInbound` against the deployment's mail // address once its supervisor spawns; the hub-link's `mail.inbound` -// handler calls `tryRoute` first so an inbound deployment-address -// message lands on the supervisor's mail-bus subscription instead of -// the legacy session path (which has no `transport` registration and -// no `sessions` entry for the deployment address). +// handler calls `tryRoute` so an inbound deployment-address message +// lands on the supervisor's mail-bus subscription. Mail for an address +// with no registered handler has no receiver and is logged-and-dropped. const multistepMailRouter = createMultistepMailRouter(); // Per-deployment-address signal handler registry the hub-link consults @@ -314,9 +309,9 @@ const multistepSubstrateEnv: Record = { SIDECAR_TOKEN: sidecarToken, PATH: requireEnv("PATH"), // Tool-loader caps the child's per-step tool materialization uses. - // Resolved once at the boot edge (same values the in-process harness - // builder receives) and threaded into the child through the substrate - // config so the child does not re-read env at a non-boundary site. + // Resolved once at the boot edge and threaded into the child through + // the substrate config so the child does not re-read env at a + // non-boundary site. SIDECAR_CACHE_MAX_BYTES: String(cacheMaxBytes), SIDECAR_REGISTRY_MAX_TARBALL_BYTES: String(registryMaxTarballBytes), // Serialize the parent's ALREADY-VALIDATED manifest (the object @@ -338,16 +333,10 @@ if (hostTmpdir !== undefined) { multistepSubstrateEnv["TMPDIR"] = hostTmpdir; } -// Hoisted so the deploy router's source-admission gate reuses the exact -// same `canBuildSource` predicate the harness builder enforces, against -// the one adapter registry, rather than a second copy of the check. -const buildHarness = createDefaultHarnessBuilder({ - cacheRoot, - cacheMaxBytes, - registryMaxTarballBytes, - adapters, - gcPolicy: agentGCPolicy, -}); +// The deploy router's source-admission gate reuses this exact +// `canBuildSource` predicate, against the one adapter registry, rather +// than a second copy of the check. +const buildHarness = createDefaultHarnessBuilder({ adapters }); // Set by the `createDeployRouter` callback below (invoked synchronously // during construction) so the boot edge can drive the router's restore pass @@ -360,8 +349,6 @@ const orchestrator = createSidecarOrchestrator({ token: sidecarToken, dataDir, transport, - buildHarness, - createAgentCrypto: createEd25519Crypto, cryptoOps: { generateKeyPair, signEd25519, diff --git a/apps/sidecar/src/workflow-host-wiring.ts b/apps/sidecar/src/workflow-host-wiring.ts index a1b50eb1..98d6f6bc 100644 --- a/apps/sidecar/src/workflow-host-wiring.ts +++ b/apps/sidecar/src/workflow-host-wiring.ts @@ -688,9 +688,9 @@ export function createSidecarDeployRouter(deps: { * mail path (`MailBusBindings.sendOutbound`) signs the agent's replies * with the AGENT's identity -- the OUTBOUND half of mailbox ownership * (§3a). Without this registration the spawned agent's address has no - * `CryptoProvider` on the transport (no `startSession` runs for it), - * and an outbound send would throw "address is not registered" rather - * than emit unsigned mail. + * `CryptoProvider` on the transport (nothing else registers one for + * it), and an outbound send would throw "address is not registered" + * rather than emit unsigned mail. */ createAgentCrypto: (keyPair: KeyPair) => CryptoProvider; /** diff --git a/packages/hub-agent/src/harness-builder.ts b/packages/hub-agent/src/harness-builder.ts index a9acafe5..3a20d535 100644 --- a/packages/hub-agent/src/harness-builder.ts +++ b/packages/hub-agent/src/harness-builder.ts @@ -1,92 +1,30 @@ // HarnessBuilder seam. // -// The package declares the shape of the per-agent harness construction -// step; the host (apps/sidecar today, any custom sidecar tomorrow) -// supplies the concrete implementation. This keeps @intx/hub-agent -// free of dependencies on the concrete tool, storage, authz, and -// inference packages the harness is wired up against. +// The package declares the shape of the host's source-admission check; +// the host (apps/sidecar today, any custom sidecar tomorrow) supplies +// the concrete implementation. This keeps @intx/hub-agent free of +// dependencies on the concrete inference packages the check consults. -import type { Harness } from "@intx/harness"; -import type { MailAuditStore } from "@intx/storage-isogit"; -import type { GrantRule } from "@intx/types/authz"; -import type { - ConnectorThreadState, - CryptoProvider, - HarnessConfig as AgentConfig, - InferenceEvent, - InferenceSource, - MessageTransport, -} from "@intx/types/runtime"; +import type { InferenceSource } from "@intx/types/runtime"; import type { DeployApplyErrorFrame } from "@intx/types/sidecar"; -/** - * Per-attempt context the builder uses to emit a deploy-apply error - * frame back to the hub. The host (typically the sidecar app's wiring - * to hub-link) translates this into the wire-level frame. - */ -export type DeployApplyErrorEmitter = ( - payload: Omit, -) => void; - -/** - * Inputs the builder receives for one agent's session-start. The package - * pre-resolves every value that depends on per-agent disk layout, transport - * registration, or crypto bootstrap so the builder itself only handles - * harness construction. - */ -export type BuildHarnessArgs = { - agentAddress: string; - agentConfig: AgentConfig; - /** Ordered inference sources; the head (the source whose id is - * `defaultSource`) starts active, the tail is the failover chain. */ - sources: InferenceSource[]; - defaultSource: string; - /** Absolute path to the per-agent directory, from AgentRepoStore. */ - storeDir: string; - /** Per-agent view of the host's message transport. */ - agentTransport: MessageTransport; - /** Per-agent crypto provider, from the host's createAgentCrypto. */ - crypto: CryptoProvider; - onEvent: (event: InferenceEvent) => void; - onConnectorStateChanged: (state: ConnectorThreadState | null) => void; +export type HarnessBuilder = { /** - * Emit a `deploy.apply.error` frame back to the hub when the tool- - * package loader rejects an apply. The builder calls this just - * before throwing to abort harness construction; the host wires it - * to the sidecar's hub-link. Hosts that do not yet support - * tool-package distribution can omit this — a builder that - * encounters a manifest while this is undefined throws with no - * dedicated frame, which lands as a generic `agent.error` via the - * existing session-start error handler. + * Throws if the supplied source cannot be built by this host. The + * sidecar deploy router calls this to admit a step's pinned inference + * source before spawning, so the operator sees rejection on the + * control plane rather than during the next inference call. */ - emitDeployApplyError?: DeployApplyErrorEmitter; + canBuildSource(source: InferenceSource): void; }; /** - * What the builder returns. The harness drives inference; the mailStore - * is owned by SessionManager for inbound/outbound audit; updateGrants - * mutates the live grant ref the harness's authz closure reads; - * disposers tear down builder-allocated resources on session end. - * - * The builder is responsible for invoking `dispose` on any resource it - * allocates that survives a build failure. The bundle's disposers are - * the contract for the *success* path; mid-construction failure must - * be cleaned up by the builder before the throw propagates. + * Callback the tool-package loader uses to emit a deploy-apply error + * frame back to the hub. The host (the sidecar's workflow-child tool + * materialization) translates this into the wire-level frame. Lives here + * as the dependency-light home the `@intx/hub-agent/paths` entry + * re-exports without pulling in the orchestrator module graph. */ -export type HarnessBundle = { - harness: Harness; - mailStore: MailAuditStore; - updateGrants(grants: GrantRule[]): void; - disposers: (() => Promise)[]; -}; - -export type HarnessBuilder = { - /** - * Throws if the supplied source cannot be built by this host. Called - * before `build` at session-start, and standalone at update-source - * time so the operator sees rejection on the control plane rather - * than during the next inference call. - */ - canBuildSource(source: InferenceSource): void; - build(args: BuildHarnessArgs): Promise; -}; +export type DeployApplyErrorEmitter = ( + payload: Omit, +) => void; diff --git a/packages/hub-agent/src/index.ts b/packages/hub-agent/src/index.ts index fc424d77..682fcb84 100644 --- a/packages/hub-agent/src/index.ts +++ b/packages/hub-agent/src/index.ts @@ -10,23 +10,11 @@ export { type AgentKeyStoreDeps, type AgentKeyEntry, } from "./agent-key-store"; -export type { - HarnessBuilder, - HarnessBundle, - BuildHarnessArgs, - DeployApplyErrorEmitter, -} from "./harness-builder"; +export type { HarnessBuilder } from "./harness-builder"; export { createSessionManager, type SessionManager, type SessionManagerConfig, - type SessionEventSink, - type ConnectorStateSink, - type AgentSession, - type AgentEventListener, - type ProvisionResult, - type RestoreResult, - type RestoredAgent, } from "./session-manager"; export { createHubLink, diff --git a/packages/hub-agent/src/session-manager.test.ts b/packages/hub-agent/src/session-manager.test.ts index 1c00b383..ed08bb2d 100644 --- a/packages/hub-agent/src/session-manager.test.ts +++ b/packages/hub-agent/src/session-manager.test.ts @@ -1,26 +1,12 @@ import { describe, test, expect, afterEach } from "bun:test"; +import fs from "node:fs"; import fsp from "node:fs/promises"; import path from "node:path"; import os from "node:os"; -import { createInMemoryTransport } from "@intx/mail-memory"; -import type { MailAuditStore } from "@intx/storage-isogit"; -import type { Harness } from "@intx/harness"; -import type { - CryptoProvider, - HarnessConfig, - InferenceEvent, - InferenceSource, - KeyPair, -} from "@intx/types/runtime"; +import git from "isomorphic-git"; -import { createAgentKeyStore } from "./agent-key-store"; -import { createAgentRepoStore, type AgentRepoStore } from "./agent-repo-store"; import { createSessionManager } from "./session-manager"; -import type { - BuildHarnessArgs, - HarnessBuilder, - HarnessBundle, -} from "./harness-builder"; +import type { AgentRepoStore } from "./agent-repo-store"; const tempDirs: string[] = []; @@ -37,604 +23,6 @@ afterEach(async () => { ); }); -function makeKeyPair(seed: number): KeyPair { - const privateKey = new Uint8Array(32); - const publicKey = new Uint8Array(32); - for (let i = 0; i < 32; i++) { - privateKey[i] = (seed + i) & 0xff; - publicKey[i] = (seed * 2 + i) & 0xff; - } - return { privateKey, publicKey }; -} - -function makeConfig(address: string): HarnessConfig { - return { - agentId: "test-agent", - agentAddress: address, - sessionId: "sess-1", - principalId: "principal-1", - tenantId: "tenant-1", - systemPrompt: "test", - tools: [], - grants: [], - sources: [ - { - id: "test:test-model", - provider: "test", - apiKey: "key", - baseURL: "http://localhost", - model: "test-model", - }, - ], - defaultSource: "test:test-model", - }; -} - -function makeCrypto(kp: KeyPair): CryptoProvider { - return { - async sign() { - return new Uint8Array(64); - }, - async signSSH() { - return "unused"; - }, - async verify() { - return true; - }, - getPublicKey: () => kp.publicKey, - }; -} - -function makeMailStoreStub(): MailAuditStore & { commits: string[] } { - const commits: string[] = []; - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- MailAuditStore from storage-isogit is a structural type whose unrelated methods are unused in these tests - const store = { - commits, - async commitMail() { - commits.push("commit"); - return null; - }, - } as unknown as MailAuditStore & { commits: string[] }; - return store; -} - -function makeHarnessStub( - onSetSources?: (sources: InferenceSource[], defaultSource: string) => void, -): Harness { - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- Harness is a library type whose unused members are not structurally satisfied by the stub - return { - start: () => { - /* no-op: harness.start is invoked by SessionManager but the tests - do not observe inference activity */ - }, - stop: () => { - /* no-op */ - }, - deliver: () => { - /* no-op */ - }, - setSource: () => { - /* no-op */ - }, - setSources: (sources: InferenceSource[], defaultSource: string) => { - onSetSources?.(sources, defaultSource); - }, - get blobReader() { - throw new Error("blobReader unused in these tests"); - }, - } as unknown as Harness; -} - -type SetSourcesRecord = { - sources: InferenceSource[]; - defaultSource: string; -}; -type RecordingBuilder = HarnessBuilder & { - buildCalls: BuildHarnessArgs[]; - grants: GrantRecord[]; - setSourcesCalls: SetSourcesRecord[]; -}; -type GrantRecord = { address: string; count: number }; - -function makeRecordingBuilder(opts: { - rejectSource?: (s: { provider: string }) => boolean; - throwOnBuild?: boolean; -}): RecordingBuilder { - const buildCalls: BuildHarnessArgs[] = []; - const grants: GrantRecord[] = []; - const setSourcesCalls: SetSourcesRecord[] = []; - - return { - buildCalls, - grants, - setSourcesCalls, - canBuildSource(source) { - if (opts.rejectSource?.(source)) { - throw new Error(`source rejected: ${source.provider}`); - } - }, - async build(args) { - buildCalls.push(args); - if (opts.throwOnBuild) { - throw new Error("builder boom"); - } - const harness = makeHarnessStub((sources, defaultSource) => { - setSourcesCalls.push({ sources, defaultSource }); - }); - const mailStore = makeMailStoreStub(); - const bundle: HarnessBundle = { - harness, - mailStore, - updateGrants(g) { - grants.push({ address: args.agentAddress, count: g.length }); - }, - disposers: [], - }; - return bundle; - }, - }; -} - -function makeManagerHarness( - dataDir: string, - opts: { - rejectSource?: (s: { provider: string }) => boolean; - throwOnBuild?: boolean; - } = {}, -): { - repoStore: ReturnType; - keyStore: ReturnType; - builder: RecordingBuilder; - transport: ReturnType; - manager: ReturnType; - events: { addr: string; sid: string }[]; -} { - const repoStore = createAgentRepoStore({ dataDir }); - const keyStore = createAgentKeyStore({ - dataDir, - generateKeyPair: async () => makeKeyPair(11), - signEd25519: async () => new Uint8Array(64), - verifySSHSig: async () => true, - }); - const builder = makeRecordingBuilder(opts); - const transport = createInMemoryTransport(); - const events: { addr: string; sid: string }[] = []; - const manager = createSessionManager({ - transport, - repoStore, - keyStore, - buildHarness: builder, - createAgentCrypto: (kp) => makeCrypto(kp), - onEvent: (addr, sid) => events.push({ addr, sid }), - onConnectorStateChanged: () => { - /* no-op: tests do not assert on connector state */ - }, - }); - return { repoStore, keyStore, builder, transport, manager, events }; -} - -describe("SessionManager.provisionAgent + startSession happy path", () => { - test("invokes the builder with the resolved per-agent context", async () => { - const dataDir = await tempDir(); - const { manager, builder } = makeManagerHarness(dataDir); - const cfg = makeConfig("agent@local"); - - await manager.provisionAgent(cfg); - await manager.startSession("agent@local"); - - expect(manager.hasSession("agent@local")).toBe(true); - expect(builder.buildCalls).toHaveLength(1); - const call = builder.buildCalls[0]; - if (call === undefined) throw new Error("unreachable"); - expect(call.agentAddress).toBe("agent@local"); - expect(call.agentConfig.agentId).toBe("test-agent"); - expect(call.defaultSource).toBe("test:test-model"); - expect(call.sources.map((s) => s.id)).toContain("test:test-model"); - expect(call.storeDir).toContain("agent_at_local"); - }); -}); - -describe("SessionManager.updateSources", () => { - const NEW_SOURCES: InferenceSource[] = [ - { - id: "test:test-model", - provider: "test", - apiKey: "rotated-key", - baseURL: "http://localhost", - model: "test-model", - }, - { - id: "test:fallback", - provider: "test", - apiKey: "fallback-key", - baseURL: "http://localhost", - model: "fallback-model", - }, - ]; - - test("installs the full ordered list on the running harness", async () => { - const dataDir = await tempDir(); - const { manager, builder } = makeManagerHarness(dataDir); - await manager.provisionAgent(makeConfig("agent@local")); - await manager.startSession("agent@local"); - - await manager.updateSources("agent@local", NEW_SOURCES, "test:test-model"); - - expect(builder.setSourcesCalls).toHaveLength(1); - const recorded = builder.setSourcesCalls[0]; - if (recorded === undefined) throw new Error("unreachable"); - expect(recorded.defaultSource).toBe("test:test-model"); - expect(recorded.sources.map((s) => s.id)).toEqual([ - "test:test-model", - "test:fallback", - ]); - }); - - test("rejects a default that names no source in the new list", async () => { - const dataDir = await tempDir(); - const { manager } = makeManagerHarness(dataDir); - await manager.provisionAgent(makeConfig("agent@local")); - await manager.startSession("agent@local"); - - await expect( - manager.updateSources("agent@local", NEW_SOURCES, "test:missing"), - ).rejects.toThrow(); - }); -}); - -describe("SessionManager.startSession rollback when builder throws", () => { - test("restores provisioned state and unregisters the transport", async () => { - const dataDir = await tempDir(); - const { manager, transport } = makeManagerHarness(dataDir, { - throwOnBuild: true, - }); - const cfg = makeConfig("agent@local"); - - await manager.provisionAgent(cfg); - await expect(manager.startSession("agent@local")).rejects.toThrow( - "builder boom", - ); - - expect(manager.hasSession("agent@local")).toBe(false); - expect(manager.isProvisioned("agent@local")).toBe(true); - // Transport's per-agent view should be gone; getTransportFor throws - // for unregistered addresses. - expect(() => transport.getTransportFor("agent@local")).toThrow(); - }); -}); - -describe("SessionManager.restoreSessions mismatch handling", () => { - test("config without a key pair is reported as failed", async () => { - const dataDir = await tempDir(); - const { manager, repoStore } = makeManagerHarness(dataDir); - - // Agent with a config but no on-disk keypair → goes to `failed`. - await fsp.mkdir(path.join(dataDir, "ghost_at_local"), { recursive: true }); - await repoStore.persistConfig("ghost@local", makeConfig("ghost@local")); - - const result = await manager.restoreSessions(); - - expect(result.failed).toContain("ghost@local"); - expect(result.restored.map((r) => r.address)).not.toContain("ghost@local"); - }); -}); - -describe("SessionManager.applyAssetPack materializes under the workspace dir", () => { - test("writes pack contents under /workspace//", async () => { - const dataDir = await tempDir(); - const { manager, repoStore } = makeManagerHarness(dataDir); - - const address = "agent@local"; - await manager.provisionAgent(makeConfig(address)); - - // Build a real pack from a tiny source repo so applyAssetPack - // can index it. Reuses isomorphic-git directly to avoid pulling - // in the asset-service surface for this test. - const sourceDir = path.join(dataDir, "asset-source"); - await fsp.mkdir(sourceDir, { recursive: true }); - const git = await import("isomorphic-git"); - const fs = await import("node:fs"); - await git.default.init({ fs, dir: sourceDir, defaultBranch: "main" }); - await fsp.writeFile(path.join(sourceDir, "hello.txt"), "hello from asset"); - await git.default.add({ fs, dir: sourceDir, filepath: "hello.txt" }); - const commitSha = await git.default.commit({ - fs, - dir: sourceDir, - message: "asset", - author: { name: "t", email: "t@t" }, - }); - const oids = new Set([commitSha]); - const { commit } = await git.default.readCommit({ - fs, - dir: sourceDir, - oid: commitSha, - }); - oids.add(commit.tree); - const { tree } = await git.default.readTree({ - fs, - dir: sourceDir, - oid: commit.tree, - }); - for (const entry of tree) oids.add(entry.oid); - const packResult = await git.default.packObjects({ - fs, - dir: sourceDir, - oids: [...oids], - write: false, - }); - if (packResult.packfile === undefined) { - throw new Error("packObjects produced no pack"); - } - - await manager.applyAssetPack( - address, - "skills/greet/", - packResult.packfile, - "refs/heads/main", - commitSha, - ); - - const expected = path.join( - repoStore.getAgentDir(address), - "workspace", - "skills", - "greet", - "hello.txt", - ); - const contents = await fsp.readFile(expected, "utf-8"); - expect(contents).toBe("hello from asset"); - }); -}); - -describe("SessionManager.updateGrants routes through the bundle", () => { - test("calls bundle.updateGrants with the new grants and persists config", async () => { - const dataDir = await tempDir(); - const { manager, builder, repoStore } = makeManagerHarness(dataDir); - const cfg = makeConfig("agent@local"); - - await manager.provisionAgent(cfg); - await manager.startSession("agent@local"); - - await manager.updateGrants("agent@local", [ - { - id: "g-1", - resource: "*", - action: "*", - effect: "allow", - origin: "system", - conditions: null, - expiresAt: null, - roleId: null, - principalId: null, - }, - ]); - - expect(builder.grants).toEqual([{ address: "agent@local", count: 1 }]); - - // Persisted config should now carry the new grants array length. - const configs = await repoStore.scanConfigs(); - const entry = configs.find((c) => c.address === "agent@local"); - expect(entry?.config.grants).toHaveLength(1); - }); -}); - -describe("SessionManager.onAgentEvent per-agent fan-out", () => { - test("delivers events only to the subscribed agent's listeners and respects the disposer", async () => { - const dataDir = await tempDir(); - const { manager, builder } = makeManagerHarness(dataDir); - - await manager.provisionAgent(makeConfig("alice@local")); - await manager.startSession("alice@local"); - await manager.provisionAgent(makeConfig("bob@local")); - await manager.startSession("bob@local"); - - const aliceBuild = builder.buildCalls.find( - (c) => c.agentAddress === "alice@local", - ); - const bobBuild = builder.buildCalls.find( - (c) => c.agentAddress === "bob@local", - ); - if (aliceBuild === undefined || bobBuild === undefined) { - throw new Error("unreachable: builder did not capture both addresses"); - } - - const aliceEvents: InferenceEvent[] = []; - const bobEvents: InferenceEvent[] = []; - const dispose = manager.onAgentEvent("alice@local", (e) => - aliceEvents.push(e), - ); - manager.onAgentEvent("bob@local", (e) => bobEvents.push(e)); - - const aliceTick: InferenceEvent = { - type: "inference.start", - seq: 0, - data: { model: "alice-m" }, - }; - const bobTick: InferenceEvent = { - type: "inference.start", - seq: 0, - data: { model: "bob-m" }, - }; - aliceBuild.onEvent(aliceTick); - bobBuild.onEvent(bobTick); - - expect(aliceEvents).toEqual([aliceTick]); - expect(bobEvents).toEqual([bobTick]); - - dispose(); - aliceBuild.onEvent(aliceTick); - expect(aliceEvents).toEqual([aliceTick]); - // bob continues to receive after alice's disposer runs. - bobBuild.onEvent(bobTick); - expect(bobEvents).toEqual([bobTick, bobTick]); - }); -}); - -async function seedAgentsOnDisk( - dataDir: string, - addresses: string[], -): Promise { - const seed = makeManagerHarness(dataDir); - for (const address of addresses) { - await seed.manager.provisionAgent(makeConfig(address)); - await seed.manager.startSession(address); - } -} - -function makeRestoredBundle(): HarnessBundle { - return { - harness: makeHarnessStub(), - mailStore: makeMailStoreStub(), - updateGrants() { - /* no-op: restore tests do not assert on grants */ - }, - disposers: [], - }; -} - -function makeRestoreStores(dataDir: string): { - repoStore: ReturnType; - keyStore: ReturnType; -} { - return { - repoStore: createAgentRepoStore({ dataDir }), - keyStore: createAgentKeyStore({ - dataDir, - generateKeyPair: async () => makeKeyPair(11), - signEd25519: async () => new Uint8Array(64), - verifySSHSig: async () => true, - }), - }; -} - -describe("SessionManager.restoreSessions bounded parallelism", () => { - test("rejects a non-positive restoreConcurrency", async () => { - const dataDir = await tempDir(); - const { repoStore, keyStore } = makeRestoreStores(dataDir); - expect(() => - createSessionManager({ - transport: createInMemoryTransport(), - repoStore, - keyStore, - buildHarness: makeRecordingBuilder({}), - createAgentCrypto: (kp) => makeCrypto(kp), - onEvent: () => { - /* no-op */ - }, - onConnectorStateChanged: () => { - /* no-op */ - }, - restoreConcurrency: 0, - }), - ).toThrow(/restoreConcurrency must be a positive integer/); - }); - - test("restores up to restoreConcurrency agents at once, no more", async () => { - const dataDir = await tempDir(); - const addresses = ["a@local", "b@local", "c@local"]; - await seedAgentsOnDisk(dataDir, addresses); - - // Fresh stores + transport stand in for a restarted process that - // restores the seeded agents from disk. - const { repoStore, keyStore } = makeRestoreStores(dataDir); - - const buildStarts: string[] = []; - let startedCount = 0; - let signalBoundReached!: () => void; - const boundReached = new Promise((resolve) => { - signalBoundReached = resolve; - }); - let releaseBuilds!: () => void; - const gate = new Promise((resolve) => { - releaseBuilds = resolve; - }); - const builder: HarnessBuilder = { - canBuildSource() { - /* accept every source */ - }, - async build(args) { - buildStarts.push(args.agentAddress); - startedCount += 1; - if (startedCount === 2) signalBoundReached(); - await gate; - return makeRestoredBundle(); - }, - }; - - const manager = createSessionManager({ - transport: createInMemoryTransport(), - repoStore, - keyStore, - buildHarness: builder, - createAgentCrypto: (kp) => makeCrypto(kp), - onEvent: () => { - /* no-op */ - }, - onConnectorStateChanged: () => { - /* no-op */ - }, - restoreConcurrency: 2, - }); - - const restorePromise = manager.restoreSessions(); - // Two builds run at once; the third cannot start until a worker - // slot frees, which needs the gate to release. A serial restore - // would hold at one started build and this await would never settle. - await boundReached; - expect(buildStarts.length).toBe(2); - - releaseBuilds(); - const result = await restorePromise; - // All three eventually build once slots free up. - expect(buildStarts.length).toBe(3); - expect(result.failed).toEqual([]); - expect(result.restored.map((r) => r.address).sort()).toEqual( - [...addresses].sort(), - ); - }); - - test("reports a failing agent as failed without blocking the rest", async () => { - const dataDir = await tempDir(); - const addresses = ["healthy1@local", "wedged@local", "healthy2@local"]; - await seedAgentsOnDisk(dataDir, addresses); - - const { repoStore, keyStore } = makeRestoreStores(dataDir); - - const builder: HarnessBuilder = { - canBuildSource() { - /* accept every source */ - }, - async build(args) { - if (args.agentAddress === "wedged@local") { - throw new Error("registry fetch exceeded the timeout"); - } - return makeRestoredBundle(); - }, - }; - - const manager = createSessionManager({ - transport: createInMemoryTransport(), - repoStore, - keyStore, - buildHarness: builder, - createAgentCrypto: (kp) => makeCrypto(kp), - onEvent: () => { - /* no-op */ - }, - onConnectorStateChanged: () => { - /* no-op */ - }, - }); - - const result = await manager.restoreSessions(); - expect(result.failed).toEqual(["wedged@local"]); - expect(result.restored.map((r) => r.address).sort()).toEqual([ - "healthy1@local", - "healthy2@local", - ]); - }); -}); - function makeStubRepoStore(opts: { dataDir: string; createStatePack: AgentRepoStore["createStatePack"]; @@ -657,29 +45,96 @@ function makeStubRepoStore(opts: { } function makeManagerWithRepoStore( - dataDir: string, repoStore: AgentRepoStore, ): ReturnType { - return createSessionManager({ - transport: createInMemoryTransport(), - repoStore, - keyStore: createAgentKeyStore({ - dataDir, - generateKeyPair: async () => makeKeyPair(11), - signEd25519: async () => new Uint8Array(64), - verifySSHSig: async () => true, - }), - buildHarness: makeRecordingBuilder({}), - createAgentCrypto: (kp) => makeCrypto(kp), - onEvent: () => { - /* no-op */ - }, - onConnectorStateChanged: () => { - /* no-op */ - }, + return createSessionManager({ repoStore }); +} + +async function buildAssetPack( + files: Record, +): Promise<{ pack: Uint8Array; commitSha: string }> { + const sourceDir = await tempDir(); + await git.init({ fs, dir: sourceDir, defaultBranch: "main" }); + + for (const [rel, content] of Object.entries(files)) { + const abs = path.join(sourceDir, rel); + await fsp.mkdir(path.dirname(abs), { recursive: true }); + await fsp.writeFile(abs, content); + await git.add({ fs, dir: sourceDir, filepath: rel }); + } + + const commitSha = await git.commit({ + fs, + dir: sourceDir, + message: "asset", + author: { name: "test", email: "test@test.dev" }, + }); + + const oids = new Set([commitSha]); + const { commit } = await git.readCommit({ + fs, + dir: sourceDir, + oid: commitSha, + }); + oids.add(commit.tree); + async function walkTree(treeOid: string): Promise { + const { tree } = await git.readTree({ fs, dir: sourceDir, oid: treeOid }); + for (const entry of tree) { + oids.add(entry.oid); + if (entry.type === "tree") await walkTree(entry.oid); + } + } + await walkTree(commit.tree); + + const result = await git.packObjects({ + fs, + dir: sourceDir, + oids: [...oids], + write: false, }); + if (result.packfile === undefined) { + throw new Error("packObjects produced no packfile"); + } + return { pack: result.packfile, commitSha }; } +describe("SessionManager.applyAssetPack", () => { + test("materializes the pack under /workspace//", async () => { + const dataDir = await tempDir(); + const { pack, commitSha } = await buildAssetPack({ + "greet/SKILL.md": "---\nname: greet\n---\nbody\n", + }); + + // The wrapper's only logic over `applyAssetPackFn` is the workspace-root + // composition: `/workspace`. Prove the pack lands there. + const repoStore = makeStubRepoStore({ + dataDir, + createStatePack: () => + Promise.reject(new Error("createStatePack not exercised by this test")), + remove: () => + Promise.reject(new Error("remove not exercised by this test")), + }); + const manager = makeManagerWithRepoStore(repoStore); + + await manager.applyAssetPack( + "agent@local", + "skills/example/", + pack, + "refs/heads/main", + commitSha, + ); + + const materialized = path.join( + dataDir, + "agent@local", + "workspace", + "skills/example", + "greet/SKILL.md", + ); + expect(fs.existsSync(materialized)).toBe(true); + }); +}); + describe("SessionManager repo-operation serialization", () => { test("deleteAgentDir removes the directory only after an in-flight state-pack read completes", async () => { const dataDir = await tempDir(); @@ -707,7 +162,7 @@ describe("SessionManager repo-operation serialization", () => { }, }); - const manager = makeManagerWithRepoStore(dataDir, repoStore); + const manager = makeManagerWithRepoStore(repoStore); const rejections: unknown[] = []; const onRejection = (reason: unknown) => rejections.push(reason); @@ -760,7 +215,7 @@ describe("SessionManager repo-operation serialization", () => { }, }); - const manager = makeManagerWithRepoStore(dataDir, repoStore); + const manager = makeManagerWithRepoStore(repoStore); const rejections: unknown[] = []; const onRejection = (reason: unknown) => rejections.push(reason); diff --git a/packages/hub-agent/src/session-manager.ts b/packages/hub-agent/src/session-manager.ts index 9bccf406..6b42aad2 100644 --- a/packages/hub-agent/src/session-manager.ts +++ b/packages/hub-agent/src/session-manager.ts @@ -1,169 +1,30 @@ -// Per-agent harness lifecycle. +// Per-agent on-disk repo operations for supervised deployments. // -// SessionManager owns the cross-agent state (provisioned/sessions maps, -// the per-agent mail-commit queue, the last-checkpoint-hash buffer used -// to thread connector reply hashes into outbound mail commits) and the -// global transport handlers (addMessageSentHandler). Per-agent harness -// construction lives behind the HarnessBuilder seam, supplied by the -// host. The package itself depends only on the lifecycle infrastructure -// (transport interface, mail-audit store type, crypto provider type) -// and the stores from this same package — it does not pin the concrete -// tool, storage, authz, or inference packages the harness is wired up -// against. The host owns those. -// -// Construction split between this module and the builder: -// - SessionManager handles: provisioned/sessions bookkeeping, -// transport.register/unregister/getTransportFor, AgentCrypto -// instantiation, the mail-commit queue, the addMessageSentHandler -// subscription, the rollback path on builder failure. -// - HarnessBuilder handles: storage + mailStore construction (from -// the per-agent signer), authz wiring (grantsRef + authorize -// closure), tool composition, harness construction. -// The boundary keeps the cross-agent state owned by SessionManager and -// the per-agent construction owned by the host. Pushing transport -// registration into the builder would break the invariant; pushing -// authz out of the builder would re-couple the package to authz. +// The in-process session runtime -- harness construction, agent +// provisioning, disk restore, and per-agent mail audit -- has been +// retired: every agent now runs as a supervised workflow-process child +// on the workflow-run substrate. What remains here is the thin +// serialization layer over the agent repo store that the deploy path +// and the hub-link still call: deploy/asset-pack applies, state-pack +// reads, deploy-ref reads, and directory teardown, each run +// one-at-a-time per agent so a teardown never races an in-flight git op. import path from "node:path"; -import { getLogger } from "@intx/log"; -import { hexEncode } from "@intx/types"; -import type { HubTransport } from "@intx/mail-memory"; -import type { GrantRule } from "@intx/types/authz"; -import type { DeployApplyErrorFrame } from "@intx/types/sidecar"; -import type { - ConnectorThreadState, - CryptoProvider, - HarnessConfig as AgentConfig, - InboundMessage, - InferenceEvent, - InferenceSource, - KeyPair, -} from "@intx/types/runtime"; -import type { Harness } from "@intx/harness"; - -import type { AgentKeyEntry, AgentKeyStore } from "./agent-key-store"; import type { AgentRepoStore } from "./agent-repo-store"; -import type { HarnessBuilder, HarnessBundle } from "./harness-builder"; import { applyAssetPack as applyAssetPackFn } from "./apply-asset-pack"; -const logger = getLogger(["interchange", "hub-agent", "session"]); - -/** - * Public session record. The grants ref and disposers live inside the - * HarnessBundle the builder produced — they are not part of this type. - */ -export type AgentSession = { - agentAddress: string; - agentId: string; - config: AgentConfig; -}; - -export type SessionEventSink = ( - agentAddress: string, - sessionId: string, - event: InferenceEvent, -) => void; - -export type ConnectorStateSink = ( - agentAddress: string, - state: ConnectorThreadState | null, -) => void; - -export type DeployApplyErrorSink = ( - agentAddress: string, - payload: Omit, -) => void; - export type SessionManagerConfig = { - transport: HubTransport; repoStore: AgentRepoStore; - keyStore: AgentKeyStore; - buildHarness: HarnessBuilder; - /** - * Per-agent crypto factory. Receives the agent's raw key pair and - * returns a CryptoProvider bound to it. Keeps the package free of - * `@intx/crypto`. - */ - createAgentCrypto: (keyPair: KeyPair) => CryptoProvider; - onEvent: SessionEventSink; - onConnectorStateChanged: ConnectorStateSink; - /** - * Optional: emit a deploy-apply error frame to the hub when the - * harness builder rejects an apply attempt. Wired by the sidecar - * app to hub-link's frame channel. Hosts without tool-package - * distribution can omit this. - */ - onDeployApplyError?: DeployApplyErrorSink; - /** - * Maximum number of agents whose sessions `restoreSessions` restores - * concurrently. Restore work is per-agent isolated (per-agent git dir, - * per-address repo lock), so agents restore in parallel up to this - * bound and a single slow or wedged agent no longer gates the rest. - * Defaults to `DEFAULT_RESTORE_CONCURRENCY`. - */ - restoreConcurrency?: number; -}; - -/** - * Default `restoreConcurrency`: how many agents `restoreSessions` - * restores in parallel when the caller does not override it. - */ -const DEFAULT_RESTORE_CONCURRENCY = 4; - -export type ProvisionResult = { - publicKey: string; - keyPair: KeyPair; -}; - -export type RestoredAgent = { - address: string; - keyPair: KeyPair; - hubPublicKey?: string; }; -export type RestoreResult = { - restored: RestoredAgent[]; - failed: string[]; -}; - -export type AgentEventListener = (event: InferenceEvent) => void; - export type SessionManager = { - provisionAgent(config: AgentConfig): Promise; /** - * Initialize the on-disk deploy-tree repo for an address without - * minting a keypair, persisting config, or entering the - * provisioned/pending bookkeeping. A single-step workflow deploy uses - * this at the head so the follow-up deploy-pack apply has a repo to - * apply into; the supervised workflow-process child mints its own - * keypair at boot, so `provisionAgent`'s keypair, `persistConfig`, and - * duplicate-address guard are neither needed nor wanted on that path. + * Initialize the on-disk deploy-tree repo for an address. A single-step + * workflow deploy uses this at the head so the follow-up deploy-pack + * apply has a repo to apply into. */ initRepo(address: string): Promise; - startSession(agentAddress: string): Promise; - destroySession(agentAddress: string): Promise; - abortSession(agentAddress: string, reason: string): Promise; - deliverMessage(agentAddress: string, message: InboundMessage): void; - updateGrants(agentAddress: string, grants: GrantRule[]): Promise; - /** - * Subscribe to InferenceEvents scoped to a specific agent address. - * Returns a disposer that removes the listener. Listeners fire - * synchronously before the global `onEvent` sink during dispatch. - * Production wires this against the workflow-host supervisor's - * trivial-launch path so the per-message reactor brackets the - * trivial workflow's run-event chain through `recordRunEvent`. - */ - onAgentEvent(agentAddress: string, listener: AgentEventListener): () => void; - updateSources( - agentAddress: string, - sources: InferenceSource[], - defaultSource: string, - ): Promise; - hasSession(agentAddress: string): boolean; - isProvisioned(agentAddress: string): boolean; - getAddresses(): string[]; - restoreSessions(): Promise; /** * Apply a deploy pack to the agent's repo. Thin wrapper around * AgentRepoStore for callers that already have a SessionManager handle. @@ -179,7 +40,7 @@ export type SessionManager = { /** * Materialize an asset pack at `//` for the * agent. The workspace root is per-agent; this is distinct from the - * agent's deploy git tree. Asset packs are unsigned in v1 — no + * agent's deploy git tree. Asset packs are unsigned in v1 -- no * `verifyCommit` parameter. */ applyAssetPack( @@ -194,97 +55,32 @@ export type SessionManager = { ): Promise<{ pack: Uint8Array; commitSha: string; ref: string }>; deleteAgentDir(agentAddress: string): Promise; getDeployRef(agentAddress: string): Promise; - persistHubPublicKey( - agentAddress: string, - hubPublicKey: string, - ): Promise; - commitInboundMail( - agentAddress: string, - rawMessage: Uint8Array, - ): Promise; + /** + * Session addresses this manager hosts. The in-process session runtime + * is retired, so this is always empty; the hub-link ships it in the + * register frame alongside the sidecar's workflow-deployment addresses. + */ + getAddresses(): string[]; + /** + * Session id for an address' outbound mail forwarding. Always undefined + * now that no in-process sessions exist; the hub-link tolerates a + * missing id and forwards the mail without one. + */ getSessionId(agentAddress: string): string | undefined; }; -type ProvisionedAgent = { - config: AgentConfig; - keyPair: KeyPair; -}; - -type LiveSession = AgentSession & { - harness: Harness; - bundle: HarnessBundle; -}; - export function createSessionManager( config: SessionManagerConfig, ): SessionManager { - const { - transport, - repoStore, - keyStore, - buildHarness, - createAgentCrypto, - onEvent, - onConnectorStateChanged, - onDeployApplyError, - restoreConcurrency = DEFAULT_RESTORE_CONCURRENCY, - } = config; - if (!Number.isInteger(restoreConcurrency) || restoreConcurrency < 1) { - throw new Error( - `createSessionManager: restoreConcurrency must be a positive integer; got ${String(restoreConcurrency)}`, - ); - } - - const sessions = new Map(); - const provisioned = new Map(); - const pending = new Set(); - - // Per-agent InferenceEvent fan-out. Subscribers register against a - // specific agentAddress; the dispatch site looks the listener set up - // by the event's owning address (the closure captured in startSession) - // and fires every listener synchronously before the global `onEvent` - // sink runs. Disposers remove a single listener and prune the set on - // emptiness so addresses without subscribers cost a single map miss. - const agentEventListeners = new Map>(); - - function onAgentEvent( - agentAddress: string, - listener: AgentEventListener, - ): () => void { - let set = agentEventListeners.get(agentAddress); - if (set === undefined) { - set = new Set(); - agentEventListeners.set(agentAddress, set); - } - set.add(listener); - return () => { - const current = agentEventListeners.get(agentAddress); - if (current === undefined) return; - current.delete(listener); - if (current.size === 0) agentEventListeners.delete(agentAddress); - }; - } - - // Checkpoint hash captured from the most recent connector.reply event for - // each agent. Consumed (deleted) by the MessageSentHandler so that only - // the connector reply mail commit receives the linkage — subsequent - // tool-initiated sends do not carry a stale hash. - // - // Ordering guarantee: the harness calls onEvent() synchronously inside - // handleEvent(), which fires before the detached transport.send() resolves. - // Because executeSend() always awaits at least once (signature generation) - // before calling MessageSentHandler, the map write in onEvent completes - // before the handler reads it. - const lastCheckpointHashes = new Map(); + const { repoStore } = config; // Per-agent promise chain that serializes the operations against an agent's - // on-disk directory that can be in flight during a live session -- mail-audit - // commits, state-pack and deploy-ref reads, and deploy/asset-pack applies all - // run one-at-a-time per agent. The chain exists for teardown: drainRepoOps - // awaits it before deleting the directory, so an operation that was valid - // when it started never runs against a path that has since vanished - // underneath it. Serializing additionally avoids corruption for the members - // that share the agent's `.git/` object store (mail commits, state-pack and + // on-disk directory -- state-pack and deploy-ref reads and deploy/asset-pack + // applies all run one-at-a-time per agent. The chain exists for teardown: + // drainRepoOps awaits it before deleting the directory, so an operation that + // was valid when it started never runs against a path that has since + // vanished underneath it. Serializing additionally avoids corruption for the + // members that share the agent's `.git/` object store (state-pack and // deploy-ref reads, deploy-pack applies), which isogit, lacking a // cross-process lock, would otherwise let interleave. Asset-pack applies are // on the chain only for the teardown reason -- they materialize into a @@ -315,16 +111,6 @@ export function createSessionManager( return result; } - function enqueueMailCommit( - agentAddress: string, - fn: () => Promise, - ): void { - void runRepoOp(agentAddress, fn).catch((err: unknown) => { - const msg = err instanceof Error ? err.message : String(err); - logger.error`Mail audit commit failed for ${agentAddress}: ${msg}`; - }); - } - // Await the agent's current operation chain so teardown removes the // directory only after in-flight git work finishes. Capturing the tail and // clearing the entry means an op enqueued AFTER this point starts a fresh @@ -340,393 +126,6 @@ export function createSessionManager( if (inflight !== undefined) await inflight; } - transport.addMessageSentHandler( - async ({ senderAddress, rawMessage, messageId }) => { - const session = sessions.get(senderAddress); - if (session === undefined) { - // Outbound mail for an address with no active session is a - // protocol violation — the transport should not have accepted - // the send. Surfacing this loudly catches the contract break - // before it propagates as silently-dropped audit records. - throw new Error( - `No active session for sender "${senderAddress}" — cannot audit outbound mail ${messageId}`, - ); - } - const mailStore = session.bundle.mailStore; - const checkpointHash = lastCheckpointHashes.get(senderAddress); - lastCheckpointHashes.delete(senderAddress); - enqueueMailCommit(senderAddress, async () => { - const result = await mailStore.commitMail(rawMessage, "out", { - ignoreDuplicate: true, - ...(checkpointHash !== undefined ? { checkpointHash } : {}), - }); - if (result !== null) { - logger.info`Committed outbound mail ${messageId} for ${senderAddress}`; - } - }); - }, - ); - - async function provisionAgent( - agentConfig: AgentConfig, - ): Promise { - const { agentAddress } = agentConfig; - - if ( - sessions.has(agentAddress) || - provisioned.has(agentAddress) || - pending.has(agentAddress) - ) { - throw new Error(`Agent already exists for address "${agentAddress}"`); - } - - pending.add(agentAddress); - - try { - const { keyPair, isNew } = await keyStore.loadOrGenerateKey(agentAddress); - - if (isNew) { - logger.info`Generated new key pair for ${agentAddress}`; - } - - await repoStore.initRepo(agentAddress); - await repoStore.persistConfig(agentAddress, agentConfig); - - provisioned.set(agentAddress, { config: agentConfig, keyPair }); - - const publicKey = hexEncode(keyPair.publicKey); - logger.info`Provisioned agent ${agentAddress}`; - return { publicKey, keyPair }; - } finally { - pending.delete(agentAddress); - } - } - - async function startSession(agentAddress: string): Promise { - const entry = provisioned.get(agentAddress); - if (entry === undefined) { - throw new Error(`No provisioned agent for address "${agentAddress}"`); - } - if (sessions.has(agentAddress)) { - throw new Error(`Session already running for agent "${agentAddress}"`); - } - - const { config: agentConfig, keyPair } = entry; - - const source = agentConfig.sources.find( - (s) => s.id === agentConfig.defaultSource, - ); - if (source === undefined) { - throw new Error( - `No source matches defaultSource "${agentConfig.defaultSource}" for agent "${agentAddress}"`, - ); - } - buildHarness.canBuildSource(source); - - provisioned.delete(agentAddress); - - try { - const crypto = createAgentCrypto(keyPair); - transport.register(agentAddress, crypto); - const agentTransport = transport.getTransportFor(agentAddress); - - const sessionId = agentConfig.sessionId; - const storeDir = repoStore.getAgentDir(agentAddress); - - const bundle = await buildHarness.build({ - agentAddress, - agentConfig, - sources: agentConfig.sources, - defaultSource: agentConfig.defaultSource, - storeDir, - agentTransport, - crypto, - onEvent(event: InferenceEvent) { - if ( - event.type === "connector.reply" && - event.data.checkpointHash !== undefined - ) { - lastCheckpointHashes.set(agentAddress, event.data.checkpointHash); - } - // Per-agent listeners fire before the global sink so an - // in-process consumer (the workflow-host trivial-launch - // subscriber) sees events at the same instant the hub - // forwarder does. Exceptions from a listener must not - // suppress the global forwarder; collect and rethrow only - // after `onEvent` has run. - let firstError: unknown; - const set = agentEventListeners.get(agentAddress); - if (set !== undefined) { - for (const listener of set) { - try { - listener(event); - } catch (err: unknown) { - if (firstError === undefined) firstError = err; - else - logger.error`agent-event listener for ${agentAddress} threw: ${String(err)}`; - } - } - } - onEvent(agentAddress, sessionId, event); - if (firstError !== undefined) throw firstError; - }, - onConnectorStateChanged(state) { - onConnectorStateChanged(agentAddress, state); - }, - ...(onDeployApplyError !== undefined - ? { - emitDeployApplyError: (payload) => { - onDeployApplyError(agentAddress, payload); - }, - } - : {}), - }); - - sessions.set(agentAddress, { - agentAddress, - agentId: agentConfig.agentId, - config: agentConfig, - harness: bundle.harness, - bundle, - }); - - // The composition-layer harness is started by `createHarness` -- - // by the time the builder returns the bundle, the agent's - // reactor is already running. No separate start() step. - logger.info`Started session for ${agentAddress} (session ${sessionId})`; - } catch (err) { - sessions.delete(agentAddress); - repoOpQueues.delete(agentAddress); - try { - transport.unregister(agentAddress); - } catch (cleanupErr) { - // Best-effort cleanup; don't mask the original error. - logger.error`Failed to unregister transport for ${agentAddress}: ${String(cleanupErr)}`; - } - provisioned.set(agentAddress, entry); - throw err; - } - } - - async function runDisposers( - session: LiveSession, - agentAddress: string, - ): Promise { - // Run every disposer even if some fail; an exception from one must - // not leak the others. Errors are collected and returned so the - // caller (destroy / abort) can decide whether to surface a - // partial-teardown condition; today the caller logs the summary - // and continues, since session teardown is also invoked from - // recovery paths where a thrown aggregate would obscure the - // original failure. - const errors: Error[] = []; - for (const disposer of session.bundle.disposers) { - try { - await disposer(); - } catch (err: unknown) { - const e = err instanceof Error ? err : new Error(String(err)); - logger.error`Disposer failed for ${agentAddress}: ${e.message}`; - errors.push(e); - } - } - return errors; - } - - async function destroySession(agentAddress: string): Promise { - if (provisioned.has(agentAddress)) { - provisioned.delete(agentAddress); - logger.info`Removed provisioned agent ${agentAddress}`; - return; - } - const session = sessions.get(agentAddress); - if (session === undefined) { - throw new Error(`No session exists for agent "${agentAddress}"`); - } - await session.harness.close(); - const disposerErrors = await runDisposers(session, agentAddress); - await drainRepoOps(agentAddress); - sessions.delete(agentAddress); - transport.unregister(agentAddress); - if (disposerErrors.length > 0) { - logger.warn`Stopped session for ${agentAddress} with ${String(disposerErrors.length)} disposer failure(s)`; - } else { - logger.info`Stopped session for ${agentAddress}`; - } - } - - async function abortSession( - agentAddress: string, - reason: string, - ): Promise { - if (provisioned.has(agentAddress)) { - provisioned.delete(agentAddress); - logger.info`Aborted provisioned agent ${agentAddress}: ${reason}`; - return; - } - const session = sessions.get(agentAddress); - if (session === undefined) { - throw new Error(`No session exists for agent "${agentAddress}"`); - } - await session.harness.close(); - const disposerErrors = await runDisposers(session, agentAddress); - await drainRepoOps(agentAddress); - sessions.delete(agentAddress); - transport.unregister(agentAddress); - if (disposerErrors.length > 0) { - logger.warn`Aborted agent ${agentAddress} (${reason}) with ${String(disposerErrors.length)} disposer failure(s)`; - } else { - logger.info`Aborted agent ${agentAddress}: ${reason}`; - } - } - - function deliverMessage(agentAddress: string, message: InboundMessage): void { - const session = sessions.get(agentAddress); - if (session === undefined) { - throw new Error(`No session exists for agent "${agentAddress}"`); - } - session.harness.deliver(message); - } - - function hasSession(agentAddress: string): boolean { - return sessions.has(agentAddress); - } - - function isProvisioned(agentAddress: string): boolean { - return provisioned.has(agentAddress); - } - - function getAddresses(): string[] { - return [...sessions.keys()]; - } - - async function restoreSessions(): Promise { - // Restore policy: an agent is restorable only when the on-disk - // state from both stores is consistent. Scan keys and configs - // independently, then join on address. A config without a key is - // surfaced as a hard failure — the agent's identity cannot be - // recovered without the key pair on disk. - // - // Key-without-config does not appear here because AgentKeyStore's - // scanKeys already requires a parseable agent.json before - // surfacing the directory, so the orphan-key case is filtered at - // the store boundary and never reaches this join. - const [keysByAddress, configEntries] = await Promise.all([ - keyStore.scanKeys().then((k) => new Map(k.map((e) => [e.address, e]))), - repoStore.scanConfigs(), - ]); - - // Restore agents with bounded parallelism. Each agent's provision + - // startSession is per-agent isolated (per-agent git dir, per-address - // repo lock), so running several at once cannot corrupt shared - // state, and one slow or wedged agent no longer gates the rest. - // Outcomes are written into per-index slots so the returned order is - // independent of completion order. - type Outcome = - | { kind: "restored"; agent: RestoredAgent } - | { kind: "failed"; address: string }; - const outcomes = new Array(configEntries.length); - - async function restoreOne(index: number): Promise { - const entry = configEntries[index]; - if (entry === undefined) return; - const keyEntry: AgentKeyEntry | undefined = keysByAddress.get( - entry.address, - ); - if (keyEntry === undefined) { - logger.error`Cannot restore "${entry.address}": agent.json exists but key pair is missing`; - outcomes[index] = { kind: "failed", address: entry.address }; - return; - } - - const agent: RestoredAgent = { - address: entry.address, - keyPair: keyEntry.keyPair, - }; - if (entry.hubPublicKey !== undefined) { - agent.hubPublicKey = entry.hubPublicKey; - } - - if (sessions.has(entry.address)) { - outcomes[index] = { kind: "restored", agent }; - return; - } - try { - await provisionAgent(entry.config); - if (entry.hubPublicKey !== undefined) { - await repoStore.persistPairing(entry.address, entry.hubPublicKey); - } - await startSession(entry.address); - outcomes[index] = { kind: "restored", agent }; - logger.info`Restored session for ${entry.address}`; - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - outcomes[index] = { kind: "failed", address: entry.address }; - logger.error`Failed to restore session for ${entry.address}: ${msg}`; - } - } - - // Worker pool: each worker pulls the next index until the list - // drains. Reading and advancing `cursor` is synchronous (no await - // between), so each index is handed to exactly one worker. - let cursor = 0; - async function worker(): Promise { - for (;;) { - const index = cursor; - cursor += 1; - if (index >= configEntries.length) return; - await restoreOne(index); - } - } - const workerCount = Math.min(restoreConcurrency, configEntries.length); - await Promise.all(Array.from({ length: workerCount }, () => worker())); - - const restored: RestoredAgent[] = []; - const failed: string[] = []; - for (const outcome of outcomes) { - if (outcome === undefined) continue; - if (outcome.kind === "restored") restored.push(outcome.agent); - else failed.push(outcome.address); - } - - return { restored, failed }; - } - - async function updateGrants( - agentAddress: string, - grants: GrantRule[], - ): Promise { - const session = sessions.get(agentAddress); - if (session === undefined) { - throw new Error(`No session exists for agent "${agentAddress}"`); - } - session.bundle.updateGrants(grants); - session.config = { ...session.config, grants }; - await repoStore.persistConfig(agentAddress, session.config); - logger.info`Updated grants for ${agentAddress} (${String(grants.length)} rules)`; - } - - async function updateSources( - agentAddress: string, - sources: InferenceSource[], - defaultSource: string, - ): Promise { - const session = sessions.get(agentAddress); - if (session === undefined) { - throw new Error(`No session exists for agent "${agentAddress}"`); - } - const source = sources.find((s) => s.id === defaultSource); - if (source === undefined) { - throw new Error( - `No source matches defaultSource "${defaultSource}" in update for agent "${agentAddress}"`, - ); - } - buildHarness.canBuildSource(source); - session.harness.setSources(sources, defaultSource); - session.config = { ...session.config, sources, defaultSource }; - await repoStore.persistConfig(agentAddress, session.config); - logger.info`Updated sources for ${agentAddress}`; - } - async function applyDeployPack( agentAddress: string, pack: Uint8Array, @@ -784,63 +183,18 @@ export function createSessionManager( await repoStore.remove(agentAddress); } - async function persistHubPublicKey( - agentAddress: string, - hubPublicKey: string, - ): Promise { - await repoStore.persistPairing(agentAddress, hubPublicKey); - } - - async function commitInboundMail( - agentAddress: string, - rawMessage: Uint8Array, - ): Promise { - const session = sessions.get(agentAddress); - if (session === undefined) { - throw new Error( - `No active session for "${agentAddress}" — cannot audit inbound mail`, - ); - } - const mailStore = session.bundle.mailStore; - enqueueMailCommit(agentAddress, async () => { - const result = await mailStore.commitMail(rawMessage, "in", { - ignoreDuplicate: true, - }); - if (result !== null) { - logger.info`Committed inbound mail ${result.messageId} for ${agentAddress}`; - } - }); - } - - function getSessionId(agentAddress: string): string | undefined { - return sessions.get(agentAddress)?.config.sessionId; - } - async function getDeployRef(agentAddress: string): Promise { return runRepoOp(agentAddress, () => repoStore.getDeployRef(agentAddress)); } return { - provisionAgent, initRepo: (address: string) => repoStore.initRepo(address), - startSession, - destroySession, - abortSession, - deliverMessage, - updateGrants, - onAgentEvent, - updateSources, - hasSession, - isProvisioned, - getAddresses, - restoreSessions, applyDeployPack, applyAssetPack, createStatePack, deleteAgentDir, getDeployRef, - persistHubPublicKey, - commitInboundMail, - getSessionId, + getAddresses: () => [], + getSessionId: () => undefined, }; } diff --git a/packages/hub-agent/src/sidecar-orchestrator.ts b/packages/hub-agent/src/sidecar-orchestrator.ts index 9d9c6735..99923d06 100644 --- a/packages/hub-agent/src/sidecar-orchestrator.ts +++ b/packages/hub-agent/src/sidecar-orchestrator.ts @@ -2,31 +2,22 @@ // of the sidecar runtime — stores, SessionManager, HubLink — and // returns a single start/close handle the host driver uses. // -// The host supplies policy (the data directory, the harness builder -// implementation, the per-agent crypto factory, the low-level crypto -// primitives, the hub credentials); the orchestrator does the -// composition. SessionManager and HubLink reference each other through -// SessionEventSink / ConnectorStateSink callbacks, but SessionManager -// is constructed first so the sinks initially point at no-op closures -// the orchestrator owns. After HubLink is constructed those closures -// are rewired to hubLink.sendEvent and hubLink.sendConnectorState, so -// the cross-reference is contained inside this module rather than +// The host supplies policy (the data directory, the low-level crypto +// primitives, the hub credentials, the deploy-router factory); the +// orchestrator does the composition. The multi-step deploy path +// forwards a spawned child's verified InferenceEvents to the hub +// through a sink the orchestrator owns: it points at a no-op closure +// until HubLink is constructed, then is rewired to hubLink.sendEvent, +// so the cross-reference is contained inside this module rather than // leaking up to the host entry point. import { getLogger } from "@intx/log"; import type { HubTransport } from "@intx/mail-memory"; -import type { DeployApplyErrorFrame } from "@intx/types/sidecar"; -import type { - ConnectorThreadState, - CryptoProvider, - InferenceEvent, - KeyPair, -} from "@intx/types/runtime"; +import type { InferenceEvent, KeyPair } from "@intx/types/runtime"; import { createAgentKeyStore, type AgentKeyStore } from "./agent-key-store"; import { createAgentRepoStore, type AgentRepoStore } from "./agent-repo-store"; import { createSessionManager, type SessionManager } from "./session-manager"; -import type { HarnessBuilder } from "./harness-builder"; import { createHubLink, type DeployRouter, @@ -53,22 +44,13 @@ export type SidecarCryptoOps = { * Factory the orchestrator invokes once `sessions` and `keyStore` * are constructed. The host returns the `DeployRouter` the link * routes every inbound `agent.deploy` through; production wires - * this against a workflow-host supervisor whose `trivialLaunch` - * closes over `sessions.provisionAgent`. The host is responsible - * for closing over any other state the router needs (transport, - * substrate handle, signing keys) at the call site. - * - * `onAgentEvent` is the per-agent InferenceEvent subscription seam - * the trivial-launch closure uses to bracket the workflow-run event - * chain against the existing reactor moments - * (`message.run.started` / `inference.start` / `message.run.ended`). - * The orchestrator hands it through unchanged so the supervisor's - * `recordRunEvent` callback fires for the right address. + * this against the sidecar's workflow-run deploy router. The host is + * responsible for closing over any other state the router needs + * (transport, substrate handle, signing keys) at the call site. */ export type CreateDeployRouter = (deps: { sessions: SessionManager; keyStore: AgentKeyStore; - onAgentEvent: SessionManager["onAgentEvent"]; /** * Per-event sink the multi-step branch routes a spawned child's * verified `InferenceEvent`s through, keyed by the deployment's agent @@ -92,8 +74,6 @@ export type SidecarOrchestratorConfig = { token: string; dataDir: string; transport: HubTransport; - buildHarness: HarnessBuilder; - createAgentCrypto: (keyPair: KeyPair) => CryptoProvider; cryptoOps: SidecarCryptoOps; /** * Host-injected `DeployRouter` factory. The orchestrator calls it @@ -161,8 +141,6 @@ export function createSidecarOrchestrator( token, dataDir, transport, - buildHarness, - createAgentCrypto, cryptoOps, createDeployRouter, mailInboundRouter, @@ -182,10 +160,10 @@ export function createSidecarOrchestrator( verifySSHSig: cryptoOps.verifySSHSig, }); - // Pre-declare the sinks. SessionManager dispatches events into them - // synchronously; the closures point at no-ops until HubLink is - // constructed below, at which point they are swapped to the link's - // sendEvent / sendConnectorState methods. + // Sink the multi-step deploy path routes a spawned child's verified + // InferenceEvents through. It points at a no-op until HubLink is + // constructed below, at which point it is swapped to the link's + // sendEvent method. let dispatchEvent: ( agentAddress: string, sessionId: string, @@ -193,40 +171,12 @@ export function createSidecarOrchestrator( ) => void = () => { /* replaced after HubLink construction */ }; - let dispatchConnectorState: ( - agentAddress: string, - state: ConnectorThreadState | null, - ) => void = () => { - /* replaced after HubLink construction */ - }; - let dispatchDeployApplyError: ( - agentAddress: string, - payload: Omit, - ) => void = () => { - /* replaced after HubLink construction */ - }; - const sessions = createSessionManager({ - transport, - repoStore, - keyStore, - buildHarness, - createAgentCrypto, - onEvent(agentAddress, sessionId, event) { - dispatchEvent(agentAddress, sessionId, event); - }, - onConnectorStateChanged(agentAddress, state) { - dispatchConnectorState(agentAddress, state); - }, - onDeployApplyError(agentAddress, payload) { - dispatchDeployApplyError(agentAddress, payload); - }, - }); + const sessions = createSessionManager({ repoStore }); const deployRouter = createDeployRouter({ sessions, keyStore, - onAgentEvent: sessions.onAgentEvent, // Route a spawned child's verified InferenceEvents up the same // hub-link `agent.event` sink the in-process path uses, so step // agent events reach the hub timeline keyed to the deploy's @@ -265,8 +215,6 @@ export function createSidecarOrchestrator( }); dispatchEvent = hubLink.sendEvent; - dispatchConnectorState = hubLink.sendConnectorState; - dispatchDeployApplyError = hubLink.sendDeployApplyError; function start(): void { hubLink.connect(); diff --git a/packages/hub-agent/src/ws/hub-link-bootstrap-prune.test.ts b/packages/hub-agent/src/ws/hub-link-bootstrap-prune.test.ts index 69e73a33..3bfebd13 100644 --- a/packages/hub-agent/src/ws/hub-link-bootstrap-prune.test.ts +++ b/packages/hub-agent/src/ws/hub-link-bootstrap-prune.test.ts @@ -31,15 +31,13 @@ import { signEd25519, verifySSHSignature } from "@intx/crypto"; import type { HarnessConfig, InboundMessage, - InferenceSource, KeyPair, } from "@intx/types/runtime"; -import type { GrantRule } from "@intx/types/authz"; import { hexDecode } from "@intx/types"; import { createHubLink, type DeployRouter } from "./hub-link"; import type { AgentKeyStore } from "../agent-key-store"; -import type { AgentEventListener, SessionManager } from "../session-manager"; +import type { SessionManager } from "../session-manager"; function createTestKeyStore(): AgentKeyStore & { registerKey(address: string, kp: KeyPair): void; @@ -85,19 +83,11 @@ function createTestKeyStore(): AgentKeyStore & { }; } -function createTestDeployRouter( - sessions: SessionManager, - keyStore: AgentKeyStore, -): DeployRouter { +function createTestDeployRouter(keyStore: AgentKeyStore): DeployRouter { return { async deploy(frame) { - const result = await sessions.provisionAgent(frame.config); keyStore.recordHubKey(frame.agentAddress, frame.hubPublicKey); - await sessions.persistHubPublicKey( - frame.agentAddress, - frame.hubPublicKey, - ); - return { publicKey: result.publicKey }; + return { publicKey: "aa".repeat(32) }; }, }; } @@ -112,55 +102,10 @@ function createMockSessionManager(): SessionManager & { provisioned: [] as HarnessConfig[], addresses: [] as string[], - async provisionAgent(config: HarnessConfig) { - mock.provisioned.push(config); - mock.addresses.push(config.agentAddress); - return { - publicKey: "deadbeef", - keyPair: { - publicKey: new Uint8Array(32), - privateKey: new Uint8Array(32), - }, - }; - }, initRepo: (_address: string) => Promise.resolve(), - async startSession(_agentAddress: string): Promise { - /* unused */ - }, - async destroySession(agentAddress: string): Promise { - mock.addresses = mock.addresses.filter((a) => a !== agentAddress); - }, - async abortSession(_agentAddress: string, _reason: string): Promise { - /* unused */ - }, - deliverMessage(_agentAddress: string, _message: InboundMessage): void { - /* unused */ - }, - async updateGrants( - _agentAddress: string, - _grants: GrantRule[], - ): Promise { - /* unused */ - }, - async updateSources( - _agentAddress: string, - _sources: InferenceSource[], - _defaultSource: string, - ): Promise { - /* unused */ - }, - hasSession(agentAddress: string): boolean { - return mock.addresses.includes(agentAddress); - }, - isProvisioned(agentAddress: string): boolean { - return mock.addresses.includes(agentAddress); - }, getAddresses(): string[] { return [...mock.addresses]; }, - async restoreSessions() { - return { restored: [], failed: [] }; - }, applyDeployPack: () => Promise.resolve(), applyAssetPack: () => Promise.resolve(), createStatePack: () => @@ -171,15 +116,7 @@ function createMockSessionManager(): SessionManager & { }), deleteAgentDir: () => Promise.resolve(), getDeployRef: (_agentAddress: string) => Promise.resolve(null), - persistHubPublicKey: (_agentAddress: string, _hubPublicKey: string) => - Promise.resolve(), - commitInboundMail: (_agentAddress: string, _rawMessage: Uint8Array) => - Promise.resolve(), getSessionId: (_agentAddress: string) => undefined, - onAgentEvent: - (_agentAddress: string, _listener: AgentEventListener) => () => { - /* unused */ - }, } satisfies SessionManager & { provisioned: HarnessConfig[]; addresses: string[]; @@ -313,7 +250,7 @@ describe("hub-link workflow-run pack bootstrap prune", () => { transport, sessions, keyStore, - deployRouter: createTestDeployRouter(sessions, keyStore), + deployRouter: createTestDeployRouter(keyStore), }); client.connect(); diff --git a/packages/hub-agent/src/ws/hub-link-mail-router.test.ts b/packages/hub-agent/src/ws/hub-link-mail-router.test.ts index c2538616..0a666ebe 100644 --- a/packages/hub-agent/src/ws/hub-link-mail-router.test.ts +++ b/packages/hub-agent/src/ws/hub-link-mail-router.test.ts @@ -25,16 +25,14 @@ import { base64Encode } from "@intx/types"; import type { HarnessConfig, InboundMessage, - InferenceSource, KeyPair, } from "@intx/types/runtime"; -import type { GrantRule } from "@intx/types/authz"; import { signEd25519, verifySSHSignature } from "@intx/crypto"; import { hexDecode } from "@intx/types"; import { createHubLink, type DeployRouter } from "./hub-link"; import type { AgentKeyStore } from "../agent-key-store"; -import type { AgentEventListener, SessionManager } from "../session-manager"; +import type { SessionManager } from "../session-manager"; function createTestKeyStore(): AgentKeyStore & { registerKey(address: string, kp: KeyPair): void; @@ -80,31 +78,23 @@ function createTestKeyStore(): AgentKeyStore & { }; } -function createTestDeployRouter( - sessions: SessionManager, - keyStore: AgentKeyStore, -): DeployRouter { +function createTestDeployRouter(keyStore: AgentKeyStore): DeployRouter { return { async deploy(frame) { - const result = await sessions.provisionAgent(frame.config); keyStore.recordHubKey(frame.agentAddress, frame.hubPublicKey); - await sessions.persistHubPublicKey( - frame.agentAddress, - frame.hubPublicKey, - ); - return { publicKey: result.publicKey }; + return { publicKey: "aa".repeat(32) }; }, }; } -function withTestDeployBindings(sessions: SessionManager): { +function withTestDeployBindings(): { keyStore: AgentKeyStore & { registerKey(address: string, kp: KeyPair): void }; deployRouter: DeployRouter; } { const keyStore = createTestKeyStore(); return { keyStore, - deployRouter: createTestDeployRouter(sessions, keyStore), + deployRouter: createTestDeployRouter(keyStore), }; } @@ -143,66 +133,10 @@ function createMockSessionManager(): SessionManager & { provisionedAddresses: [] as string[], shouldThrow: null as string | null, - async provisionAgent(config: HarnessConfig) { - if (mock.shouldThrow !== null) throw new Error(mock.shouldThrow); - mock.provisioned.push(config); - mock.provisionedAddresses.push(config.agentAddress); - return { - publicKey: "deadbeef", - keyPair: { - publicKey: new Uint8Array(32), - privateKey: new Uint8Array(32), - }, - }; - }, initRepo: (_address: string) => Promise.resolve(), - async startSession(agentAddress: string): Promise { - if (mock.shouldThrow !== null) throw new Error(mock.shouldThrow); - mock.started.push(agentAddress); - mock.provisionedAddresses = mock.provisionedAddresses.filter( - (a) => a !== agentAddress, - ); - mock.addresses.push(agentAddress); - }, - async destroySession(agentAddress: string): Promise { - if (mock.shouldThrow !== null) throw new Error(mock.shouldThrow); - mock.destroyed.push(agentAddress); - mock.addresses = mock.addresses.filter((a) => a !== agentAddress); - }, - async abortSession(agentAddress: string, reason: string): Promise { - if (mock.shouldThrow !== null) throw new Error(mock.shouldThrow); - mock.aborted.push({ address: agentAddress, reason }); - mock.addresses = mock.addresses.filter((a) => a !== agentAddress); - }, - deliverMessage(agentAddress: string, message: InboundMessage): void { - if (mock.shouldThrow !== null) throw new Error(mock.shouldThrow); - mock.delivered.push({ agentAddress, message }); - }, - async updateGrants( - _agentAddress: string, - _grants: GrantRule[], - ): Promise { - if (mock.shouldThrow !== null) throw new Error(mock.shouldThrow); - }, - async updateSources( - _agentAddress: string, - _sources: InferenceSource[], - _defaultSource: string, - ): Promise { - if (mock.shouldThrow !== null) throw new Error(mock.shouldThrow); - }, - hasSession(agentAddress: string): boolean { - return mock.addresses.includes(agentAddress); - }, - isProvisioned(agentAddress: string): boolean { - return mock.provisionedAddresses.includes(agentAddress); - }, getAddresses(): string[] { return [...mock.addresses]; }, - async restoreSessions() { - return { restored: [], failed: [] }; - }, applyDeployPack: () => Promise.resolve(), applyAssetPack: () => Promise.resolve(), createStatePack: () => @@ -213,15 +147,7 @@ function createMockSessionManager(): SessionManager & { }), deleteAgentDir: () => Promise.resolve(), getDeployRef: (_agentAddress: string) => Promise.resolve(null), - persistHubPublicKey: (_agentAddress: string, _hubPublicKey: string) => - Promise.resolve(), - commitInboundMail: (_agentAddress: string, _rawMessage: Uint8Array) => - Promise.resolve(), getSessionId: (_agentAddress: string) => undefined, - onAgentEvent: - (_agentAddress: string, _listener: AgentEventListener) => () => { - /* no-op disposer: this test does not exercise per-agent events */ - }, }; return mock; } @@ -320,7 +246,7 @@ describe("hub-link mail.inbound throwing router", () => { token: "test-token", transport, sessions, - ...withTestDeployBindings(sessions), + ...withTestDeployBindings(), mailInboundRouter, }); diff --git a/packages/hub-agent/src/ws/hub-link.test.ts b/packages/hub-agent/src/ws/hub-link.test.ts index 674f4372..0aac036b 100644 --- a/packages/hub-agent/src/ws/hub-link.test.ts +++ b/packages/hub-agent/src/ws/hub-link.test.ts @@ -10,12 +10,7 @@ import { import { createInMemoryTransport } from "@intx/mail-memory"; import { signEd25519, verifySSHSignature } from "@intx/crypto"; import { base64Encode, hexEncode } from "@intx/types"; -import type { - HarnessConfig, - InboundMessage, - InferenceSource, -} from "@intx/types/runtime"; -import type { GrantRule } from "@intx/types/authz"; +import type { HarnessConfig } from "@intx/types/runtime"; import { createHubLink, @@ -23,29 +18,20 @@ import { type ReconnectScheduler, } from "./hub-link"; import type { AgentKeyStore } from "../agent-key-store"; -import type { AgentEventListener, SessionManager } from "../session-manager"; +import type { SessionManager } from "../session-manager"; /** - * Test-only deploy router that mirrors the pre-supervisor inline - * flow: provisionAgent + recordHubKey + persistHubPublicKey. The - * production wiring lifts this same closure into a workflow-host - * supervisor's `trivialLaunch`; the tests here exercise the link's - * surface against the closure directly so the SessionManager mock - * assertions are unchanged. + * Test-only deploy router modelling the current deploy path: record the + * hub pairing key so `verifyDeployCommit` can accept the deployment's + * packs, and surface a public key on the ack. Production stages the + * deploy through the workflow-run substrate; the tests here exercise the + * link's surface against the router directly. */ -function createTestDeployRouter( - sessions: SessionManager, - keyStore: AgentKeyStore, -): DeployRouter { +function createTestDeployRouter(keyStore: AgentKeyStore): DeployRouter { return { async deploy(frame) { - const result = await sessions.provisionAgent(frame.config); keyStore.recordHubKey(frame.agentAddress, frame.hubPublicKey); - await sessions.persistHubPublicKey( - frame.agentAddress, - frame.hubPublicKey, - ); - return { publicKey: result.publicKey }; + return { publicKey: "aa".repeat(32) }; }, }; } @@ -57,14 +43,14 @@ function createTestDeployRouter( * call site does not have to name a temporary binding for the * keyStore-router pairing. */ -function withTestDeployBindings(sessions: SessionManager): { +function withTestDeployBindings(): { keyStore: AgentKeyStore & { registerKey(address: string, kp: KeyPair): void }; deployRouter: DeployRouter; } { const keyStore = createTestKeyStore(); return { keyStore, - deployRouter: createTestDeployRouter(sessions, keyStore), + deployRouter: createTestDeployRouter(keyStore), }; } import type { KeyPair } from "@intx/types/runtime"; @@ -136,88 +122,9 @@ async function waitFor( } } -type DeliveredMessage = { agentAddress: string; message: InboundMessage }; - -function createMockSessionManager(): SessionManager & { - provisioned: HarnessConfig[]; - started: string[]; - destroyed: string[]; - aborted: { address: string; reason: string }[]; - delivered: DeliveredMessage[]; - addresses: string[]; - provisionedAddresses: string[]; - shouldThrow: string | null; -} { - const mock = { - provisioned: [] as HarnessConfig[], - started: [] as string[], - destroyed: [] as string[], - aborted: [] as { address: string; reason: string }[], - delivered: [] as DeliveredMessage[], - addresses: [] as string[], - provisionedAddresses: [] as string[], - shouldThrow: null as string | null, - - async provisionAgent(config: HarnessConfig) { - if (mock.shouldThrow !== null) throw new Error(mock.shouldThrow); - mock.provisioned.push(config); - mock.provisionedAddresses.push(config.agentAddress); - return { - publicKey: "deadbeef", - keyPair: { - publicKey: new Uint8Array(32), - privateKey: new Uint8Array(32), - }, - }; - }, +function createMockSessionManager(): SessionManager { + return { initRepo: (_address: string) => Promise.resolve(), - async startSession(agentAddress: string): Promise { - if (mock.shouldThrow !== null) throw new Error(mock.shouldThrow); - mock.started.push(agentAddress); - mock.provisionedAddresses = mock.provisionedAddresses.filter( - (a) => a !== agentAddress, - ); - mock.addresses.push(agentAddress); - }, - async destroySession(agentAddress: string): Promise { - if (mock.shouldThrow !== null) throw new Error(mock.shouldThrow); - mock.destroyed.push(agentAddress); - mock.addresses = mock.addresses.filter((a) => a !== agentAddress); - }, - async abortSession(agentAddress: string, reason: string): Promise { - if (mock.shouldThrow !== null) throw new Error(mock.shouldThrow); - mock.aborted.push({ address: agentAddress, reason }); - mock.addresses = mock.addresses.filter((a) => a !== agentAddress); - }, - deliverMessage(agentAddress: string, message: InboundMessage): void { - if (mock.shouldThrow !== null) throw new Error(mock.shouldThrow); - mock.delivered.push({ agentAddress, message }); - }, - async updateGrants( - _agentAddress: string, - _grants: GrantRule[], - ): Promise { - if (mock.shouldThrow !== null) throw new Error(mock.shouldThrow); - }, - async updateSources( - _agentAddress: string, - _sources: InferenceSource[], - _defaultSource: string, - ): Promise { - if (mock.shouldThrow !== null) throw new Error(mock.shouldThrow); - }, - hasSession(agentAddress: string): boolean { - return mock.addresses.includes(agentAddress); - }, - isProvisioned(agentAddress: string): boolean { - return mock.provisionedAddresses.includes(agentAddress); - }, - getAddresses(): string[] { - return [...mock.addresses]; - }, - async restoreSessions() { - return { restored: [], failed: [] }; - }, applyDeployPack: () => Promise.resolve(), applyAssetPack: () => Promise.resolve(), createStatePack: () => @@ -228,17 +135,9 @@ function createMockSessionManager(): SessionManager & { }), deleteAgentDir: () => Promise.resolve(), getDeployRef: (_agentAddress: string) => Promise.resolve(null), - persistHubPublicKey: (_agentAddress: string, _hubPublicKey: string) => - Promise.resolve(), - commitInboundMail: (_agentAddress: string, _rawMessage: Uint8Array) => - Promise.resolve(), + getAddresses: () => [], getSessionId: (_agentAddress: string) => undefined, - onAgentEvent: - (_agentAddress: string, _listener: AgentEventListener) => () => { - /* no-op disposer: the link tests do not exercise per-agent events */ - }, }; - return mock; } const TEST_CONFIG: HarnessConfig = { @@ -363,7 +262,7 @@ describe("sidecar↔hub integration", () => { transport, sessions, - ...withTestDeployBindings(sessions), + ...withTestDeployBindings(), }); client.connect(); @@ -380,7 +279,7 @@ describe("sidecar↔hub integration", () => { } }); - test("hub sends session.create and sidecar acks", async () => { + test("hub sends a deploy and the sidecar acks", async () => { const transport = createInMemoryTransport(); const sessions = createMockSessionManager(); const client = createHubLink({ @@ -390,7 +289,7 @@ describe("sidecar↔hub integration", () => { transport, sessions, - ...withTestDeployBindings(sessions), + ...withTestDeployBindings(), }); client.connect(); @@ -401,10 +300,9 @@ describe("sidecar↔hub integration", () => { await env.router.sendAgentDeploy("agent-1@test.interchange", TEST_CONFIG); - expect(sessions.provisioned).toHaveLength(1); - expect(sessions.provisioned[0]?.agentAddress).toBe( - "agent-1@test.interchange", - ); + // The deploy resolves against the ack; the sidecar stays connected + // after handling it. + expect(env.router.getConnectedSidecars()).toContain("sc-create"); } finally { client.close(); await waitFor( @@ -413,37 +311,6 @@ describe("sidecar↔hub integration", () => { } }); - test("hub receives session.error when create fails", async () => { - const transport = createInMemoryTransport(); - const sessions = createMockSessionManager(); - sessions.shouldThrow = "provider not configured"; - const client = createHubLink({ - hubURL: `ws://localhost:${env.server.port}/ws`, - sidecarId: "sc-fail", - token: "test-token", - - transport, - sessions, - ...withTestDeployBindings(sessions), - }); - - client.connect(); - try { - await waitFor(() => - env.router.getConnectedSidecars().includes("sc-fail"), - ); - - await expect( - env.router.sendAgentDeploy("fail-agent@test", TEST_CONFIG), - ).rejects.toThrow("provider not configured"); - } finally { - client.close(); - await waitFor( - () => !env.router.getConnectedSidecars().includes("sc-fail"), - ); - } - }); - test("sidecar forwards agent events to hub", async () => { const transport = createInMemoryTransport(); const sessions = createMockSessionManager(); @@ -455,7 +322,7 @@ describe("sidecar↔hub integration", () => { transport, sessions, - ...withTestDeployBindings(sessions), + ...withTestDeployBindings(), }); client.connect(); @@ -487,59 +354,6 @@ describe("sidecar↔hub integration", () => { } }); - test("hub routes mail inbound to sidecar", async () => { - const transport = createInMemoryTransport(); - const sessions = createMockSessionManager(); - // The agent must be in the session manager's address list so the - // register frame includes it in the routing table. - sessions.addresses.push("agent-1@test.interchange"); - const { generateKeyPair, createEd25519Crypto } = await import( - "@intx/crypto" - ); - const kp = await generateKeyPair(); - transport.register("agent-1@test.interchange", createEd25519Crypto(kp)); - - const client = createHubLink({ - hubURL: `ws://localhost:${env.server.port}/ws`, - sidecarId: "sc-mail-in", - token: "test-token", - - transport, - sessions, - ...withTestDeployBindings(sessions), - }); - - client.connect(); - try { - await waitFor(() => - env.router.getRoutableAddresses().includes("agent-1@test.interchange"), - ); - - const encoded = base64Encode(VALID_MESSAGE); - const routed = env.router.routeMail("agent-1@test.interchange", encoded); - expect(routed).toBe(true); - - // Wait for the message to be delivered to the agent's INBOX. - const agentTransport = transport.getTransportFor( - "agent-1@test.interchange", - ); - await waitFor(async () => { - const refs = await agentTransport.search("INBOX", {}); - return refs.length > 0; - }); - - const refs = await agentTransport.search("INBOX", {}); - expect(refs).toHaveLength(1); - const headers = await agentTransport.fetchHeaders(refs[0]!); - expect(headers.from).toBe("external@remote.interchange"); - } finally { - client.close(); - await waitFor( - () => !env.router.getConnectedSidecars().includes("sc-mail-in"), - ); - } - }); - test("sidecar forwards outbound mail to hub", async () => { const transport = createInMemoryTransport(); const sessions = createMockSessionManager(); @@ -557,7 +371,7 @@ describe("sidecar↔hub integration", () => { transport, sessions, - ...withTestDeployBindings(sessions), + ...withTestDeployBindings(), }); client.connect(); @@ -586,89 +400,9 @@ describe("sidecar↔hub integration", () => { } }); - test("mail routes between two sidecars via hub", async () => { - const { generateKeyPair, createEd25519Crypto } = await import( - "@intx/crypto" - ); - - // Sidecar A - const transportA = createInMemoryTransport(); - const sessionsA = createMockSessionManager(); - sessionsA.addresses.push("alice@test.interchange"); - const kpA = await generateKeyPair(); - transportA.register("alice@test.interchange", createEd25519Crypto(kpA)); - - // Sidecar B - const transportB = createInMemoryTransport(); - const sessionsB = createMockSessionManager(); - sessionsB.addresses.push("bob@test.interchange"); - const kpB = await generateKeyPair(); - transportB.register("bob@test.interchange", createEd25519Crypto(kpB)); - - const clientA = createHubLink({ - hubURL: `ws://localhost:${env.server.port}/ws`, - sidecarId: "sc-alice", - token: "test-token", - transport: transportA, - sessions: sessionsA, - ...withTestDeployBindings(sessionsA), - }); - const clientB = createHubLink({ - hubURL: `ws://localhost:${env.server.port}/ws`, - sidecarId: "sc-bob", - token: "test-token", - transport: transportB, - sessions: sessionsB, - ...withTestDeployBindings(sessionsB), - }); - - clientA.connect(); - clientB.connect(); - try { - await waitFor(() => - env.router.getRoutableAddresses().includes("alice@test.interchange"), - ); - await waitFor(() => - env.router.getRoutableAddresses().includes("bob@test.interchange"), - ); - - // Alice sends a message to Bob. - const aliceTransport = transportA.getTransportFor( - "alice@test.interchange", - ); - await aliceTransport.send({ - to: "bob@test.interchange", - type: "conversation.message", - content: "Hello Bob", - }); - - // Bob should receive it in his INBOX. - const bobTransport = transportB.getTransportFor("bob@test.interchange"); - await waitFor(async () => { - const refs = await bobTransport.search("INBOX", {}); - return refs.length > 0; - }); - - const refs = await bobTransport.search("INBOX", {}); - expect(refs).toHaveLength(1); - const headers = await bobTransport.fetchHeaders(refs[0]!); - expect(headers.from).toBe("alice@test.interchange"); - } finally { - clientA.close(); - clientB.close(); - await waitFor( - () => !env.router.getConnectedSidecars().includes("sc-alice"), - ); - await waitFor( - () => !env.router.getConnectedSidecars().includes("sc-bob"), - ); - } - }); - test("disconnect cleans up routing table", async () => { const transport = createInMemoryTransport(); const sessions = createMockSessionManager(); - sessions.addresses.push("tracked@test.interchange"); const client = createHubLink({ hubURL: `ws://localhost:${env.server.port}/ws`, @@ -677,7 +411,8 @@ describe("sidecar↔hub integration", () => { transport, sessions, - ...withTestDeployBindings(sessions), + ...withTestDeployBindings(), + getWorkflowAddresses: () => ["tracked@test.interchange"], }); client.connect(); @@ -706,7 +441,7 @@ describe("sidecar↔hub integration", () => { token: "test-token", transport, sessions, - ...withTestDeployBindings(sessions), + ...withTestDeployBindings(), }); client.connect(); @@ -749,7 +484,7 @@ describe("sidecar↔hub integration", () => { token: "test-token", transport, sessions, - ...withTestDeployBindings(sessions), + ...withTestDeployBindings(), }); client.connect(); @@ -834,7 +569,7 @@ describe("sidecar↔hub integration", () => { token: "test-token", transport, sessions, - ...withTestDeployBindings(sessions), + ...withTestDeployBindings(), }); client.connect(); @@ -853,34 +588,25 @@ describe("sidecar↔hub integration", () => { } }); - test("reconnect restores hubPublicKey into hubKeys map", async () => { + test("a deploy records the hub key so verifyCommit is bound to it", async () => { const { generateKeyPair, createSSHSignature } = await import( "@intx/crypto" ); - // Agent keypair — used for challenge/response signing. - const agentKp = await generateKeyPair(); // Hub keypair — the key whose public half the sidecar stores to verify - // deploy commit signatures. Distinct from the agent keypair. + // deploy-commit signatures. const hubKp = await generateKeyPair(); - const fakeAddress = "restored@test.interchange"; - - const agentPublicKeyHex = hexEncode(agentKp.publicKey); + const deployedAddress = "deployed@test.interchange"; const hubPublicKeyHex = hexEncode(hubKp.publicKey); - // Stand up a hub that supports reconnect (lookupPublicKey configured). - const reconnectRouter = createSidecarRouter({ + const deployHubRouter = createSidecarRouter({ requestTimeoutMs: 5000, challengeTimeoutMs: 5000, hubPublicKey: hubPublicKeyHex, - lookups: { - lookupPublicKey: async (addr) => - addr === fakeAddress ? agentPublicKeyHex : null, - }, }); - const reconnectApp = new Hono(); - reconnectApp.get( + const deployApp = new Hono(); + deployApp.get( "/ws", upgradeWebSocket((_c) => { let handle: WsHandle; @@ -894,69 +620,54 @@ describe("sidecar↔hub integration", () => { ws.close(); }, }; - reconnectRouter.handleOpen(handle); + deployHubRouter.handleOpen(handle); }, onMessage(evt, _ws) { if (typeof evt.data === "string") { - reconnectRouter.handleMessage(handle, evt.data); + deployHubRouter.handleMessage(handle, evt.data); } }, onClose(_evt, _ws) { - reconnectRouter.handleClose(handle); + deployHubRouter.handleClose(handle); }, }; }), ); - const reconnectServer = Bun.serve({ - fetch: reconnectApp.fetch, + const deployServer = Bun.serve({ + fetch: deployApp.fetch, websocket, port: 0, }); const transport = createInMemoryTransport(); const sessions = createMockSessionManager(); - - sessions.addresses.push(fakeAddress); - sessions.restoreSessions = async () => ({ - restored: [ - { - address: fakeAddress, - keyPair: agentKp, - config: { ...TEST_CONFIG, agentAddress: fakeAddress }, - hubPublicKey: hubPublicKeyHex, - }, - ], - failed: [], - }); - sessions.getDeployRef = async () => "a".repeat(40); - - // In production, SessionManager.restoreSessions populates - // AgentKeyStore's in-memory cache via scanKeys before HubLink - // iterates the restored agents. The mock SessionManager here does - // not exercise that path, so we seed the keyStore directly. const keyStore = createTestKeyStore(); - keyStore.registerKey(fakeAddress, agentKp); const client = createHubLink({ - hubURL: `ws://localhost:${reconnectServer.port}/ws`, - sidecarId: "sc-restore-key", + hubURL: `ws://localhost:${deployServer.port}/ws`, + sidecarId: "sc-deploy-key", token: "test-token", transport, sessions, keyStore, - deployRouter: createTestDeployRouter(sessions, keyStore), + deployRouter: createTestDeployRouter(keyStore), }); client.connect(); try { - await waitFor( - () => reconnectRouter.getRoutableAddresses().includes(fakeAddress), - 5000, + await waitFor(() => + deployHubRouter.getConnectedSidecars().includes("sc-deploy-key"), ); - // Capture the verifyCommit callback to prove the correct hub key - // was restored — not just any key. + // The hub's deploy frame carries hubPublicKeyHex; the deploy router + // records it via keyStore.recordHubKey, so a later pack's verifyCommit + // is bound to the hub key. + await deployHubRouter.sendAgentDeploy(deployedAddress, { + ...TEST_CONFIG, + agentAddress: deployedAddress, + }); + let capturedVerifyCommit: | ((p: string, s: string) => Promise) | undefined; @@ -971,8 +682,8 @@ describe("sidecar↔hub integration", () => { capturedVerifyCommit = verifyCommit; }; - await reconnectRouter.sendPack( - fakeAddress, + await deployHubRouter.sendPack( + deployedAddress, new Uint8Array([1, 2, 3]), "refs/heads/deploy", "b".repeat(40), @@ -980,8 +691,8 @@ describe("sidecar↔hub integration", () => { expect(capturedVerifyCommit).toBeFunction(); - // Create a real signature with the hub's private key and verify - // it round-trips through the restored verifyCommit callback. + // A signature from the hub's key round-trips through the recorded + // verifyCommit callback. const payload = "tree abc\nauthor t 0 +0000\n\ntest\n"; const sig = await createSSHSignature( payload, @@ -990,8 +701,8 @@ describe("sidecar↔hub integration", () => { ); expect(await capturedVerifyCommit!(payload, sig)).toBe(true); - // A signature from a different key must fail, proving the callback - // is bound to the specific hub key that was restored. + // A signature from a different key fails, proving the callback is bound + // to the specific hub key the deploy recorded. const wrongKp = await generateKeyPair(); const wrongSig = await createSSHSignature( payload, @@ -1001,7 +712,7 @@ describe("sidecar↔hub integration", () => { expect(await capturedVerifyCommit!(payload, wrongSig)).toBe(false); } finally { client.close(); - await reconnectServer.stop(true); + await deployServer.stop(true); } }); @@ -1018,7 +729,7 @@ describe("sidecar↔hub integration", () => { transport, sessions, - ...withTestDeployBindings(sessions), + ...withTestDeployBindings(), pingIntervalMs: 100, }); @@ -1059,7 +770,7 @@ describe("sidecar↔hub integration", () => { token: "test-token", transport, sessions, - ...withTestDeployBindings(sessions), + ...withTestDeployBindings(), }); client.connect(); @@ -1119,7 +830,7 @@ describe("sidecar↔hub integration", () => { token: "test-token", transport, sessions, - ...withTestDeployBindings(sessions), + ...withTestDeployBindings(), }); client.connect(); @@ -1173,7 +884,7 @@ describe("sidecar↔hub integration", () => { transport, sessions, - ...withTestDeployBindings(sessions), + ...withTestDeployBindings(), scheduleReconnect: fakeScheduleReconnect, }); @@ -1212,7 +923,6 @@ describe("sidecar↔hub integration", () => { // must consult mailInboundRouter first and -- on a `true` return // -- skip transport.deliver and sessions.commitInboundMail. const deploymentAddress = "dep_multistep-1@integration.interchange"; - sessions.addresses.push(deploymentAddress); const routed: { address: string; bytes: Uint8Array }[] = []; const mailInboundRouter = { @@ -1228,8 +938,9 @@ describe("sidecar↔hub integration", () => { token: "test-token", transport, sessions, - ...withTestDeployBindings(sessions), + ...withTestDeployBindings(), mailInboundRouter, + getWorkflowAddresses: () => [deploymentAddress], }); client.connect(); @@ -1247,21 +958,6 @@ describe("sidecar↔hub integration", () => { expect(routed).toHaveLength(1); expect(routed[0]?.address).toBe(deploymentAddress); expect(routed[0]?.bytes).toEqual(VALID_MESSAGE); - - // Legacy fallback paths must not have been reached. The mock - // sessions tracks delivered messages and `commitInboundMail` - // returns a resolved promise without recording, so the sentinel - // we check is the transport-side delivery. The transport's - // `deliver` would throw because no address is registered, and - // the link's `deliverLocalMail` catches and logs that throw. - // The cleaner check: sessions.delivered stays empty. (Even if - // the mock's commitInboundMail did nothing, `deliverLocalMail` - // would attempt transport.deliver -- but the test transport has - // no registered mailbox for `deploymentAddress`, so that would - // surface as a logged warn rather than a delivery. Asserting - // sessions.delivered.length === 0 makes the no-fallback - // intent explicit.) - expect(sessions.delivered).toHaveLength(0); } finally { client.close(); await waitFor( @@ -1270,62 +966,10 @@ describe("sidecar↔hub integration", () => { } }); - test("mailInboundRouter that returns false falls through to the legacy path", async () => { - const transport = createInMemoryTransport(); - const sessions = createMockSessionManager(); - const address = "agent-fallback@test.interchange"; - sessions.addresses.push(address); - const { generateKeyPair, createEd25519Crypto } = await import( - "@intx/crypto" - ); - const kp = await generateKeyPair(); - transport.register(address, createEd25519Crypto(kp)); - - const consulted: string[] = []; - const mailInboundRouter = { - tryRoute(addr: string, _message: Uint8Array): boolean { - consulted.push(addr); - return false; - }, - }; - - const client = createHubLink({ - hubURL: `ws://localhost:${env.server.port}/ws`, - sidecarId: "sc-fallback-mail", - token: "test-token", - transport, - sessions, - ...withTestDeployBindings(sessions), - mailInboundRouter, - }); - - client.connect(); - try { - await waitFor(() => env.router.getRoutableAddresses().includes(address)); - - const encoded = base64Encode(VALID_MESSAGE); - expect(env.router.routeMail(address, encoded)).toBe(true); - - const agentTransport = transport.getTransportFor(address); - await waitFor(async () => { - const refs = await agentTransport.search("INBOX", {}); - return refs.length > 0; - }); - - expect(consulted).toContain(address); - } finally { - client.close(); - await waitFor( - () => !env.router.getConnectedSidecars().includes("sc-fallback-mail"), - ); - } - }); - test("drainInboundRouter dispatches an inbound drain.deliver frame", async () => { const transport = createInMemoryTransport(); const sessions = createMockSessionManager(); const deploymentAddress = "dep_drain-1@integration.interchange"; - sessions.addresses.push(deploymentAddress); const routed: { agentAddress: string; deadlineMs: number }[] = []; const drainInboundRouter = { @@ -1347,8 +991,9 @@ describe("sidecar↔hub integration", () => { token: "test-token", transport, sessions, - ...withTestDeployBindings(sessions), + ...withTestDeployBindings(), drainInboundRouter, + getWorkflowAddresses: () => [deploymentAddress], }); client.connect(); @@ -1476,7 +1121,7 @@ describe("sidecar↔hub integration", () => { token: "test-token", transport, sessions, - ...withTestDeployBindings(sessions), + ...withTestDeployBindings(), }); client.connect(); @@ -1548,8 +1193,8 @@ describe("sidecar↔hub integration", () => { }); }); -describe("routability decoupled from restore", () => { - test("empty register ships before restore resolves; reconnect-with-addresses only after", async () => { +describe("register frame on connect", () => { + test("ships a single register carrying the sidecar's workflow addresses", async () => { const frames: string[] = []; const app = new Hono(); app.get( @@ -1566,75 +1211,40 @@ describe("routability decoupled from restore", () => { const transport = createInMemoryTransport(); const sessions = createMockSessionManager(); - - type RestoreResult = Awaited>; - let releaseRestore!: (result: RestoreResult) => void; - const pendingRestore = new Promise((resolve) => { - releaseRestore = resolve; - }); - sessions.restoreSessions = () => pendingRestore; - sessions.getDeployRef = async () => "a".repeat(40); + const workflowAddresses = ["ins_dep-1@integration.interchange"]; const client = createHubLink({ hubURL: `ws://localhost:${server.port}/ws`, - sidecarId: "sc-early-register", + sidecarId: "sc-register", token: "test-token", transport, sessions, - ...withTestDeployBindings(sessions), + ...withTestDeployBindings(), + getWorkflowAddresses: () => workflowAddresses, }); client.connect(); try { - // The empty-address register reaches the hub while restoreSessions is - // still pending: routability no longer waits on restore. await waitFor(() => frames .map((s) => JSON.parse(s)) .some((f: { type: string }) => f.type === "register"), ); - const beforeResolve = frames.map((s) => JSON.parse(s)); - const registerFrames = beforeResolve.filter( + const parsed = frames.map((s) => JSON.parse(s)); + const registerFrames = parsed.filter( (f: { type: string }) => f.type === "register", ); + // Exactly one register: the in-process session runtime is retired, so + // there is no empty-register-then-reconnect dance. expect(registerFrames).toHaveLength(1); + // Session addresses are always empty now; the sidecar's workflow + // deployments ride the same register frame so the hub re-registers + // their keyless routes without a challenge. expect(registerFrames[0].agentAddresses).toEqual([]); - // No reconnect yet: addresses enter addressIndex only after restore. - expect( - beforeResolve.some((f: { type: string }) => f.type === "reconnect"), - ).toBe(false); - - releaseRestore({ - restored: [ - { - address: "agent-1@test.interchange", - keyPair: { - publicKey: new Uint8Array(32), - privateKey: new Uint8Array(32), - }, - }, - ], - failed: [], - }); - - // The challenged reconnect frame, carrying the restored address, ships - // only after restore resolves. - await waitFor(() => - frames - .map((s) => JSON.parse(s)) - .some((f: { type: string }) => f.type === "reconnect"), - ); - const parsed = frames.map((s) => JSON.parse(s)); - const reconnectFrame = parsed.find( - (f: { type: string }) => f.type === "reconnect", - ); - expect(reconnectFrame.agentAddresses).toEqual([ - "agent-1@test.interchange", - ]); - expect( - parsed.findIndex((f: { type: string }) => f.type === "register"), - ).toBeLessThan( - parsed.findIndex((f: { type: string }) => f.type === "reconnect"), + expect(registerFrames[0].workflowAddresses).toEqual(workflowAddresses); + // No reconnect frame: disk-restored sessions no longer exist. + expect(parsed.some((f: { type: string }) => f.type === "reconnect")).toBe( + false, ); } finally { client.close(); diff --git a/packages/hub-agent/src/ws/hub-link.ts b/packages/hub-agent/src/ws/hub-link.ts index 65470f64..177b4e6d 100644 --- a/packages/hub-agent/src/ws/hub-link.ts +++ b/packages/hub-agent/src/ws/hub-link.ts @@ -1,8 +1,8 @@ // HubLink: the sidecar-side WebSocket protocol. // -// Connects to the hub, sends the register or reconnect frame, forwards -// outbound mail and inference events, and handles inbound agent -// lifecycle commands. Per-agent key material lives on AgentKeyStore; +// Connects to the hub, sends the register frame, forwards outbound +// mail and inference events, and handles inbound agent lifecycle +// commands. Per-agent key material lives on AgentKeyStore; // the link calls into the store for challenge signing, deploy-commit // verification, and hub-key bookkeeping. The wire layer itself never // touches raw key bytes. @@ -17,9 +17,6 @@ import { type AgentUndeployFrame, type ChallengeFrame, type ChallengeFailedFrame, - type SessionAbortFrame, - type GrantsUpdateFrame, - type SourcesUpdateFrame, type PackPushFrame, type PackDoneFrame, type PackAckFrame, @@ -28,17 +25,23 @@ import { type SignalDeliverFrame, type DrainDeliverFrame, type SyncRequestFrame, - type DeployApplyErrorFrame, } from "@intx/types/sidecar"; import { createPackReceiver, createPackSender } from "@intx/pack-transport"; import { base64Decode, base64Encode, hexDecode, hexEncode } from "@intx/types"; +import type { InferenceEvent } from "@intx/types/runtime"; import type { AgentKeyStore } from "../agent-key-store"; -import type { - ConnectorStateSink, - SessionEventSink, - SessionManager, -} from "../session-manager"; +import type { SessionManager } from "../session-manager"; + +/** + * Sink the link exposes for forwarding a spawned child's verified + * InferenceEvents to the hub timeline, keyed by the deploy's session id. + */ +export type SessionEventSink = ( + agentAddress: string, + sessionId: string, + event: InferenceEvent, +) => void; const logger = getLogger(["interchange", "hub-agent", "ws"]); @@ -64,10 +67,10 @@ const defaultScheduleReconnect: ReconnectScheduler = (callback, delayMs) => { }; /** - * Result the deploy router returns to the link after the trivial - * branch completes. Carries the values the link folds into the - * outbound `agent.deploy.ack` frame; the link itself stays out of - * the provisioning details. + * Result the deploy router returns to the link once a deploy has + * staged. Carries the values the link folds into the outbound + * `agent.deploy.ack` frame; the link itself stays out of the deploy + * details. */ export type DeployRouterResult = { /** Hex-encoded agent public key the hub records for verification. */ @@ -76,11 +79,11 @@ export type DeployRouterResult = { /** * Single-ingress deploy contract the link routes every `agent.deploy` - * frame through. The workflow-host supervisor is the production - * implementation -- the supervisor decides between the trivial - * (1-step) passthrough and the multi-step IPC-backed branch. The - * shape lives on hub-agent so the package boundary stays one-way - * (`@intx/hub-agent` does not import `@intx/workflow-host`). + * frame through. The sidecar's workflow-run deploy router is the + * production implementation -- it stages every deploy through the + * workflow-run substrate. The shape lives on hub-agent so the package + * boundary stays one-way (`@intx/hub-agent` does not import + * `@intx/workflow-host`). */ export interface DeployRouter { deploy(frame: AgentDeployFrame): Promise; @@ -90,30 +93,26 @@ export interface DeployRouter { * per-deployment registrations the deploy path installed * (`MultistepMailRouter`, `MultistepSignalRouter`, * `MultistepDrainRouter`, `DeploymentAddressRegistry`). Optional - * so test routers and the inline trivial test fixture can omit - * the implementation. + * so test routers can omit the implementation. */ undeploy?: (frame: AgentUndeployFrame) => Promise; } /** * Per-address mail handler registry the link consults on every - * `mail.inbound` frame before falling back to `transport.deliver` / - * `sessions.commitInboundMail`. Production wires this against the - * sidecar's `createMultistepMailRouter` so a multi-step deployment's - * supervisor receives the bytes through its mail-bus subscription. - * Trivial deploys never register a handler; their mail flows through - * the legacy session path unchanged. The shape lives on hub-agent so - * the link does not import the sidecar host's wiring module, and so - * tests can substitute a stub. + * `mail.inbound` frame. Production wires this against the sidecar's + * `createMultistepMailRouter` so a supervised deployment's supervisor + * receives the bytes through its mail-bus subscription. Mail for an + * address with no registered handler has no receiver and is dropped. + * The shape lives on hub-agent so the link does not import the sidecar + * host's wiring module, and so tests can substitute a stub. */ export interface MailInboundRouter { /** * Attempt to dispatch `message` to a handler registered against * `agentAddress`. Returns `true` if a handler claimed the message; - * `false` if no handler is registered. A `true` return causes the - * link to skip the legacy `transport.deliver` / - * `sessions.commitInboundMail` fallback entirely. + * `false` if no handler is registered, in which case the link logs + * and drops the mail. */ tryRoute(agentAddress: string, message: Uint8Array): boolean; } @@ -123,10 +122,10 @@ export interface MailInboundRouter { * every inbound `signal.deliver` frame. Production wires this against * the sidecar's multi-step deploy registry so the frame flows into the * deployment's supervisor (which forwards `signal.deliver` over the - * control IPC to the workflow-process child). Trivial deployments do - * not register a handler; the link logs and drops a frame whose - * `agentAddress` is unknown so the wire surface fails loudly rather - * than silently absorbing a misrouted delivery. + * control IPC to the workflow-process child). The link logs and drops + * a frame whose `agentAddress` matches no registered handler so the + * wire surface fails loudly rather than silently absorbing a misrouted + * delivery. * * The shape lives on hub-agent so the link does not import the sidecar * host's wiring module, and so tests can substitute a stub. @@ -150,10 +149,9 @@ export interface SignalInboundRouter { * the sidecar's multi-step deploy registry so the frame flows into the * deployment's supervisor (which forwards a `drain` control IPC frame * to the workflow-process child and arms one drainTimeout accumulator - * per in-flight run). Trivial deployments do not register a handler; - * the link logs and drops a frame whose `agentAddress` is unknown so - * the wire surface fails loudly rather than silently absorbing a - * misrouted delivery. + * per in-flight run). The link logs and drops a frame whose + * `agentAddress` matches no registered handler so the wire surface + * fails loudly rather than silently absorbing a misrouted delivery. * * The shape lives on hub-agent so the link does not import the sidecar * host's wiring module, and so tests can substitute a stub. @@ -193,37 +191,32 @@ export type HubLinkConfig = { */ deployRouter: DeployRouter; /** - * Optional pre-fallback mail dispatcher. When present, the link - * consults this router on every inbound `mail.inbound` frame; a - * `true` return takes the bytes off the legacy - * `transport.deliver` + `sessions.commitInboundMail` path entirely. - * Production wires this against the sidecar's multi-step deploy - * registry so a deployment-address inbound flows into the - * supervisor's mail-bus subscription. Trivial deployments (and - * tests that exercise the legacy path) omit it; an absent router - * is treated as "no handler ever claims" so behaviour is unchanged. + * Optional inbound mail dispatcher. When present, the link consults + * this router on every inbound `mail.inbound` frame. Production wires + * this against the sidecar's multi-step deploy registry so a + * deployment-address inbound flows into the supervisor's mail-bus + * subscription. Absent (or a `false` return) means no handler claims + * the mail, so the link logs and drops it. */ mailInboundRouter?: MailInboundRouter; /** - * Optional pre-fallback signal dispatcher. When present, the link - * routes every inbound `signal.deliver` frame through this router. - * Production wires this against the sidecar's multi-step deploy - * registry so a deployment-address signal flows into the - * supervisor's `deliverSignal`. Trivial deployments (and tests that - * do not exercise the signal-delivery surface) omit it; an absent - * router causes inbound signal frames to be logged-and-dropped so a - * misrouted delivery is observable rather than silent. + * Optional inbound signal dispatcher. When present, the link routes + * every inbound `signal.deliver` frame through this router. Production + * wires this against the sidecar's multi-step deploy registry so a + * deployment-address signal flows into the supervisor's + * `deliverSignal`. Absent (or a `false` return) causes inbound signal + * frames to be logged-and-dropped so a misrouted delivery is + * observable rather than silent. */ signalInboundRouter?: SignalInboundRouter; /** - * Optional pre-fallback drain dispatcher. When present, the link - * routes every inbound `drain.deliver` frame through this router. - * Production wires this against the sidecar's multi-step deploy - * registry so a deployment-address drain flows into the supervisor's - * `drain`. Trivial deployments (and tests that do not exercise the - * drain surface) omit it; an absent router causes inbound drain - * frames to be logged-and-dropped so a misrouted delivery is - * observable rather than silent. + * Optional inbound drain dispatcher. When present, the link routes + * every inbound `drain.deliver` frame through this router. Production + * wires this against the sidecar's multi-step deploy registry so a + * deployment-address drain flows into the supervisor's `drain`. Absent + * (or a `false` return) causes inbound drain frames to be + * logged-and-dropped so a misrouted delivery is observable rather than + * silent. */ drainInboundRouter?: DrainInboundRouter; /** @@ -249,16 +242,6 @@ export type HubLink = { connect(): void; close(): void; sendEvent: SessionEventSink; - sendConnectorState: ConnectorStateSink; - /** - * Ship a deploy.apply.error frame to the hub. Caller supplies the - * agentAddress separately so the frame's other fields stay close to - * the loader's failure-site description. - */ - sendDeployApplyError: ( - agentAddress: string, - payload: Omit, - ) => void; /** * Ship a workflow-run pack to the hub. Streams the supplied pack as * `repo.pack.push` chunks followed by a `repo.pack.done`, then @@ -373,18 +356,18 @@ export function createHubLink(config: HubLinkConfig): HubLink { async function handleAgentDeploy(frame: AgentDeployFrame): Promise { try { - // The deploy router (production: workflow-host supervisor) - // owns the agent.deploy framing decision -- trivial vs - // multi-step -- and returns the deploy public key the link - // folds into the outbound ack. The link itself does not - // re-decide; routing lives on the supervisor side of the seam. + // The deploy router (production: the sidecar's workflow-run deploy + // router) stages the deploy through the substrate and returns the + // deploy public key the link folds into the outbound ack. The link + // itself does not re-decide; routing lives on the router side of + // the seam. const result = await deployRouter.deploy(frame); send({ type: "agent.deploy.ack", agentAddress: frame.agentAddress, publicKey: result.publicKey, }); - logger.info`Provisioned agent ${frame.agentAddress}`; + logger.info`Deployed agent ${frame.agentAddress}`; } catch (err) { const message = err instanceof Error ? err.message : String(err); send({ @@ -404,9 +387,8 @@ export function createHubLink(config: HubLinkConfig): HubLink { // the registrations released, any in-flight `signal.deliver` / // `drain.deliver` / `mail.inbound` frame that lands during teardown // is rejected by the router rather than dispatched into a - // soon-to-be-orphaned supervisor handler. Trivial-deploy routers - // (and test stubs) omit the hook; an absent hook means there was - // nothing to release. + // soon-to-be-orphaned supervisor handler. Test stubs omit the hook; + // an absent hook means there was nothing to release. if (deployRouter.undeploy !== undefined) { try { await deployRouter.undeploy(frame); @@ -434,14 +416,6 @@ export function createHubLink(config: HubLinkConfig): HubLink { workflowRunPackBootstrappedByAddress.delete(frame.agentAddress); } - // Stop the harness first. - try { - await sessions.destroySession(frame.agentAddress); - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - logger.warn`Failed to stop session for ${frame.agentAddress}: ${msg}`; - } - // Best-effort state push to the hub before deleting the directory. // statePushed reflects whether we sent the pack frames, not whether // the hub acknowledged them. We intentionally skip waiting for @@ -523,71 +497,13 @@ export function createHubLink(config: HubLinkConfig): HubLink { async function handleChallengeFailed( frame: ChallengeFailedFrame, ): Promise { - // The hub rejected this agent during reconnect — tear it down so - // the address is freed for future deploys. The agent may not have - // an active session (provisioned but never started, or already - // destroyed by a concurrent path), so any error from destroySession - // is logged but does not abort the wire layer. Destroy first so any - // disposer or mail-commit code still has the agent's crypto handle; - // forget the key material only once the session lifecycle methods - // have returned. - try { - await sessions.destroySession(frame.address); - } catch (err: unknown) { - const msg = err instanceof Error ? err.message : String(err); - logger.warn`destroySession during challenge.failed for ${frame.address}: ${msg}`; - } + // The hub rejected this agent during reconnect -- forget its key + // material so the address is freed for future deploys. keyStore.forgetAgent(frame.address); logger.warn`Challenge failed for ${frame.address}, agent torn down: ${frame.reason}`; } - async function handleSessionAbort(frame: SessionAbortFrame): Promise { - try { - await sessions.abortSession(frame.agentAddress, frame.reason); - send({ type: "session.ack", requestId: frame.requestId }); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - send({ - type: "session.error", - requestId: frame.requestId, - error: message, - }); - } - } - - async function handleGrantsUpdate(frame: GrantsUpdateFrame): Promise { - try { - await sessions.updateGrants(frame.agentAddress, frame.grants); - send({ type: "session.ack", requestId: frame.requestId }); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - send({ - type: "session.error", - requestId: frame.requestId, - error: message, - }); - } - } - - async function handleSourcesUpdate(frame: SourcesUpdateFrame): Promise { - try { - await sessions.updateSources( - frame.agentAddress, - frame.sources, - frame.defaultSource, - ); - send({ type: "session.ack", requestId: frame.requestId }); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - send({ - type: "session.error", - requestId: frame.requestId, - error: message, - }); - } - } - function handlePackPush(frame: PackPushFrame): void { const reason = packReceiver.handlePush(frame); if (reason !== null) { @@ -882,17 +798,13 @@ export function createHubLink(config: HubLinkConfig): HubLink { switch (frame.type) { case "mail.inbound": { const rawBytes = base64Decode(frame.rawMessage); - // Multi-step deployments register the deployment-level mail - // address on `mailInboundRouter` once their supervisor spawns. - // For those addresses the legacy session path is not the right - // receiver -- the transport mailbox is never registered for the - // deployment address (no `startSession` ever runs against it), - // and there is no `sessions` entry to satisfy - // `commitInboundMail`. Routing through the registered handler - // delivers the bytes to the supervisor's mail-bus subscription, - // which is what the workflow-host's `awaitSignal` listens on. - // Trivial deployments do not register a handler; the fallback - // path is the legacy single-agent provisioning surface. + // Supervised deployments register the deployment-level mail + // address on `mailInboundRouter` once their supervisor spawns; + // that handler delivers the bytes to the supervisor's mail-bus + // subscription, which is what the workflow-host's `awaitSignal` + // listens on. Mail for an address with no registered handler has + // no receiver -- the in-process session runtime that once backed + // it is retired -- so it is logged and dropped. // // Guard the router call with try/catch so a throwing handler // does not reject this `handleMessage` promise and wedge the @@ -909,11 +821,9 @@ export function createHubLink(config: HubLinkConfig): HubLink { logger.warn`mail.inbound router threw for ${frame.agentAddress}: ${msg}`; } } - if (routed) { - break; + if (!routed) { + logger.warn`Dropping mail.inbound for ${frame.agentAddress}: no registered handler`; } - deliverLocalMail(frame.agentAddress, rawBytes); - void sessions.commitInboundMail(frame.agentAddress, rawBytes); break; } case "agent.deploy": @@ -931,15 +841,6 @@ export function createHubLink(config: HubLinkConfig): HubLink { case "challenge.failed": await handleChallengeFailed(frame); break; - case "session.abort": - await handleSessionAbort(frame); - break; - case "grants.update": - await handleGrantsUpdate(frame); - break; - case "sources.update": - await handleSourcesUpdate(frame); - break; case "repo.pack.push": handlePackPush(frame); break; @@ -966,15 +867,6 @@ export function createHubLink(config: HubLinkConfig): HubLink { } } - function deliverLocalMail(agentAddress: string, message: Uint8Array): void { - try { - transport.deliver(agentAddress, message); - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - logger.warn`Failed to deliver inbound mail to ${agentAddress}: ${msg}`; - } - } - function connect(): void { // Reconnect cancellation in close() is the load-bearing protection // against post-close reconnect attempts. A caller invoking connect() @@ -1005,93 +897,23 @@ export function createHubLink(config: HubLinkConfig): HubLink { packReceiver.reset(); packSender.cancelAll("Connection lost"); - // Register the connection for routing before the restore loop runs. The - // hub learns of a sidecar only from a register/reconnect frame, and - // connections is the map sendAgentDeploy consults to route a new - // provision. An empty-address register here populates that map - // immediately, so a provision arriving during restore routes instead of - // hitting an empty map and failing with "No sidecar available". The - // address list must stay empty: carrying addresses through register - // writes addressIndex with no challenge and discards disconnect queues, - // so restored sessions enter addressIndex through the challenged - // reconnect frame below instead. + // Announce this sidecar to the hub for routing. The hub learns of a + // sidecar only from a register frame, and `connections` is the map + // `sendAgentDeploy` consults to route a deploy. Session addresses are + // always empty -- the in-process session runtime is retired -- but the + // frame still establishes the sidecar in that map. Workflow deployments + // restored at boot (before this connect) re-register here WITHOUT a + // challenge: they are hub-minted and keyless, so the hub would otherwise + // drop their route on a WS reconnect. + const workflowAddresses = getWorkflowAddresses(); send({ type: "register", sidecarId, token, - agentAddresses: [], + agentAddresses: sessions.getAddresses(), + ...(workflowAddresses.length > 0 ? { workflowAddresses } : {}), }); - - void (async () => { - try { - const { restored, failed } = await sessions.restoreSessions(); - - // The live workflow-substrate deployment addresses this sidecar - // hosts, re-read AFTER restore so any deploy that arrived during the - // restore window is included. These are hub-minted (no per-address - // key) and re-register for routing WITHOUT a challenge -- the hub - // otherwise drops their route on a WS reconnect. Carried on whichever - // terminal frame fires below; the frame carries the complete current - // set, so a plain register does not wipe them. - const workflowAddresses = getWorkflowAddresses(); - - if (restored.length > 0) { - const deployRefs: Record = {}; - for (const entry of restored) { - // SessionManager.restoreSessions populated AgentKeyStore's - // in-memory keypair cache via scanKeys. Replay the - // hub-side pairing record here so verifyDeployCommit can - // accept incoming packs without re-running an agent.deploy. - if (entry.hubPublicKey !== undefined) { - keyStore.recordHubKey(entry.address, entry.hubPublicKey); - } - const ref = await sessions.getDeployRef(entry.address); - if (ref !== null) { - deployRefs[entry.address] = ref; - } - } - send({ - type: "reconnect", - sidecarId, - token, - agentAddresses: restored.map((e) => e.address), - ...(workflowAddresses.length > 0 ? { workflowAddresses } : {}), - deployRefs, - }); - if (failed.length > 0) { - logger.warn`Reconnected ${String(restored.length)} agent(s), ${String(failed.length)} failed to restore`; - } else { - logger.info`Sent reconnect with ${String(restored.length)} agent(s)`; - } - } else { - // Nothing was restored from disk. The empty register on open - // already established routability for session addresses. Send a - // register when there are session addresses to re-announce OR - // workflow deployments to (re-)register -- a workflow-only sidecar - // restores nothing here yet must still announce its deployments. - // Session addresses present here belong to agents provisioned - // during the restore window (already in addressIndex from their - // deploy); disk-restored sessions take the challenged reconnect - // branch above. - const addresses = sessions.getAddresses(); - if (addresses.length > 0 || workflowAddresses.length > 0) { - send({ - type: "register", - sidecarId, - token, - agentAddresses: addresses, - ...(workflowAddresses.length > 0 ? { workflowAddresses } : {}), - }); - } - } - - flush(); - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - logger.error`Session restore failed, closing connection: ${msg}`; - ws?.close(); - } - })(); + flush(); }); ws.addEventListener("message", (event) => { @@ -1166,34 +988,10 @@ export function createHubLink(config: HubLinkConfig): HubLink { }); }; - const sendConnectorState: ConnectorStateSink = ( - agentAddress, - connectorState, - ) => { - send({ - type: "connector.state.changed", - agentAddress, - connectorState, - }); - }; - - const sendDeployApplyError: HubLink["sendDeployApplyError"] = ( - agentAddress, - payload, - ) => { - send({ - type: "deploy.apply.error", - agentAddress, - ...payload, - }); - }; - return { connect, close, sendEvent, - sendConnectorState, - sendDeployApplyError, pushWorkflowRunPack, }; } diff --git a/packages/tool-packaging/src/loader.ts b/packages/tool-packaging/src/loader.ts index 6cce817e..ba7f9aae 100644 --- a/packages/tool-packaging/src/loader.ts +++ b/packages/tool-packaging/src/loader.ts @@ -140,8 +140,8 @@ export interface LoaderConfig { /** * Deadline in milliseconds for a single HTTP-registry tarball fetch, * spanning the request and the streamed body read. A stalled registry - * cannot block the fetch -- and the restoring `startSession` awaiting - * it -- past this bound. Defaults to + * cannot block the fetch -- and the deploy's tool materialization + * awaiting it -- past this bound. Defaults to * `DEFAULT_REGISTRY_FETCH_TIMEOUT_MS`. Asset-sourced tarballs read from * the local filesystem and are not subject to it. */ @@ -175,8 +175,8 @@ export const DEFAULT_MAX_REGISTRY_TARBALL_BYTES = 10 * 1024 * 1024; * both the request and the streamed body read. `readResponseWithLimit` * consumes the body through a manual reader loop, so the byte cap bounds * size but nothing bounds time: a registry that accepts the connection - * and then stalls mid-stream would block the fetch -- and the - * `startSession` awaiting it during a sidecar restore -- indefinitely. + * and then stalls mid-stream would block the fetch -- and the deploy's + * tool materialization awaiting it -- indefinitely. * The deadline is generous so a legitimately large tarball on a slow * link still completes within it. Callers that need a different bound * pass `registryFetchTimeoutMs` to `createToolLoader`. @@ -726,7 +726,7 @@ export function createToolLoader(config: LoaderConfig): ToolLoader { entry.tarballUrl ?? defaultTarballUrl(registry.url, entry.name, entry.version); // Bound the whole fetch -- request and streamed body read -- so a - // stalled registry cannot block the awaiting startSession forever. + // stalled registry cannot block the awaiting deploy forever. // npm-registry-fetch honors the signal for the request phase; // readResponseWithLimit honors it for the manual body read. The // timer spans both phases and is cleared only once the read settles. diff --git a/packages/workflow-host/src/mail-bus/hub-transport-adapter.ts b/packages/workflow-host/src/mail-bus/hub-transport-adapter.ts index f4caf971..26a78007 100644 --- a/packages/workflow-host/src/mail-bus/hub-transport-adapter.ts +++ b/packages/workflow-host/src/mail-bus/hub-transport-adapter.ts @@ -10,9 +10,9 @@ // adapter fans the bytes out to every subscribed handler. The // transport itself is treated as a sink the host already owns -- the // adapter does not register addresses on the transport on the -// supervisor's behalf (production sidecars already register via -// their own `provisionAgent` flow) and does not double-deliver -// messages the transport itself routes. +// supervisor's behalf (production sidecars register the head's address +// through the deploy router before the supervisor spawns) and does not +// double-deliver messages the transport itself routes. import type { OutboundMessage, SendReceipt } from "@intx/types/runtime"; @@ -44,10 +44,9 @@ export interface HubTransportMailBusAdapter extends MailBusBindings { * Wrap an existing `HubTransport` instance as the supervisor-facing * `MailBusBindings` shape. The `transport` argument is held only as * a sink-side reference the adapter does not actively reach into - * today -- production wiring uses the existing `provisionAgent` - * flow to register agent addresses on the transport, and the - * adapter delivers inbound bytes through `routeInbound` directly - * into the per-address subscriber map below. + * today -- the sidecar's deploy router registers the head's address + * on the transport, and the adapter delivers inbound bytes through + * `routeInbound` directly into the per-address subscriber map below. */ export function wrapHubTransportAsMailBus( transport: HubTransport, @@ -55,11 +54,10 @@ export function wrapHubTransportAsMailBus( const subscribers = new Map void>>(); return { registerAddress(address: string) { - // The supervisor's address registration is the seam for the - // multi-step branch where the workflow-process child owns - // its own mailbox; production sidecars register the trivial - // address through `SessionManager.provisionAgent` before any - // workflow-host hook runs, so this method is intentionally + // The supervisor's address registration is the seam where the + // workflow-process child owns its own mailbox; production + // sidecars register the head's address through the deploy router + // before the supervisor spawns, so this method is intentionally // inert. The `transport` reference is retained so a future // routing change that wants the bus to own registration has // the handle in scope. From add1b9192aa50be2e2aa2f67f9bb120591c4f803 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Sat, 4 Jul 2026 06:47:25 -0500 Subject: [PATCH 043/101] Rename the survivors the deletions left misnamed The trivial and warm-harness removals left several live symbols carrying names that no longer describe anything. `deriveTrivialDeploymentId` is the sole deployment-id derivation for every deploy path, so it becomes `deriveDeploymentId`. `wrapHarnessAsTrivialAgent` wraps a harness config as a single-step workflow, so it becomes `wrapHarnessAsSingleStepWorkflow`, and the `makeTrivialWorkflow` test fixture becomes `makeSingleStepWorkflow`. The supervisor's run-event module no longer signs anything -- it only folds a terminated run's per-event blobs into one combined log -- so `run-event-signing.ts` becomes `run-event-compaction.ts` and its runtime error prefix follows. Comments and test identifiers that named the removed trivial branch are updated to match. --- .../agent-key-registration-lifecycle.test.ts | 4 +- ...ow-host-wiring-undeploy-supervisor.test.ts | 4 +- apps/sidecar/src/workflow-host-wiring.test.ts | 22 +++++------ apps/sidecar/src/workflow-host-wiring.ts | 14 +++---- docs/IMPLEMENTATION.md | 2 +- docs/unified-execution-host-design.md | 8 ++-- packages/hub-api/src/routes/workflows.test.ts | 2 +- packages/hub-sessions/src/session-service.ts | 9 +++-- .../src/capability-walk.test.ts | 4 +- packages/workflow-deploy/src/index.ts | 2 +- .../workflow-deploy/src/orchestrator.test.ts | 22 +++++------ packages/workflow-deploy/src/orchestrator.ts | 8 ++-- ...> single-step-wrap-grant-coverage.test.ts} | 39 ++++++++++--------- ...g.test.ts => run-event-compaction.test.ts} | 2 +- ...ent-signing.ts => run-event-compaction.ts} | 4 +- .../src/supervisor/supervisor.ts | 2 +- .../child-workflow-roundtrip.test.ts | 8 ++-- .../cross-process-custom-adapter.test.ts | 4 +- tests/workflow-deploy/drain-roundtrip.test.ts | 6 +-- tests/workflow-deploy/fifo-mail-load.test.ts | 4 +- tests/workflow-deploy/fifo-mail.test.ts | 4 +- .../instance-failover-real-agent.test.ts | 12 +++--- .../instance-reroute-real-agent.test.ts | 14 +++---- .../latency-d2-attribution.bench.ts | 4 +- tests/workflow-deploy/latency-gate.bench.ts | 4 +- tests/workflow-deploy/mail-edge-cases.test.ts | 4 +- .../workflow-deploy/multistep-signal.test.ts | 6 +-- .../multistep-signed-send.test.ts | 4 +- .../single-step-full-lifecycle.test.ts | 4 +- .../single-step-grants-bridge.test.ts | 6 +-- .../single-step-message-input.test.ts | 4 +- .../single-step-posix-tool.test.ts | 4 +- .../single-step-real-agent.test.ts | 4 +- 33 files changed, 123 insertions(+), 121 deletions(-) rename packages/workflow-deploy/src/{trivial-wrap-grant-coverage.test.ts => single-step-wrap-grant-coverage.test.ts} (75%) rename packages/workflow-host/src/supervisor/{run-event-signing.test.ts => run-event-compaction.test.ts} (99%) rename packages/workflow-host/src/supervisor/{run-event-signing.ts => run-event-compaction.ts} (96%) diff --git a/apps/sidecar/src/agent-key-registration-lifecycle.test.ts b/apps/sidecar/src/agent-key-registration-lifecycle.test.ts index 1af4dd6f..4cab2388 100644 --- a/apps/sidecar/src/agent-key-registration-lifecycle.test.ts +++ b/apps/sidecar/src/agent-key-registration-lifecycle.test.ts @@ -33,7 +33,7 @@ import { type } from "arktype"; import { createSidecarDeployRouter, - deriveTrivialDeploymentId, + deriveDeploymentId, } from "./workflow-host-wiring"; import { WorkflowDeploymentRecord } from "./workflow-deployment-record"; import { @@ -326,7 +326,7 @@ describe("agent signing-key registration lifecycle on the host transport", () => // (b') The deploy persisted a schema-valid restore record for the // deployment, carrying the head address so a boot-time restore can // re-establish it. - const deploymentId = deriveTrivialDeploymentId(AGENT_ADDRESS); + const deploymentId = deriveDeploymentId(AGENT_ADDRESS); const recordFile = path.join( dataDir, "workflow-runs", diff --git a/apps/sidecar/src/workflow-host-wiring-undeploy-supervisor.test.ts b/apps/sidecar/src/workflow-host-wiring-undeploy-supervisor.test.ts index ebff0abf..fddbb51b 100644 --- a/apps/sidecar/src/workflow-host-wiring-undeploy-supervisor.test.ts +++ b/apps/sidecar/src/workflow-host-wiring-undeploy-supervisor.test.ts @@ -31,7 +31,7 @@ import type { AgentDeployFrame } from "@intx/types/sidecar"; import { createSidecarDeployRouter, - deriveTrivialDeploymentId, + deriveDeploymentId, } from "./workflow-host-wiring"; import { createMultistepDrainRouter, @@ -357,7 +357,7 @@ describe("createSidecarDeployRouter multi-step undeploy shuts the supervisor dow // one-per-message); a stale cold `runs//` subtree models a // multi-step leftover the per-run cleanup did not drop. An unrelated // deployment's step-state subtree must survive the undeploy sweep. - const deploymentId = deriveTrivialDeploymentId(frame.agentAddress); + const deploymentId = deriveDeploymentId(frame.agentAddress); const stepStateRoot = path.join(dataDir, "workflow-step-state"); const warmWorkspaceFile = path.join( stepStateRoot, diff --git a/apps/sidecar/src/workflow-host-wiring.test.ts b/apps/sidecar/src/workflow-host-wiring.test.ts index b180525f..3bacaf28 100644 --- a/apps/sidecar/src/workflow-host-wiring.test.ts +++ b/apps/sidecar/src/workflow-host-wiring.test.ts @@ -22,7 +22,7 @@ import { computeWireDefinitionHash, createSidecarDeployRouter, createSidecarWorkflowSupervisor, - deriveTrivialDeploymentId, + deriveDeploymentId, STEP_INFERENCE_SOURCES_ENV_KEY, validateWorkflowProjection, } from "./workflow-host-wiring"; @@ -896,7 +896,7 @@ describe("createSidecarDeployRouter multi-step branch", () => { const recordFile = path.join( dataDir, "workflow-runs", - deriveTrivialDeploymentId(agentAddress), + deriveDeploymentId(agentAddress), "deployment.json", ); expect( @@ -1579,15 +1579,13 @@ describe("createSidecarDeployRouter multi-step branch", () => { expect(isRegistered(freshTransport, badHead)).toBe(false); // The failed record is KEPT on disk -- never deleted, unlike a // soft-failed deploy -- so a later boot can retry it. - expect( - await recordExists(dataDir, deriveTrivialDeploymentId(badHead)), - ).toBe(true); + expect(await recordExists(dataDir, deriveDeploymentId(badHead))).toBe(true); }); test("restore applies validateWorkflowProjection: a stepOrder entry with no matching steps is skipped", async () => { const dataDir = await createTempBaseDir("sidecar-restore-validator-data-"); const head = "ins_validator@example.com"; - const deploymentId = deriveTrivialDeploymentId(head); + const deploymentId = deriveDeploymentId(head); // Hand-write a record plus a workflow.json whose `stepOrder` names a step // `steps` does not define. This clears the wire arktype @@ -1664,7 +1662,7 @@ describe("createSidecarDeployRouter multi-step branch", () => { test("a second deploy for a live address is rejected without orphaning its restore record", async () => { const dataDir = await createTempBaseDir("sidecar-restore-dup-data-"); const head = "ins_dup@example.com"; - const deploymentId = deriveTrivialDeploymentId(head); + const deploymentId = deriveDeploymentId(head); const spawner = makeReadyDrivingSpawner(9700); const { router, transport } = await buildMultistepFixture({ @@ -1727,7 +1725,7 @@ describe("createSidecarDeployRouter multi-step branch", () => { "sidecar-restore-unbuildable-data-", ); const head = "ins_unbuildable_restore@example.com"; - const deploymentId = deriveTrivialDeploymentId(head); + const deploymentId = deriveDeploymentId(head); // First process: a permissive gate lets the deploy through, persisting // the record and its workflow.json. @@ -1768,7 +1766,7 @@ describe("createSidecarDeployRouter multi-step branch", () => { test("a deploy whose child never signals ready times out and rejects", async () => { const dataDir = await createTempBaseDir("sidecar-ready-timeout-data-"); const head = "ins_readytimeout@example.com"; - const deploymentId = deriveTrivialDeploymentId(head); + const deploymentId = deriveDeploymentId(head); // A spawner whose child is created but never driven through the `ready` // handshake. With a small threaded readyTimeoutMs the supervisor times @@ -1823,8 +1821,8 @@ describe("createSidecarDeployRouter multi-step branch", () => { ); }); - test("two addresses whose deriveTrivialDeploymentId slugs collide are rejected at the second deploy", async () => { - // deriveTrivialDeploymentId substitutes every disallowed character with + test("two addresses whose deriveDeploymentId slugs collide are rejected at the second deploy", async () => { + // deriveDeploymentId substitutes every disallowed character with // `-`, so two distinct addresses can collapse to the same slug. The slug // IS the workflow-run repoId, so a silent collision would let the second // deploy overwrite the first deploy's repo state. claimSlug rejects the @@ -1847,7 +1845,7 @@ describe("createSidecarDeployRouter multi-step branch", () => { await expect( router.deploy(singleStepFrame("ins_col-a@example.com", "wf-collide")), - ).rejects.toThrow(/deriveTrivialDeploymentId collision/); + ).rejects.toThrow(/deriveDeploymentId collision/); expect(spawner.spawnCount()).toBe(1); }); }); diff --git a/apps/sidecar/src/workflow-host-wiring.ts b/apps/sidecar/src/workflow-host-wiring.ts index 98d6f6bc..890eccf5 100644 --- a/apps/sidecar/src/workflow-host-wiring.ts +++ b/apps/sidecar/src/workflow-host-wiring.ts @@ -84,7 +84,7 @@ const logger = getLogger(["interchange", "sidecar", "workflow-host-wiring"]); * the sidecar's call sites readable while the rationale and the * substrate `SAFE_REPO_ID` contract live with the shared function. */ -export function deriveTrivialDeploymentId(agentAddress: string): string { +export function deriveDeploymentId(agentAddress: string): string { return deriveWorkflowRunRepoId(agentAddress); } @@ -907,7 +907,7 @@ export function createSidecarDeployRouter(deps: { // with the deployment. const activeSupervisors = new Map(); - // Slug-collision tracking. `deriveTrivialDeploymentId` substitutes + // Slug-collision tracking. `deriveDeploymentId` substitutes // disallowed characters with `-`, which is deterministic but lossy: // two distinct agent addresses can collapse to the same slug, and // a collision would let the second deploy silently overwrite the @@ -921,7 +921,7 @@ export function createSidecarDeployRouter(deps: { const existing = slugClaims.get(deploymentId); if (existing !== undefined && existing !== agentAddress) { throw new Error( - `deriveTrivialDeploymentId collision: agent addresses ${JSON.stringify(existing)} and ${JSON.stringify(agentAddress)} both project to deploymentId ${JSON.stringify(deploymentId)}`, + `deriveDeploymentId collision: agent addresses ${JSON.stringify(existing)} and ${JSON.stringify(agentAddress)} both project to deploymentId ${JSON.stringify(deploymentId)}`, ); } // A same-address re-claim is a defensive no-op: the `activeSupervisors` @@ -1068,7 +1068,7 @@ export function createSidecarDeployRouter(deps: { `sidecar deploy router: a supervisor is already active for ${spec.agentAddress}; refusing to spawn a second`, ); } - const deploymentId = deriveTrivialDeploymentId(spec.agentAddress); + const deploymentId = deriveDeploymentId(spec.agentAddress); // Single-step launched-agent deploy vs. derived multi-step deploy. A // one-step deployment keeps the deployment's own (legacy) mail address @@ -1381,7 +1381,7 @@ export function createSidecarDeployRouter(deps: { ); } - const deploymentId = deriveTrivialDeploymentId(frame.agentAddress); + const deploymentId = deriveDeploymentId(frame.agentAddress); // Single-step launched-agent deploy vs. derived multi-step deploy. // @@ -1514,7 +1514,7 @@ export function createSidecarDeployRouter(deps: { // boundary rather than dispatched into a supervisor that is in // the middle of tearing its child down. The pattern is: drop // racing frames first, then unwind the underlying resource. - const deploymentId = deriveTrivialDeploymentId(frame.agentAddress); + const deploymentId = deriveDeploymentId(frame.agentAddress); deps.multistepMailRouter?.unregister(frame.agentAddress); deps.multistepSignalRouter?.unregister(frame.agentAddress); deps.multistepDrainRouter?.unregister(frame.agentAddress); @@ -1588,7 +1588,7 @@ export function createSidecarDeployRouter(deps: { // Integrity: the stored address must re-derive to its own directory // name. A mismatch means a corrupt or misplaced record; skip it // rather than restore a deployment under the wrong slug. - const derived = deriveTrivialDeploymentId(record.agentAddress); + const derived = deriveDeploymentId(record.agentAddress); if (derived !== deploymentId) { logger.warn`skipping workflow deployment restore: ${record.agentAddress} derives slug ${derived}, not its directory ${deploymentId}`; continue; diff --git a/docs/IMPLEMENTATION.md b/docs/IMPLEMENTATION.md index c4e16eb8..7d4e6acb 100644 --- a/docs/IMPLEMENTATION.md +++ b/docs/IMPLEMENTATION.md @@ -807,7 +807,7 @@ The workflow-run repo's substrate `repoId.id` is constrained to agent-address shape (`ins_@`) does not satisfy. The sidecar wiring derives the trivial branch's deployment id by substituting disallowed characters with `-`; see -`deriveTrivialDeploymentId` in +`deriveDeploymentId` in `apps/sidecar/src/workflow-host-wiring.ts`. The supervisor principal's `deploymentId` and the workflow-run `repoId.id` are kept equal so the workflow-run kind handler's principal-vs-repo authz diff --git a/docs/unified-execution-host-design.md b/docs/unified-execution-host-design.md index 3aa20783..9bc27c9b 100644 --- a/docs/unified-execution-host-design.md +++ b/docs/unified-execution-host-design.md @@ -1013,7 +1013,7 @@ Every surviving "trivial"-named symbol is renamed to reflect that the single-step identity path is the canonical single-agent deploy, not a "trivial" special case — or deleted where the in-process branch it served is gone: -- `wrapHarnessAsTrivialAgent` -> `wrapHarnessAsSingleStepAgent` +- `wrapHarnessAsSingleStepWorkflow` -> `wrapHarnessAsSingleStepAgent` (`packages/workflow-deploy/src/orchestrator.ts`), still used to build the one-step definition. - `buildTrivialApprovalSet` -> `buildSingleStepApprovalSet` @@ -1021,7 +1021,7 @@ special case — or deleted where the in-process branch it served is gone: - `isTrivialDeploy` / `trivialBindings` / the trivial orchestrator branch: **deleted** if single-step routes through the multi-step branch (likely); otherwise renamed `isSingleStepDeploy`. Confirm during build. -- `deriveTrivialDeploymentId` (`apps/sidecar/src/workflow-host-wiring.ts`): +- `deriveDeploymentId` (`apps/sidecar/src/workflow-host-wiring.ts`): deleted; call `deriveWorkflowRunRepoId` directly at the call sites (it is a thin delegator). - `TrivialLaunch` / `trivialLaunch` / `trivialClaimedSlugSucceeded`: deleted @@ -1495,7 +1495,7 @@ are explicitly **not** a go-live gate for INTR-209. `runWorkflowChildFromProcessEnv`). - Deploy router / wiring: `apps/sidecar/src/workflow-host-wiring.ts` (`createSidecarDeployRouter`, `deployMultiStep`, `activeSupervisors`, - `deriveTrivialDeploymentId`, `driveTrivialRunChain`, `TrivialRunCell`, + `deriveDeploymentId`, `driveTrivialRunChain`, `TrivialRunCell`, `publishWorkflowInferenceEvent`). - Child-launch / sandbox-boundary seam (Decision 1): `apps/sidecar/src/workflow-host-wiring.ts` (`SubprocessSpawner`, @@ -1521,7 +1521,7 @@ are explicitly **not** a go-live gate for INTR-209. `WorkflowRunWorkflowProcessPrincipal`. - Identity: `packages/workflow-deploy/src/orchestrator.ts` (`deriveWorkflowRunRepoId`, `isWorkflowDerivedAddress`, - `wrapHarnessAsTrivialAgent`); `packages/types/src/agent-address.ts` + `wrapHarnessAsSingleStepWorkflow`); `packages/types/src/agent-address.ts` (`formatAgentAddress`, `parseAgentAddress`); `packages/hub-sessions/src/hub-session-orchestrator.ts` (deploy-ack listener); `packages/hub-sessions/src/hub-session-lookups.ts` (`findInstance`, diff --git a/packages/hub-api/src/routes/workflows.test.ts b/packages/hub-api/src/routes/workflows.test.ts index 5d2ecf00..31945ae4 100644 --- a/packages/hub-api/src/routes/workflows.test.ts +++ b/packages/hub-api/src/routes/workflows.test.ts @@ -35,7 +35,7 @@ const DEPLOYMENT_ID = "dep_abc"; // The sidecar's deploy router keys the workflow-run repo by the // sanitized deployment address, NOT the bare deployment id (see -// `deriveTrivialDeploymentId` -> `deriveWorkflowRunRepoId` in +// `deriveDeploymentId` -> `deriveWorkflowRunRepoId` in // apps/sidecar/src/workflow-host-wiring.ts). The read routes must // reconstruct the same id from `(deploymentId, tenantDomain)`; the // run-observe tests build their on-disk repo under this derived id so a diff --git a/packages/hub-sessions/src/session-service.ts b/packages/hub-sessions/src/session-service.ts index 5b863b76..c9631e3e 100644 --- a/packages/hub-sessions/src/session-service.ts +++ b/packages/hub-sessions/src/session-service.ts @@ -50,7 +50,7 @@ import { createWorkflowDeployOrchestrator, deriveDeploymentAddress, walkCapabilities, - wrapHarnessAsTrivialAgent, + wrapHarnessAsSingleStepWorkflow, type ApprovalSet, type DeployContent as OrchestratorDeployContent, type DeployWorkflowArgs, @@ -1013,10 +1013,13 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { }): Promise<{ publicKey: string }> { const { agentAddress, agentId, instanceId, config, deployContent } = params; - const trivialAgent = wrapHarnessAsTrivialAgent({ config, deployContent }); + const singleStepAgent = wrapHarnessAsSingleStepWorkflow({ + config, + deployContent, + }); const workflow = defineWorkflow({ id: `wf_${agentId}`, - agent: trivialAgent, + agent: singleStepAgent, trigger: { type: "mail", to: agentAddress }, }); diff --git a/packages/workflow-deploy/src/capability-walk.test.ts b/packages/workflow-deploy/src/capability-walk.test.ts index 04961593..4bc1dee4 100644 --- a/packages/workflow-deploy/src/capability-walk.test.ts +++ b/packages/workflow-deploy/src/capability-walk.test.ts @@ -41,7 +41,7 @@ function makeTrivialAgent(): AgentDefinition { }); } -function makeTrivialWorkflow( +function makeSingleStepWorkflow( agent: AgentDefinition, ): WorkflowDefinition { return defineWorkflow({ @@ -84,7 +84,7 @@ describe("walkCapabilities", () => { test("trivial workflow grants match the implicit agent-deploy grants", () => { const registry = createDefaultDirectorRegistry(); const agent = makeTrivialAgent(); - const workflow = makeTrivialWorkflow(agent); + const workflow = makeSingleStepWorkflow(agent); const walk = walkCapabilities(workflow, registry); diff --git a/packages/workflow-deploy/src/index.ts b/packages/workflow-deploy/src/index.ts index f1ffa2f4..6bb9f570 100644 --- a/packages/workflow-deploy/src/index.ts +++ b/packages/workflow-deploy/src/index.ts @@ -32,7 +32,7 @@ export { deriveStepAgentId, deriveWorkflowRunRepoId, isWorkflowDerivedAddress, - wrapHarnessAsTrivialAgent, + wrapHarnessAsSingleStepWorkflow, CapabilityApprovalDeniedError, MultiStepDeployHandoffMissingError, MultiStepDeploymentArgsMissingError, diff --git a/packages/workflow-deploy/src/orchestrator.test.ts b/packages/workflow-deploy/src/orchestrator.test.ts index 54f01a19..3b702fed 100644 --- a/packages/workflow-deploy/src/orchestrator.test.ts +++ b/packages/workflow-deploy/src/orchestrator.test.ts @@ -29,7 +29,7 @@ import { MultiStepDeployHandoffMissingError, MultiStepDeploymentArgsMissingError, WorkflowDefinitionInvalidError, - wrapHarnessAsTrivialAgent, + wrapHarnessAsSingleStepWorkflow, type DeployContent, type DeploySingleStepFn, type LaunchSessionFn, @@ -64,7 +64,7 @@ function makeAgent( }); } -function makeTrivialWorkflow( +function makeSingleStepWorkflow( agent: AgentDefinition, ): WorkflowDefinition { return defineWorkflow({ @@ -562,7 +562,7 @@ describe("createWorkflowDeployOrchestrator", () => { test("single-step workflow deploys once at the head", async () => { const agent = makeAgent("only"); - const workflow = makeTrivialWorkflow(agent); + const workflow = makeSingleStepWorkflow(agent); const directorRegistry = createDefaultDirectorRegistry(); const workflowRepo = createRecordingWorkflowRepoWriter(); const launch = createRecordingLaunch(); @@ -673,7 +673,7 @@ describe("createWorkflowDeployOrchestrator", () => { describe("approval failures", () => { test("unapproved grant throws CapabilityApprovalDeniedError naming the step", async () => { const agent = makeAgent("legacy-agent"); - const workflow = makeTrivialWorkflow(agent); + const workflow = makeSingleStepWorkflow(agent); const directorRegistry = createDefaultDirectorRegistry(); const workflowRepo = createRecordingWorkflowRepoWriter(); const launch = createRecordingLaunch(); @@ -713,7 +713,7 @@ describe("createWorkflowDeployOrchestrator", () => { test("zero approved sources throws with the offending step and missing source", async () => { const agent = makeAgent("legacy-agent"); - const workflow = makeTrivialWorkflow(agent); + const workflow = makeSingleStepWorkflow(agent); const directorRegistry = createDefaultDirectorRegistry(); const workflowRepo = createRecordingWorkflowRepoWriter(); const launch = createRecordingLaunch(); @@ -831,9 +831,9 @@ describe("createWorkflowDeployOrchestrator", () => { }); }); -describe("wrapHarnessAsTrivialAgent", () => { +describe("wrapHarnessAsSingleStepWorkflow", () => { test("derives id from config.agentId", () => { - const agent = wrapHarnessAsTrivialAgent({ + const agent = wrapHarnessAsSingleStepWorkflow({ config: HARNESS_CONFIG_BASE, deployContent: DEPLOY_CONTENT_BASE, }); @@ -844,7 +844,7 @@ describe("wrapHarnessAsTrivialAgent", () => { const customContent: DeployContent = { systemPrompt: "you are the trivial agent", }; - const agent = wrapHarnessAsTrivialAgent({ + const agent = wrapHarnessAsSingleStepWorkflow({ config: HARNESS_CONFIG_BASE, deployContent: customContent, }); @@ -871,7 +871,7 @@ describe("wrapHarnessAsTrivialAgent", () => { }, ], }; - const agent = wrapHarnessAsTrivialAgent({ + const agent = wrapHarnessAsSingleStepWorkflow({ config, deployContent: DEPLOY_CONTENT_BASE, }); @@ -882,7 +882,7 @@ describe("wrapHarnessAsTrivialAgent", () => { }); test("empty toolFactories and capabilities (deploy tree is the source of truth)", () => { - const agent = wrapHarnessAsTrivialAgent({ + const agent = wrapHarnessAsSingleStepWorkflow({ config: HARNESS_CONFIG_BASE, deployContent: DEPLOY_CONTENT_BASE, }); @@ -891,7 +891,7 @@ describe("wrapHarnessAsTrivialAgent", () => { }); test("no director ref (caller carries no director state in the trivial shape)", () => { - const agent = wrapHarnessAsTrivialAgent({ + const agent = wrapHarnessAsSingleStepWorkflow({ config: HARNESS_CONFIG_BASE, deployContent: DEPLOY_CONTENT_BASE, }); diff --git a/packages/workflow-deploy/src/orchestrator.ts b/packages/workflow-deploy/src/orchestrator.ts index bb662be1..5d1b3626 100644 --- a/packages/workflow-deploy/src/orchestrator.ts +++ b/packages/workflow-deploy/src/orchestrator.ts @@ -922,7 +922,7 @@ function serializeWalk(walk: CapabilityWalkResult): unknown { * projection would let the gate admit every deploy regardless of what * `HarnessConfig.tools` named, weakening the approval gate. */ -export function wrapHarnessAsTrivialAgent(args: { +export function wrapHarnessAsSingleStepWorkflow(args: { config: HarnessConfig; deployContent: DeployContent; }): AgentDefinition { @@ -955,8 +955,8 @@ export function wrapHarnessAsTrivialAgent(args: { * `validateNamespacedId` (the constructor `defineTool` runs) is * deliberately skipped: `HarnessConfig.tools[i].name` is the existing * wire shape downstream consumers gate against, and re-validating it - * here would diverge the trivial-wrap's surface from what the harness - * actually loads. The walk and the gate only consult `.id`, so a bare + * here would diverge the single-step wrap's surface from what the + * harness actually loads. The walk and the gate only consult `.id`, so a bare * name still produces a stable grant string. */ function synthesizeWalkToolFactory( @@ -964,7 +964,7 @@ function synthesizeWalkToolFactory( ): AnnotatedToolFactory { const factory = (_env: BaseEnv): never => { throw new Error( - `wrapHarnessAsTrivialAgent synthesized tool factory for ${JSON.stringify(tool.name)} is walk-only; do not instantiate the trivial-wrap agent`, + `wrapHarnessAsSingleStepWorkflow synthesized tool factory for ${JSON.stringify(tool.name)} is walk-only; do not instantiate the single-step wrap agent`, ); }; return Object.assign(factory, { diff --git a/packages/workflow-deploy/src/trivial-wrap-grant-coverage.test.ts b/packages/workflow-deploy/src/single-step-wrap-grant-coverage.test.ts similarity index 75% rename from packages/workflow-deploy/src/trivial-wrap-grant-coverage.test.ts rename to packages/workflow-deploy/src/single-step-wrap-grant-coverage.test.ts index 74e83968..daa3f6ba 100644 --- a/packages/workflow-deploy/src/trivial-wrap-grant-coverage.test.ts +++ b/packages/workflow-deploy/src/single-step-wrap-grant-coverage.test.ts @@ -1,11 +1,11 @@ -// Regression coverage for H-D1: the trivial-wrap path must surface +// Regression coverage for H-D1: the single-step wrap path must surface // the HarnessConfig's tool surface to the capability walk so the // operator-approval gate gates `tool:` grants. Before the fix, -// `wrapHarnessAsTrivialAgent` emitted `toolFactories: []` regardless +// `wrapHarnessAsSingleStepWorkflow` emitted `toolFactories: []` regardless // of `HarnessConfig.tools`, and the walk emitted zero `tool:` grants. -// The gate then trivially admitted every deploy whose only path was -// the trivial branch, defeating the agent-deploy uniformity claim's -// substance at the approval layer. +// The gate then admitted every single-step deploy unconditionally, +// defeating the agent-deploy uniformity claim's substance at the +// approval layer. // // The fix widens the wrap to propagate `HarnessConfig.tools[i].name` // into the synthesized agent's `toolFactories[i].id`, so the existing @@ -22,7 +22,7 @@ import type { HarnessConfig } from "@intx/types/runtime"; import { defineWorkflow } from "@intx/workflow/definition"; import { walkCapabilities } from "./capability-walk"; -import { wrapHarnessAsTrivialAgent } from "./orchestrator"; +import { wrapHarnessAsSingleStepWorkflow } from "./orchestrator"; const HARNESS_CONFIG_WITH_TOOLS: HarnessConfig = { sessionId: "ses-1", @@ -56,20 +56,21 @@ const HARNESS_CONFIG_WITH_TOOLS: HarnessConfig = { defaultSource: "src-1", }; -describe("H-D1: trivial wrap surfaces HarnessConfig.tools to the walk", () => { +describe("H-D1: single-step wrap surfaces HarnessConfig.tools to the walk", () => { test("walk emits a tool: grant per HarnessConfig.tools entry", () => { - const trivialAgent: AgentDefinition = wrapHarnessAsTrivialAgent({ - config: HARNESS_CONFIG_WITH_TOOLS, - deployContent: { systemPrompt: "legacy" }, - }); - expect(trivialAgent.toolFactories.length).toBe(2); - const factoryIds = trivialAgent.toolFactories.map((f) => f.id); + const singleStepAgent: AgentDefinition = + wrapHarnessAsSingleStepWorkflow({ + config: HARNESS_CONFIG_WITH_TOOLS, + deployContent: { systemPrompt: "legacy" }, + }); + expect(singleStepAgent.toolFactories.length).toBe(2); + const factoryIds = singleStepAgent.toolFactories.map((f) => f.id); expect(factoryIds).toContain("sketchy_tool"); expect(factoryIds).toContain("other_tool"); const workflow = defineWorkflow({ - id: "wf_trivial", - agent: trivialAgent, + id: "wf_single_step", + agent: singleStepAgent, trigger: { type: "mail", to: HARNESS_CONFIG_WITH_TOOLS.agentAddress }, }); const walk = walkCapabilities(workflow, createDefaultDirectorRegistry()); @@ -88,15 +89,15 @@ describe("H-D1: trivial wrap surfaces HarnessConfig.tools to the walk", () => { }); test("walk emits no tool: grants when HarnessConfig.tools is empty", () => { - const trivialAgent = wrapHarnessAsTrivialAgent({ + const singleStepAgent = wrapHarnessAsSingleStepWorkflow({ config: { ...HARNESS_CONFIG_WITH_TOOLS, tools: [] }, deployContent: { systemPrompt: "legacy" }, }); - expect(trivialAgent.toolFactories).toEqual([]); + expect(singleStepAgent.toolFactories).toEqual([]); const workflow = defineWorkflow({ - id: "wf_trivial_empty", - agent: trivialAgent, + id: "wf_single_step_empty", + agent: singleStepAgent, trigger: { type: "mail", to: HARNESS_CONFIG_WITH_TOOLS.agentAddress }, }); const walk = walkCapabilities(workflow, createDefaultDirectorRegistry()); diff --git a/packages/workflow-host/src/supervisor/run-event-signing.test.ts b/packages/workflow-host/src/supervisor/run-event-compaction.test.ts similarity index 99% rename from packages/workflow-host/src/supervisor/run-event-signing.test.ts rename to packages/workflow-host/src/supervisor/run-event-compaction.test.ts index a3f135c4..fa8163fc 100644 --- a/packages/workflow-host/src/supervisor/run-event-signing.test.ts +++ b/packages/workflow-host/src/supervisor/run-event-compaction.test.ts @@ -17,7 +17,7 @@ import type { WorkflowRunSupervisorPrincipal, } from "@intx/hub-sessions"; -import { compactRunEvents } from "./run-event-signing"; +import { compactRunEvents } from "./run-event-compaction"; const REF = "refs/heads/main"; const allowAll: AuthorizeFn = () => ({ allowed: true }); diff --git a/packages/workflow-host/src/supervisor/run-event-signing.ts b/packages/workflow-host/src/supervisor/run-event-compaction.ts similarity index 96% rename from packages/workflow-host/src/supervisor/run-event-signing.ts rename to packages/workflow-host/src/supervisor/run-event-compaction.ts index 3662c2b0..a28a22cd 100644 --- a/packages/workflow-host/src/supervisor/run-event-signing.ts +++ b/packages/workflow-host/src/supervisor/run-event-compaction.ts @@ -52,7 +52,7 @@ export type CompactRunEventsOpts = { * or one whose latest event is not terminal is left untouched, so the call * is safe to repeat. The live caller fires it once per run, right after the * run terminates; a bounded recovery sweep that would re-fire it to seal a - * run whose fold a crash interrupted is tracked separately (INTR-229). + * run whose fold a crash interrupted is not yet implemented. * * The combined file is the verbatim byte concatenation of the per-event * blobs in seq order (`encodeCombinedEventLog`), the exact shape the @@ -124,7 +124,7 @@ export async function compactRunEvents( const match = EVENT_FILENAME_RE.exec(name); if (match === null || match[1] === undefined) { throw new Error( - `supervisor run-event-signing: unexpected non-event file ${filepath} under run ${opts.runId}; refusing to compact`, + `supervisor run-event-compaction: unexpected non-event file ${filepath} under run ${opts.runId}; refusing to compact`, ); } entries.push({ seq: Number.parseInt(match[1], 10), bytes }); diff --git a/packages/workflow-host/src/supervisor/supervisor.ts b/packages/workflow-host/src/supervisor/supervisor.ts index 9d27a407..f0a34567 100644 --- a/packages/workflow-host/src/supervisor/supervisor.ts +++ b/packages/workflow-host/src/supervisor/supervisor.ts @@ -84,7 +84,7 @@ import { type CredentialsSnapshot, } from "./credentials"; import { commitCancelRequested } from "./cancel-signing"; -import { compactRunEvents } from "./run-event-signing"; +import { compactRunEvents } from "./run-event-compaction"; import { createDrainTimeoutAccumulator, DEFAULT_DRAIN_TIMEOUT_MS, diff --git a/tests/workflow-deploy/child-workflow-roundtrip.test.ts b/tests/workflow-deploy/child-workflow-roundtrip.test.ts index c6830ed8..6dec40da 100644 --- a/tests/workflow-deploy/child-workflow-roundtrip.test.ts +++ b/tests/workflow-deploy/child-workflow-roundtrip.test.ts @@ -37,7 +37,7 @@ import { type SendMultiStepDeployFn, type WorkflowRepoWriter, } from "@intx/workflow-deploy"; -import { deriveTrivialDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; +import { deriveDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; import type { RepoId, WorkflowRunHubPrincipal } from "@intx/hub-sessions"; import { DEFAULT_ASSET_REF } from "@intx/hub-sessions"; @@ -287,7 +287,7 @@ describe("parent -> child workflow round-trip", () => { const parentWorkflowRunRepoId: RepoId = { kind: "workflow-run", - id: deriveTrivialDeploymentId(parentMailAddress), + id: deriveDeploymentId(parentMailAddress), }; env.registerDeployment({ deploymentId: PARENT_DEPLOYMENT_ID, @@ -687,7 +687,7 @@ describe("parent -> child workflow round-trip", () => { const parentWorkflowRunRepoId: RepoId = { kind: "workflow-run", - id: deriveTrivialDeploymentId(parentMailAddress), + id: deriveDeploymentId(parentMailAddress), }; env.registerDeployment({ deploymentId: NESTED_PARENT_DEPLOYMENT_ID, @@ -1033,7 +1033,7 @@ describe("parent -> child workflow round-trip", () => { const parentWorkflowRunRepoId: RepoId = { kind: "workflow-run", - id: deriveTrivialDeploymentId(parentMailAddress), + id: deriveDeploymentId(parentMailAddress), }; env.registerDeployment({ deploymentId: SIBLINGS_PARENT_DEPLOYMENT_ID, diff --git a/tests/workflow-deploy/cross-process-custom-adapter.test.ts b/tests/workflow-deploy/cross-process-custom-adapter.test.ts index 68557df0..87a31ed9 100644 --- a/tests/workflow-deploy/cross-process-custom-adapter.test.ts +++ b/tests/workflow-deploy/cross-process-custom-adapter.test.ts @@ -40,7 +40,7 @@ import { type SendMultiStepDeployFn, type WorkflowRepoWriter, } from "@intx/workflow-deploy"; -import { deriveTrivialDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; +import { deriveDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; import type { RepoId, WorkflowRunHubPrincipal } from "@intx/hub-sessions"; import { DEFAULT_ASSET_REF } from "@intx/hub-sessions"; @@ -228,7 +228,7 @@ async function deployCustomProviderWorkflow( const workflowRunRepoId: RepoId = { kind: "workflow-run", - id: deriveTrivialDeploymentId(deploymentMailAddress), + id: deriveDeploymentId(deploymentMailAddress), }; env.registerDeployment({ deploymentId, diff --git a/tests/workflow-deploy/drain-roundtrip.test.ts b/tests/workflow-deploy/drain-roundtrip.test.ts index 5d0c0ca4..63022bb4 100644 --- a/tests/workflow-deploy/drain-roundtrip.test.ts +++ b/tests/workflow-deploy/drain-roundtrip.test.ts @@ -56,7 +56,7 @@ import { type SendMultiStepDeployFn, type WorkflowRepoWriter, } from "@intx/workflow-deploy"; -import { deriveTrivialDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; +import { deriveDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; import type { RepoId, WorkflowRunHubPrincipal } from "@intx/hub-sessions"; import { DEFAULT_ASSET_REF } from "@intx/hub-sessions"; @@ -243,7 +243,7 @@ describe("drain round-trip", () => { const workflowRunRepoId: RepoId = { kind: "workflow-run", - id: deriveTrivialDeploymentId(deploymentMailAddress), + id: deriveDeploymentId(deploymentMailAddress), }; env.registerDeployment({ deploymentId: DEPLOYMENT_ID, @@ -473,7 +473,7 @@ describe("drain round-trip", () => { const workflowRunRepoId: RepoId = { kind: "workflow-run", - id: deriveTrivialDeploymentId(deploymentMailAddress), + id: deriveDeploymentId(deploymentMailAddress), }; env.registerDeployment({ deploymentId: waitDeploymentId, diff --git a/tests/workflow-deploy/fifo-mail-load.test.ts b/tests/workflow-deploy/fifo-mail-load.test.ts index 3c44d712..5c33a337 100644 --- a/tests/workflow-deploy/fifo-mail-load.test.ts +++ b/tests/workflow-deploy/fifo-mail-load.test.ts @@ -36,7 +36,7 @@ import { type SendMultiStepDeployFn, type WorkflowRepoWriter, } from "@intx/workflow-deploy"; -import { deriveTrivialDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; +import { deriveDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; import type { RepoId, WorkflowRunHubPrincipal } from "@intx/hub-sessions"; import { DEFAULT_ASSET_REF } from "@intx/hub-sessions"; @@ -276,7 +276,7 @@ describe("FIFO mail-trigger serialization under load", () => { const workflowRunRepoId: RepoId = { kind: "workflow-run", - id: deriveTrivialDeploymentId(deploymentMailAddress), + id: deriveDeploymentId(deploymentMailAddress), }; env.registerDeployment({ deploymentId: DEPLOYMENT_ID_LOAD, diff --git a/tests/workflow-deploy/fifo-mail.test.ts b/tests/workflow-deploy/fifo-mail.test.ts index bced36ab..4846a116 100644 --- a/tests/workflow-deploy/fifo-mail.test.ts +++ b/tests/workflow-deploy/fifo-mail.test.ts @@ -66,7 +66,7 @@ import { type SendMultiStepDeployFn, type WorkflowRepoWriter, } from "@intx/workflow-deploy"; -import { deriveTrivialDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; +import { deriveDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; import type { RepoId, WorkflowRunHubPrincipal } from "@intx/hub-sessions"; import { DEFAULT_ASSET_REF } from "@intx/hub-sessions"; @@ -259,7 +259,7 @@ describe("FIFO mail-trigger serialization", () => { const workflowRunRepoId: RepoId = { kind: "workflow-run", - id: deriveTrivialDeploymentId(deploymentMailAddress), + id: deriveDeploymentId(deploymentMailAddress), }; env.registerDeployment({ deploymentId: DEPLOYMENT_ID, diff --git a/tests/workflow-deploy/instance-failover-real-agent.test.ts b/tests/workflow-deploy/instance-failover-real-agent.test.ts index 6fd12a2b..1d348453 100644 --- a/tests/workflow-deploy/instance-failover-real-agent.test.ts +++ b/tests/workflow-deploy/instance-failover-real-agent.test.ts @@ -24,9 +24,9 @@ import { afterAll, beforeAll, describe, expect, test } from "bun:test"; import type { HarnessConfig } from "@intx/types/runtime"; -import { wrapHarnessAsTrivialAgent } from "@intx/workflow-deploy"; +import { wrapHarnessAsSingleStepWorkflow } from "@intx/workflow-deploy"; import { defineWorkflow, type WorkflowDefinition } from "@intx/workflow"; -import { deriveTrivialDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; +import { deriveDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; import type { RepoId } from "@intx/hub-sessions"; import { @@ -119,7 +119,7 @@ describe("instance inference-source failover real-agent round-trip", () => { // the run-observation handle. Same deterministic inputs -> same wrap. const workflow: WorkflowDefinition = defineWorkflow({ id: `wf_${AGENT_ID}`, - agent: wrapHarnessAsTrivialAgent({ config, deployContent }), + agent: wrapHarnessAsSingleStepWorkflow({ config, deployContent }), trigger: { type: "mail", to: agentAddress }, }); @@ -134,7 +134,7 @@ describe("instance inference-source failover real-agent round-trip", () => { const workflowRunRepoId: RepoId = { kind: "workflow-run", - id: deriveTrivialDeploymentId(agentAddress), + id: deriveDeploymentId(agentAddress), }; env.registerDeployment({ deploymentId: INSTANCE_ID, @@ -220,7 +220,7 @@ describe("instance inference-source failover real-agent round-trip", () => { const workflow: WorkflowDefinition = defineWorkflow({ id: `wf_${soleDeadAgentId}`, - agent: wrapHarnessAsTrivialAgent({ config, deployContent }), + agent: wrapHarnessAsSingleStepWorkflow({ config, deployContent }), trigger: { type: "mail", to: agentAddress }, }); @@ -234,7 +234,7 @@ describe("instance inference-source failover real-agent round-trip", () => { const workflowRunRepoId: RepoId = { kind: "workflow-run", - id: deriveTrivialDeploymentId(agentAddress), + id: deriveDeploymentId(agentAddress), }; env.registerDeployment({ deploymentId: soleDeadInstanceId, diff --git a/tests/workflow-deploy/instance-reroute-real-agent.test.ts b/tests/workflow-deploy/instance-reroute-real-agent.test.ts index 730baca1..c21bd023 100644 --- a/tests/workflow-deploy/instance-reroute-real-agent.test.ts +++ b/tests/workflow-deploy/instance-reroute-real-agent.test.ts @@ -2,11 +2,11 @@ // // The proof that a single-agent INSTANCE deploy, routed through // `SessionService.deployInstanceAtHead` (which wraps the harness as a -// one-step workflow via `wrapHarnessAsTrivialAgent` and deploys it at the +// one-step workflow via `wrapHarnessAsSingleStepWorkflow` and deploys it at the // head), runs a REAL agent in the spawned workflow-process child -- against // the real hub + real sidecar subprocess + mock inference fixture. This // closes the gap that unit tests (mock spawner) leave open: that the -// walk-only trivial-wrap agent instantiates and runs in a real child. +// walk-only single-step wrap agent instantiates and runs in a real child. // // The child never invokes the wrap's walk-only tool factories: it // materializes tools from the deploy tree and calls the real `createAgent` @@ -20,9 +20,9 @@ import { afterAll, beforeAll, describe, expect, test } from "bun:test"; import type { HarnessConfig } from "@intx/types/runtime"; -import { wrapHarnessAsTrivialAgent } from "@intx/workflow-deploy"; +import { wrapHarnessAsSingleStepWorkflow } from "@intx/workflow-deploy"; import { defineWorkflow, type WorkflowDefinition } from "@intx/workflow"; -import { deriveTrivialDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; +import { deriveDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; import type { RepoId } from "@intx/hub-sessions"; import { @@ -91,7 +91,7 @@ describe("instance-reroute real-agent round-trip", () => { // the run-observation handle. Same deterministic inputs -> same wrap. const workflow: WorkflowDefinition = defineWorkflow({ id: `wf_${AGENT_ID}`, - agent: wrapHarnessAsTrivialAgent({ config, deployContent }), + agent: wrapHarnessAsSingleStepWorkflow({ config, deployContent }), trigger: { type: "mail", to: agentAddress }, }); @@ -107,7 +107,7 @@ describe("instance-reroute real-agent round-trip", () => { const workflowRunRepoId: RepoId = { kind: "workflow-run", - id: deriveTrivialDeploymentId(agentAddress), + id: deriveDeploymentId(agentAddress), }; env.registerDeployment({ deploymentId: INSTANCE_ID, @@ -146,7 +146,7 @@ describe("instance-reroute real-agent round-trip", () => { } // The proof: the step output is the REAL agent reply from `agent.send` - // (the mock provider's deterministic output) -- the wrapped trivial agent + // (the mock provider's deterministic output) -- the wrapped single-step agent // instantiated and ran in the child rather than throwing on its walk-only // tool factories. const reply = readStepReply(stepCompleted.body); diff --git a/tests/workflow-deploy/latency-d2-attribution.bench.ts b/tests/workflow-deploy/latency-d2-attribution.bench.ts index ea7472a3..fdf38d1d 100644 --- a/tests/workflow-deploy/latency-d2-attribution.bench.ts +++ b/tests/workflow-deploy/latency-d2-attribution.bench.ts @@ -64,7 +64,7 @@ import { type SendMultiStepDeployFn, type WorkflowRepoWriter, } from "@intx/workflow-deploy"; -import { deriveTrivialDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; +import { deriveDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; import type { RepoId, WorkflowRunHubPrincipal } from "@intx/hub-sessions"; import { DEFAULT_ASSET_REF } from "@intx/hub-sessions"; @@ -470,7 +470,7 @@ async function runUnifiedD2(opts: { const workflowRunRepoId: RepoId = { kind: "workflow-run", - id: deriveTrivialDeploymentId(deploymentMailAddress), + id: deriveDeploymentId(deploymentMailAddress), }; env.registerDeployment({ deploymentId: DEPLOYMENT_ID, diff --git a/tests/workflow-deploy/latency-gate.bench.ts b/tests/workflow-deploy/latency-gate.bench.ts index 7a511b82..a5d24af7 100644 --- a/tests/workflow-deploy/latency-gate.bench.ts +++ b/tests/workflow-deploy/latency-gate.bench.ts @@ -71,7 +71,7 @@ import { type SendMultiStepDeployFn, type WorkflowRepoWriter, } from "@intx/workflow-deploy"; -import { deriveTrivialDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; +import { deriveDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; import type { RepoId, WorkflowRunHubPrincipal } from "@intx/hub-sessions"; import { DEFAULT_ASSET_REF } from "@intx/hub-sessions"; @@ -433,7 +433,7 @@ async function runUnified(opts: { const workflowRunRepoId: RepoId = { kind: "workflow-run", - id: deriveTrivialDeploymentId(deploymentMailAddress), + id: deriveDeploymentId(deploymentMailAddress), }; env.registerDeployment({ deploymentId: DEPLOYMENT_ID, diff --git a/tests/workflow-deploy/mail-edge-cases.test.ts b/tests/workflow-deploy/mail-edge-cases.test.ts index be3c6f37..d68315c5 100644 --- a/tests/workflow-deploy/mail-edge-cases.test.ts +++ b/tests/workflow-deploy/mail-edge-cases.test.ts @@ -65,7 +65,7 @@ import { type SendMultiStepDeployFn, type WorkflowRepoWriter, } from "@intx/workflow-deploy"; -import { deriveTrivialDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; +import { deriveDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; import type { RepoId, WorkflowRunHubPrincipal } from "@intx/hub-sessions"; import { DEFAULT_ASSET_REF } from "@intx/hub-sessions"; @@ -477,7 +477,7 @@ async function deployEdgeWorkflow( const workflowRunRepoId: RepoId = { kind: "workflow-run", - id: deriveTrivialDeploymentId(deploymentMailAddress), + id: deriveDeploymentId(deploymentMailAddress), }; env.registerDeployment({ deploymentId, diff --git a/tests/workflow-deploy/multistep-signal.test.ts b/tests/workflow-deploy/multistep-signal.test.ts index 77cd6cd8..a25c82aa 100644 --- a/tests/workflow-deploy/multistep-signal.test.ts +++ b/tests/workflow-deploy/multistep-signal.test.ts @@ -45,7 +45,7 @@ import { type SendMultiStepDeployFn, type WorkflowRepoWriter, } from "@intx/workflow-deploy"; -import { deriveTrivialDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; +import { deriveDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; import type { RepoId, WorkflowRunHubPrincipal } from "@intx/hub-sessions"; import { DEFAULT_ASSET_REF } from "@intx/hub-sessions"; @@ -244,12 +244,12 @@ describe("multi-step workflow round-trip with signal-await", () => { expect(result.kind).toBe("multi-step"); // The sidecar's deploy router slugs the deployment mail address - // into the workflow-run repo id via `deriveTrivialDeploymentId`; + // into the workflow-run repo id via `deriveDeploymentId`; // the helper queries `runs//events/.json` against the // same id. const workflowRunRepoId: RepoId = { kind: "workflow-run", - id: deriveTrivialDeploymentId(deploymentMailAddress), + id: deriveDeploymentId(deploymentMailAddress), }; env.registerDeployment({ deploymentId: DEPLOYMENT_ID, diff --git a/tests/workflow-deploy/multistep-signed-send.test.ts b/tests/workflow-deploy/multistep-signed-send.test.ts index e56b7bad..44ba5a13 100644 --- a/tests/workflow-deploy/multistep-signed-send.test.ts +++ b/tests/workflow-deploy/multistep-signed-send.test.ts @@ -39,7 +39,7 @@ import { type SendMultiStepDeployFn, type WorkflowRepoWriter, } from "@intx/workflow-deploy"; -import { deriveTrivialDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; +import { deriveDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; import type { RepoId, WorkflowRunHubPrincipal } from "@intx/hub-sessions"; import { DEFAULT_ASSET_REF } from "@intx/hub-sessions"; @@ -248,7 +248,7 @@ describe("multi-step signed outbound send", () => { const workflowRunRepoId: RepoId = { kind: "workflow-run", - id: deriveTrivialDeploymentId(deploymentMailAddress), + id: deriveDeploymentId(deploymentMailAddress), }; env.registerDeployment({ deploymentId: DEPLOYMENT_ID, diff --git a/tests/workflow-deploy/single-step-full-lifecycle.test.ts b/tests/workflow-deploy/single-step-full-lifecycle.test.ts index 02f9a180..a3716821 100644 --- a/tests/workflow-deploy/single-step-full-lifecycle.test.ts +++ b/tests/workflow-deploy/single-step-full-lifecycle.test.ts @@ -83,7 +83,7 @@ import { type SendMultiStepDeployFn, type WorkflowRepoWriter, } from "@intx/workflow-deploy"; -import { deriveTrivialDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; +import { deriveDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; import { createDurableConversationRegistry, reconstructDurableConversation, @@ -334,7 +334,7 @@ describe("single-step full lifecycle on the unified child path (Phase 4.6)", () const workflowRunRepoId: RepoId = { kind: "workflow-run", - id: deriveTrivialDeploymentId(deploymentMailAddress), + id: deriveDeploymentId(deploymentMailAddress), }; env.registerDeployment({ deploymentId: DEPLOYMENT_ID, diff --git a/tests/workflow-deploy/single-step-grants-bridge.test.ts b/tests/workflow-deploy/single-step-grants-bridge.test.ts index 617d0cd1..af715457 100644 --- a/tests/workflow-deploy/single-step-grants-bridge.test.ts +++ b/tests/workflow-deploy/single-step-grants-bridge.test.ts @@ -58,7 +58,7 @@ import { type SendMultiStepDeployFn, type WorkflowRepoWriter, } from "@intx/workflow-deploy"; -import { deriveTrivialDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; +import { deriveDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; import { generateKeyPair } from "@intx/crypto"; import { assembleCredentialsSnapshot, @@ -295,7 +295,7 @@ describe("single-step launched-agent grants bridge via spawned child", () => { const workflowRunRepoId: RepoId = { kind: "workflow-run", - id: deriveTrivialDeploymentId(deploymentMailAddress), + id: deriveDeploymentId(deploymentMailAddress), }; env.registerDeployment({ deploymentId: DEPLOYMENT_ID, @@ -340,7 +340,7 @@ describe("single-step launched-agent grants bridge via spawned child", () => { repoStore: readbackRepoStore, principal: { kind: "hub" }, stepOrder: [STEP_ID], - deploymentId: deriveTrivialDeploymentId(deploymentMailAddress), + deploymentId: deriveDeploymentId(deploymentMailAddress), deriveStepAddress: () => deploymentMailAddress, deriveStepRepoId: () => legacyAgentStateRepoId, }); diff --git a/tests/workflow-deploy/single-step-message-input.test.ts b/tests/workflow-deploy/single-step-message-input.test.ts index 1f90779f..9993cd19 100644 --- a/tests/workflow-deploy/single-step-message-input.test.ts +++ b/tests/workflow-deploy/single-step-message-input.test.ts @@ -34,7 +34,7 @@ import { type SendMultiStepDeployFn, type WorkflowRepoWriter, } from "@intx/workflow-deploy"; -import { deriveTrivialDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; +import { deriveDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; import type { RepoId, WorkflowRunHubPrincipal } from "@intx/hub-sessions"; import { DEFAULT_ASSET_REF } from "@intx/hub-sessions"; @@ -207,7 +207,7 @@ describe("single-step message-input round-trip", () => { const workflowRunRepoId: RepoId = { kind: "workflow-run", - id: deriveTrivialDeploymentId(deploymentMailAddress), + id: deriveDeploymentId(deploymentMailAddress), }; env.registerDeployment({ deploymentId: DEPLOYMENT_ID, diff --git a/tests/workflow-deploy/single-step-posix-tool.test.ts b/tests/workflow-deploy/single-step-posix-tool.test.ts index 0f6c4a06..f3754392 100644 --- a/tests/workflow-deploy/single-step-posix-tool.test.ts +++ b/tests/workflow-deploy/single-step-posix-tool.test.ts @@ -37,7 +37,7 @@ import { type SendMultiStepDeployFn, type WorkflowRepoWriter, } from "@intx/workflow-deploy"; -import { deriveTrivialDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; +import { deriveDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; import { sanitizeAddress } from "@intx/hub-agent"; import type { RepoId, WorkflowRunHubPrincipal } from "@intx/hub-sessions"; import { DEFAULT_ASSET_REF } from "@intx/hub-sessions"; @@ -239,7 +239,7 @@ describe("single-step posix-tool in-child execution", () => { const workflowRunRepoId: RepoId = { kind: "workflow-run", - id: deriveTrivialDeploymentId(deploymentMailAddress), + id: deriveDeploymentId(deploymentMailAddress), }; env.registerDeployment({ deploymentId: DEPLOYMENT_ID, diff --git a/tests/workflow-deploy/single-step-real-agent.test.ts b/tests/workflow-deploy/single-step-real-agent.test.ts index 96b3e379..9124d325 100644 --- a/tests/workflow-deploy/single-step-real-agent.test.ts +++ b/tests/workflow-deploy/single-step-real-agent.test.ts @@ -38,7 +38,7 @@ import { type SendMultiStepDeployFn, type WorkflowRepoWriter, } from "@intx/workflow-deploy"; -import { deriveTrivialDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; +import { deriveDeploymentId } from "@intx/sidecar-app/src/workflow-host-wiring"; import { reconstructDurableConversation } from "@intx/sidecar-app/src/conversation-state"; import type { RepoId, WorkflowRunHubPrincipal } from "@intx/hub-sessions"; import { DEFAULT_ASSET_REF } from "@intx/hub-sessions"; @@ -215,7 +215,7 @@ describe("single-step real-agent round-trip", () => { const workflowRunRepoId: RepoId = { kind: "workflow-run", - id: deriveTrivialDeploymentId(deploymentMailAddress), + id: deriveDeploymentId(deploymentMailAddress), }; env.registerDeployment({ deploymentId: DEPLOYMENT_ID, From c4c1900e78b256f57adf6ff25a3b1f6236551442 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Sat, 4 Jul 2026 07:01:02 -0500 Subject: [PATCH 044/101] Remove the dead agent-metadata persistence layer The in-process session runtime persisted each agent's config and hub pairing key to an on-disk `agent.json` so the restore path could rehydrate agents across a sidecar restart. That restore path is gone: every agent now runs as a supervised workflow-process child, the deploy pack carries the config, and the deploy path re-records the hub key on every deploy. Nothing reads `agent.json` anymore. Drop the whole metadata layer. AgentRepoStore loses persistConfig, persistPairing, scanConfigs and their AgentMeta/AgentConfigEntry types and read/write helpers, keeping the repo and pack surface the deploy path uses. AgentKeyStore loses scanKeys, the directory-walk of key files that the restore path drove; loadOrGenerateKey and the per-frame crypto operations stay. The agent-paths module drops the now-unused metadata filename and path helper. Tests that covered only the removed surface go with it. --- .../sidecar/src/workflow-deployment-record.ts | 3 +- ...orkflow-host-wiring-deploy-failure.test.ts | 3 - .../hub-agent/src/agent-key-store.test.ts | 33 ----- packages/hub-agent/src/agent-key-store.ts | 99 +------------ packages/hub-agent/src/agent-paths.ts | 17 +-- .../hub-agent/src/agent-repo-store.test.ts | 124 ---------------- packages/hub-agent/src/agent-repo-store.ts | 140 +----------------- packages/hub-agent/src/index.ts | 2 - .../hub-agent/src/session-manager.test.ts | 3 - .../src/ws/hub-link-bootstrap-prune.test.ts | 6 - .../src/ws/hub-link-mail-router.test.ts | 6 - packages/hub-agent/src/ws/hub-link.test.ts | 6 - packages/storage-isogit/src/pack-receive.ts | 4 +- packages/types/src/grant-wire.ts | 2 +- 14 files changed, 15 insertions(+), 433 deletions(-) delete mode 100644 packages/hub-agent/src/agent-repo-store.test.ts diff --git a/apps/sidecar/src/workflow-deployment-record.ts b/apps/sidecar/src/workflow-deployment-record.ts index ed7da6b8..10a2485b 100644 --- a/apps/sidecar/src/workflow-deployment-record.ts +++ b/apps/sidecar/src/workflow-deployment-record.ts @@ -76,8 +76,7 @@ export async function writeWorkflowDeploymentRecord( await mkdir(dirname(path), { recursive: true }); // Owner-only (0o600): the record embeds each source's `apiKey`, so it must // not be world-readable on a shared host. This matches the private-key - // writes elsewhere on the sidecar and is stricter than the legacy - // `agent.json`, which persists the same credentials at the default mode. + // writes elsewhere on the sidecar. await writeFile(path, JSON.stringify(record, null, 2), { encoding: "utf8", mode: 0o600, diff --git a/apps/sidecar/src/workflow-host-wiring-deploy-failure.test.ts b/apps/sidecar/src/workflow-host-wiring-deploy-failure.test.ts index 7853a977..77924950 100644 --- a/apps/sidecar/src/workflow-host-wiring-deploy-failure.test.ts +++ b/apps/sidecar/src/workflow-host-wiring-deploy-failure.test.ts @@ -44,9 +44,6 @@ function stubKeyStore(): Parameters< // exercises. return { keyPair: await generateKeyPair(), isNew: false }; }, - async scanKeys() { - return []; - }, signChallenge() { return null; }, diff --git a/packages/hub-agent/src/agent-key-store.test.ts b/packages/hub-agent/src/agent-key-store.test.ts index e114a1c7..6a7d76c8 100644 --- a/packages/hub-agent/src/agent-key-store.test.ts +++ b/packages/hub-agent/src/agent-key-store.test.ts @@ -108,36 +108,3 @@ describe("AgentKeyStore — load-or-generate", () => { ); }); }); - -describe("AgentKeyStore — scanKeys", () => { - test("returns key entries whose directory has both files plus agent.json", async () => { - const dataDir = await tempDir(); - const store = createAgentKeyStore({ - dataDir, - generateKeyPair: async () => makeKeyPair(3), - ...stubCrypto, - }); - await store.loadOrGenerateKey("agent@local"); - await fs.writeFile( - path.join(dataDir, "agent_at_local", "agent.json"), - JSON.stringify({ version: 1, address: "agent@local" }), - ); - - const entries = await store.scanKeys(); - expect(entries).toHaveLength(1); - expect(entries[0]?.address).toBe("agent@local"); - }); - - test("skips an agent directory with no agent.json", async () => { - const dataDir = await tempDir(); - const store = createAgentKeyStore({ - dataDir, - generateKeyPair: async () => makeKeyPair(5), - ...stubCrypto, - }); - await store.loadOrGenerateKey("agent@local"); - - const entries = await store.scanKeys(); - expect(entries).toEqual([]); - }); -}); diff --git a/packages/hub-agent/src/agent-key-store.ts b/packages/hub-agent/src/agent-key-store.ts index 35d91a7f..62523f36 100644 --- a/packages/hub-agent/src/agent-key-store.ts +++ b/packages/hub-agent/src/agent-key-store.ts @@ -13,27 +13,10 @@ // for incoming deploy packs. import fsp from "node:fs/promises"; -import path from "node:path"; -import { getLogger } from "@intx/log"; import { hasCode, hexDecode } from "@intx/types"; import type { KeyPair } from "@intx/types/runtime"; -import { - KEYS_DIR_NAME, - META_FILE, - PRIVATE_KEY_FILE, - PUBLIC_KEY_FILE, - keysDir, - privateKeyPath, - publicKeyPath, -} from "./agent-paths"; - -const logger = getLogger(["interchange", "hub-agent", "key-store"]); - -export type AgentKeyEntry = { - address: string; - keyPair: KeyPair; -}; +import { keysDir, privateKeyPath, publicKeyPath } from "./agent-paths"; export type AgentKeyStoreDeps = { dataDir: string; @@ -67,14 +50,6 @@ export type AgentKeyStore = { loadOrGenerateKey( address: string, ): Promise<{ keyPair: KeyPair; isNew: boolean }>; - /** - * Read every persisted keypair in the data directory and warm the - * in-memory cache with each one. Used by the sidecar's restore path - * to recover agents across restarts. Directories with a partial - * keypair (one of the two files present) are skipped with a warning; - * the operator can re-pair or remove them manually. - */ - scanKeys(): Promise; /** * Sign the challenge payload with the agent's cached private key. * Returns null when no key is cached for the address — the caller @@ -85,9 +60,9 @@ export type AgentKeyStore = { payload: Uint8Array, ): Promise; /** - * Record the hub public key the agent has been paired with. Cached - * in memory only; on-disk persistence of the pairing record lives in - * AgentRepoStore.persistPairing. + * Record the hub public key the agent has been paired with. Cached in + * memory only; the deploy path re-records it on every deploy, so it + * does not need to survive a restart. */ recordHubKey(address: string, hexHubPublicKey: string): void; /** @@ -154,62 +129,6 @@ export function createAgentKeyStore(deps: AgentKeyStoreDeps): AgentKeyStore { return { keyPair, isNew: true }; } - async function scanKeys(): Promise { - let entries; - try { - entries = await fsp.readdir(dataDir, { withFileTypes: true }); - } catch (err: unknown) { - if (hasCode(err) && err.code === "ENOENT") return []; - throw err; - } - - const results: AgentKeyEntry[] = []; - for (const entry of entries) { - if (!entry.isDirectory()) continue; - const dir = path.join(dataDir, entry.name); - const privPath = path.join(dir, KEYS_DIR_NAME, PRIVATE_KEY_FILE); - const pubPath = path.join(dir, KEYS_DIR_NAME, PUBLIC_KEY_FILE); - const [privOk, pubOk] = await Promise.all([ - fileExists(privPath), - fileExists(pubPath), - ]); - if (!privOk && !pubOk) continue; - if (!privOk || !pubOk) { - logger.warn`Skipping ${entry.name}: incomplete key pair (private=${String(privOk)}, public=${String(pubOk)})`; - continue; - } - const metaRaw = await readOptional(path.join(dir, META_FILE)); - if (metaRaw === null) { - // A key pair without a metadata file means the agent was - // half-provisioned and crashed. The restore composer logs - // and skips it; the operator can pair the key with a new - // config or remove the directory. - continue; - } - let parsed: unknown; - try { - parsed = JSON.parse(metaRaw); - } catch { - continue; - } - if (typeof parsed !== "object" || parsed === null) continue; - if (!("address" in parsed)) continue; - const address = parsed.address; - if (typeof address !== "string") continue; - const [privateKey, publicKey] = await Promise.all([ - fsp.readFile(privPath), - fsp.readFile(pubPath), - ]); - const keyPair: KeyPair = { - privateKey: new Uint8Array(privateKey), - publicKey: new Uint8Array(publicKey), - }; - agentKeys.set(address, keyPair); - results.push({ address, keyPair }); - } - return results; - } - async function signChallenge( address: string, payload: Uint8Array, @@ -244,7 +163,6 @@ export function createAgentKeyStore(deps: AgentKeyStoreDeps): AgentKeyStore { return { loadOrGenerateKey, - scanKeys, signChallenge, recordHubKey, verifyDeployCommit, @@ -264,12 +182,3 @@ async function fileExists(filePath: string): Promise { throw err; } } - -async function readOptional(filePath: string): Promise { - try { - return await fsp.readFile(filePath, "utf-8"); - } catch (err: unknown) { - if (hasCode(err) && err.code === "ENOENT") return null; - throw err; - } -} diff --git a/packages/hub-agent/src/agent-paths.ts b/packages/hub-agent/src/agent-paths.ts index 2a553300..5cf836ec 100644 --- a/packages/hub-agent/src/agent-paths.ts +++ b/packages/hub-agent/src/agent-paths.ts @@ -9,15 +9,10 @@ import path from "node:path"; -// The single source of truth for the per-agent layout filenames. The -// scan paths in agent-repo-store and agent-key-store cannot use the -// address-keyed helpers below because they walk the data directory -// before any agent.json has been parsed — those callers join these -// constants against the on-disk directory name directly. -export const KEYS_DIR_NAME = "keys"; -export const PRIVATE_KEY_FILE = "id_ed25519"; -export const PUBLIC_KEY_FILE = "id_ed25519.pub"; -export const META_FILE = "agent.json"; +// The per-agent key-file layout the address-keyed helpers below build on. +const KEYS_DIR_NAME = "keys"; +const PRIVATE_KEY_FILE = "id_ed25519"; +const PUBLIC_KEY_FILE = "id_ed25519.pub"; export function sanitizeAddress(address: string): string { return address.replace(/@/g, "_at_").replace(/[^a-zA-Z0-9_-]/g, "_"); @@ -38,7 +33,3 @@ export function privateKeyPath(dataDir: string, address: string): string { export function publicKeyPath(dataDir: string, address: string): string { return path.join(keysDir(dataDir, address), PUBLIC_KEY_FILE); } - -export function metaPath(dataDir: string, address: string): string { - return path.join(agentDir(dataDir, address), META_FILE); -} diff --git a/packages/hub-agent/src/agent-repo-store.test.ts b/packages/hub-agent/src/agent-repo-store.test.ts deleted file mode 100644 index ef596406..00000000 --- a/packages/hub-agent/src/agent-repo-store.test.ts +++ /dev/null @@ -1,124 +0,0 @@ -import { describe, test, expect, afterEach } from "bun:test"; -import fs from "node:fs/promises"; -import path from "node:path"; -import os from "node:os"; -import type { HarnessConfig } from "@intx/types/runtime"; - -import { createAgentRepoStore } from "./agent-repo-store"; - -const tempDirs: string[] = []; - -async function tempDir(): Promise { - const d = await fs.mkdtemp(path.join(os.tmpdir(), "agent-repo-store-test-")); - tempDirs.push(d); - return d; -} - -afterEach(async () => { - const dirs = tempDirs.splice(0); - await Promise.all( - dirs.map((d) => fs.rm(d, { recursive: true, force: true })), - ); -}); - -function makeConfig(address: string): HarnessConfig { - return { - agentId: "test-agent", - agentAddress: address, - sessionId: "sess-1", - principalId: "principal-1", - tenantId: "tenant-1", - systemPrompt: "test", - tools: [], - grants: [], - sources: [ - { - id: "test:test-model", - provider: "test", - apiKey: "key", - baseURL: "http://localhost", - model: "test-model", - }, - ], - defaultSource: "test:test-model", - }; -} - -// Provision the on-disk shape `scanConfigs` looks for: a per-agent -// directory containing the keypair files (presence is checked, contents -// are not used here — repo-store tests do not need a real keypair). -async function provisionAgentDir( - dataDir: string, - agentName: string, -): Promise { - const dir = path.join(dataDir, agentName); - await fs.mkdir(path.join(dir, "keys"), { recursive: true }); - await fs.writeFile(path.join(dir, "keys", "id_ed25519"), new Uint8Array()); - await fs.writeFile( - path.join(dir, "keys", "id_ed25519.pub"), - new Uint8Array(), - ); - return dir; -} - -describe("AgentRepoStore — config + pairing persistence", () => { - test("persistPairing then persistConfig preserves the pairing key", async () => { - const dataDir = await tempDir(); - const address = "agent@local"; - await provisionAgentDir(dataDir, "agent_at_local"); - - const repo = createAgentRepoStore({ dataDir }); - await repo.persistConfig(address, makeConfig(address)); - await repo.persistPairing(address, "aabbcc"); - - const entries = await repo.scanConfigs(); - expect(entries).toHaveLength(1); - const entry = entries[0]; - if (entry === undefined) throw new Error("unreachable"); - expect(entry.hubPublicKey).toBe("aabbcc"); - }); - - test("persistConfig after persistPairing keeps the pairing key intact", async () => { - const dataDir = await tempDir(); - const address = "agent@local"; - await provisionAgentDir(dataDir, "agent_at_local"); - - const repo = createAgentRepoStore({ dataDir }); - await repo.persistConfig(address, makeConfig(address)); - await repo.persistPairing(address, "aabbcc"); - - const updatedConfig = { ...makeConfig(address), systemPrompt: "updated" }; - await repo.persistConfig(address, updatedConfig); - - const entries = await repo.scanConfigs(); - expect(entries).toHaveLength(1); - const entry = entries[0]; - if (entry === undefined) throw new Error("unreachable"); - expect(entry.hubPublicKey).toBe("aabbcc"); - expect(entry.config.systemPrompt).toBe("updated"); - }); - - test("absent pairing key returns undefined from scanConfigs", async () => { - const dataDir = await tempDir(); - const address = "agent@local"; - await provisionAgentDir(dataDir, "agent_at_local"); - - const repo = createAgentRepoStore({ dataDir }); - await repo.persistConfig(address, makeConfig(address)); - - const entries = await repo.scanConfigs(); - expect(entries).toHaveLength(1); - const entry = entries[0]; - if (entry === undefined) throw new Error("unreachable"); - expect(entry.hubPublicKey).toBeUndefined(); - }); - - test("persistPairing on an unknown address throws", async () => { - const dataDir = await tempDir(); - const repo = createAgentRepoStore({ dataDir }); - - await expect(repo.persistPairing("nobody@local", "aabbcc")).rejects.toThrow( - "no existing agent.json", - ); - }); -}); diff --git a/packages/hub-agent/src/agent-repo-store.ts b/packages/hub-agent/src/agent-repo-store.ts index e0e410a9..3c0ccc09 100644 --- a/packages/hub-agent/src/agent-repo-store.ts +++ b/packages/hub-agent/src/agent-repo-store.ts @@ -1,18 +1,14 @@ // Per-agent on-disk repository layout. // -// Owns the isogit repo wrapper, the deploy-pack apply / state-pack -// produce flow, and the persistence of the per-agent `agent.json` -// metadata blob. Key custody lives in AgentKeyStore alongside this +// Owns the isogit repo wrapper and the deploy-pack apply / state-pack +// produce flow. Key custody lives in AgentKeyStore alongside this // store; both share the directory-layout helpers in agent-paths. import fs from "node:fs"; import fsp from "node:fs/promises"; -import path from "node:path"; import git from "isomorphic-git"; -import { type } from "arktype"; import { getLogger } from "@intx/log"; import { hasCode } from "@intx/types"; -import { HarnessConfig } from "@intx/types/runtime"; import { initAgentRepo, applyPack, @@ -21,30 +17,10 @@ import { type CommitVerifier, } from "@intx/storage-isogit"; -import { META_FILE, agentDir, metaPath } from "./agent-paths"; +import { agentDir } from "./agent-paths"; const logger = getLogger(["interchange", "hub-agent", "repo-store"]); -const AgentMeta = type({ - version: "1", - address: "string", - config: HarnessConfig, - "hubPublicKey?": "string", -}); -type AgentMeta = typeof AgentMeta.infer; - -/** - * A per-agent metadata record as it appears on disk. `hubPublicKey` - * — the hub identity this agent has been paired with — is set once - * by `persistPairing` and is otherwise preserved across `persistConfig` - * updates. - */ -export type AgentConfigEntry = { - address: string; - config: HarnessConfig; - hubPublicKey?: string; -}; - export type ApplyDeployPackArgs = { address: string; pack: Uint8Array; @@ -68,25 +44,6 @@ export type AgentRepoStore = { ): Promise<{ pack: Uint8Array; commitSha: string; ref: string }>; getDeployRef(address: string): Promise; remove(address: string): Promise; - /** - * Write the agent's HarnessConfig to disk. Preserves the existing - * `hubPublicKey` field if one was previously recorded by - * `persistPairing`. To update the pairing key, call `persistPairing` - * separately. - */ - persistConfig(address: string, config: HarnessConfig): Promise; - /** - * Record the hub public key this agent has been paired with. Set-once - * in normal operation; rewriting an existing value is permitted but - * indicates the agent has been re-paired with a different hub. - */ - persistPairing(address: string, hubPublicKey: string): Promise; - /** - * Scan the data directory for agent.json files and return the - * persisted config + pairing key for each. Files that fail to - * parse are skipped with a warning. - */ - scanConfigs(): Promise; }; export function createAgentRepoStore(config: { @@ -142,94 +99,6 @@ export function createAgentRepoStore(config: { logger.info`Deleted agent directory for ${address}`; } - async function readMeta(address: string): Promise { - try { - const raw = await fsp.readFile(metaPath(dataDir, address), "utf-8"); - const parsed: unknown = JSON.parse(raw); - const validated = AgentMeta(parsed); - if (validated instanceof type.errors) { - logger.warn`agent.json invalid for ${address}: ${validated.summary}`; - return null; - } - return validated; - } catch (err: unknown) { - if (hasCode(err) && err.code === "ENOENT") { - return null; - } - throw err; - } - } - - async function writeMeta(meta: AgentMeta): Promise { - await fsp.writeFile(metaPath(dataDir, meta.address), JSON.stringify(meta)); - } - - async function persistConfig( - address: string, - config: HarnessConfig, - ): Promise { - const existing = await readMeta(address); - const meta: AgentMeta = { version: 1, address, config }; - if (existing?.hubPublicKey !== undefined) { - meta.hubPublicKey = existing.hubPublicKey; - } - await writeMeta(meta); - } - - async function persistPairing( - address: string, - hubPublicKey: string, - ): Promise { - const existing = await readMeta(address); - if (existing === null) { - throw new Error( - `Cannot persist hub pairing for "${address}": no existing agent.json`, - ); - } - await writeMeta({ ...existing, hubPublicKey }); - } - - async function scanConfigs(): Promise { - let entries: fs.Dirent[]; - try { - entries = await fsp.readdir(dataDir, { withFileTypes: true }); - } catch (err: unknown) { - if (hasCode(err) && err.code === "ENOENT") return []; - throw err; - } - - const results: AgentConfigEntry[] = []; - for (const entry of entries) { - if (!entry.isDirectory()) continue; - // Read agent.json directly by directory name — we cannot reverse - // sanitizeAddress, so the address comes from the file's `address` - // field, not the directory name. - const metaFile = path.join(dataDir, entry.name, META_FILE); - let raw: string; - try { - raw = await fsp.readFile(metaFile, "utf-8"); - } catch (err: unknown) { - if (hasCode(err) && err.code === "ENOENT") continue; - throw err; - } - const parsed: unknown = JSON.parse(raw); - const validated = AgentMeta(parsed); - if (validated instanceof type.errors) { - logger.warn`Skipping ${entry.name}: invalid agent.json: ${validated.summary}`; - continue; - } - const result: AgentConfigEntry = { - address: validated.address, - config: validated.config, - }; - if (validated.hubPublicKey !== undefined) { - result.hubPublicKey = validated.hubPublicKey; - } - results.push(result); - } - return results; - } - return { getAgentDir, initRepo, @@ -237,8 +106,5 @@ export function createAgentRepoStore(config: { createStatePack, getDeployRef, remove, - persistConfig, - persistPairing, - scanConfigs, }; } diff --git a/packages/hub-agent/src/index.ts b/packages/hub-agent/src/index.ts index 682fcb84..000e182e 100644 --- a/packages/hub-agent/src/index.ts +++ b/packages/hub-agent/src/index.ts @@ -1,14 +1,12 @@ export { createAgentRepoStore, type AgentRepoStore, - type AgentConfigEntry, type ApplyDeployPackArgs, } from "./agent-repo-store"; export { createAgentKeyStore, type AgentKeyStore, type AgentKeyStoreDeps, - type AgentKeyEntry, } from "./agent-key-store"; export type { HarnessBuilder } from "./harness-builder"; export { diff --git a/packages/hub-agent/src/session-manager.test.ts b/packages/hub-agent/src/session-manager.test.ts index ed08bb2d..00a2a359 100644 --- a/packages/hub-agent/src/session-manager.test.ts +++ b/packages/hub-agent/src/session-manager.test.ts @@ -38,9 +38,6 @@ function makeStubRepoStore(opts: { createStatePack: opts.createStatePack, getDeployRef: unused("getDeployRef"), remove: opts.remove, - persistConfig: unused("persistConfig"), - persistPairing: unused("persistPairing"), - scanConfigs: unused("scanConfigs"), }; } diff --git a/packages/hub-agent/src/ws/hub-link-bootstrap-prune.test.ts b/packages/hub-agent/src/ws/hub-link-bootstrap-prune.test.ts index 3bfebd13..6b6ec28c 100644 --- a/packages/hub-agent/src/ws/hub-link-bootstrap-prune.test.ts +++ b/packages/hub-agent/src/ws/hub-link-bootstrap-prune.test.ts @@ -53,12 +53,6 @@ function createTestKeyStore(): AgentKeyStore & { if (existing !== undefined) return { keyPair: existing, isNew: false }; throw new Error(`No key registered for ${address} in test store`); }, - async scanKeys() { - return [...agentKeys.entries()].map(([address, keyPair]) => ({ - address, - keyPair, - })); - }, async signChallenge(address, payload) { const kp = agentKeys.get(address); if (kp === undefined) return null; diff --git a/packages/hub-agent/src/ws/hub-link-mail-router.test.ts b/packages/hub-agent/src/ws/hub-link-mail-router.test.ts index 0a666ebe..93259c5a 100644 --- a/packages/hub-agent/src/ws/hub-link-mail-router.test.ts +++ b/packages/hub-agent/src/ws/hub-link-mail-router.test.ts @@ -48,12 +48,6 @@ function createTestKeyStore(): AgentKeyStore & { if (existing !== undefined) return { keyPair: existing, isNew: false }; throw new Error(`No key registered for ${address} in test store`); }, - async scanKeys() { - return [...agentKeys.entries()].map(([address, keyPair]) => ({ - address, - keyPair, - })); - }, async signChallenge(address, payload) { const kp = agentKeys.get(address); if (kp === undefined) return null; diff --git a/packages/hub-agent/src/ws/hub-link.test.ts b/packages/hub-agent/src/ws/hub-link.test.ts index 0aac036b..40627e7f 100644 --- a/packages/hub-agent/src/ws/hub-link.test.ts +++ b/packages/hub-agent/src/ws/hub-link.test.ts @@ -75,12 +75,6 @@ function createTestKeyStore(): AgentKeyStore & { if (existing !== undefined) return { keyPair: existing, isNew: false }; throw new Error(`No key registered for ${address} in test store`); }, - async scanKeys() { - return [...agentKeys.entries()].map(([address, keyPair]) => ({ - address, - keyPair, - })); - }, async signChallenge(address, payload) { const kp = agentKeys.get(address); if (kp === undefined) return null; diff --git a/packages/storage-isogit/src/pack-receive.ts b/packages/storage-isogit/src/pack-receive.ts index 421414a0..a0a14158 100644 --- a/packages/storage-isogit/src/pack-receive.ts +++ b/packages/storage-isogit/src/pack-receive.ts @@ -138,8 +138,8 @@ async function topLevelNames(dir: string, oid: string): Promise> { * * Derives the set of deploy-managed top-level entries from the union of the * old and new commit trees, removes those entries, then writes the new tree. - * Paths that never appear in any commit tree (e.g. .git, agent.json, state, - * keys) are never touched. + * Paths that never appear in any commit tree (e.g. .git, state, keys) are + * never touched. * * NOTE: The rm-then-write sequence is not atomic. If writeTree fails after rm * succeeds (e.g. disk full), the working tree will be missing the cleared diff --git a/packages/types/src/grant-wire.ts b/packages/types/src/grant-wire.ts index 0074f824..a5e5f389 100644 --- a/packages/types/src/grant-wire.ts +++ b/packages/types/src/grant-wire.ts @@ -3,7 +3,7 @@ // GrantRule.expiresAt is a Date | null at runtime, but JSON round-trips // turn it into a string | null. This validator accepts either form and // coerces strings back to Date instances, making it safe to use when -// deserializing persisted configs (e.g. agent.json on sidecar restart). +// deserializing grants that have round-tripped through JSON. import { type } from "arktype"; From d1fe915201f0c5fbfbad038e2ee62a02ce3df6a6 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Sat, 4 Jul 2026 07:13:38 -0500 Subject: [PATCH 045/101] Rewrite the deploy narrative for the substrate-only flow The implementation doc described the retired deploy protocol: a provision-then-session-start handshake with `session.start` frames, a supervisor-owned trivial-vs-multi-step routing decision, and an in-supervisor run-event write path. None of that exists. Rewrite the deploy-routing, deploy-flow, deployment-procedure, and reconnect-sequencing sections to describe what runs now: the sidecar's deploy router stages every deploy through the workflow-run substrate, priming a per-step repo for a provision-step frame or spawning the supervised workflow-process child for a workflow frame; a single agent deploys as a single-step workflow; the child begins inference itself with no session-start step and commits its own run events; and on reconnect the sidecar re-announces its keyless workflow-deployment addresses in one register frame. Drop the session-start wire frames from the frame table. --- docs/IMPLEMENTATION.md | 210 ++++++++++++++++------------------------- 1 file changed, 82 insertions(+), 128 deletions(-) diff --git a/docs/IMPLEMENTATION.md b/docs/IMPLEMENTATION.md index 7d4e6acb..41234ed3 100644 --- a/docs/IMPLEMENTATION.md +++ b/docs/IMPLEMENTATION.md @@ -249,19 +249,17 @@ The sidecar WebSocket protocol includes frames for agent deployment, reconnectio **Agent deployment:** -| Direction | Frame | Purpose | -| ------------- | -------------------- | ------------------------------------------------------ | -| Hub → Sidecar | `agent.deploy` | Provision an agent (keys, directory, ephemeral config) | -| Sidecar → Hub | `agent.deploy.ack` | Confirm provisioning, provide agent public key | -| Hub → Sidecar | `session.start` | Start the inference harness for a provisioned agent | -| Sidecar → Hub | `session.start.ack` | Confirm the harness is running | -| Hub → Sidecar | `agent.undeploy` | Remove an agent from the sidecar | -| Sidecar → Hub | `agent.undeploy.ack` | Confirm teardown (includes state push status) | -| Sidecar → Hub | `agent.error` | Report a failure at any stage | +| Direction | Frame | Purpose | +| ------------- | -------------------- | ------------------------------------------------- | +| Hub → Sidecar | `agent.deploy` | Stage a deploy through the workflow-run substrate | +| Sidecar → Hub | `agent.deploy.ack` | Confirm the deploy staged, provide the public key | +| Hub → Sidecar | `agent.undeploy` | Remove an agent from the sidecar | +| Sidecar → Hub | `agent.undeploy.ack` | Confirm teardown (includes state push status) | +| Sidecar → Hub | `agent.error` | Report a failure at any stage | -Agent deployment is a three-phase operation: provision, pack delivery, session start. The hub sends `agent.deploy` to provision the agent (generate keys, create directory, persist config). After receiving `agent.deploy.ack` with the agent's public key, the hub streams the deploy tree via pack frames. After `pack.ack`, the hub sends `session.start` to start the inference harness. This ordering ensures the deploy tree (prompt, skills) is on disk before the harness reads it. +Agent deployment stages through the workflow-run substrate rather than a separate provision-then-start handshake. The hub sends `agent.deploy`; the sidecar's deploy router primes the per-step repo (for a provision-step frame) or spawns the supervised workflow-process child (for a workflow frame), then acks with `agent.deploy.ack` carrying the public key. The deploy tree (prompt, skills) rides in on the follow-up deploy pack, which the child reads from the substrate. The workflow-process child starts inference itself once spawned -- there is no separate `session.start` step. -Undeploy is an acknowledged operation. The sidecar stops the harness, pushes state to the hub (best-effort), deletes the agent directory, and responds with `agent.undeploy.ack`. The `statePushed` field indicates whether the state push was attempted. The hub defers routing table cleanup until the ack arrives. +Undeploy is an acknowledged operation. The sidecar shuts the deployment's supervisor down, pushes state to the hub (best-effort), deletes the agent directory, and responds with `agent.undeploy.ack`. The `statePushed` field indicates whether the state push was attempted. The hub defers routing table cleanup until the ack arrives. **Reconnection:** @@ -738,88 +736,59 @@ live `subscribeKind` tail, but a resume-rehydrated queued signal needs the runtime body's `RunState` reader plumbing, which lands when the per-run state-machine reader is wired into the child. -### Deploy Routing (Option Z) - -The supervisor is the single ingress for inbound `agent.deploy` -frames. The host-side handler calls `supervisor.deploy(frame)` and -the supervisor decides between the trivial (1-step) passthrough -and the multi-step IPC-backed spawn. Routing lives on the -supervisor side of the seam; the host does not re-decide. The -sidecar's hub-link surface lives at -`packages/hub-agent/src/ws/hub-link.ts`; it consumes a -`DeployRouter` binding whose production implementation is the -workflow-host supervisor. - -Four invariants on the routing model are locked: - -1. **Supervisor owns `agent.deploy` framing.** The hub-side handler - in `packages/hub-agent/src/ws/hub-link.ts` calls - `deployRouter.deploy(frame)`; the production router is the - workflow-host supervisor's `supervisor.deploy(frame)`. No routing - logic leaks back into `hub-agent` or `hub-sessions`. -2. **Trivial branch is a process-topology passthrough.** The - supervisor invokes the host-injected `trivialLaunch` callback - directly. No IPC channel opens. No workflow-process child is - spawned. No mail-bus subscription registers. The bytes flowing - through the deploy-flow gate path are bit-identical to the - pre-supervisor surface. -3. **Trivial deploys emit the canonical workflow-run event chain.** - The supervisor commits `RunStarted` / `StepStarted` / - `StepCompleted` / `RunCompleted` to the workflow-run repo from - the supervisor process itself, signed via the host's - `signAsPrincipal("supervisor", ...)` callback. The chain fires - per inbound mail trigger (one run per fire) and is driven by - the host calling `bindings.recordRunEvent(...)` from the - trivialLaunch callback's reactor / harness lifecycle moments - (`message.run.started` / `message.run.ended` brackets). The - on-disk envelope is byte-identical to the one a multi-step - deployment's workflow-process child produces. Trivial and - multi-step deployments differ in process topology, not in - observability. The supervisor's signing key never leaves the - supervisor's address space; `signAsPrincipal` returns the raw - signature bytes for the canonical payload only after the - supervisor has serialized them. -4. **`credentialsSnapshot` is multi-step-only.** The trivial branch - does not assemble a snapshot; `getCredentialsSnapshot()` - continues to return `null` after a trivial deploy. The - multi-step branch (`steps.length >= 2`) provisions per-step - `agent-state` repos, mints keys, spawns the workflow-process - child via `subprocessSpawner`, registers the deployment's mail - address, waits for `ready`, and assembles the - `credentialsSnapshot` -- this is the body of `spawn(opts)`, - which the multi-step branch is the worker for. The - `agent.deploy` wire frame today carries only a `HarnessConfig` - (no workflow definition); every frame is therefore trivial. The - seam exists now so the frame-format extension that carries a - `WorkflowDefinition` lands as a pure data-shape change. - -The supervisor's per-event commit primitive lives in -`packages/workflow-host/src/supervisor/run-event-signing.ts` -alongside the analogous `commitCancelRequested` path. The trivial -branch's `recordRunEvent` is a thin closure over that primitive that -the supervisor hands into the trivialLaunch bindings; the multi-step -branch composes the same primitive from inside its in-supervisor -event-channel receiver. +### Deploy Routing + +The sidecar's deploy router is the single ingress for inbound +`agent.deploy` frames. The hub-link surface at +`packages/hub-agent/src/ws/hub-link.ts` consumes a `DeployRouter` +binding whose production implementation is `createSidecarDeployRouter` +in `apps/sidecar/src/workflow-host-wiring.ts`. Routing lives on the +router side of the seam; the hub-link hands the frame off and folds the +returned public key into the outbound `agent.deploy.ack`. + +Every deploy stages through the workflow-run substrate. The router +branches on the frame shape: + +1. **Provision-step frame (`provisionStep: true`).** The router primes + the frame's per-step `agent-state` repo and records the hub key, + without constructing a supervisor or spawning a child. The follow-up + full-closure deploy pack then applies into the primed repo and + verifies against the recorded key. +2. **Workflow frame (carries a `WorkflowDefinition`).** The router + constructs a fresh per-deployment workflow-host supervisor and drives + its `spawn(opts)` lifecycle: per-step `agent-state` repo + provisioning, key minting, workflow-process child spawn via + `subprocessSpawner`, mail-bus registration, the IPC ready handshake, + and `credentialsSnapshot` assembly. + +A frame carrying neither shape is rejected -- there is no in-process +deploy path. A single agent deploys as a single-step workflow: the +instance-deploy path wraps its `HarnessConfig` with +`wrapHarnessAsSingleStepWorkflow` +(`packages/workflow-deploy/src/orchestrator.ts`) and deploys it at the +workflow's head, so single- and multi-step deployments run the same +supervised-child topology and differ only in step count. + +Run-lifecycle events (`RunStarted` / `StepStarted` / `StepCompleted` / +`RunCompleted`) are committed by the workflow-process child from its own +address space against the workflow-run repo; the supervisor does not +commit them in-process. The supervisor's surviving run-event work is +compaction: once a run terminates it folds that run's per-event blobs +into one combined log +(`packages/workflow-host/src/supervisor/run-event-compaction.ts`). The workflow-run repo's substrate `repoId.id` is constrained to `/^[a-zA-Z0-9_-]+$/` (`SAFE_REPO_ID` in `packages/hub-sessions/src/repo-store/types.ts`), which the -agent-address shape (`ins_@`) does not satisfy. The -sidecar wiring derives the trivial branch's deployment id by -substituting disallowed characters with `-`; see -`deriveDeploymentId` in -`apps/sidecar/src/workflow-host-wiring.ts`. The supervisor -principal's `deploymentId` and the workflow-run `repoId.id` are kept -equal so the workflow-run kind handler's principal-vs-repo authz -check holds for every supervisor-authored event commit. - -The sidecar production wiring lives in -`apps/sidecar/src/workflow-host-wiring.ts`: -`createSidecarDeployRouter` constructs a fresh per-deployment -supervisor on every inbound frame whose `trivialLaunch` closes -over `SessionManager.provisionAgent` plus the hub-pairing-key -recording the legacy handler performed inline. The `HubTransport` -mail-bus adapter the supervisor consumes lives in the +agent-address shape (`ins_@`) does not satisfy. The sidecar +wiring derives the deployment id by substituting disallowed characters +with `-`; see `deriveDeploymentId` in +`apps/sidecar/src/workflow-host-wiring.ts`. The supervisor principal's +`deploymentId` and the workflow-run `repoId.id` are kept equal so the +workflow-run kind handler's principal-vs-repo authz check holds for +every supervisor-authored event commit. + +The `HubTransport` mail-bus adapter the supervisor consumes lives in the `@intx/workflow-host` package proper (`packages/workflow-host/src/mail-bus/`) so an alternative-sidecar implementation can reuse it without forking the wiring. @@ -844,8 +813,8 @@ snapshot. The implementation lives in returns a `CredentialsSnapshot` with per-step `address`, `grants`, and `contentHash`. The hash is stable across processes so the child can detect a stale snapshot pushed after a fresher one. -- A missing grants file is treated as an empty grant array (so the - trivial path with no operator-supplied grants does not crash); a +- A missing grants file is treated as an empty grant array (so a + single-step deploy with no operator-supplied grants does not crash); a malformed file fails loudly at the boundary. ### Signed CancelRequested Authority @@ -1104,14 +1073,13 @@ Health can also be reported via periodic heartbeat messages to the control plane ### Deployment Procedure -1. **Provision** - Hub sends `agent.deploy` with ephemeral config; sidecar creates directory, keys, and returns the public key +1. **Stage** - Hub sends `agent.deploy`; the sidecar's deploy router stages the deploy through the workflow-run substrate (priming the per-step repo or spawning the supervised workflow-process child) and returns the public key 2. **Assembly** - Hub resolves skill dependencies and assembles the deploy tree -3. **Pack transfer** - Hub streams the packfile to the sidecar (see Agent Deployment) -4. **Session start** - Hub sends `session.start`; sidecar reads deploy tree and starts the harness -5. **Health gate** - New version must pass health checks before receiving traffic -6. **Traffic shift** - Registry updates discovery to point to new version -7. **Drain** - Old version stops accepting new work, completes in-flight operations -8. **Retirement** - Old version shuts down; deploy tag remains for rollback +3. **Pack transfer** - Hub streams the packfile to the sidecar; the workflow-process child reads the deploy tree from the substrate and begins inference (see Agent Deployment) +4. **Health gate** - New version must pass health checks before receiving traffic +5. **Traffic shift** - Registry updates discovery to point to new version +6. **Drain** - Old version stops accepting new work, completes in-flight operations +7. **Retirement** - Old version shuts down; deploy tag remains for rollback ### Rollback @@ -1222,16 +1190,14 @@ Pack transfers share the WebSocket with live session traffic. To prevent interfe ### Deploy Flow -Deployment is a three-phase operation: provision, pack delivery, session start. +Deployment is a two-phase operation: stage and pack delivery. There is no separate session-start phase -- the workflow-process child begins inference once it is spawned and its deploy tree has landed. -**Phase 1: Provision** +**Phase 1: Stage** -1. Hub sends `agent.deploy` with ephemeral config (credentials, materialized grants, providers, session ID) -2. Sidecar creates the agent directory, generates an Ed25519 key pair, and persists the config -3. Sidecar responds with `agent.deploy.ack` containing the agent's public key (hex-encoded) -4. Hub stores the public key for future challenge/response verification - -The sidecar is now provisioned but not running. It can receive pack data. +1. Hub sends `agent.deploy` with the deploy config (credentials, materialized grants, providers, session ID) and, for a multi-step deploy, a workflow definition +2. The sidecar's deploy router stages the deploy through the workflow-run substrate: a provision-step frame primes the per-step `agent-state` repo and records the hub key; a workflow frame spawns the supervised workflow-process child +3. Sidecar responds with `agent.deploy.ack` containing the public key (hex-encoded) +4. Hub stores the public key for future deploy-commit verification **Phase 2: Pack delivery** @@ -1239,19 +1205,12 @@ The sidecar is now provisioned but not running. It can receive pack data. 6. Hub sends `pack.push` frames (chunked, interleaved with other traffic) 7. Hub sends `pack.done` with target ref (`refs/heads/deploy`) and commit SHA 8. Sidecar validates the packfile integrity -9. Sidecar verifies the deploy commit signature against the hub's public key +9. Sidecar verifies the deploy commit signature against the recorded hub public key 10. Sidecar unpacks objects into the git object store 11. Sidecar updates `refs/heads/deploy` to the new commit -12. Sidecar merges `deploy` into its agent branch (trivial merge) +12. Sidecar merges `deploy` into its agent branch (fast-forward) 13. Sidecar checks out only `deploy/` paths via partial checkout (`filepaths: ["deploy/"], noUpdateHead: true`) — `state/` working-tree files are untouched because `filepaths` restricts the scope, `noUpdateHead` prevents moving the branch pointer -14. Sidecar responds with `pack.ack` - -**Phase 3: Session start** - -15. Hub sends `session.start` -16. Sidecar reads the deploy tree from disk (prompt, skills/tools) -17. Sidecar creates the inference harness with the deploy tree and ephemeral config -18. Sidecar starts the harness and responds with `session.start.ack` +14. Sidecar responds with `pack.ack`; the workflow-process child reads the deploy tree from the substrate and runs The agent is now running and can receive messages. @@ -1260,13 +1219,13 @@ On first deploy a full packfile is sent. On subsequent deploys, if the sidecar a ### Undeploy Flow 1. Hub sends `agent.undeploy` with a reason string -2. Sidecar stops the inference harness (or removes the agent from the provisioned set if session has not started) +2. Sidecar shuts the deployment's supervisor down, releasing the workflow-process child and its per-deployment routing state (a no-op if no supervisor is live for the address) 3. Sidecar pushes state to the hub via `pack.push`/`pack.done` (best-effort — the `statePushed` field in the ack indicates whether this was attempted, not whether the hub confirmed receipt) 4. Sidecar deletes the agent directory 5. Sidecar responds with `agent.undeploy.ack` 6. Hub removes the agent from the routing table -If the sidecar disconnects before sending the ack, the hub removes the agent from the routing table on disconnect. If `startSession` fails, the agent remains provisioned and can be retried or undeployed. +If the sidecar disconnects before sending the ack, the hub removes the agent from the routing table on disconnect. ### State Push Flow @@ -1284,18 +1243,13 @@ If the WebSocket disconnects mid-transfer, no git state is corrupted: the sideca ### Reconnect Sequencing -Pack transfers are sequenced after identity verification: +On reconnect the sidecar restores its workflow deployments from local disk (see the deployment restore path) and re-announces them so the hub restores their routes: -1. Sidecar sends `reconnect` with `agentAddresses` and `deployRefs` -2. Hub sends `challenge` per address -3. Sidecar sends `challenge.response` per address -4. Hub verifies signatures — only verified agents proceed -5. Hub sends `grants.update` per verified agent (sidecar must have current grants before processing messages) -6. For agents whose deploy ref is behind: hub initiates pack transfer, waits for `pack.ack` -7. Hub sends `session.start` per agent -8. After `session.start.ack`: hub flushes queued messages +1. Sidecar sends a single `register` frame carrying its live workflow-deployment addresses (`workflowAddresses`) +2. Hub re-registers those addresses for routing. Workflow-deployment addresses are hub-minted and keyless, so they re-register without the challenge/response flow a per-agent key would require +3. For a deployment whose deploy ref the hub tracks as behind, the hub initiates a pack transfer and waits for `pack.ack` -Pack content is never sent to a sidecar that has not proved ownership of the agent's key. +The sidecar verifies every inbound deploy pack's commit signature against the hub key it recorded at deploy time before applying it, so pack content is never applied on an unverified signature. On first deploy (no prior key exists), the sidecar is authenticated by its registration token but cannot prove agent key ownership (the key does not exist yet). The hub sends `agent.deploy` to provision the agent, and the sidecar generates the key and returns it in `agent.deploy.ack`. The registration token and the authenticated WebSocket channel bound the trust for first-deploy; challenge/response protects all subsequent interactions. From 59ab2fa83359995bf2be8ef69e9a5fcd6840495f Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Sat, 4 Jul 2026 07:27:41 -0500 Subject: [PATCH 046/101] Stop pushing grants and sources to the sidecar on reconnect The in-process session runtime removal deleted the sidecar handlers for the `grants.update` and `sources.update` frames, but the hub's `agent.reconnected` handler still pushed both over the wire -- an unguarded `sendGrantsUpdate` awaited a reply the sidecar no longer sends. Every reconnect of a supervised instance therefore hung on the request timeout and then rejected the instance from the routing table with its queued mail unflushed. Reconnect is a normal event, so this broke the supervised deployments the migration produces on the happy path. Drop both pushes from the reconnect handler. A supervised deployment carries its grants and sources in the deploy pack and refreshes them over the supervisor's IPC credentials snapshot at spawn and recycle, so the wire push is redundant -- removing it loses no supervised behaviour and un-breaks reconnect. Narrow the orchestrator's router facade to the methods it still drives and drop the now-unused grant store from its dependencies. --- apps/hub/src/index.ts | 4 +- packages/hub-sessions/README.md | 6 +-- .../src/hub-session-orchestrator.test.ts | 54 +++---------------- .../src/hub-session-orchestrator.ts | 46 ++++------------ 4 files changed, 22 insertions(+), 88 deletions(-) diff --git a/apps/hub/src/index.ts b/apps/hub/src/index.ts index 3feda98f..bedc5d5d 100644 --- a/apps/hub/src/index.ts +++ b/apps/hub/src/index.ts @@ -1,4 +1,4 @@ -import { createDB, createGrantStore } from "@intx/db"; +import { createDB } from "@intx/db"; import { createApp, createAuth } from "@intx/hub-api"; import { createAgentRepoStore, @@ -35,7 +35,6 @@ const { db } = createDB({ }); const auth = createAuth(db); -const grantStore = createGrantStore(db); const hubDataDir = process.env["HUB_DATA_DIR"]; if (!hubDataDir) { @@ -141,7 +140,6 @@ createHubSessionOrchestrator({ router: sidecarRouter, db, eventCollectors, - grantStore, agentRepoStore, }); diff --git a/packages/hub-sessions/README.md b/packages/hub-sessions/README.md index 1bca7946..dcd0734f 100644 --- a/packages/hub-sessions/README.md +++ b/packages/hub-sessions/README.md @@ -16,7 +16,7 @@ feed agent events back to the HTTP layer for observability. `assetService` paired with a `db` handle for the asset manifest inserts. `createHubSessionOrchestrator` takes a `HubSessionOrchestratorDeps` of `events`, `router`, `db`, -`eventCollectors`, `grantStore`, and `agentRepoStore`. See the +`eventCollectors`, and `agentRepoStore`. See the exported types in `src/session-service.ts` and `src/hub-session-orchestrator.ts` for the authoritative shapes; `@intx/hub-api` is the in-tree consumer that wires these @@ -37,8 +37,8 @@ for the authoritative type shapes. provisions and tears down agent sessions on the connected sidecar. - `createHubSessionOrchestrator` / `HubSessionOrchestrator`, `HubSessionOrchestratorDeps`, `HubSessionRouterFacade` — the - higher-level orchestrator that wires the router, event collectors, - and grant store together. + higher-level orchestrator that wires the router and event + collectors together. - `createHubSessionLookups` / `HubSessionLookupsDeps` — builds the lookup callbacks the sidecar router needs to resolve sessions. diff --git a/packages/hub-sessions/src/hub-session-orchestrator.test.ts b/packages/hub-sessions/src/hub-session-orchestrator.test.ts index 5c3aeebb..8ab2d01a 100644 --- a/packages/hub-sessions/src/hub-session-orchestrator.test.ts +++ b/packages/hub-sessions/src/hub-session-orchestrator.test.ts @@ -1,8 +1,7 @@ import { describe, test, expect, beforeEach } from "bun:test"; import type { DB } from "@intx/db"; -import type { GrantRule, GrantStore } from "@intx/types/authz"; -import type { InferenceEvent, InferenceSource } from "@intx/types/runtime"; +import type { InferenceEvent } from "@intx/types/runtime"; import type { AgentRepoStore, DeployContent } from "./agent-repo"; import type { RepoStore } from "./repo-store"; @@ -119,13 +118,6 @@ function createMockDB(opts: MockDBOpts) { } type RouterCall = - | { kind: "sendGrantsUpdate"; addr: string; grants: GrantRule[] } - | { - kind: "sendSourcesUpdate"; - addr: string; - sources: InferenceSource[]; - defaultSource: string; - } | { kind: "sendPack"; addr: string; @@ -143,12 +135,6 @@ function createRouterFacade(): { return { calls, facade: { - async sendGrantsUpdate(addr, grants) { - calls.push({ kind: "sendGrantsUpdate", addr, grants }); - }, - async sendSourcesUpdate(addr, sources, defaultSource) { - calls.push({ kind: "sendSourcesUpdate", addr, sources, defaultSource }); - }, async sendPack(addr, pack, ref, sha) { calls.push({ kind: "sendPack", addr, pack, ref, sha }); }, @@ -197,12 +183,6 @@ function createCollectorRegistry( }; } -function createGrantStoreStub(grants: GrantRule[] = []): GrantStore { - return { - collectGrants: async () => grants, - }; -} - type RepoCall = { kind: "createDeployPack"; agentId: string }; function createRepoStoreStub(): { @@ -275,21 +255,19 @@ type Harness = { dispose: () => void; }; -function setup(opts: MockDBOpts & { grants?: GrantRule[] } = {}): Harness { +function setup(opts: MockDBOpts = {}): Harness { const updates: UpdateCall[] = []; const db = createMockDB({ ...opts, recordUpdates: updates }); const events = createSidecarEmitter(); const router = createRouterFacade(); const collectors = createCollectorRegistry(); const repo = createRepoStoreStub(); - const grantStore = createGrantStoreStub(opts.grants); const orchestrator = createHubSessionOrchestrator({ events, router: router.facade, db, eventCollectors: collectors.registry, - grantStore, agentRepoStore: repo.store, }); @@ -386,33 +364,17 @@ describe("createHubSessionOrchestrator", () => { }); describe("agent.reconnected", () => { - test("refreshes grants and skips status update when already running", async () => { - const grant: GrantRule = { - id: "grant-1", - resource: "x:y", - action: "read", - effect: "allow", - origin: "system", - conditions: null, - expiresAt: null, - roleId: null, - principalId: PRINCIPAL_ID, - }; - harness = setup({ instance: makeInstance(), grants: [grant] }); + test("skips the status update when already running", async () => { + harness = setup({ instance: makeInstance() }); await harness.events.emitAndAwait("agent.reconnected", { agentAddress: AGENT_ADDRESS, }); - const grantsCall = harness.router.calls.find( - (c) => c.kind === "sendGrantsUpdate", - ); - expect(grantsCall).toBeDefined(); - if (grantsCall?.kind === "sendGrantsUpdate") { - expect(grantsCall.grants).toEqual([grant]); - } - - // status was already "running", no update should fire + // A supervised deployment refreshes grants/sources over the + // supervisor IPC snapshot, not a reconnect wire push, so no router + // call fires here. status was already "running", so no update fires. + expect(harness.router.calls).toHaveLength(0); expect(harness.updates).toHaveLength(0); }); diff --git a/packages/hub-sessions/src/hub-session-orchestrator.ts b/packages/hub-sessions/src/hub-session-orchestrator.ts index e913c07e..fcff263f 100644 --- a/packages/hub-sessions/src/hub-session-orchestrator.ts +++ b/packages/hub-sessions/src/hub-session-orchestrator.ts @@ -2,9 +2,9 @@ // // Subscribes to the sidecar router's events and runs the host-side // reactions: dispatching inference events to the event collector, -// refreshing grants and credentials on reconnect, ingesting state -// packs, re-deploying stale agents, persisting mail, and forwarding -// mail.delivered notifications back to subscribers. +// restoring the event collector on reconnect, re-deploying stale +// agents, and forwarding mail.delivered notifications back to +// subscribers. // // The orchestrator depends on a narrow `HubSessionRouterFacade` rather // than the full SidecarRouter, so tests can drive subscriber behavior @@ -14,11 +14,8 @@ import { eq } from "drizzle-orm"; import { type } from "arktype"; import type { DB } from "@intx/db"; import { agentInstance } from "@intx/db/schema"; -import { pushInstanceSourceUpdate } from "./credential-push"; import { parseMailToEmail } from "@intx/mime"; import { parseInferenceEvent } from "@intx/types/runtime"; -import type { GrantRule, GrantStore } from "@intx/types/authz"; -import type { InferenceSource } from "@intx/types/runtime"; import { getLogger } from "@intx/log"; import { isWorkflowDerivedAddress } from "@intx/workflow-deploy"; @@ -37,12 +34,6 @@ const log = getLogger(["hub", "orchestrator"]); * narrow surface keeps tests honest and decouples the orchestrator * from the rest of the router API. */ export type HubSessionRouterFacade = { - sendGrantsUpdate(agentAddress: string, grants: GrantRule[]): Promise; - sendSourcesUpdate( - agentAddress: string, - sources: InferenceSource[], - defaultSource: string, - ): Promise; sendPack( agentAddress: string, pack: Uint8Array, @@ -57,7 +48,6 @@ export type HubSessionOrchestratorDeps = { router: HubSessionRouterFacade; db: DB["db"]; eventCollectors: EventCollectorRegistry; - grantStore: GrantStore; agentRepoStore: AgentRepoStore; }; @@ -71,8 +61,7 @@ export type HubSessionOrchestrator = { export function createHubSessionOrchestrator( deps: HubSessionOrchestratorDeps, ): HubSessionOrchestrator { - const { events, router, db, eventCollectors, grantStore, agentRepoStore } = - deps; + const { events, router, db, eventCollectors, agentRepoStore } = deps; const unsubscribers: (() => void)[] = []; @@ -148,27 +137,12 @@ export function createHubSessionOrchestrator( } const sessionId = instance.sessionId; - // Refresh grants before creating any local state. If this - // fails, the address is rejected and nothing needs cleanup. - const grants = await grantStore.collectGrants( - instance.principalId, - instance.tenantId, - ); - await router.sendGrantsUpdate(agentAddress, grants); - - // Re-resolve and push sources so the agent picks up any catalog or - // credential changes that happened while the sidecar was - // disconnected. Fail-open: a stale source causes runtime 401s, not a - // security escalation, so we log rather than reject the reconnect. - try { - await pushInstanceSourceUpdate(db, router, instance); - } catch (err: unknown) { - const msg = err instanceof Error ? err.message : String(err); - log.warn( - "Failed to push sources on reconnect for {agentAddress}: {msg}", - { agentAddress, msg }, - ); - } + // A supervised deployment carries its grants and sources in the + // deploy pack and refreshes them over the supervisor's IPC + // credentials snapshot at spawn and recycle, so reconnect does not + // push them over the wire. The legacy grants.update / sources.update + // frames are no longer handled sidecar-side; pushing them here would + // hang on the sidecar's dropped frame and reject the reconnect. const now = new Date(); if (instance.status !== "running") { From 6298f1836147c27fbb5fcd9fa09b43d727d1ff3b Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Sat, 4 Jul 2026 07:36:47 -0500 Subject: [PATCH 047/101] Remove the unused abort instance route `POST /:instanceId/abort` aborted the agent's current inference or tool execution through the in-process session runtime's `session.abort` frame. That runtime is retired: the sidecar no longer handles the frame, so the route hung on the request timeout and then 502'd for every instance. Nothing calls it -- no UI, client, or test -- so it is dead surface rather than a capability worth re-pointing. Delete the route and its request schema, and regenerate the API doc. If aborting a running operation becomes a supervised feature, it can be added fresh against the workflow-host supervisor's cancel/drain signal. --- docs/API.md | 13 ---- packages/hub-api/src/routes/instances.ts | 92 ------------------------ 2 files changed, 105 deletions(-) diff --git a/docs/API.md b/docs/API.md index cf44e133..d0b692e1 100644 --- a/docs/API.md +++ b/docs/API.md @@ -51,7 +51,6 @@ | GET | /api/tenants/:tenantId/agents/instances/:instanceId/health | Get instance health | | GET | /api/tenants/:tenantId/agents/instances/:instanceId/offerings | List instance offerings | | GET | /api/tenants/:tenantId/agents/instances/:instanceId/events | SSE event stream | -| POST | /api/tenants/:tenantId/agents/instances/:instanceId/abort | Abort current operation | | POST | /api/tenants/:tenantId/agents/instances/:instanceId/mail | Send mail to the agent | | GET | /api/tenants/:tenantId/agents/instances/:instanceId/mail | List mail for an instance | | GET | /api/tenants/:tenantId/agents/instances/:instanceId/turns | List inference turns for an instance | @@ -603,18 +602,6 @@ Server-Sent Events stream for agent events. Use POST .../messages for client-to- 404: ErrorResponse -- Instance not found 410: ErrorResponse -- Instance stopped -### POST /api/tenants/:tenantId/agents/instances/:instanceId/abort -Abort current operation - -Aborts the agent's current inference or tool execution. - -Body: unknown - -204: (no content) -- Abort signal sent -404: ErrorResponse -- Instance not found -409: ErrorResponse -- Instance not running -502: ErrorResponse -- Sidecar unavailable - ### POST /api/tenants/:tenantId/agents/instances/:instanceId/mail Send mail to the agent diff --git a/packages/hub-api/src/routes/instances.ts b/packages/hub-api/src/routes/instances.ts index 1ed505d4..0c32972e 100644 --- a/packages/hub-api/src/routes/instances.ts +++ b/packages/hub-api/src/routes/instances.ts @@ -72,11 +72,6 @@ const GrantRequirements = GrantRequirement.array(); // rejected here before the JSON parser allocates a giant string. const MAX_MAIL_BODY_BYTES = 44 * 1024 * 1024; -const AbortBody = type({ - "reason?": - "'user_disconnect' | 'wallet_exhaustion' | 'admin_kill' | 'session_timeout' | 'credential_revocation'", -}); - function formatInstance( row: typeof agentInstance.$inferSelect, agentName: string, @@ -1232,93 +1227,6 @@ export function createInstanceRoutes({ }, ); - app.post( - "/:instanceId/abort", - requireGrant(idResource("instance", "instanceId"), "manage"), - describeRoute({ - tags: ["Instances"], - summary: "Abort current operation", - description: "Aborts the agent's current inference or tool execution.", - responses: { - 204: { - description: "Abort signal sent", - }, - 404: { - description: "Instance not found", - content: { - "application/json": { schema: resolver(ErrorResponse) }, - }, - }, - 409: { - description: "Instance not running", - content: { - "application/json": { schema: resolver(ErrorResponse) }, - }, - }, - 502: { - description: "Sidecar unavailable", - content: { - "application/json": { schema: resolver(ErrorResponse) }, - }, - }, - }, - }), - validator("json", AbortBody), - async (c) => { - const tenantCtx = c.get("tenant"); - const instanceId = c.req.param("instanceId"); - const body = c.req.valid("json"); - - const row = await db.query.agentInstance.findFirst({ - where: and( - eq(agentInstance.id, instanceId), - eq(agentInstance.tenantId, tenantCtx.id), - ), - }); - - if (!row) { - return c.json( - { error: { code: "not_found", message: "Instance not found" } }, - 404, - ); - } - - if (row.status !== "running") { - return c.json( - { - error: { - code: "conflict", - message: `Instance is not running (status: ${row.status})`, - }, - }, - 409, - ); - } - - try { - await sidecarRouter.sendSessionAbort( - row.address, - body.reason ?? "user_disconnect", - ); - } catch (err) { - return c.json( - { - error: { - code: "sidecar_unavailable", - message: - err instanceof Error - ? err.message - : "Failed to reach sidecar for abort", - }, - }, - 502, - ); - } - - return c.body(null, 204); - }, - ); - // Crypto providers for signing outbound messages, keyed by instance ID. // Evicted when an instance is stopped. The cache is per-factory call, // not per-process; two createInstanceRoutes() calls in the same process From dfdb5337d82d7ced52e9d6198b436830a1a6c3f5 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Sat, 4 Jul 2026 12:42:35 -0500 Subject: [PATCH 048/101] Advance the mirror turn count by the persisted delta The substrate mirror sliced its new-turn delta from the reactor's live array before the WAL-append await, then set the mirrored turn count to that array's length after it. peekTurns returns the array by reference, so a turn the reactor appended during the await was counted as mirrored though it was never written; the next mirror sliced past it and dropped it from the durable WAL for good. Advance the count by the length of the delta actually persisted, which is a pre-await snapshot immune to the concurrent append. Also set the isogit store's last-turns marker only after its durable write succeeds, so peekTurns never surfaces an array that failed to persist. Cover the concurrent append with a test that mutates the live array during the WAL write and asserts the turn lands in the next WAL entry and in the reconstruction. --- apps/sidecar/src/conversation-state.ts | 7 +- packages/storage-isogit/src/store.ts | 5 +- .../conversation-state-wal.test.ts | 88 +++++++++++++++++++ 3 files changed, 98 insertions(+), 2 deletions(-) diff --git a/apps/sidecar/src/conversation-state.ts b/apps/sidecar/src/conversation-state.ts index 1d2a64b1..ce0d6529 100644 --- a/apps/sidecar/src/conversation-state.ts +++ b/apps/sidecar/src/conversation-state.ts @@ -543,7 +543,12 @@ export async function createDurableConversationStore( const boundarySeq = mirroredBoundaryCount; await appendWalEntry(boundarySeq, newTurns, metadata); mirroredBoundaryCount = boundarySeq + 1; - mirroredTurnCount = turns.length; + // Advance by the count actually persisted -- newTurns is a pre-await + // snapshot -- not by turns.length. `turns` is the reactor's live array + // by reference; reading its length after the await would count any turn + // appended during appendWalEntry as mirrored, so the next mirror would + // slice past it and drop it from the WAL permanently. + mirroredTurnCount = mirroredTurnCount + newTurns.length; // Compact once the live WAL reaches the interval (measured in mirror // boundaries = WAL entries, which bounds both the bucket fan-out and the diff --git a/packages/storage-isogit/src/store.ts b/packages/storage-isogit/src/store.ts index 5650e76b..02dfa284 100644 --- a/packages/storage-isogit/src/store.ts +++ b/packages/storage-isogit/src/store.ts @@ -506,11 +506,14 @@ export class IsogitStore turns: ConversationTurn[], _signal?: AbortSignal, ): Promise { - this.lastTurns = turns; await fs.promises.writeFile( path.join(this.dir, TURNS_FILE), encodeJsonlLines(turns), ); + // Advance the in-memory marker only after the durable write succeeds. + // peekTurns must never surface an array that failed to persist -- a + // write failure leaves it pointing at the last array that did. + this.lastTurns = turns; } peekTurns(): ConversationTurn[] { diff --git a/tests/workflow-deploy/conversation-state-wal.test.ts b/tests/workflow-deploy/conversation-state-wal.test.ts index 38372349..dbfdf22f 100644 --- a/tests/workflow-deploy/conversation-state-wal.test.ts +++ b/tests/workflow-deploy/conversation-state-wal.test.ts @@ -467,4 +467,92 @@ describe("durable conversation store WAL + checkpoint (Phase D1)", () => { reconstructDurableConversation(h.agentStateDir, AGENT_KEY), ).rejects.toThrow(/seq gap/); }); + + test("a turn appended during the WAL write is not skipped by the next mirror", async () => { + // Regression guard: mirrorToSubstrate slices its new-turn delta from the + // reactor's live array BEFORE the appendWalEntry await, then advances the + // mirrored turn count AFTER it. peekTurns returns that array by reference, + // so a turn the reactor appends DURING the await must be counted as the + // count actually persisted -- not the post-await live length -- or the + // next mirror slices past it and drops it from the WAL permanently. + const liveTurns: ConversationTurn[] = [userTurn("a")]; + let injected = false; + + // Wrap the substrate so the first WAL append (boundary 0) appends a turn + // to the reactor's live array mid-write, reproducing the concurrent + // append in the between-slice-and-count window. + const writeTreePreservingPrefix: RepoStore["writeTreePreservingPrefix"] = ( + principal, + repoId, + ref, + args, + ) => { + if (!injected && args.preservePrefix.includes("/wal/")) { + injected = true; + liveTurns.push(userTurn("b")); + } + return h.substrate.writeTreePreservingPrefix( + principal, + repoId, + ref, + args, + ); + }; + const wrappedSubstrate = new Proxy(h.substrate, { + get(target, prop, receiver): unknown { + if (prop === "writeTreePreservingPrefix") { + return writeTreePreservingPrefix; + } + return Reflect.get(target, prop, receiver); + }, + }); + + const store = await createDurableConversationStore({ + localStoreDir: localDir, + signer: h.signer, + substrate: wrappedSubstrate, + workflowRunRepoId: h.workflowRunRepoId, + workflowRunRef: WORKFLOW_RUN_REF, + principal: PRINCIPAL, + agentKey: AGENT_KEY, + }); + + // Boundary 0: the local store holds [a]; during its WAL append the + // wrapper appends b to the reactor's live array. + await store.storage.writeTurns(liveTurns); + await store.storage.writeMetadata({ + pendingOperations: [], + tokenUsage: tokenUsage(0), + }); + await store.storage.commit({ message: "turn-a" }); + await store.mirrorToSubstrate(); + expect(injected).toBe(true); + + // Boundary 1: the reactor has since persisted [a, b] locally. The mirror + // must pick up b. The pre-fix code counted b as already mirrored at + // boundary 0 (reading the live array length after the await) and sliced + // past it here, so boundary 1's WAL entry was an empty delta and b was + // lost from the durable log. + await store.storage.writeTurns(liveTurns); + await store.storage.writeMetadata({ + pendingOperations: [], + tokenUsage: tokenUsage(1), + }); + await store.storage.commit({ message: "turn-b" }); + await store.mirrorToSubstrate(); + + const entry = WalEntryShape(readWalEntry(h.agentStateDir, 1)); + if (entry instanceof type.errors) { + throw new Error(`WAL entry 1 failed validation: ${entry.summary}`); + } + expect(entry.turns.length).toBe(1); + + // Reconstruction yields both turns; under the bug it would yield only [a]. + const reconstructed = await reconstructDurableConversation( + h.agentStateDir, + AGENT_KEY, + ); + if (reconstructed === null) throw new Error("expected a reconstruction"); + expect(reconstructed.turns).toEqual([userTurn("a"), userTurn("b")]); + }); }); From 7aa4da6c18c97fc5945a398dec80d7d61cdb67c3 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Sat, 4 Jul 2026 12:55:02 -0500 Subject: [PATCH 049/101] Drop the vestigial deploy-result discriminant The deploy result collapsed to a single-member union when the in-process trivial deploy path was removed, leaving a kind field that discriminates nothing and labels every single-step deploy as multi-step. No production caller reads it: the session service takes only the public key. Reduce the result to just the public key and convert the tests that asserted the constant discriminant into public-key smoke checks, which exercise the actual load-bearing output instead of a literal. --- .../src/orchestrator-fallback-source.test.ts | 4 ++-- .../src/orchestrator-nonagent-source.test.ts | 2 +- packages/workflow-deploy/src/orchestrator.test.ts | 3 +-- packages/workflow-deploy/src/orchestrator.ts | 5 ++--- .../child-workflow-roundtrip.test.ts | 14 +++++++------- .../cross-process-custom-adapter.test.ts | 2 +- tests/workflow-deploy/drain-roundtrip.test.ts | 4 ++-- tests/workflow-deploy/fifo-mail-load.test.ts | 2 +- tests/workflow-deploy/fifo-mail.test.ts | 2 +- .../latency-d2-attribution.bench.ts | 6 ++---- tests/workflow-deploy/latency-gate.bench.ts | 6 ++---- tests/workflow-deploy/mail-edge-cases.test.ts | 4 +--- tests/workflow-deploy/multistep-signal.test.ts | 2 +- .../workflow-deploy/multistep-signed-send.test.ts | 2 +- .../single-step-full-lifecycle.test.ts | 2 +- .../single-step-grants-bridge.test.ts | 2 +- .../single-step-message-input.test.ts | 2 +- .../workflow-deploy/single-step-posix-tool.test.ts | 2 +- .../workflow-deploy/single-step-real-agent.test.ts | 2 +- 19 files changed, 30 insertions(+), 38 deletions(-) diff --git a/packages/workflow-deploy/src/orchestrator-fallback-source.test.ts b/packages/workflow-deploy/src/orchestrator-fallback-source.test.ts index 257119cf..762b19c1 100644 --- a/packages/workflow-deploy/src/orchestrator-fallback-source.test.ts +++ b/packages/workflow-deploy/src/orchestrator-fallback-source.test.ts @@ -236,7 +236,7 @@ describe("pickStepInferenceSource (agent step)", () => { operatorApprovals: approvals, }); - expect(result.kind).toBe("multi-step"); + expect(result.publicKey).toBeTruthy(); expect(deps.sources).toHaveLength(1); const sources = deps.sources[0]; if (sources === undefined) throw new Error("missing sources"); @@ -304,7 +304,7 @@ describe("pickStepInferenceSource (agent step)", () => { operatorApprovals: approvals, }); - expect(result.kind).toBe("multi-step"); + expect(result.publicKey).toBeTruthy(); expect(deps.sources).toHaveLength(1); const sources = deps.sources[0]; if (sources === undefined) throw new Error("missing sources"); diff --git a/packages/workflow-deploy/src/orchestrator-nonagent-source.test.ts b/packages/workflow-deploy/src/orchestrator-nonagent-source.test.ts index 012c29a1..044b3094 100644 --- a/packages/workflow-deploy/src/orchestrator-nonagent-source.test.ts +++ b/packages/workflow-deploy/src/orchestrator-nonagent-source.test.ts @@ -308,7 +308,7 @@ describe("pickStepInferenceSource (non-agent step)", () => { operatorApprovals: approvals, }); - expect(result.kind).toBe("multi-step"); + expect(result.publicKey).toBeTruthy(); expect(deps.sources).toHaveLength(1); const sources = deps.sources[0]; if (sources === undefined) throw new Error("missing sources"); diff --git a/packages/workflow-deploy/src/orchestrator.test.ts b/packages/workflow-deploy/src/orchestrator.test.ts index 3b702fed..ca2d0c50 100644 --- a/packages/workflow-deploy/src/orchestrator.test.ts +++ b/packages/workflow-deploy/src/orchestrator.test.ts @@ -379,7 +379,6 @@ describe("createWorkflowDeployOrchestrator", () => { expect(handoff.sources.plan).toEqual([baseSource]); expect(handoff.sources.execute).toEqual([baseSource]); expect(result).toEqual({ - kind: "multi-step", publicKey: "ff".repeat(32), }); }); @@ -603,7 +602,7 @@ describe("createWorkflowDeployOrchestrator", () => { throw new Error("missing step id"); } expect(call.sources[expectedStepId]).toBeDefined(); - expect(result.kind).toBe("multi-step"); + expect(result.publicKey).toBeTruthy(); }); test("source-pin failure carries workflow.id and names the offending provider+model", async () => { diff --git a/packages/workflow-deploy/src/orchestrator.ts b/packages/workflow-deploy/src/orchestrator.ts index 5d1b3626..8aa35a36 100644 --- a/packages/workflow-deploy/src/orchestrator.ts +++ b/packages/workflow-deploy/src/orchestrator.ts @@ -153,7 +153,6 @@ export interface MultiStepDeployResult { * it alongside the deployment record. */ export type DeployWorkflowResult = { - readonly kind: "multi-step"; readonly publicKey: string; }; @@ -401,7 +400,7 @@ export function createWorkflowDeployOrchestrator( args, deploySingleStepAtHead, }); - return { kind: "multi-step", publicKey: result.publicKey }; + return { publicKey: result.publicKey }; } const result = await runMultiStepBranch({ @@ -409,7 +408,7 @@ export function createWorkflowDeployOrchestrator( launchSession, sendMultiStepDeploy, }); - return { kind: "multi-step", publicKey: result.publicKey }; + return { publicKey: result.publicKey }; }, }; } diff --git a/tests/workflow-deploy/child-workflow-roundtrip.test.ts b/tests/workflow-deploy/child-workflow-roundtrip.test.ts index 6dec40da..d11f3c28 100644 --- a/tests/workflow-deploy/child-workflow-roundtrip.test.ts +++ b/tests/workflow-deploy/child-workflow-roundtrip.test.ts @@ -266,7 +266,7 @@ describe("parent -> child workflow round-trip", () => { deploymentDomain: DEPLOYMENT_DOMAIN, hubPublicKey: "00".repeat(32), }); - expect(childResult.kind).toBe("multi-step"); + expect(childResult.publicKey).toBeTruthy(); const parentResult = await orchestrator.deployWorkflow({ workflow: parentWorkflowDefinition, @@ -283,7 +283,7 @@ describe("parent -> child workflow round-trip", () => { deploymentDomain: DEPLOYMENT_DOMAIN, hubPublicKey: "00".repeat(32), }); - expect(parentResult.kind).toBe("multi-step"); + expect(parentResult.publicKey).toBeTruthy(); const parentWorkflowRunRepoId: RepoId = { kind: "workflow-run", @@ -649,7 +649,7 @@ describe("parent -> child workflow round-trip", () => { deploymentDomain: DEPLOYMENT_DOMAIN, hubPublicKey: "00".repeat(32), }); - expect(grandchildResult.kind).toBe("multi-step"); + expect(grandchildResult.publicKey).toBeTruthy(); const childResult = await orchestrator.deployWorkflow({ workflow: childWorkflowDefinition, @@ -666,7 +666,7 @@ describe("parent -> child workflow round-trip", () => { deploymentDomain: DEPLOYMENT_DOMAIN, hubPublicKey: "00".repeat(32), }); - expect(childResult.kind).toBe("multi-step"); + expect(childResult.publicKey).toBeTruthy(); const parentResult = await orchestrator.deployWorkflow({ workflow: parentWorkflowDefinition, @@ -683,7 +683,7 @@ describe("parent -> child workflow round-trip", () => { deploymentDomain: DEPLOYMENT_DOMAIN, hubPublicKey: "00".repeat(32), }); - expect(parentResult.kind).toBe("multi-step"); + expect(parentResult.publicKey).toBeTruthy(); const parentWorkflowRunRepoId: RepoId = { kind: "workflow-run", @@ -1011,7 +1011,7 @@ describe("parent -> child workflow round-trip", () => { deploymentDomain: DEPLOYMENT_DOMAIN, hubPublicKey: "00".repeat(32), }); - expect(r.kind).toBe("multi-step"); + expect(r.publicKey).toBeTruthy(); } const parentResult = await orchestrator.deployWorkflow({ @@ -1029,7 +1029,7 @@ describe("parent -> child workflow round-trip", () => { deploymentDomain: DEPLOYMENT_DOMAIN, hubPublicKey: "00".repeat(32), }); - expect(parentResult.kind).toBe("multi-step"); + expect(parentResult.publicKey).toBeTruthy(); const parentWorkflowRunRepoId: RepoId = { kind: "workflow-run", diff --git a/tests/workflow-deploy/cross-process-custom-adapter.test.ts b/tests/workflow-deploy/cross-process-custom-adapter.test.ts index 87a31ed9..b8cd1e7d 100644 --- a/tests/workflow-deploy/cross-process-custom-adapter.test.ts +++ b/tests/workflow-deploy/cross-process-custom-adapter.test.ts @@ -224,7 +224,7 @@ async function deployCustomProviderWorkflow( deploymentDomain: DEPLOYMENT_DOMAIN, hubPublicKey: "00".repeat(32), }); - expect(result.kind).toBe("multi-step"); + expect(result.publicKey).toBeTruthy(); const workflowRunRepoId: RepoId = { kind: "workflow-run", diff --git a/tests/workflow-deploy/drain-roundtrip.test.ts b/tests/workflow-deploy/drain-roundtrip.test.ts index 63022bb4..935d8482 100644 --- a/tests/workflow-deploy/drain-roundtrip.test.ts +++ b/tests/workflow-deploy/drain-roundtrip.test.ts @@ -239,7 +239,7 @@ describe("drain round-trip", () => { { cause }, ); } - expect(result.kind).toBe("multi-step"); + expect(result.publicKey).toBeTruthy(); const workflowRunRepoId: RepoId = { kind: "workflow-run", @@ -469,7 +469,7 @@ describe("drain round-trip", () => { deploymentDomain: DEPLOYMENT_DOMAIN, hubPublicKey: "00".repeat(32), }); - expect(result.kind).toBe("multi-step"); + expect(result.publicKey).toBeTruthy(); const workflowRunRepoId: RepoId = { kind: "workflow-run", diff --git a/tests/workflow-deploy/fifo-mail-load.test.ts b/tests/workflow-deploy/fifo-mail-load.test.ts index 5c33a337..14a87605 100644 --- a/tests/workflow-deploy/fifo-mail-load.test.ts +++ b/tests/workflow-deploy/fifo-mail-load.test.ts @@ -272,7 +272,7 @@ describe("FIFO mail-trigger serialization under load", () => { { cause }, ); } - expect(result.kind).toBe("multi-step"); + expect(result.publicKey).toBeTruthy(); const workflowRunRepoId: RepoId = { kind: "workflow-run", diff --git a/tests/workflow-deploy/fifo-mail.test.ts b/tests/workflow-deploy/fifo-mail.test.ts index 4846a116..44310714 100644 --- a/tests/workflow-deploy/fifo-mail.test.ts +++ b/tests/workflow-deploy/fifo-mail.test.ts @@ -255,7 +255,7 @@ describe("FIFO mail-trigger serialization", () => { { cause }, ); } - expect(result.kind).toBe("multi-step"); + expect(result.publicKey).toBeTruthy(); const workflowRunRepoId: RepoId = { kind: "workflow-run", diff --git a/tests/workflow-deploy/latency-d2-attribution.bench.ts b/tests/workflow-deploy/latency-d2-attribution.bench.ts index fdf38d1d..4e8c4dae 100644 --- a/tests/workflow-deploy/latency-d2-attribution.bench.ts +++ b/tests/workflow-deploy/latency-d2-attribution.bench.ts @@ -462,10 +462,8 @@ async function runUnifiedD2(opts: { deploymentDomain: DEPLOYMENT_DOMAIN, hubPublicKey: "00".repeat(32), }); - if (result.kind !== "multi-step") { - throw new Error( - `d2 bench: expected multi-step deploy, got ${result.kind}`, - ); + if (!result.publicKey) { + throw new Error("d2 bench: deploy did not return a public key"); } const workflowRunRepoId: RepoId = { diff --git a/tests/workflow-deploy/latency-gate.bench.ts b/tests/workflow-deploy/latency-gate.bench.ts index a5d24af7..00d22a9a 100644 --- a/tests/workflow-deploy/latency-gate.bench.ts +++ b/tests/workflow-deploy/latency-gate.bench.ts @@ -425,10 +425,8 @@ async function runUnified(opts: { deploymentDomain: DEPLOYMENT_DOMAIN, hubPublicKey: "00".repeat(32), }); - if (result.kind !== "multi-step") { - throw new Error( - `unified bench: expected multi-step deploy, got ${result.kind}`, - ); + if (!result.publicKey) { + throw new Error("unified bench: deploy did not return a public key"); } const workflowRunRepoId: RepoId = { diff --git a/tests/workflow-deploy/mail-edge-cases.test.ts b/tests/workflow-deploy/mail-edge-cases.test.ts index d68315c5..677510d6 100644 --- a/tests/workflow-deploy/mail-edge-cases.test.ts +++ b/tests/workflow-deploy/mail-edge-cases.test.ts @@ -471,9 +471,7 @@ async function deployEdgeWorkflow( deploymentDomain: DEPLOYMENT_DOMAIN, hubPublicKey: "00".repeat(32), }); - if (result.kind !== "multi-step") { - throw new Error(`expected multi-step deploy; got ${result.kind}`); - } + expect(result.publicKey).toBeTruthy(); const workflowRunRepoId: RepoId = { kind: "workflow-run", diff --git a/tests/workflow-deploy/multistep-signal.test.ts b/tests/workflow-deploy/multistep-signal.test.ts index a25c82aa..2f46d322 100644 --- a/tests/workflow-deploy/multistep-signal.test.ts +++ b/tests/workflow-deploy/multistep-signal.test.ts @@ -241,7 +241,7 @@ describe("multi-step workflow round-trip with signal-await", () => { { cause }, ); } - expect(result.kind).toBe("multi-step"); + expect(result.publicKey).toBeTruthy(); // The sidecar's deploy router slugs the deployment mail address // into the workflow-run repo id via `deriveDeploymentId`; diff --git a/tests/workflow-deploy/multistep-signed-send.test.ts b/tests/workflow-deploy/multistep-signed-send.test.ts index 44ba5a13..0804c52f 100644 --- a/tests/workflow-deploy/multistep-signed-send.test.ts +++ b/tests/workflow-deploy/multistep-signed-send.test.ts @@ -244,7 +244,7 @@ describe("multi-step signed outbound send", () => { { cause }, ); } - expect(result.kind).toBe("multi-step"); + expect(result.publicKey).toBeTruthy(); const workflowRunRepoId: RepoId = { kind: "workflow-run", diff --git a/tests/workflow-deploy/single-step-full-lifecycle.test.ts b/tests/workflow-deploy/single-step-full-lifecycle.test.ts index a3716821..94f30254 100644 --- a/tests/workflow-deploy/single-step-full-lifecycle.test.ts +++ b/tests/workflow-deploy/single-step-full-lifecycle.test.ts @@ -320,7 +320,7 @@ describe("single-step full lifecycle on the unified child path (Phase 4.6)", () { cause }, ); } - expect(result.kind).toBe("multi-step"); + expect(result.publicKey).toBeTruthy(); // 4.1 identity preserved: the deploy-ack fired for the legacy // `ins_` address and persisted a public key. diff --git a/tests/workflow-deploy/single-step-grants-bridge.test.ts b/tests/workflow-deploy/single-step-grants-bridge.test.ts index af715457..21d9978a 100644 --- a/tests/workflow-deploy/single-step-grants-bridge.test.ts +++ b/tests/workflow-deploy/single-step-grants-bridge.test.ts @@ -277,7 +277,7 @@ describe("single-step launched-agent grants bridge via spawned child", () => { { cause }, ); } - expect(result.kind).toBe("multi-step"); + expect(result.publicKey).toBeTruthy(); // (a) identity: the deploy-ack fired for the legacy `ins_` // address and persisted a public key. The ack listener keys on the diff --git a/tests/workflow-deploy/single-step-message-input.test.ts b/tests/workflow-deploy/single-step-message-input.test.ts index 9993cd19..957ff826 100644 --- a/tests/workflow-deploy/single-step-message-input.test.ts +++ b/tests/workflow-deploy/single-step-message-input.test.ts @@ -203,7 +203,7 @@ describe("single-step message-input round-trip", () => { { cause }, ); } - expect(result.kind).toBe("multi-step"); + expect(result.publicKey).toBeTruthy(); const workflowRunRepoId: RepoId = { kind: "workflow-run", diff --git a/tests/workflow-deploy/single-step-posix-tool.test.ts b/tests/workflow-deploy/single-step-posix-tool.test.ts index f3754392..d2eea3c9 100644 --- a/tests/workflow-deploy/single-step-posix-tool.test.ts +++ b/tests/workflow-deploy/single-step-posix-tool.test.ts @@ -235,7 +235,7 @@ describe("single-step posix-tool in-child execution", () => { { cause }, ); } - expect(result.kind).toBe("multi-step"); + expect(result.publicKey).toBeTruthy(); const workflowRunRepoId: RepoId = { kind: "workflow-run", diff --git a/tests/workflow-deploy/single-step-real-agent.test.ts b/tests/workflow-deploy/single-step-real-agent.test.ts index 9124d325..db0f9075 100644 --- a/tests/workflow-deploy/single-step-real-agent.test.ts +++ b/tests/workflow-deploy/single-step-real-agent.test.ts @@ -211,7 +211,7 @@ describe("single-step real-agent round-trip", () => { { cause }, ); } - expect(result.kind).toBe("multi-step"); + expect(result.publicKey).toBeTruthy(); const workflowRunRepoId: RepoId = { kind: "workflow-run", From 1e2d591a774d95065acad1ca79302a6b4cd9d999 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Sat, 4 Jul 2026 12:59:47 -0500 Subject: [PATCH 050/101] Register the deployment address after resolving the ack key The spawn core registered the deployment-address mapping and only then derived the ack public key, so a derivation failure -- unreachable but deterministic -- would have leaked the boot-edge registry entry, which the failure unwind never reversed. The comment nonetheless claimed registration was the last step. Resolve the ack key first and register the address after it, with nothing fallible in between, so registration genuinely is last and the unwind has no registry entry to reverse. --- apps/sidecar/src/workflow-host-wiring.ts | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/apps/sidecar/src/workflow-host-wiring.ts b/apps/sidecar/src/workflow-host-wiring.ts index 890eccf5..ebe30007 100644 --- a/apps/sidecar/src/workflow-host-wiring.ts +++ b/apps/sidecar/src/workflow-host-wiring.ts @@ -1262,24 +1262,25 @@ export function createSidecarDeployRouter(deps: { }); routersRegistered = true; + // Resolve the ack public key BEFORE registering the deployment + // address so an (unreachable, deterministic) derivation failure + // unwinds the spawn without having touched the boot-edge + // `DeploymentAddressRegistry`. A single-step head acks its agent key + // (captured above); a multi-step deployment acks the supervisor + // principal key its workflow-run events are signed with. + const publicKey = + headAgentPublicKey ?? + (await derivePrincipalPublicKeyHex(deps.signingKeySeed)); + // Register the deployment-address mapping last so a failure in any // earlier step leaves the boot-edge `DeploymentAddressRegistry` - // untouched. + // untouched. Nothing fallible runs after it, so the finally unwind + // has no registry entry to reverse. deps.registerDeployment({ deploymentId, agentAddress: spec.agentAddress, }); - // Resolve the ack public key BEFORE marking the spawn succeeded so an - // (unreachable, deterministic) derivation failure unwinds the spawn - // rather than leaving a live-but-unacked deployment whose slug the - // caller then frees. Once `succeeded` is set the finally is a no-op - // and the deployment is retained. A single-step head acks its agent - // key (captured above); a multi-step deployment acks the supervisor - // principal key its workflow-run events are signed with. - const publicKey = - headAgentPublicKey ?? - (await derivePrincipalPublicKeyHex(deps.signingKeySeed)); succeeded = true; return { publicKey }; } finally { From 376bd5e50f957f38d7dae941c156181538dac471 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Sat, 4 Jul 2026 13:08:59 -0500 Subject: [PATCH 051/101] Cover the unwired single-step deploy handoff error The orchestrator throws SingleStepDeployHandoffMissingError when a single-step workflow is deployed without the deploySingleStepAtHead dependency wired, the single-step parallel to the already-covered multi-step handoff error. Only the multi-step case had a test. Add the matching single-step case so both fail-fast paths are exercised. --- .../workflow-deploy/src/orchestrator.test.ts | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/packages/workflow-deploy/src/orchestrator.test.ts b/packages/workflow-deploy/src/orchestrator.test.ts index ca2d0c50..44d5475b 100644 --- a/packages/workflow-deploy/src/orchestrator.test.ts +++ b/packages/workflow-deploy/src/orchestrator.test.ts @@ -28,6 +28,7 @@ import { isWorkflowDerivedAddress, MultiStepDeployHandoffMissingError, MultiStepDeploymentArgsMissingError, + SingleStepDeployHandoffMissingError, WorkflowDefinitionInvalidError, wrapHarnessAsSingleStepWorkflow, type DeployContent, @@ -497,6 +498,32 @@ describe("createWorkflowDeployOrchestrator", () => { ).rejects.toBeInstanceOf(MultiStepDeployHandoffMissingError); }); + test("throws when deploySingleStepAtHead dep is unwired", async () => { + const agent = makeAgent("only"); + const workflow = makeSingleStepWorkflow(agent); + const directorRegistry = createDefaultDirectorRegistry(); + const workflowRepo = createRecordingWorkflowRepoWriter(); + const launch = createRecordingLaunch(); + const orchestrator = createWorkflowDeployOrchestrator({ + directorRegistry, + workflowRepo, + launchSession: launch.fn, + }); + const approvals = approvedGrantsForWorkflow(workflow, [agent]); + + await expect( + orchestrator.deployWorkflow({ + workflow, + deploymentId: "dep_xyz", + deploymentDomain: "workflow.interchange", + config: HARNESS_CONFIG_BASE, + deployContent: DEPLOY_CONTENT_BASE, + hubPublicKey: "00".repeat(32), + operatorApprovals: approvals, + }), + ).rejects.toBeInstanceOf(SingleStepDeployHandoffMissingError); + }); + test("throws when deploymentId is missing", async () => { const workflow = makeMultiStepWorkflow(); const planAgent = workflow.steps.plan; From df1d97359f3bec19471636779b8169484b7fba0e Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Sat, 4 Jul 2026 13:16:11 -0500 Subject: [PATCH 052/101] Refresh comments the renames and deletions left stale Three comments drifted when the in-process trivial deploy path and its session runtime were removed and the index-free tree assembly landed. The session-service asset doc named the removed launchSession method; the sidecar mail-registry doc named the deleted commitInboundMail fallback and the gone trivial deploy path; and the repo-store cache comment claimed every git call threads the dir cache, which the index-free writeTree and writeBlob calls do not. Update each to describe the code as it stands. --- apps/sidecar/src/workflow-run-pack-client.ts | 8 +++----- packages/hub-sessions/src/repo-store/store.ts | 15 +++++++++------ packages/hub-sessions/src/session-service.ts | 6 +++--- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/apps/sidecar/src/workflow-run-pack-client.ts b/apps/sidecar/src/workflow-run-pack-client.ts index 51bc37ec..ba109065 100644 --- a/apps/sidecar/src/workflow-run-pack-client.ts +++ b/apps/sidecar/src/workflow-run-pack-client.ts @@ -118,11 +118,9 @@ export type MultistepMailHandler = (message: Uint8Array) => void; /** * Per-deployment-address mail handler registry the sidecar hub-link - * consults before falling back to `transport.deliver` / - * `sessions.commitInboundMail`. The trivial deploy path never registers - * a handler -- its mail flows through the legacy session path. The - * multi-step deploy router registers a handler against the deployment's - * mail address after `wired.supervisor.spawn` succeeds, so an inbound + * consults before falling back to `transport.deliver`. The multi-step + * deploy router registers a handler against the deployment's mail + * address after `wired.supervisor.spawn` succeeds, so an inbound * `mail.inbound` frame for that address dispatches into the * supervisor's mail-bus subscription rather than the * never-provisioned-for-this-address transport mailbox. diff --git a/packages/hub-sessions/src/repo-store/store.ts b/packages/hub-sessions/src/repo-store/store.ts index 905a7584..d669b346 100644 --- a/packages/hub-sessions/src/repo-store/store.ts +++ b/packages/hub-sessions/src/repo-store/store.ts @@ -127,12 +127,15 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { const { dataDir, signingKey, handlers, authorize, signingCallback, gc } = config; - // Per-repo-directory isomorphic-git memoization cache. Every git.* - // call the store makes against a `dir` threads that dir's cache, so - // isomorphic-git reuses the parsed on-disk index across the repo's - // serialized write sequence instead of re-reading and re-parsing it - // on every git.add / remove / updateIndex / commit / listFiles, and - // reuses parsed packfile indexes across object reads. The store is + // Per-repo-directory isomorphic-git memoization cache. The store + // threads that dir's cache through every index-touching and + // object-reading git.* call, so isomorphic-git reuses the parsed + // on-disk index across the repo's serialized write sequence instead + // of re-reading and re-parsing it on every git.add / remove / + // updateIndex / commit / listFiles, and reuses parsed packfile + // indexes across object reads. The index-free tree assembly + // (git.writeTree / git.writeBlob) uses no index and threads no cache. + // The store is // the single writer under withRepoLock, so a long-lived per-repo // cache never races a concurrent mutator. // diff --git a/packages/hub-sessions/src/session-service.ts b/packages/hub-sessions/src/session-service.ts index c9631e3e..60bc76f9 100644 --- a/packages/hub-sessions/src/session-service.ts +++ b/packages/hub-sessions/src/session-service.ts @@ -240,11 +240,11 @@ export type SessionServiceDeps = { sidecarRouter: SidecarRouter; agentRepoStore: AgentRepoStore; /** - * Optional asset attachment integration. When set, `launchSession` + * Optional asset attachment integration. When set, the deploy flow * fans out per-attachment packs after the deploy pack lands and * inserts a `session_asset` row per attachment. When unset, only - * the deploy pack is sent — the legacy single-pack path is - * preserved bit-for-bit. + * the deploy pack is sent — the single-pack path is preserved + * bit-for-bit. */ assetService?: AssetService; /** DB handle used for `session_asset` manifest inserts. Required From f928d20a81b65f5da939f02d76f7ab4079bebc72 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Sat, 4 Jul 2026 17:54:16 -0500 Subject: [PATCH 053/101] Reject a same-address deploy that is still in flight The deploy path records a supervisor in activeSupervisors only after spawn succeeds, so two same-address deploy frames could both pass the liveness check while the first was mid-spawn. The loser then reached the spawn-core guard, threw, and its unwind deleted the winner's live deployment record and released its slug, stranding a running deployment with no persisted record. Reserve the address synchronously before the first durable write so a concurrent frame is rejected before it can touch that state. --- apps/sidecar/src/workflow-host-wiring.test.ts | 114 ++++++++++++++++++ apps/sidecar/src/workflow-host-wiring.ts | 53 ++++++-- 2 files changed, 155 insertions(+), 12 deletions(-) diff --git a/apps/sidecar/src/workflow-host-wiring.test.ts b/apps/sidecar/src/workflow-host-wiring.test.ts index 3bacaf28..cbcef2d6 100644 --- a/apps/sidecar/src/workflow-host-wiring.test.ts +++ b/apps/sidecar/src/workflow-host-wiring.test.ts @@ -742,6 +742,120 @@ describe("createSidecarDeployRouter multi-step branch", () => { void supervisorIpcKeyPair; }); + test("a second same-address deploy is rejected mid-spawn and never deletes the live deployment record", async () => { + // Pins the synchronous single-flight reservation guard. The first + // deploy runs its durable writes and then suspends inside + // supervisor.spawn awaiting the child's `ready` handshake -- the window + // in which its reservation is held but `activeSupervisors` is not yet + // populated. A second same-address frame arriving in that window must be + // rejected at the reservation guard (its own message, distinct from the + // spawn-core backstop) before it touches durable state, so it cannot + // delete the first deploy's live record via the soft-fail catch. + const childIpcKeyPair = await generateKeyPair(); + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const eventChildToSupervisor = createMemoryFrameStream(); + let resolveExit: ((code: number) => void) | undefined; + const exited = new Promise((resolve) => { + resolveExit = resolve; + }); + let spawnCount = 0; + let observedEnv: Record | undefined; + const spawner: SubprocessSpawner = ({ env }) => { + spawnCount += 1; + observedEnv = env; + const handle: SubprocessHandle = { + pid: 4242, + controlWriter: supervisorToChild.writer, + controlReader: childToSupervisor.reader, + eventReader: eventChildToSupervisor.reader, + kill: () => { + childToSupervisor.close(); + eventChildToSupervisor.close(); + resolveExit?.(0); + }, + exited, + }; + return handle; + }; + + const multiDataDir = await createTempBaseDir("sidecar-concurrent-deploy-"); + const registered: string[] = []; + const { router } = await buildMultistepFixture({ + spawner, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: multiDataDir }, + registerDeployment: ({ agentAddress }) => { + registered.push(agentAddress); + }, + }); + + const sources = defaultMultistepSources(); + const definition = { + id: "wf-concurrent", + triggers: [{ type: "manual" }], + stepOrder: ["step-1", "step-2"], + steps: { "step-1": { kind: "step" }, "step-2": { kind: "step" } }, + }; + const frame = makeMultistepFrame({ definition, sources }); + const deploymentId = deriveDeploymentId(frame.agentAddress); + const recordFile = path.join( + multiDataDir, + "workflow-runs", + deploymentId, + "deployment.json", + ); + + const firstDeploy = router.deploy(frame); + // Wait until the first deploy has spawned: its record is on disk and it + // is now suspended in the ready handshake with the reservation held. + while (observedEnv === undefined) { + await new Promise((r) => setTimeout(r, 1)); + } + const recordBefore = await fs.readFile(recordFile, "utf8"); + expect(recordBefore.length).toBeGreaterThan(0); + + // The loser is rejected at the reservation guard, not the spawn-core + // backstop -- the guard's message is the one asserted here. + await expect(router.deploy(frame)).rejects.toThrow( + /is already deployed; undeploy it before redeploying/, + ); + // It never reached the spawner and never deleted the live record. + expect(spawnCount).toBe(1); + const recordAfter = await fs.readFile(recordFile, "utf8"); + expect(recordAfter).toBe(recordBefore); + + // Drive the first deploy's ready handshake so it completes, then confirm + // the winner is the live, registered deployment. + const channelId = observedEnv.IPC_CHANNEL_ID; + if (channelId === undefined) { + throw new Error("IPC_CHANNEL_ID not set in spawn-time env"); + } + const childSender = createControlChannelSender({ + privateKeySeed: childIpcKeyPair.privateKey, + channelId, + writer: { + write(line: string) { + childToSupervisor.inject(line); + return Promise.resolve(); + }, + }, + }); + await childSender.send({ + type: "ready", + data: { + childPid: 4242, + childPublicKey: Buffer.from(childIpcKeyPair.publicKey).toString("hex"), + }, + }); + + const result = await firstDeploy; + expect(result.publicKey).toMatch(/^[0-9a-f]{64}$/); + expect(registered).toEqual([frame.agentAddress]); + expect(router.activeAddresses()).toEqual([frame.agentAddress]); + + void supervisorToChild; + }); + test("registers a multistepMailRouter handler against the deployment address once spawn succeeds", async () => { // Drives the spawn handshake the same way the first multi-step // test does, but injects a `multistepMailRouter` and asserts the diff --git a/apps/sidecar/src/workflow-host-wiring.ts b/apps/sidecar/src/workflow-host-wiring.ts index ebe30007..db9c0eb1 100644 --- a/apps/sidecar/src/workflow-host-wiring.ts +++ b/apps/sidecar/src/workflow-host-wiring.ts @@ -907,6 +907,19 @@ export function createSidecarDeployRouter(deps: { // with the deployment. const activeSupervisors = new Map(); + // Synchronous single-flight guard for the deploy path. The real supervisor + // does not exist until inside `spawnWorkflowDeployment`, so `deployMultiStep` + // cannot reserve its `activeSupervisors` slot up front; instead it records + // the address here synchronously, before its first await, and clears it in a + // finally once the deploy settles. `activeSupervisors` is populated only + // after `spawn` succeeds, so the has-check alone leaves a window in which two + // same-address frames both pass and the loser's unwind deletes the winner's + // live deployment record. This set closes that window: a second frame that + // arrives while the first is mid-deploy is rejected before it touches any + // durable state. Only the live deploy path reserves; the boot restore path + // is serial and relies on the `activeSupervisors` backstop instead. + const reservingDeployAddresses = new Set(); + // Slug-collision tracking. `deriveDeploymentId` substitutes // disallowed characters with `-`, which is deterministic but lossy: // two distinct agent addresses can collapse to the same slug, and @@ -1364,19 +1377,23 @@ export function createSidecarDeployRouter(deps: { } } - // Reject a re-deploy of an address already live in this process BEFORE - // touching any durable state. The durable writes below (the restore - // record, workflow.json, step grants) are destructive overwrites of state - // owned by whatever deployment currently holds the address; overwriting is - // only legal when this deploy owns the address. `spawnWorkflowDeployment` - // carries the same guard as its single-owner backstop, but it throws only - // after this method has already overwritten -- so the guard must also sit - // ahead of the writes, or a duplicate frame would clobber a live - // deployment's record (and the catch below would then delete it). A + // Reject a re-deploy of an address already live OR mid-deploy in this + // process BEFORE touching any durable state. The durable writes below (the + // restore record, workflow.json, step grants) are destructive overwrites of + // state owned by whatever deployment currently holds the address; + // overwriting is only legal when this deploy owns the address. + // `activeSupervisors` catches an address whose deploy has completed; + // `reservingDeployAddresses` catches one whose deploy is still in flight. + // The map is populated only after `spawn` succeeds, so the has-check alone + // leaves a window in which two frames both pass and the loser's catch below + // deletes the winner's live record; the reservation set closes it. A // re-deploy after `undeploy` passes: `undeploy` drops the - // `activeSupervisors` entry; a failed deploy that unwound is not in the - // map either. - if (activeSupervisors.has(frame.agentAddress)) { + // `activeSupervisors` entry, and a failed or completed deploy has already + // cleared its reservation. + if ( + activeSupervisors.has(frame.agentAddress) || + reservingDeployAddresses.has(frame.agentAddress) + ) { throw new Error( `sidecar deploy router: ${frame.agentAddress} is already deployed; undeploy it before redeploying`, ); @@ -1447,6 +1464,13 @@ export function createSidecarDeployRouter(deps: { }; claimSlug(deploymentId, frame.agentAddress); + // Hold the single-flight reservation across the async body below and clear + // it in the finally. Everything above is synchronous and throws before any + // durable write, so the reservation is only needed from the first await + // here onward; the top-of-method guard already consults this set for a + // concurrent frame, and claimSlug/deploymentId derivation above cannot + // yield control before this point. + reservingDeployAddresses.add(frame.agentAddress); try { // Persist the deployment record BEFORE the spawn so a crash mid-spawn // leaves a record the boot scan re-drives (an idempotent re-spawn; the @@ -1483,6 +1507,11 @@ export function createSidecarDeployRouter(deps: { await deleteWorkflowDeploymentRecord(dataDir, deploymentId); releaseSlug(deploymentId, frame.agentAddress); throw cause; + } finally { + // Release the single-flight reservation whether the deploy succeeded or + // threw. On success the address is now in `activeSupervisors`, which the + // guard also consults, so a later re-deploy is still rejected. + reservingDeployAddresses.delete(frame.agentAddress); } } From b768b3d7bfc64c9006ed3bf67dec0273dcb3c3d8 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Sat, 4 Jul 2026 18:14:22 -0500 Subject: [PATCH 054/101] Reverse the recorded hub key when a single-step spawn fails A failed single-step spawn left the head's recorded hub key in the key store's in-memory cache, so a failed deploy leaked residue the success path would have owned. Clear it in the unwind. The head's on-disk repo is deliberately left intact: it is idempotent and re-pushed on every redeploy, and its directory also holds the durable identity keypair a rerouted head must keep across a failed redeploy. --- ...orkflow-host-wiring-deploy-failure.test.ts | 146 ++++++++++++++++++ apps/sidecar/src/workflow-host-wiring.test.ts | 4 + apps/sidecar/src/workflow-host-wiring.ts | 17 ++ 3 files changed, 167 insertions(+) diff --git a/apps/sidecar/src/workflow-host-wiring-deploy-failure.test.ts b/apps/sidecar/src/workflow-host-wiring-deploy-failure.test.ts index 77924950..b2fd9220 100644 --- a/apps/sidecar/src/workflow-host-wiring-deploy-failure.test.ts +++ b/apps/sidecar/src/workflow-host-wiring-deploy-failure.test.ts @@ -249,4 +249,150 @@ describe("deploy-failure registry leak", () => { const slug = "ins_mstep-x-example"; expect(registry.resolve(slug)).toBeNull(); }); + + test("single-step spawn failure reverses the recorded hub key but preserves the head repo", async () => { + const { registry, mailRouter, signalRouter, drainRouter, transport } = + makeRouterDeps(); + + const initedRepos: string[] = []; + const removedRepos: string[] = []; + const recordedHubKeys: string[] = []; + const forgottenAgents: string[] = []; + + // A sessions stub whose `initRepo` succeeds (so the single-step head + // proceeds to record the hub key and then fail at spawn) and whose + // `deleteAgentDir` is spied: the unwind must NOT call it, because the + // head repo directory also holds the durable identity keypair. + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub: only initRepo runs on the happy path; deleteAgentDir is spied to prove it is NOT called + const sessions = { + initRepo: async (address: string) => { + initedRepos.push(address); + }, + deleteAgentDir: async (address: string) => { + removedRepos.push(address); + }, + } as unknown as Parameters[0]["sessions"]; + + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub + const keyStore = { + async loadOrGenerateKey() { + return { keyPair: await generateKeyPair(), isNew: false }; + }, + signChallenge() { + return null; + }, + recordHubKey(address: string) { + recordedHubKeys.push(address); + }, + verifyDeployCommit() { + return true; + }, + forgetAgent(address: string) { + forgottenAgents.push(address); + }, + } as unknown as Parameters[0]["keyStore"]; + + const failingSpawner: SubprocessSpawner = () => { + throw new Error("spawner forced failure"); + }; + + const tmpDir = mkdtempSync(pathJoin(tmpdir(), "single-step-unwind-")); + + const repoStoreStub: Partial = { + getRepoDir(repoId: RepoId): string { + return pathJoin(tmpDir, repoId.kind, repoId.id); + }, + writeTree(_p, repoId, _ref, content) { + const dir = pathJoin(tmpDir, repoId.kind, repoId.id); + for (const [relPath, contents] of Object.entries(content.files)) { + const full = pathJoin(dir, relPath); + mkdirSync(dirname(full), { recursive: true }); + writeFileSync(full, contents); + } + return Promise.resolve({ + commitSha: "stub-sha", + newlyTerminalRuns: [], + }); + }, + }; + + const router = createSidecarDeployRouter({ + sessions, + keyStore, + transport, + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub: only getRepoDir + writeTree are exercised before the spawn-time failure + repoStore: repoStoreStub as RepoStore, + signingKeySeed: new Uint8Array(32), + createAgentCrypto: createEd25519Crypto, + assertSourceBuildable: () => undefined, + registerDeployment: ({ deploymentId, agentAddress }) => { + registry.record(deploymentId, agentAddress); + }, + unregisterDeployment: ({ deploymentId }) => { + registry.unregister(deploymentId); + }, + multistepMailRouter: mailRouter, + multistepSignalRouter: signalRouter, + multistepDrainRouter: drainRouter, + multistepSubstrateEnv: { + SIDECAR_DATA_DIR: tmpDir, + SIDECAR_SIGNING_PUBLIC_KEY: "00".repeat(32), + SIDECAR_SIGNING_PRIVATE_KEY: "00".repeat(32), + HUB_WS_URL: "ws://test", + SIDECAR_ID: "sc", + SIDECAR_TOKEN: "tok", + PATH: "/usr/bin", + }, + multistepSubprocessSpawner: failingSpawner, + }); + + const frame: AgentDeployFrame = { + type: "agent.deploy", + agentAddress: "ins_single@x.example", + agentId: "single", + hubPublicKey: "00".repeat(32), + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- single-step branch does not consult config before failing at spawn + config: { + agentAddress: "ins_single@x.example", + agentId: "single", + sessionId: "s", + sources: [], + defaultSource: "primary", + grants: [], + } as unknown as AgentDeployFrame["config"], + workflow: { + definition: { + id: "wf-single", + triggers: [{ type: "manual" }], + stepOrder: ["s1"], + steps: { s1: { kind: "step" } }, + }, + sources: { + s1: [ + { + id: "primary", + provider: "anthropic", + baseURL: "https://api.anthropic.com", + apiKey: "sk-x", + model: "claude-3-5", + }, + ], + }, + }, + }; + + await expect(router.deploy(frame)).rejects.toThrow(); + + // The single-step head initialized its repo and recorded the hub key + // before the spawn failed. + expect(initedRepos).toEqual(["ins_single@x.example"]); + expect(recordedHubKeys).toEqual(["ins_single@x.example"]); + // The unwind reversed the in-memory hub key... + expect(forgottenAgents).toEqual(["ins_single@x.example"]); + // ...but did NOT remove the head repo directory (it holds the durable + // identity keypair a rerouted head must keep across a failed redeploy). + expect(removedRepos).toEqual([]); + // Registry untouched. + expect(registry.resolve("ins_single-x-example")).toBeNull(); + }); }); diff --git a/apps/sidecar/src/workflow-host-wiring.test.ts b/apps/sidecar/src/workflow-host-wiring.test.ts index cbcef2d6..ef0d9401 100644 --- a/apps/sidecar/src/workflow-host-wiring.test.ts +++ b/apps/sidecar/src/workflow-host-wiring.test.ts @@ -588,6 +588,10 @@ describe("createSidecarDeployRouter multi-step branch", () => { keyPair: opts.headKeyPair ?? (await generateKeyPair()), isNew: false, }), + // A single-step spawn failure unwinds the recorded hub key via + // forgetAgent; the fixture exercises that unwind, so the stub must + // honor the call. + forgetAgent: () => undefined, } as unknown as Parameters< typeof createSidecarDeployRouter >[0]["keyStore"], diff --git a/apps/sidecar/src/workflow-host-wiring.ts b/apps/sidecar/src/workflow-host-wiring.ts index db9c0eb1..1cf858c6 100644 --- a/apps/sidecar/src/workflow-host-wiring.ts +++ b/apps/sidecar/src/workflow-host-wiring.ts @@ -1107,6 +1107,7 @@ export function createSidecarDeployRouter(deps: { let supervisorRegistered = false; let routersRegistered = false; let agentTransportRegistered = false; + let hubKeyRecorded = false; try { const definitionHash = await computeWireDefinitionHash(spec.definition); @@ -1214,6 +1215,7 @@ export function createSidecarDeployRouter(deps: { ); } deps.keyStore.recordHubKey(spec.agentAddress, spec.hubPublicKey); + hubKeyRecorded = true; } const stepOrder = [...spec.definition.stepOrder]; @@ -1320,6 +1322,21 @@ export function createSidecarDeployRouter(deps: { // not leave the address live with a dangling `CryptoProvider`. deps.transport.unregister(spec.agentAddress); } + if (hubKeyRecorded) { + // Reverse the single-step head's `recordHubKey` so a failed deploy + // leaves no in-memory hub key behind. `forgetAgent` also drops the + // agent keypair cache `loadOrGenerateKey` populated, which is safe: + // the transport registration is already unwound above, nothing reads + // that cache after unwind, and a redeploy reloads the keypair from + // disk. The on-disk deploy-tree repo `initRepo` created is + // deliberately NOT reversed. It is idempotent and the hub re-pushes + // the deploy pack on every redeploy, so it is benign residue; and + // decisively, the durable Ed25519 identity keypair lives inside that + // same directory (`keys/` nests under the agent repo dir), so + // removing the repo would destroy an identity a rerouted head must + // keep across a failed redeploy. + deps.keyStore.forgetAgent(spec.agentAddress); + } } } } From 23b76fa080a86747cc11360288f3963ce6eb17c1 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Sat, 4 Jul 2026 18:35:33 -0500 Subject: [PATCH 055/101] Widen the sources-updated control frame to carry the source list The frame previously carried only an optional sourceHashes map that no code wrote or read. Replace it with the inline source list and default source id the frame exists to convey. No hash rides along: a source list is flat, with no per-item pin for a receiver to cross-check, unlike the per-step grants snapshot whose stepHashes verify each step's contentHash. --- packages/workflow-host/src/child/run-child.ts | 4 +- .../workflow-host/src/ipc/control-channel.ts | 15 +++- packages/workflow-host/src/ipc/ipc.test.ts | 78 +++++++++++++++++++ 3 files changed, 94 insertions(+), 3 deletions(-) diff --git a/packages/workflow-host/src/child/run-child.ts b/packages/workflow-host/src/child/run-child.ts index ca3ec430..bcb59b35 100644 --- a/packages/workflow-host/src/child/run-child.ts +++ b/packages/workflow-host/src/child/run-child.ts @@ -831,7 +831,9 @@ async function handleControlPayload( return true; } case "sources-updated": { - logger.info`workflow-child sources-updated: ${JSON.stringify(payload.data)}`; + // Each source carries an apiKey, so log only the shape, never the + // payload contents. + logger.info`workflow-child sources-updated: ${String(payload.data.sources.length)} sources, default ${payload.data.defaultSource}`; return false; } case "ready": { diff --git a/packages/workflow-host/src/ipc/control-channel.ts b/packages/workflow-host/src/ipc/control-channel.ts index 92bbafc3..ff930556 100644 --- a/packages/workflow-host/src/ipc/control-channel.ts +++ b/packages/workflow-host/src/ipc/control-channel.ts @@ -37,7 +37,7 @@ import { type } from "arktype"; import { hexDecode, hexEncode } from "@intx/types"; -import { InterchangeType } from "@intx/types/runtime"; +import { InferenceSource, InterchangeType } from "@intx/types/runtime"; import { decodeEnvelope, @@ -177,7 +177,18 @@ export const ControlPayload = type( .or({ type: "'sources-updated'", data: { - "sourceHashes?": "Record", + /** + * The full ordered inference-source failover chain plus the default + * source id. Carried inline like grants-updated's snapshot: a + * single-producer, single-consumer supervisor->child push, so a + * substrate round-trip would only add latency. Constrained non-empty + * because a rotation always has at least one source; an empty list + * is a malformed frame rejected at the wire boundary. No per-source + * hash rides along -- sources are a flat list with no per-item pin + * for a receiver to cross-check, unlike the per-step grants snapshot. + */ + sources: InferenceSource.array().atLeastLength(1), + defaultSource: "string > 0", }, }) .or({ diff --git a/packages/workflow-host/src/ipc/ipc.test.ts b/packages/workflow-host/src/ipc/ipc.test.ts index 5fda6e35..e8e07270 100644 --- a/packages/workflow-host/src/ipc/ipc.test.ts +++ b/packages/workflow-host/src/ipc/ipc.test.ts @@ -1442,6 +1442,84 @@ describe("substrate.write.request payload validation", () => { }); }); +describe("sources-updated payload validation", () => { + const source = { + id: "primary", + provider: "anthropic", + baseURL: "https://api.anthropic.com", + apiKey: "sk-x", + model: "claude-3-5", + }; + + test("accepts a well-formed sources-updated frame", () => { + const payload = { + type: "sources-updated", + data: { sources: [source], defaultSource: "primary" }, + }; + const validated = ControlPayload(payload); + expect(validated instanceof type.errors).toBe(false); + }); + + test("rejects an empty sources list", () => { + const payload = { + type: "sources-updated", + data: { sources: [], defaultSource: "primary" }, + }; + const validated = ControlPayload(payload); + expect(validated instanceof type.errors).toBe(true); + }); + + test("rejects a missing sources list", () => { + const payload = { + type: "sources-updated", + data: { defaultSource: "primary" }, + }; + const validated = ControlPayload(payload); + expect(validated instanceof type.errors).toBe(true); + }); + + test("rejects a missing defaultSource", () => { + const payload = { + type: "sources-updated", + data: { sources: [source] }, + }; + const validated = ControlPayload(payload); + expect(validated instanceof type.errors).toBe(true); + }); + + test("rejects an empty-string defaultSource", () => { + const payload = { + type: "sources-updated", + data: { sources: [source], defaultSource: "" }, + }; + const validated = ControlPayload(payload); + expect(validated instanceof type.errors).toBe(true); + }); + + test("accepts a defaultSource that is not a member of sources", () => { + // Constraint ownership: the wire boundary does not enforce + // `defaultSource in sources`. The consumer (the agent's source + // registry, via indexOfDefault) owns that invariant and throws when + // it does not hold, so re-checking it here would duplicate the rule. + const payload = { + type: "sources-updated", + data: { sources: [source], defaultSource: "not-in-list" }, + }; + const validated = ControlPayload(payload); + expect(validated instanceof type.errors).toBe(false); + }); + + test("rejects a source element missing a required field", () => { + const { apiKey: _apiKey, ...sourceWithoutApiKey } = source; + const payload = { + type: "sources-updated", + data: { sources: [sourceWithoutApiKey], defaultSource: "primary" }, + }; + const validated = ControlPayload(payload); + expect(validated instanceof type.errors).toBe(true); + }); +}); + describe("substrate.merge.request payload validation", () => { test("accepts a well-formed substrate.merge.request", () => { const payload = { From b2014a2fe7b6e9a11405aac1534e4387ca61c5f7 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Sun, 5 Jul 2026 08:51:40 -0500 Subject: [PATCH 056/101] Read single-step build sources through a mutable reference The step build resolved its inference sources from a table fixed when the build closure was constructed. Resolve them instead from a mutable reference the child's run loop owns and reads at each build, so a build reflects the table's current contents rather than a snapshot frozen at construction. Behavior is unchanged here: the reference is seeded once from the spawn env and has no runtime writer, so every build reads the seeded table. --- ...low-substrate-factory-step-storage.test.ts | 77 ++++++++++++++++--- .../sidecar/src/workflow-substrate-factory.ts | 44 +++++++---- packages/workflow-host/src/child/index.ts | 1 + packages/workflow-host/src/child/run-child.ts | 33 ++++++++ packages/workflow-host/src/index.ts | 1 + 5 files changed, 130 insertions(+), 26 deletions(-) diff --git a/apps/sidecar/src/workflow-substrate-factory-step-storage.test.ts b/apps/sidecar/src/workflow-substrate-factory-step-storage.test.ts index 326437ab..28d96e76 100644 --- a/apps/sidecar/src/workflow-substrate-factory-step-storage.test.ts +++ b/apps/sidecar/src/workflow-substrate-factory-step-storage.test.ts @@ -31,7 +31,11 @@ import type { } from "@intx/types/runtime"; import type { RepoId } from "@intx/hub-sessions"; import { createBuiltinRegistry } from "@intx/inference/providers"; -import type { ChildOutboundMailBridge, StepEnvBase } from "@intx/workflow-host"; +import type { + ChildOutboundMailBridge, + SourcesSnapshotRef, + StepEnvBase, +} from "@intx/workflow-host"; import type { StepInvokeRequest } from "@intx/workflow"; import { @@ -86,10 +90,8 @@ function stubDurableConversationRegistry(): DurableConversationRegistry { function buildDeps(opts: { dataDir: string; durableConversation?: DurableConversationRegistry; - sourceChain?: InferenceSource[]; }): SidecarStepBuildEnvDeps { return { - table: { [STEP_ID]: opts.sourceChain ?? [SOURCE] }, dataDir: opts.dataDir, workflowRunRepoId: WORKFLOW_RUN_REPO_ID, signer: (payload: string) => Promise.resolve(`sig:${payload.length}`), @@ -104,6 +106,12 @@ function buildDeps(opts: { }; } +function sourcesRefFor( + chain: InferenceSource[] = [SOURCE], +): SourcesSnapshotRef { + return { current: { [STEP_ID]: chain } }; +} + function requestForRun(runId: string): StepInvokeRequest { return { // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- buildEnv reads only authzContext; the agent definition is never consulted here @@ -128,12 +136,19 @@ describe("createSidecarStepBuildEnv per-step scratch keying", () => { }), ); - const env1: StepEnvBase = await buildEnv(requestForRun("run-1")); + const sourcesRef = sourcesRefFor(); + const env1: StepEnvBase = await buildEnv( + requestForRun("run-1"), + sourcesRef, + ); // A file the agent would write during run-1's turn. const marker = path.join(env1.workdir, "turn-1.txt"); await fs.writeFile(marker, "carried"); - const env2: StepEnvBase = await buildEnv(requestForRun("run-2")); + const env2: StepEnvBase = await buildEnv( + requestForRun("run-2"), + sourcesRef, + ); // Stable keying: a different runId resolves to the SAME workdir, so // the warm case is bounded to one dir per agent (not one-per-message) @@ -154,9 +169,16 @@ describe("createSidecarStepBuildEnv per-step scratch keying", () => { test("cold path keys a distinct per-run workdir under that run's subtree", async () => { const dataDir = await makeTempDir(); const buildEnv = createSidecarStepBuildEnv(buildDeps({ dataDir })); + const sourcesRef = sourcesRefFor(); - const env1: StepEnvBase = await buildEnv(requestForRun("run-1")); - const env2: StepEnvBase = await buildEnv(requestForRun("run-2")); + const env1: StepEnvBase = await buildEnv( + requestForRun("run-1"), + sourcesRef, + ); + const env2: StepEnvBase = await buildEnv( + requestForRun("run-2"), + sourcesRef, + ); // Per-run keying: each run gets its own workdir, rooted under that // run's `runs//` subtree -- exactly what the run-completion @@ -188,15 +210,48 @@ describe("createSidecarStepBuildEnv per-step scratch keying", () => { }; const chain = [SOURCE, failoverSource]; const dataDir = await makeTempDir(); - const buildEnv = createSidecarStepBuildEnv( - buildDeps({ dataDir, sourceChain: chain }), - ); + const buildEnv = createSidecarStepBuildEnv(buildDeps({ dataDir })); - const env: StepEnvBase = await buildEnv(requestForRun("run-1")); + const env: StepEnvBase = await buildEnv( + requestForRun("run-1"), + sourcesRefFor(chain), + ); // The whole chain reaches the reactor, ordered, with element 0 as the // initial source. expect(env.sources).toEqual(chain); expect(env.defaultSource).toBe(SOURCE.id); }); + + test("resolves sources from the mutable ref at build time", async () => { + // The build reads the source table through the ref, so a rotation that + // writes the ref before a build is reflected in the agent that build + // constructs (the cold-window path; a warm already-built agent swaps + // sources through the warm cache, not here). + const dataDir = await makeTempDir(); + const buildEnv = createSidecarStepBuildEnv(buildDeps({ dataDir })); + const sourcesRef = sourcesRefFor(); + + const before: StepEnvBase = await buildEnv( + requestForRun("run-1"), + sourcesRef, + ); + expect(before.sources).toEqual([SOURCE]); + + const rotated: InferenceSource = { + id: "rotated", + provider: "openai", + baseURL: "https://api.openai.com", + apiKey: "sk-rotated", + model: "gpt-rotated", + }; + sourcesRef.current = { [STEP_ID]: [rotated] }; + + const after: StepEnvBase = await buildEnv( + requestForRun("run-2"), + sourcesRef, + ); + expect(after.sources).toEqual([rotated]); + expect(after.defaultSource).toBe(rotated.id); + }); }); diff --git a/apps/sidecar/src/workflow-substrate-factory.ts b/apps/sidecar/src/workflow-substrate-factory.ts index a76cf71d..941d7560 100644 --- a/apps/sidecar/src/workflow-substrate-factory.ts +++ b/apps/sidecar/src/workflow-substrate-factory.ts @@ -66,6 +66,7 @@ import { type GrantEvaluator, type RunChildWorkflow, type RunWorkflowChildBindings, + type SourcesSnapshotRef, type StepEnvBase, type SubstrateFactory, type SubstrateFactoryEnv, @@ -183,8 +184,9 @@ function parseByteCap(raw: string, name: string): number { * `STEP_INFERENCE_SOURCES` env entry. The deploy router serializes * `frame.workflow.sources` (a `Record`) as * JSON and threads it through the supervisor's `substrateEnv`; the - * factory parses and validates the table once at construction time - * and pins it for `buildEnv` lookups. Each value is the step's ordered + * factory parses and validates the table once at construction time and + * seeds it into the run loop's mutable sources reference, which each + * `buildEnv` reads. Each value is the step's ordered * failover chain -- element 0 is the active source, the tail are the * reactor's forward-only failover targets -- so the list is non-empty. */ @@ -262,8 +264,8 @@ export function parseAdapterManifest(raw: string): AdapterManifest { } /** - * Resolve the per-step inference-source failover chain pinned at - * factory construction. The supervisor's multi-step branch only invokes + * Resolve the per-step inference-source failover chain from the table a + * build reads. The supervisor's multi-step branch only invokes * a step whose `stepId` appears in `frame.workflow.sources`; a lookup * miss here is a programmer error in the supervisor, not a wire-side * failure, and the resolver surfaces it with the missing `stepId` @@ -431,7 +433,6 @@ function warmStepStorageRoot(args: { } export interface SidecarStepBuildEnvDeps { - table: StepInferenceSourceTable; dataDir: string; workflowRunRepoId: RepoId; signer: (payload: string) => Promise; @@ -498,9 +499,10 @@ export interface SidecarStepBuildEnvDeps { * the per-step env construction is observable without standing up the * full substrate. * - * The closure pins the parsed per-step source table, derives the - * `stepId` / `runId` / `attempt` from the runtime's `AuthorizeContext`, - * resolves the per-step `InferenceSource` from the table, and stands up + * The closure reads the per-step source table from the mutable + * reference passed per build, derives the `stepId` / `runId` / `attempt` + * from the runtime's `AuthorizeContext`, resolves the per-step + * `InferenceSource` from that table, and stands up * a per-step isogit `ContextStore` (also serving as the audit store) * plus a per-step workspace directory rooted under the run. A * construction failure (mkdir, isogit init) surfaces here rather than @@ -509,11 +511,22 @@ export interface SidecarStepBuildEnvDeps { */ export function createSidecarStepBuildEnv( deps: SidecarStepBuildEnvDeps, -): (req: StepInvokeRequest) => Promise { - const resolveStepInferenceSource = createStepInferenceSourceResolver( - deps.table, - ); - return async (req: StepInvokeRequest): Promise => { +): ( + req: StepInvokeRequest, + sourcesRef: SourcesSnapshotRef, +) => Promise { + return async ( + req: StepInvokeRequest, + sourcesRef: SourcesSnapshotRef, + ): Promise => { + // Resolve against the live table each build so a source rotation that + // wrote `sourcesRef.current` before this build is reflected in the + // agent this build constructs. A warm agent that is already built does + // not pass through here again, so a rotation does not reach it through + // this path -- this ref covers only a build that has not happened yet. + const resolveStepInferenceSource = createStepInferenceSourceResolver( + sourcesRef.current, + ); const { stepId, runId, attempt } = req.authzContext; if (stepId === undefined) { throw new Error( @@ -1039,7 +1052,6 @@ export function createSidecarSubstrateFactory( : undefined; const buildStepEnv = createSidecarStepBuildEnv({ - table: stepInferenceSources, dataDir: validated.SIDECAR_DATA_DIR, workflowRunRepoId, signer: conversationSigner, @@ -1127,10 +1139,11 @@ export function createSidecarSubstrateFactory( onEvent, authorize, warmCache, + sourcesRef, ) => createWorkflowStepInvoker({ workflowAuthorize: authorize, - buildEnv: buildStepEnv, + buildEnv: (buildReq) => buildStepEnv(buildReq, sourcesRef), agentFactory: stepAgentFactory, onEvent, ...(warmCache !== undefined ? { warmCache } : {}), @@ -1211,6 +1224,7 @@ export function createSidecarSubstrateFactory( workflowDefinitionRepoId, workflowDefinitionRef: validated.WORKFLOW_DEFINITION_REF, invokeStep, + initialSources: stepInferenceSources, spawnChild, scheduler, evaluateGrants: evaluateGrantsAdapter, diff --git a/packages/workflow-host/src/child/index.ts b/packages/workflow-host/src/child/index.ts index 7cccf3ef..0d785d26 100644 --- a/packages/workflow-host/src/child/index.ts +++ b/packages/workflow-host/src/child/index.ts @@ -9,6 +9,7 @@ export { type RunWorkflowChildBindings, type RunWorkflowChildOpts, type RunWorkflowChildResult, + type SourcesSnapshotRef, type SubstrateWriteResponseSink, } from "./run-child"; diff --git a/packages/workflow-host/src/child/run-child.ts b/packages/workflow-host/src/child/run-child.ts index bcb59b35..891ae6c4 100644 --- a/packages/workflow-host/src/child/run-child.ts +++ b/packages/workflow-host/src/child/run-child.ts @@ -91,6 +91,8 @@ import { type WorkflowHostDrainController, } from "../drain-controller"; +import type { InferenceSource } from "@intx/types/runtime"; + import { createWorkflowRunRepoStore } from "../adapters/repo-store"; import { createWorkflowRunBlobSubstrate } from "../adapters/blob-substrate"; import { @@ -135,6 +137,19 @@ export type CredentialsSnapshotRef = { current: CredentialsSnapshot | null; }; +/** + * Per-step inference-source table the build path reads through a mutable + * reference, keyed by stepId. Each value is the step's ordered failover + * chain (element 0 is the active source). The single-step build resolves + * its sources from `current` at build time, so a rotation that writes + * `current` before the first build is reflected in the built agent. A + * warm agent that is already built does not re-read this ref, so rotating + * its live sources is out of this ref's scope. + */ +export type SourcesSnapshotRef = { + current: Record; +}; + /** * Construct the workflow-level authorize closure backed by a * mutable credentialsSnapshot reference. @@ -228,6 +243,7 @@ export type ChildStepInvoker = ( onEvent: (event: EventPayload) => void, authorize: WorkflowAuthorizeFn, warmCache: WarmAgentCache | undefined, + sourcesRef: SourcesSnapshotRef, ) => Promise; /** @@ -302,6 +318,14 @@ export interface RunWorkflowChildBindings { * value defers to the first `grants-updated` control frame. */ initialCredentialsSnapshot?: CredentialsSnapshot; + /** + * Bootstrap per-step inference-source table (keyed by stepId), parsed + * from the spawn env by the host's substrate factory. Seeds the + * mutable `sourcesRef` the build path reads. Absent value defers to an + * empty table, so a step with no pinned source fails loudly at build + * rather than resolving a default. + */ + initialSources?: Record; /** * Optional override for the child's Ed25519 keypair factory. The * child mints a fresh keypair at startup, holds the private half @@ -424,6 +448,9 @@ export async function runWorkflowChild( const credentialsRef: CredentialsSnapshotRef = { current: opts.bindings.initialCredentialsSnapshot ?? null, }; + const sourcesRef: SourcesSnapshotRef = { + current: opts.bindings.initialSources ?? {}, + }; const directors = opts.bindings.directors ?? createDefaultDirectorRegistry(); const clock = opts.bindings.clock ?? defaultClock; const newId = opts.bindings.newId ?? defaultNewId; @@ -508,6 +535,7 @@ export async function runWorkflowChild( newId, drainController, warmCache, + sourcesRef, onEvent: (event) => { void eventSender.send(event).catch((cause) => { logger.error`event-channel send failed during resume run ${run.runId}: ${String(cause)}`; @@ -590,6 +618,7 @@ export async function runWorkflowChild( drainController, triggeredRunIds, warmCache, + sourcesRef, ...(opts.substrateWriteBridge !== undefined ? { substrateWriteBridge: opts.substrateWriteBridge } : {}), @@ -662,6 +691,7 @@ async function handleControlPayload( drainController: DrainController; triggeredRunIds: string[]; warmCache: WarmAgentCache | undefined; + sourcesRef: SourcesSnapshotRef; substrateWriteBridge?: SubstrateWriteResponseSink; outboundMailBridge?: ChildOutboundMailBridge; }, @@ -695,6 +725,7 @@ async function handleControlPayload( newId: ctx.newId, drainController: ctx.drainController, warmCache: ctx.warmCache, + sourcesRef: ctx.sourcesRef, onEvent: (event) => { void ctx.eventSender.send(event).catch((cause) => { logger.error`event-channel send failed during run ${payload.data.runId}: ${String(cause)}`; @@ -942,6 +973,7 @@ function buildRuntimeEnv(args: { newId: (prefix: string) => string; drainController: DrainController; warmCache: WarmAgentCache | undefined; + sourcesRef: SourcesSnapshotRef; onEvent: (event: EventPayload) => void; }): WorkflowRuntimeEnv { const signalChannel = createWorkflowHostSignalChannel({ @@ -974,6 +1006,7 @@ function buildRuntimeEnv(args: { args.onEvent, args.authorize, args.warmCache, + args.sourcesRef, ); }; return { diff --git a/packages/workflow-host/src/index.ts b/packages/workflow-host/src/index.ts index 339d0c1e..bc3881a5 100644 --- a/packages/workflow-host/src/index.ts +++ b/packages/workflow-host/src/index.ts @@ -148,6 +148,7 @@ export { type RunWorkflowChildFromProcessEnvOpts, type RunWorkflowChildOpts, type RunWorkflowChildResult, + type SourcesSnapshotRef, type SpawnTimeEnv, type SubstrateFactory, type SubstrateFactoryEnv, From d1ad3ed093133ad9fe31eb3f3bd050ba971fdb27 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Sun, 5 Jul 2026 09:30:40 -0500 Subject: [PATCH 057/101] Rotate a warm single-step agent's sources on a sources-updated frame A sources-updated frame now rotates the warm single-step agent's inference sources instead of being logged and dropped. The handler swaps the sources on the built warm agent in place through the agent's setSources, then updates the table the next cold build reads, so a rotation takes effect whether or not the agent has been built yet. The wire boundary enforces the frame's structural invariants -- a non-empty list, unique source ids, and the head element as the default -- so the handler trusts the frame and re-validates nothing. A sources-updated for a multi-step deployment or one arriving with no warm cache is a routing bug and throws rather than silently doing nothing. --- .../workflow-host/src/child/run-child.test.ts | 312 +++++++++++++++++- packages/workflow-host/src/child/run-child.ts | 43 ++- .../src/child/warm-agent-cache.test.ts | 57 ++++ .../src/child/warm-agent-cache.ts | 25 +- .../workflow-host/src/ipc/control-channel.ts | 53 ++- packages/workflow-host/src/ipc/ipc.test.ts | 24 +- 6 files changed, 485 insertions(+), 29 deletions(-) create mode 100644 packages/workflow-host/src/child/warm-agent-cache.test.ts diff --git a/packages/workflow-host/src/child/run-child.test.ts b/packages/workflow-host/src/child/run-child.test.ts index e31db82f..266d1785 100644 --- a/packages/workflow-host/src/child/run-child.test.ts +++ b/packages/workflow-host/src/child/run-child.test.ts @@ -185,6 +185,10 @@ function createStubRepoStore(baseDir: string): RepoStore { async function seedWorkflowDefinition( baseDir: string, repoId: RepoId, + workflow: { steps: Record; stepOrder: string[] } = { + steps: {}, + stepOrder: [], + }, ): Promise { const dir = path.join(baseDir, repoId.kind, repoId.id); await fs.mkdir(dir, { recursive: true }); @@ -193,8 +197,8 @@ async function seedWorkflowDefinition( JSON.stringify({ id: "test-workflow", triggers: [], - steps: {}, - stepOrder: [], + steps: workflow.steps, + stepOrder: workflow.stepOrder, }), ); } @@ -906,6 +910,128 @@ describe("runWorkflowChild", () => { ]); }); + test("sources-updated on a multi-step deployment is rejected", async () => { + const baseDir = await makeTempDir("child-sources-multistep-"); + const supervisorKeyPair = await generateKeyPair(); + const childKeyPair = await generateKeyPair(); + const channelId = generateChannelId(); + const hmacKey = generateHmacKey(); + await seedWorkflowDefinition( + baseDir, + { kind: "workflow", id: "workflow-asset" }, + { + steps: { "step-1": { kind: "step" }, "step-2": { kind: "step" } }, + stepOrder: ["step-1", "step-2"], + }, + ); + + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const eventStream = createMemoryFrameStream(); + + const env = parseSpawnTimeEnv( + makeSpawnEnv({ + channelId, + hmacKeyHex: hexEncode(hmacKey), + hostPubKeyHex: hexEncode(supervisorKeyPair.publicKey), + }), + ); + const bindings = buildBindings({ baseDir, childKeyPair }); + const supervisorSender = createControlChannelSender({ + privateKeySeed: supervisorKeyPair.privateKey, + channelId, + writer: supervisorToChild.writer, + }); + + const runPromise = runWorkflowChild({ + env, + controlReader: supervisorToChild.reader, + controlWriter: childToSupervisor.writer, + eventWriter: eventStream.writer, + bindings, + }); + + await supervisorSender.send({ + type: "sources-updated", + data: { + sources: [ + { + id: "primary", + provider: "anthropic", + baseURL: "https://api.anthropic.com", + apiKey: "sk-x", + model: "claude-test", + }, + ], + defaultSource: "primary", + }, + }); + + await expect(runPromise).rejects.toThrow(/single-step deployment/); + supervisorToChild.close(); + }); + + test("sources-updated with no warm cache is rejected as a routing bug", async () => { + // A single-step definition with WARM_KEEP off leaves the child with no + // warm cache. A sources-updated must never silently no-op there; it is + // a routing bug, so the handler throws. + const baseDir = await makeTempDir("child-sources-nowarm-"); + const supervisorKeyPair = await generateKeyPair(); + const childKeyPair = await generateKeyPair(); + const channelId = generateChannelId(); + const hmacKey = generateHmacKey(); + await seedWorkflowDefinition( + baseDir, + { kind: "workflow", id: "workflow-asset" }, + { steps: { "step-1": { kind: "step" } }, stepOrder: ["step-1"] }, + ); + + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const eventStream = createMemoryFrameStream(); + + const env = parseSpawnTimeEnv( + makeSpawnEnv({ + channelId, + hmacKeyHex: hexEncode(hmacKey), + hostPubKeyHex: hexEncode(supervisorKeyPair.publicKey), + }), + ); + const bindings = buildBindings({ baseDir, childKeyPair }); + const supervisorSender = createControlChannelSender({ + privateKeySeed: supervisorKeyPair.privateKey, + channelId, + writer: supervisorToChild.writer, + }); + + const runPromise = runWorkflowChild({ + env, + controlReader: supervisorToChild.reader, + controlWriter: childToSupervisor.writer, + eventWriter: eventStream.writer, + bindings, + }); + + await supervisorSender.send({ + type: "sources-updated", + data: { + sources: [ + { + id: "primary", + provider: "anthropic", + baseURL: "https://api.anthropic.com", + apiKey: "sk-x", + model: "claude-test", + }, + ], + defaultSource: "primary", + }, + }); + + await expect(runPromise).rejects.toThrow(/no warm cache/); + supervisorToChild.close(); + }); + test("child ready frame is signed by the child's own keypair and bootstraps the supervisor's verification key", async () => { const baseDir = await makeTempDir("child-ready-"); const supervisorKeyPair = await generateKeyPair(); @@ -1108,6 +1234,10 @@ interface WarmAgentSpy { closeCount: number; readonly conversation: string[]; readonly replies: string[]; + readonly sourceRotations: { + sources: InferenceSource[]; + defaultSource: string; + }[]; } /** @@ -1125,6 +1255,7 @@ function buildWarmAgentSpy(): { agent: Agent; spy: WarmAgentSpy } { closeCount: 0, conversation: [], replies: [], + sourceRotations: [], }; let endStream: () => void = () => undefined; const streamEnded = new Promise((resolve) => { @@ -1167,8 +1298,11 @@ function buildWarmAgentSpy(): { agent: Agent; spy: WarmAgentSpy } { setSource(_source: InferenceSource) { throw new Error("stub setSource() not used"); }, - setSources(_sources: InferenceSource[], _defaultSource: string) { - throw new Error("stub setSources() not used"); + setSources(sources: InferenceSource[], defaultSource: string) { + // Live source rotation lands here on the warm agent. Record it so the + // sources-updated round-trip can assert the swap reached the agent + // in place rather than rebuilding it. + spy.sourceRotations.push({ sources, defaultSource }); }, async history() { return []; @@ -1506,6 +1640,176 @@ describe("warm-agent round-trip (Phase 4.4)", () => { expect(spy.closeCount).toBe(1); expect(spy.lspAlive).toBe(false); }); + + test("a sources-updated frame swaps the built warm agent's sources in place", async () => { + const baseDir = await makeTempDir("warm-sources-"); + const supervisorKeyPair = await generateKeyPair(); + const childKeyPair = await generateKeyPair(); + const channelId = generateChannelId(); + const hmacKey = generateHmacKey(); + const stepId = "step-1"; + const deploymentId = "deployment-x"; + const workflowRunRepoId: RepoId = { + kind: "workflow-run", + id: deploymentId, + }; + const workflowDefinitionRepoId: RepoId = { + kind: "workflow-run", + id: "workflow-asset", + }; + + const signingKey: KeyPair = await generateKeyPair(); + const allowAll: AuthorizeFn = () => ({ allowed: true }); + const substrate = createRepoStore({ + dataDir: baseDir, + signingKey, + handlers: { "workflow-run": workflowRunKindHandler }, + authorize: allowAll, + }); + const principalShape = { kind: "workflow-process", deploymentId }; + const principal: Principal = principalShape; + + await substrate.writeTree( + { kind: "hub" }, + workflowRunRepoId, + "refs/heads/main", + { files: { [WORKFLOW_RUN_GITIGNORE_PATH]: "" }, message: "genesis" }, + ); + const runRepoDir = substrate.getRepoDir(workflowRunRepoId); + await seedProcessingEntryInDir(runRepoDir, { + address: "deployment-x@example.com", + messageId: "msg-1", + receivedAt: 1, + text: "alpha body", + }); + await substrate.writeTree( + { kind: "hub" }, + workflowDefinitionRepoId, + "refs/heads/main", + { files: { [WORKFLOW_RUN_GITIGNORE_PATH]: "" }, message: "genesis" }, + ); + await seedOneStepWorkflowDir( + substrate.getRepoDir(workflowDefinitionRepoId), + stepId, + ); + + const { agent, spy } = buildWarmAgentSpy(); + let factoryCalls = 0; + const invokeStep: ChildStepInvoker = async ( + req, + onEvent, + authorize, + warmCache, + ) => + createWorkflowStepInvoker({ + workflowAuthorize: authorize, + buildEnv: async () => stubStepEnv(), + agentFactory: async () => { + factoryCalls += 1; + return agent; + }, + onEvent: (event) => onEvent(event), + ...(warmCache !== undefined ? { warmCache } : {}), + })(req); + + const bindings: RunWorkflowChildBindings = { + substrate, + workflowRunRepoId, + workflowRunRef: "refs/heads/main", + principal, + workflowDefinitionRepoId, + workflowDefinitionRef: "refs/heads/main", + invokeStep, + spawnChild: async () => ({ terminalStatus: "completed" }), + scheduler: { scheduleIn: () => () => undefined }, + evaluateGrants: async () => ({ + effect: "allow" as const, + matchingGrants: [], + resolvedBy: null, + }), + ipcChildKeyPairFactory: () => Promise.resolve(childKeyPair), + initialCredentialsSnapshot: { + steps: [ + { + stepId, + address: "deployment-x@example.com", + grants: [], + contentHash: "deadbeef", + }, + ], + }, + }; + + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const eventStream = createMemoryFrameStream(); + + const env = parseSpawnTimeEnv({ + ...makeSpawnEnv({ + channelId, + hmacKeyHex: hexEncode(hmacKey), + hostPubKeyHex: hexEncode(supervisorKeyPair.publicKey), + }), + WARM_KEEP: "true", + }); + + const supervisorSender = createControlChannelSender({ + privateKeySeed: supervisorKeyPair.privateKey, + channelId, + writer: supervisorToChild.writer, + }); + + const runPromise = runWorkflowChild({ + env, + controlReader: supervisorToChild.reader, + controlWriter: childToSupervisor.writer, + eventWriter: eventStream.writer, + bindings, + }); + + await waitForTriggeredRun(childToSupervisor, (lines) => lines.length > 0); + + // Build the warm agent with one message so the cache holds an entry. + await supervisorSender.send({ + type: "trigger.fire", + data: { runId: "run-1", messageId: "msg-1", receivedAt: 1 }, + }); + await waitForTriggeredRun(childToSupervisor, () => spy.replies.length >= 1); + expect(factoryCalls).toBe(1); + + // Rotate the sources on the live warm agent. + const rotated: InferenceSource[] = [ + { + id: "rotated", + provider: "openai", + baseURL: "https://api.openai.com", + apiKey: "sk-rotated", + model: "gpt-rotated", + }, + ]; + await supervisorSender.send({ + type: "sources-updated", + data: { sources: rotated, defaultSource: "rotated" }, + }); + for (let i = 0; i < 400; i += 1) { + if (spy.sourceRotations.length >= 1) break; + await new Promise((r) => setTimeout(r, 5)); + } + + // The swap reached the built agent in place, carrying the frame's list + // and default, and did NOT rebuild the agent. + expect(spy.sourceRotations).toHaveLength(1); + expect(spy.sourceRotations[0]?.sources).toEqual(rotated); + expect(spy.sourceRotations[0]?.defaultSource).toBe("rotated"); + expect(factoryCalls).toBe(1); + + await supervisorSender.send({ + type: "shutdown", + data: { reason: "undeploy" }, + }); + supervisorToChild.close(); + await runPromise; + }); }); describe("event channel writer wiring", () => { diff --git a/packages/workflow-host/src/child/run-child.ts b/packages/workflow-host/src/child/run-child.ts index 891ae6c4..372049c8 100644 --- a/packages/workflow-host/src/child/run-child.ts +++ b/packages/workflow-host/src/child/run-child.ts @@ -862,9 +862,46 @@ async function handleControlPayload( return true; } case "sources-updated": { - // Each source carries an apiKey, so log only the shape, never the - // payload contents. - logger.info`workflow-child sources-updated: ${String(payload.data.sources.length)} sources, default ${payload.data.defaultSource}`; + // Live inference-source rotation for the warm single-step agent. The + // wire boundary (`SourcesUpdatedData`) already guaranteed the list is + // non-empty, its ids are unique, and its head is the default, so this + // trusts the frame and does not re-validate it. + // + // Only a single-step deployment rotates sources: its sole step's id + // is the sole key in the sources table, so the whole table is + // replaced. A multi-step deployment has no single per-agent source + // identity to swap and is never routed a sources-updated frame; + // assert it so a mis-route fails loudly rather than corrupting the + // table. + if (ctx.definition.stepOrder.length !== 1) { + throw new Error( + `workflow-child sources-updated: only a single-step deployment can rotate sources; got ${String(ctx.definition.stepOrder.length)} steps`, + ); + } + const stepId = ctx.definition.stepOrder[0]; + if (stepId === undefined) { + throw new Error( + "workflow-child sources-updated: single-step deployment has no step id", + ); + } + // A sources-updated only reaches a warm single-step deployment, which + // always builds a warm cache. An absent cache is a routing bug, not a + // silent no-op. + if (ctx.warmCache === undefined) { + throw new Error( + "workflow-child sources-updated: no warm cache; a sources rotation must target a warm single-step deployment", + ); + } + // Swap the built warm agent first (a no-op when none is built yet), + // then update the table the next cold build reads. Applying to the + // agent first means a rotation racing eviction -- a closed-agent + // `setSources` throw -- leaves the table untouched rather than ahead + // of a half-applied swap. + ctx.warmCache.applySources( + payload.data.sources, + payload.data.defaultSource, + ); + ctx.sourcesRef.current = { [stepId]: payload.data.sources }; return false; } case "ready": { diff --git a/packages/workflow-host/src/child/warm-agent-cache.test.ts b/packages/workflow-host/src/child/warm-agent-cache.test.ts new file mode 100644 index 00000000..7746096f --- /dev/null +++ b/packages/workflow-host/src/child/warm-agent-cache.test.ts @@ -0,0 +1,57 @@ +import { describe, test, expect } from "bun:test"; +import type { Agent } from "@intx/agent"; +import type { InferenceSource } from "@intx/types/runtime"; + +import { + createWarmAgentCache, + type WarmEventSinkRef, +} from "./warm-agent-cache"; + +function makeSource(id: string): InferenceSource { + return { + id, + provider: "anthropic", + baseURL: "https://api.anthropic.com", + apiKey: `sk-${id}`, + model: "claude-test", + }; +} + +function stubAgent(): { + agent: Agent; + calls: { sources: InferenceSource[]; defaultSource: string }[]; +} { + const calls: { sources: InferenceSource[]; defaultSource: string }[] = []; + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub: applySources only calls setSources on the entry's agent + const agent = { + setSources(sources: InferenceSource[], defaultSource: string) { + calls.push({ sources, defaultSource }); + }, + } as unknown as Agent; + return { agent, calls }; +} + +describe("warm-agent cache applySources", () => { + test("applies the rotated sources to a built warm agent", () => { + const cache = createWarmAgentCache(); + const { agent, calls } = stubAgent(); + const sinkRef: WarmEventSinkRef = { current: null }; + cache.store("step-1", agent, sinkRef, Promise.resolve()); + + const sources = [makeSource("primary"), makeSource("failover")]; + cache.applySources(sources, "primary"); + + expect(calls).toHaveLength(1); + expect(calls[0]?.sources).toEqual(sources); + expect(calls[0]?.defaultSource).toBe("primary"); + }); + + test("is a no-op when no warm agent is built yet", () => { + const cache = createWarmAgentCache(); + // The pre-first-build window: a rotation arriving before the agent is + // built must not throw; the ref carries it to the next build. + expect(() => + cache.applySources([makeSource("primary")], "primary"), + ).not.toThrow(); + }); +}); diff --git a/packages/workflow-host/src/child/warm-agent-cache.ts b/packages/workflow-host/src/child/warm-agent-cache.ts index bd60ca2f..6e5a7c69 100644 --- a/packages/workflow-host/src/child/warm-agent-cache.ts +++ b/packages/workflow-host/src/child/warm-agent-cache.ts @@ -35,7 +35,7 @@ import { getLogger } from "@intx/log"; import type { Agent } from "@intx/agent"; -import type { InferenceEvent } from "@intx/types/runtime"; +import type { InferenceEvent, InferenceSource } from "@intx/types/runtime"; const logger = getLogger(["workflow-host", "child", "warm-agent-cache"]); @@ -102,6 +102,19 @@ export interface WarmAgentCache { * no-op: the agent may already have been evicted. */ clearEventSink(key: string): void; + /** + * Apply a rotated inference-source list to every retained warm agent in + * place, via `Agent.setSources`. A single-step warm cache holds 0 or 1 + * entry, so this rotates the one built agent -- or is a no-op when none + * is built yet (the pre-first-build window). The swap mutates the + * agent's shared active-source object in place and takes effect on the + * reactor's next inference call; the single-threaded control loop means + * there is no torn read against a concurrent send. `setSources` validates + * the list and throws on an invalid source (or a closed agent, if a + * rotation races eviction), so a bad rotation surfaces rather than being + * swallowed. + */ + applySources(sources: InferenceSource[], defaultSource: string): void; /** * Tear down every cached warm agent: run the wrapped `agent.close()` * (disposing plugins and killing the LSP subprocess) and drain the @@ -160,6 +173,15 @@ export function createWarmAgentCache(): WarmAgentCache { entry.eventSinkRef.current = null; } + function applySources( + sources: InferenceSource[], + defaultSource: string, + ): void { + for (const entry of entries.values()) { + entry.agent.setSources(sources, defaultSource); + } + } + async function evictAll(reason: string): Promise { if (entries.size === 0) return; const toEvict = [...entries.values()]; @@ -195,6 +217,7 @@ export function createWarmAgentCache(): WarmAgentCache { store, setEventSink, clearEventSink, + applySources, evictAll, }; } diff --git a/packages/workflow-host/src/ipc/control-channel.ts b/packages/workflow-host/src/ipc/control-channel.ts index ff930556..e5ab218c 100644 --- a/packages/workflow-host/src/ipc/control-channel.ts +++ b/packages/workflow-host/src/ipc/control-channel.ts @@ -68,6 +68,44 @@ export const CredentialsSnapshotPayload = type({ steps: CredentialsSnapshotStepPayload.array(), }); +/** + * Wire shape of a `sources-updated` frame's `data`: the full ordered + * inference-source failover chain plus the default source id. Carried + * inline like the grants snapshot -- a single-producer, single-consumer + * supervisor->child push, so a substrate round-trip would only add + * latency. No per-source hash rides along; a source list is flat, with no + * per-item pin for a receiver to cross-check. + * + * The `narrow` pins two frame-structural invariants at this boundary so + * every consumer can trust them without re-checking: source ids are + * unique, and the first element is the default source. The head-is-default + * rule is what keeps the two rotation paths in agreement -- a warm agent's + * `setSources` activates the matched default index, while a cold rebuild + * pins element 0 -- so they pick the same active source only when the + * default is the head. + */ +export const SourcesUpdatedData = type({ + sources: InferenceSource.array().atLeastLength(1), + defaultSource: "string > 0", +}).narrow((data, ctx) => { + const seen = new Set(); + for (const source of data.sources) { + if (seen.has(source.id)) { + return ctx.mustBe( + `a source list with unique ids; "${source.id}" appears more than once`, + ); + } + seen.add(source.id); + } + const head = data.sources[0]; + if (head === undefined || head.id !== data.defaultSource) { + return ctx.mustBe( + "a source list whose first element is the default source", + ); + } + return true; +}); + /** * Wire projection of an attachment on an outbound mail message. The * runtime `MessageAttachment.data` is raw bytes; the NDJSON control @@ -176,20 +214,7 @@ export const ControlPayload = type( }) .or({ type: "'sources-updated'", - data: { - /** - * The full ordered inference-source failover chain plus the default - * source id. Carried inline like grants-updated's snapshot: a - * single-producer, single-consumer supervisor->child push, so a - * substrate round-trip would only add latency. Constrained non-empty - * because a rotation always has at least one source; an empty list - * is a malformed frame rejected at the wire boundary. No per-source - * hash rides along -- sources are a flat list with no per-item pin - * for a receiver to cross-check, unlike the per-step grants snapshot. - */ - sources: InferenceSource.array().atLeastLength(1), - defaultSource: "string > 0", - }, + data: SourcesUpdatedData, }) .or({ type: "'ready'", diff --git a/packages/workflow-host/src/ipc/ipc.test.ts b/packages/workflow-host/src/ipc/ipc.test.ts index e8e07270..e11afa8b 100644 --- a/packages/workflow-host/src/ipc/ipc.test.ts +++ b/packages/workflow-host/src/ipc/ipc.test.ts @@ -1496,17 +1496,27 @@ describe("sources-updated payload validation", () => { expect(validated instanceof type.errors).toBe(true); }); - test("accepts a defaultSource that is not a member of sources", () => { - // Constraint ownership: the wire boundary does not enforce - // `defaultSource in sources`. The consumer (the agent's source - // registry, via indexOfDefault) owns that invariant and throws when - // it does not hold, so re-checking it here would duplicate the rule. + test("rejects a defaultSource that is not the head source", () => { + // The wire boundary owns the head-is-default invariant: the first + // element must be the default source, so the warm-swap and cold-build + // rotation paths agree on the active source. A default that is present + // but not first is still rejected here. + const second = { ...source, id: "secondary" }; const payload = { type: "sources-updated", - data: { sources: [source], defaultSource: "not-in-list" }, + data: { sources: [source, second], defaultSource: "secondary" }, }; const validated = ControlPayload(payload); - expect(validated instanceof type.errors).toBe(false); + expect(validated instanceof type.errors).toBe(true); + }); + + test("rejects duplicate source ids", () => { + const payload = { + type: "sources-updated", + data: { sources: [source, { ...source }], defaultSource: "primary" }, + }; + const validated = ControlPayload(payload); + expect(validated instanceof type.errors).toBe(true); }); test("rejects a source element missing a required field", () => { From 4ad92b6dc46825c9150a75af6c0d942a1ba09721 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Sun, 5 Jul 2026 09:59:12 -0500 Subject: [PATCH 058/101] Push an inference-source rotation to the child from the supervisor The supervisor gains a deliverSources method that sends a sources-updated control frame to the child, mirroring deliverSignal. Delivery is phase-guarded to starting/running: during a recycle the control sender still points at the dying child, so a frame would buffer behind the SIGTERM or write into a closed pipe and be lost. Rejecting instead surfaces the race to the caller, which can retry once the recycle completes. --- packages/workflow-host/src/index.ts | 1 + .../workflow-host/src/supervisor/index.ts | 1 + .../src/supervisor/supervisor.test.ts | 153 ++++++++++++++++++ .../src/supervisor/supervisor.ts | 43 ++++- 4 files changed, 197 insertions(+), 1 deletion(-) diff --git a/packages/workflow-host/src/index.ts b/packages/workflow-host/src/index.ts index bc3881a5..abfe6b7c 100644 --- a/packages/workflow-host/src/index.ts +++ b/packages/workflow-host/src/index.ts @@ -42,6 +42,7 @@ export { type CredentialsSnapshot, type CredentialsSnapshotStep, type DeliverSignalOpts, + type DeliverSourcesOpts, type DeriveMailAuditRef, type DeriveStepAddress, type DeriveStepRepoId, diff --git a/packages/workflow-host/src/supervisor/index.ts b/packages/workflow-host/src/supervisor/index.ts index ae20d0bf..0fe3b511 100644 --- a/packages/workflow-host/src/supervisor/index.ts +++ b/packages/workflow-host/src/supervisor/index.ts @@ -4,6 +4,7 @@ export { type CancelCommitInfo, type CancelRequestOpts, type DeliverSignalOpts, + type DeliverSourcesOpts, type DrainOpts, type RecycleOpts, type SpawnOpts, diff --git a/packages/workflow-host/src/supervisor/supervisor.test.ts b/packages/workflow-host/src/supervisor/supervisor.test.ts index 15db9552..c6113642 100644 --- a/packages/workflow-host/src/supervisor/supervisor.test.ts +++ b/packages/workflow-host/src/supervisor/supervisor.test.ts @@ -7,6 +7,7 @@ import { type } from "arktype"; import { generateKeyPair } from "@intx/crypto"; import { hexDecode, hexEncode } from "@intx/types"; +import type { InferenceSource } from "@intx/types/runtime"; import type { RepoId, RepoStore } from "@intx/hub-sessions"; import { @@ -62,6 +63,26 @@ function parseTriggerFireRunIds(lines: readonly string[]): string[] { return ids; } +function parseSourcesUpdatedFrames( + lines: readonly string[], +): { sources: InferenceSource[]; defaultSource: string }[] { + const out: { sources: InferenceSource[]; defaultSource: string }[] = []; + for (const line of lines) { + if (!line.includes("sources-updated")) continue; + const raw: unknown = JSON.parse(line); + const signed = SignedEnvelope(raw); + if (signed instanceof type.errors) continue; + const payload = ControlPayload(signed.envelope.payload); + if (payload instanceof type.errors) continue; + if (payload.type !== "sources-updated") continue; + out.push({ + sources: payload.data.sources, + defaultSource: payload.data.defaultSource, + }); + } + return out; +} + const CancelRequestedBlob = type({ type: "string", seq: "number", @@ -1726,6 +1747,138 @@ describe("createWorkflowSupervisor", () => { }), ).rejects.toThrow(/deliverSignal called in phase idle/); }); + + test("deliverSources() rejects when the supervisor is idle (no spawn has run)", async () => { + // Same phase-guard contract as deliverSignal: a sources rotation + // landing against a supervisor that is not starting/running throws so + // the sidecar router's rejection surfaces to the hub-link rather than + // writing into a dead child's pipe. + const baseDir = await makeTempDir("supervisor-deliver-sources-idle-"); + const bindings = await buildBindings({ + baseDir, + spawner: () => { + throw new Error("spawner must not be invoked on the idle sources path"); + }, + signSpy: () => ({ + sig: new Uint8Array(64), + principalKind: "supervisor", + }), + mailBus: createMockMailBus(), + }); + const supervisor = createWorkflowSupervisor(bindings); + await expect( + supervisor.deliverSources({ + sources: [ + { + id: "primary", + provider: "anthropic", + baseURL: "https://api.anthropic.com", + apiKey: "sk-x", + model: "claude-test", + }, + ], + defaultSource: "primary", + }), + ).rejects.toThrow(/deliverSources called in phase idle/); + }); + + test("deliverSources() sends a sources-updated frame when running", async () => { + const baseDir = await makeTempDir("supervisor-deliver-sources-running-"); + await seedStepGrants( + baseDir, + defaultStepRepoId({ deploymentId: "deployment-x", stepId: "step-1" }), + [{ resource: "thing", action: "read" }], + ); + + const supervisorIpcKeyPair = await generateKeyPair(); + const childIpcKeyPair = await generateKeyPair(); + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const eventChildToSupervisor = createMemoryFrameStream(); + let resolveExit: ((code: number) => void) | undefined; + const exited = new Promise((resolve) => { + resolveExit = resolve; + }); + + let observedEnv: Record | undefined; + const spawner: SubprocessSpawner = ({ env }) => { + observedEnv = env; + const handle: SubprocessHandle = { + pid: 4321, + controlWriter: supervisorToChild.writer, + controlReader: childToSupervisor.reader, + eventReader: eventChildToSupervisor.reader, + kill: () => { + childToSupervisor.close(); + eventChildToSupervisor.close(); + resolveExit?.(0); + }, + exited, + }; + return handle; + }; + + const baseBindings = await buildBindings({ + baseDir, + spawner, + signSpy: () => ({ sig: new Uint8Array(64), principalKind: "supervisor" }), + mailBus: createMockMailBus(), + }); + const bindings: WorkflowSupervisorBindings = { + ...baseBindings, + ipcKeyPairFactory: () => Promise.resolve(supervisorIpcKeyPair), + }; + const supervisor = createWorkflowSupervisor(bindings); + + const spawnPromise = supervisor.spawn({ + stepOrder: ["step-1"], + definitionHash: "def-hash-abc", + warmKeep: true, + onInferenceEvent: () => undefined, + }); + while (observedEnv === undefined) { + await new Promise((r) => setTimeout(r, 1)); + } + const channelId = observedEnv.IPC_CHANNEL_ID; + if (channelId === undefined) { + throw new Error("IPC_CHANNEL_ID not set in spawn-time env"); + } + const childSender = createControlChannelSender({ + privateKeySeed: childIpcKeyPair.privateKey, + channelId, + writer: { + write(line: string) { + childToSupervisor.inject(line); + }, + }, + }); + await childSender.send({ + type: "ready", + data: { + childPid: 4321, + childPublicKey: Buffer.from(childIpcKeyPair.publicKey).toString("hex"), + }, + }); + await spawnPromise; + + const sources: InferenceSource[] = [ + { + id: "primary", + provider: "anthropic", + baseURL: "https://api.anthropic.com", + apiKey: "sk-primary", + model: "claude-test", + }, + ]; + await supervisor.deliverSources({ sources, defaultSource: "primary" }); + + const frames = parseSourcesUpdatedFrames(supervisorToChild.flushed()); + expect(frames).toHaveLength(1); + expect(frames[0]?.sources).toEqual(sources); + expect(frames[0]?.defaultSource).toBe("primary"); + + await supervisor.shutdown(); + }); }); describe("assembleCredentialsSnapshot", () => { diff --git a/packages/workflow-host/src/supervisor/supervisor.ts b/packages/workflow-host/src/supervisor/supervisor.ts index f0a34567..736f5800 100644 --- a/packages/workflow-host/src/supervisor/supervisor.ts +++ b/packages/workflow-host/src/supervisor/supervisor.ts @@ -64,7 +64,7 @@ import { } from "@intx/hub-sessions/substrate"; import { base64Decode, base64Encode, hexEncode } from "@intx/types"; import { RepoId } from "@intx/types/sidecar"; -import type { OutboundMessage } from "@intx/types/runtime"; +import type { InferenceSource, OutboundMessage } from "@intx/types/runtime"; import type { CancelOrigin } from "@intx/workflow"; import { @@ -213,6 +213,14 @@ export interface WorkflowSupervisor { * for serializing delivery against `spawn` completion. */ deliverSignal(opts: DeliverSignalOpts): Promise; + /** + * Push a rotated inference-source list to the child's warm single-step + * agent. Mirrors `deliverSignal`: the supervisor is the single producer + * of `sources-updated` control frames, and delivery is phase-guarded to + * starting/running so a frame is never written into a recycling child's + * closing pipe. Throws otherwise. + */ + deliverSources(opts: DeliverSourcesOpts): Promise; /** * Current snapshot of the credentials pushed to the child. Surfaced * so the host can audit the per-step contentHash without @@ -293,6 +301,17 @@ export type DeliverSignalOpts = { payload: unknown; }; +export type DeliverSourcesOpts = { + /** + * The rotated ordered inference-source failover chain; element 0 is the + * active source. The wire boundary enforces a non-empty list with unique + * ids whose head is the default. + */ + sources: InferenceSource[]; + /** The default source id; the wire boundary requires it to equal `sources[0].id`. */ + defaultSource: string; +}; + export type RecycleOpts = { reason: string; /** @@ -2372,6 +2391,27 @@ export function createWorkflowSupervisor( }); } + async function deliverSources(opts: DeliverSourcesOpts): Promise { + // The supervisor is the single producer of `sources-updated` control + // frames. `recycling` is rejected for the same reason as + // `deliverSignal`: `state.controlSender` still points at the dying + // child, so a frame would either buffer behind the SIGTERM or write + // into a closed pipe and be lost. Rejecting surfaces the race so the + // caller can retry once the recycle completes. + if (state.phase !== "running" && state.phase !== "starting") { + throw new Error( + `supervisor: deliverSources called in phase ${state.phase}; expected starting/running`, + ); + } + await state.controlSender.send({ + type: "sources-updated", + data: { + sources: opts.sources, + defaultSource: opts.defaultSource, + }, + }); + } + function getCredentialsSnapshot(): CredentialsSnapshot | null { if (state.phase === "starting" || state.phase === "running") { return state.credentialsSnapshot; @@ -2386,6 +2426,7 @@ export function createWorkflowSupervisor( drain, recycle, deliverSignal, + deliverSources, getCredentialsSnapshot, }; } From 242d0ad351c3bda7e399d198eac2f69bbf698dd2 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Sun, 5 Jul 2026 10:23:56 -0500 Subject: [PATCH 059/101] Route inbound source rotations to single-step deployments Add a MultistepSourcesRouter: a per-deployment-address registry that maps a sources rotation onto the supervisor hosting the target agent. Only a single-step warm deployment registers a handler once its supervisor spawns, and the handler forwards the rotation into the supervisor's deliverSources. A multi-step deployment has no single warm agent to rotate, so it registers none and tryRoute reports its address as unrouted. The registry is constructed at the sidecar boot edge and injected into the deploy router alongside the mail, signal, and drain routers it mirrors. --- apps/sidecar/src/index.ts | 11 +++ apps/sidecar/src/workflow-host-wiring.test.ts | 74 +++++++++++++++++++ apps/sidecar/src/workflow-host-wiring.ts | 39 ++++++++++ .../src/workflow-run-pack-client.test.ts | 54 ++++++++++++++ apps/sidecar/src/workflow-run-pack-client.ts | 61 +++++++++++++++ 5 files changed, 239 insertions(+) diff --git a/apps/sidecar/src/index.ts b/apps/sidecar/src/index.ts index 99a7a105..0b3d1703 100644 --- a/apps/sidecar/src/index.ts +++ b/apps/sidecar/src/index.ts @@ -39,6 +39,7 @@ import { createMultistepDrainRouter, createMultistepMailRouter, createMultistepSignalRouter, + createMultistepSourcesRouter, createWorkflowRunPackClient, createWorkflowRunPackPushingRepoStore, } from "./workflow-run-pack-client"; @@ -248,6 +249,15 @@ const multistepSignalRouter = createMultistepSignalRouter(); // workflow-run repo when the deadline expires. const multistepDrainRouter = createMultistepDrainRouter(); +// Per-deployment-address sources-rotation handler registry. Only a +// single-step warm deployment registers a handler once its supervisor +// spawns; the handler forwards the rotated list into the supervisor's +// `deliverSources`, which sends a `sources-updated` control IPC frame to +// the child, where the warm agent's live sources are swapped in place. A +// multi-step deployment registers none, so a rotation resolved against +// its address is unrouted. +const multistepSourcesRouter = createMultistepSourcesRouter(); + const transport = createInMemoryTransport(); // The pack-push client closes over the substrate (for `createPack`) @@ -393,6 +403,7 @@ const orchestrator = createSidecarOrchestrator({ multistepMailRouter, multistepSignalRouter, multistepDrainRouter, + multistepSourcesRouter, multistepSubstrateEnv, publishWorkflowInferenceEvent, ...(onDispatchTiming !== undefined ? { onDispatchTiming } : {}), diff --git a/apps/sidecar/src/workflow-host-wiring.test.ts b/apps/sidecar/src/workflow-host-wiring.test.ts index ef0d9401..18285632 100644 --- a/apps/sidecar/src/workflow-host-wiring.test.ts +++ b/apps/sidecar/src/workflow-host-wiring.test.ts @@ -28,7 +28,9 @@ import { } from "./workflow-host-wiring"; import { createMultistepMailRouter, + createMultistepSourcesRouter, type MultistepMailRouter, + type MultistepSourcesRouter, } from "./workflow-run-pack-client"; import { writeWorkflowDeploymentRecord, @@ -519,6 +521,7 @@ describe("createSidecarDeployRouter multi-step branch", () => { multistepBinaryPath?: string; multistepSubstrateEnv?: Record; multistepMailRouter?: MultistepMailRouter; + multistepSourcesRouter?: MultistepSourcesRouter; registerDeployment?: (args: { deploymentId: string; agentAddress: string; @@ -617,6 +620,9 @@ describe("createSidecarDeployRouter multi-step branch", () => { ...(opts.multistepMailRouter !== undefined ? { multistepMailRouter: opts.multistepMailRouter } : {}), + ...(opts.multistepSourcesRouter !== undefined + ? { multistepSourcesRouter: opts.multistepSourcesRouter } + : {}), ...(opts.readyTimeoutMs !== undefined ? { readyTimeoutMs: opts.readyTimeoutMs } : {}), @@ -1939,6 +1945,74 @@ describe("createSidecarDeployRouter multi-step branch", () => { ); }); + test("registers a sources-rotation handler for a single-step deployment", async () => { + const sourcesRouter = createMultistepSourcesRouter(); + const spawner = makeReadyDrivingSpawner(10600); + const { router } = await buildMultistepFixture({ + spawner: spawner.spawner, + multistepSourcesRouter: sourcesRouter, + multistepSubstrateEnv: { + SIDECAR_DATA_DIR: await createTempBaseDir("sidecar-sources-single-"), + }, + }); + + const deployPromise = router.deploy( + singleStepFrame("ins_srcsingle@example.com", "wf-srcsingle"), + ); + await spawner.driveReadyFor(0); + await deployPromise; + + // The single-step deploy registered a rotation handler, so an inbound + // sources.update for its address routes. + expect( + await sourcesRouter.tryRoute({ + type: "sources.update", + agentAddress: "ins_srcsingle@example.com", + sources: [makeInferenceSource("primary")], + defaultSource: "primary", + }), + ).toBe(true); + }); + + test("does not register a sources-rotation handler for a multi-step deployment", async () => { + const sourcesRouter = createMultistepSourcesRouter(); + const spawner = makeReadyDrivingSpawner(10700); + const { router } = await buildMultistepFixture({ + spawner: spawner.spawner, + multistepSourcesRouter: sourcesRouter, + multistepSubstrateEnv: { + SIDECAR_DATA_DIR: await createTempBaseDir("sidecar-sources-multi-"), + }, + }); + + const frame = makeMultistepFrame({ + definition: { + id: "wf-srcmulti", + triggers: [{ type: "manual" }], + stepOrder: ["step-1", "step-2"], + steps: { "step-1": { kind: "step" }, "step-2": { kind: "step" } }, + }, + sources: { + "step-1": [makeInferenceSource("step-1")], + "step-2": [makeInferenceSource("step-2")], + }, + }); + const deployPromise = router.deploy(frame); + await spawner.driveReadyFor(0); + await deployPromise; + + // A multi-step deployment has no single warm agent to rotate, so no + // handler is registered and the inbound rotation is unrouted. + expect( + await sourcesRouter.tryRoute({ + type: "sources.update", + agentAddress: frame.agentAddress, + sources: [makeInferenceSource("primary")], + defaultSource: "primary", + }), + ).toBe(false); + }); + test("two addresses whose deriveDeploymentId slugs collide are rejected at the second deploy", async () => { // deriveDeploymentId substitutes every disallowed character with // `-`, so two distinct addresses can collapse to the same slug. The slug diff --git a/apps/sidecar/src/workflow-host-wiring.ts b/apps/sidecar/src/workflow-host-wiring.ts index 1cf858c6..a787f6c2 100644 --- a/apps/sidecar/src/workflow-host-wiring.ts +++ b/apps/sidecar/src/workflow-host-wiring.ts @@ -64,6 +64,7 @@ import type { MultistepDrainRouter, MultistepMailRouter, MultistepSignalRouter, + MultistepSourcesRouter, } from "./workflow-run-pack-client"; import { deleteWorkflowDeploymentRecord, @@ -831,6 +832,20 @@ export function createSidecarDeployRouter(deps: { * the wiring is plumbed. */ multistepDrainRouter?: MultistepDrainRouter; + /** + * Per-deployment-address sources-rotation handler registry. Only a + * single-step warm deployment registers a handler (against the + * deployment's mail address once `supervisor.spawn` succeeds) so a + * rotation resolved for its address flows into + * `wired.supervisor.deliverSources` and on to the child's warm agent. A + * multi-step deployment registers none -- it has no single warm agent to + * rotate -- so `tryRoute` reports its address as unrouted. + * + * Optional so tests that exercise deploys without a rotation loop can + * omit the binding; an absent registry means no rotation handler is + * installed for any deployment. + */ + multistepSourcesRouter?: MultistepSourcesRouter; /** * Optional per-message dispatch-timing observer the multi-step branch * forwards to each supervisor it constructs. Resolved at the sidecar @@ -1275,6 +1290,22 @@ export function createSidecarDeployRouter(deps: { deps.multistepDrainRouter?.register(spec.agentAddress, async (args) => { await wired.supervisor.drain({ deadlineMs: args.deadlineMs }); }); + // Register the sources-rotation handler ONLY for a single-step warm + // deployment: it has one long-lived agent whose sources can be + // swapped in place. A multi-step deployment has no single warm agent, + // so it registers no handler and `tryRoute` reports its address as + // unrouted. + if (warmKeep) { + deps.multistepSourcesRouter?.register( + spec.agentAddress, + async (args) => { + await wired.supervisor.deliverSources({ + sources: args.sources, + defaultSource: args.defaultSource, + }); + }, + ); + } routersRegistered = true; // Resolve the ack public key BEFORE registering the deployment @@ -1306,6 +1337,11 @@ export function createSidecarDeployRouter(deps: { deps.multistepMailRouter?.unregister(spec.agentAddress); deps.multistepSignalRouter?.unregister(spec.agentAddress); deps.multistepDrainRouter?.unregister(spec.agentAddress); + // Unregister unconditionally: the sources handler was registered + // only for a single-step deploy, but `unregister` is a no-op for + // an address that never registered one, so a multi-step unwind + // safely calls it too. + deps.multistepSourcesRouter?.unregister(spec.agentAddress); } if (supervisorRegistered) { activeSupervisors.delete(spec.agentAddress); @@ -1565,6 +1601,9 @@ export function createSidecarDeployRouter(deps: { deps.multistepMailRouter?.unregister(frame.agentAddress); deps.multistepSignalRouter?.unregister(frame.agentAddress); deps.multistepDrainRouter?.unregister(frame.agentAddress); + // Unregister unconditionally (a no-op for a multi-step address that + // registered no sources handler), matching the sibling routers. + deps.multistepSourcesRouter?.unregister(frame.agentAddress); // Shut the per-deployment supervisor down so the workflow-process // child, its IPC pipes, and its event-channel fd are released. // The supervisor's `shutdown()` is idempotent (returns early when diff --git a/apps/sidecar/src/workflow-run-pack-client.test.ts b/apps/sidecar/src/workflow-run-pack-client.test.ts index eac38f09..54872a84 100644 --- a/apps/sidecar/src/workflow-run-pack-client.test.ts +++ b/apps/sidecar/src/workflow-run-pack-client.test.ts @@ -1,5 +1,6 @@ import { describe, test, expect } from "bun:test"; +import type { InferenceSource } from "@intx/types/runtime"; import type { RepoId, RepoStore } from "@intx/hub-sessions"; import { @@ -7,6 +8,7 @@ import { createMultistepDrainRouter, createMultistepMailRouter, createMultistepSignalRouter, + createMultistepSourcesRouter, createWorkflowRunPackClient, createWorkflowRunPackPushingRepoStore, } from "./workflow-run-pack-client"; @@ -449,6 +451,58 @@ describe("createMultistepMailRouter", () => { }); }); +describe("createMultistepSourcesRouter", () => { + const source: InferenceSource = { + id: "primary", + provider: "anthropic", + baseURL: "https://api.anthropic.com", + apiKey: "sk-x", + model: "claude-test", + }; + const frame = { + type: "sources.update" as const, + agentAddress: "dep@integration.interchange", + sources: [source], + defaultSource: "primary", + }; + + test("tryRoute returns false when no handler is registered", async () => { + const router = createMultistepSourcesRouter(); + expect(await router.tryRoute(frame)).toBe(false); + }); + + test("a registered handler receives the rotation and tryRoute returns true", async () => { + const router = createMultistepSourcesRouter(); + const received: { sources: InferenceSource[]; defaultSource: string }[] = + []; + router.register("dep@integration.interchange", async (args) => { + received.push(args); + }); + expect(await router.tryRoute(frame)).toBe(true); + expect(received).toHaveLength(1); + expect(received[0]?.sources).toEqual([source]); + expect(received[0]?.defaultSource).toBe("primary"); + }); + + test("registration is per-address; an unrelated address falls through", async () => { + const router = createMultistepSourcesRouter(); + router.register("dep-a@integration.interchange", async () => undefined); + expect( + await router.tryRoute({ + ...frame, + agentAddress: "dep-b@integration.interchange", + }), + ).toBe(false); + }); + + test("unregister removes the handler", async () => { + const router = createMultistepSourcesRouter(); + router.register("dep@integration.interchange", async () => undefined); + router.unregister("dep@integration.interchange"); + expect(await router.tryRoute(frame)).toBe(false); + }); +}); + describe("createMultistepDrainRouter", () => { test("tryRoute resolves to false when no handler is registered", async () => { const router = createMultistepDrainRouter(); diff --git a/apps/sidecar/src/workflow-run-pack-client.ts b/apps/sidecar/src/workflow-run-pack-client.ts index ba109065..b9b8a17b 100644 --- a/apps/sidecar/src/workflow-run-pack-client.ts +++ b/apps/sidecar/src/workflow-run-pack-client.ts @@ -14,6 +14,7 @@ // transferId is minted inside `HubLink.pushWorkflowRunPack`. import { getLogger } from "@intx/log"; +import type { InferenceSource } from "@intx/types/runtime"; import type { RepoId, RepoStore, @@ -281,6 +282,66 @@ export function createMultistepDrainRouter(): MultistepDrainRouter { }; } +/** + * Per-deployment sources-rotation handler the deploy router installs + * against the `MultistepSourcesRouter` after a supervisor's `spawn` + * succeeds -- but ONLY for a single-step (warm launched-agent) + * deployment. The handler hands the rotated list off to the supervisor's + * `deliverSources`, which sends a `sources-updated` control IPC frame to + * the workflow-process child, where the warm agent's live sources are + * swapped in place. A multi-step deployment has no single warm agent to + * rotate, so the router registers no handler for it and an inbound + * `sources.update` for a multi-step address is unrouted. + */ +export type MultistepSourcesHandler = (args: { + sources: InferenceSource[]; + defaultSource: string; +}) => Promise; + +/** + * Per-deployment-address sources-rotation handler registry. Only a + * single-step warm deployment registers a handler (after + * `wired.supervisor.spawn` succeeds); the trivial and multi-step paths + * never do, so `tryRoute` resolves a rotation only for a registered + * single-step address and returns `false` for any other. + * + * The registry lives at the sidecar's host layer for the same boundary + * reason as the mail/signal/drain routers: the routing decision is a + * concrete sidecar host concern, and the workflow-host package stays + * agnostic to which transport surface its supervisor handle rides on. + */ +export type MultistepSourcesRouter = { + register(address: string, handler: MultistepSourcesHandler): void; + unregister(address: string): void; + tryRoute(frame: { + type: "sources.update"; + agentAddress: string; + sources: InferenceSource[]; + defaultSource: string; + }): Promise; +}; + +export function createMultistepSourcesRouter(): MultistepSourcesRouter { + const handlers = new Map(); + return { + register(address, handler) { + handlers.set(address, handler); + }, + unregister(address) { + handlers.delete(address); + }, + async tryRoute(frame) { + const handler = handlers.get(frame.agentAddress); + if (handler === undefined) return false; + await handler({ + sources: frame.sources, + defaultSource: frame.defaultSource, + }); + return true; + }, + }; +} + /** * Boot-edge facade around the substrate-shaped `RepoStore`. Forwards * every method to the underlying store; intercepts the From 379af7977247f031a9f789bbb1e98696387f2dc2 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Sun, 5 Jul 2026 11:02:19 -0500 Subject: [PATCH 060/101] Answer inbound source rotations over the sidecar hub link The sidecar's `sources.update` receive handler was absent, so the hub's sendSourcesUpdate rejected on its request timeout. Rebuild it as a request/ack handler: route the frame through an injected SourcesInboundRouter and answer session.ack on accept, or session.error on an unrouted address, an invalid rotation, or a delivery failure. A missing answer would hang the hub's request, so once a frame reaches the handler every path replies. The sources router validates the rotation before dispatch, unlike the mail, signal, and drain routers. A bad list -- duplicate ids, or a default that is not the head -- would crash the child's control-channel receiver on its narrow, so rejecting at the router turns a producer bug into a session.error rather than a child crash paired with a lying ack. --- apps/sidecar/src/index.ts | 1 + .../src/workflow-run-pack-client.test.ts | 44 +++++ apps/sidecar/src/workflow-run-pack-client.ts | 22 +++ .../hub-agent/src/sidecar-orchestrator.ts | 12 ++ packages/hub-agent/src/ws/hub-link.test.ts | 178 ++++++++++++++++++ packages/hub-agent/src/ws/hub-link.ts | 75 ++++++++ .../src/hub-session-orchestrator.ts | 4 +- packages/types/src/sidecar.ts | 9 +- packages/workflow-host/src/index.ts | 1 + packages/workflow-host/src/ipc/index.ts | 1 + 10 files changed, 340 insertions(+), 7 deletions(-) diff --git a/apps/sidecar/src/index.ts b/apps/sidecar/src/index.ts index 0b3d1703..fd1466ed 100644 --- a/apps/sidecar/src/index.ts +++ b/apps/sidecar/src/index.ts @@ -367,6 +367,7 @@ const orchestrator = createSidecarOrchestrator({ mailInboundRouter: multistepMailRouter, signalInboundRouter: multistepSignalRouter, drainInboundRouter: multistepDrainRouter, + sourcesInboundRouter: multistepSourcesRouter, // The hub link calls this on every (re)connect to announce the workflow // deployments this sidecar hosts so the hub re-registers their routes. // `createDeployRouter` runs synchronously during construction (below), so diff --git a/apps/sidecar/src/workflow-run-pack-client.test.ts b/apps/sidecar/src/workflow-run-pack-client.test.ts index 54872a84..1b328cbf 100644 --- a/apps/sidecar/src/workflow-run-pack-client.test.ts +++ b/apps/sidecar/src/workflow-run-pack-client.test.ts @@ -501,6 +501,50 @@ describe("createMultistepSourcesRouter", () => { router.unregister("dep@integration.interchange"); expect(await router.tryRoute(frame)).toBe(false); }); + + test("rejects a rotation with duplicate ids for a registered address without dispatching", async () => { + const router = createMultistepSourcesRouter(); + let called = false; + router.register("dep@integration.interchange", async () => { + called = true; + }); + // Duplicate ids would crash the child's control-channel receiver on + // its narrow, so the router rejects before dispatch. + await expect( + router.tryRoute({ ...frame, sources: [source, { ...source }] }), + ).rejects.toThrow(/unique ids/); + expect(called).toBe(false); + }); + + test("rejects a rotation whose default is not the head source", async () => { + const router = createMultistepSourcesRouter(); + let called = false; + router.register("dep@integration.interchange", async () => { + called = true; + }); + const second = { ...source, id: "secondary" }; + await expect( + router.tryRoute({ + ...frame, + sources: [source, second], + defaultSource: "secondary", + }), + ).rejects.toThrow(/first element is the default/); + expect(called).toBe(false); + }); + + test("an invalid rotation for an unregistered address is unrouted, not rejected", async () => { + const router = createMultistepSourcesRouter(); + // Registration is checked before validation: an unregistered address + // reports `false` and its (here invalid) payload is never inspected. + expect( + await router.tryRoute({ + ...frame, + agentAddress: "unregistered@integration.interchange", + sources: [source, { ...source }], + }), + ).toBe(false); + }); }); describe("createMultistepDrainRouter", () => { diff --git a/apps/sidecar/src/workflow-run-pack-client.ts b/apps/sidecar/src/workflow-run-pack-client.ts index b9b8a17b..7216d435 100644 --- a/apps/sidecar/src/workflow-run-pack-client.ts +++ b/apps/sidecar/src/workflow-run-pack-client.ts @@ -13,7 +13,10 @@ // shape the supervisor uses for `writeTreePreservingPrefix`; the // transferId is minted inside `HubLink.pushWorkflowRunPack`. +import { type } from "arktype"; + import { getLogger } from "@intx/log"; +import { SourcesUpdatedData } from "@intx/workflow-host"; import type { InferenceSource } from "@intx/types/runtime"; import type { RepoId, @@ -332,7 +335,26 @@ export function createMultistepSourcesRouter(): MultistepSourcesRouter { }, async tryRoute(frame) { const handler = handlers.get(frame.agentAddress); + // Registration check first: an unregistered (multi-step or torn-down) + // address is unrouted -- reported as `false`, its payload never + // inspected, because it would not be acted on regardless. if (handler === undefined) return false; + // Validate the rotation BEFORE dispatch. This is the only inbound + // router that validates its frame, and deliberately so: a bad list + // (duplicate ids, or a default that is not the head element) would + // reach the child's control-channel receiver and crash it on + // `SourcesUpdatedData`'s narrow -- the sources-updated frame is the + // only inbound frame carrying a crash-on-invalid narrow downstream, + // and the only one that is request/ack. Rejecting here throws, and + // the hub-link turns the throw into a truthful `session.error` + // instead of acking and detonating the child. + const validated = SourcesUpdatedData({ + sources: frame.sources, + defaultSource: frame.defaultSource, + }); + if (validated instanceof type.errors) { + throw new Error(validated.summary); + } await handler({ sources: frame.sources, defaultSource: frame.defaultSource, diff --git a/packages/hub-agent/src/sidecar-orchestrator.ts b/packages/hub-agent/src/sidecar-orchestrator.ts index 99923d06..33d5b554 100644 --- a/packages/hub-agent/src/sidecar-orchestrator.ts +++ b/packages/hub-agent/src/sidecar-orchestrator.ts @@ -25,6 +25,7 @@ import { type MailInboundRouter, type SignalInboundRouter, type DrainInboundRouter, + type SourcesInboundRouter, type ReconnectScheduler, } from "./ws/hub-link"; @@ -107,6 +108,15 @@ export type SidecarOrchestratorConfig = { * orchestrator forwards the binding unchanged to `createHubLink`. */ drainInboundRouter?: DrainInboundRouter; + /** + * Optional inbound sources-rotation dispatcher the link consults on + * every inbound `sources.update` frame. Production wires this against + * the sidecar's single-step deployment sources handler registry so a + * deployment-address rotation flows into the supervisor's + * `deliverSources`. The orchestrator forwards the binding unchanged to + * `createHubLink`. + */ + sourcesInboundRouter?: SourcesInboundRouter; /** * Returns the workflow-substrate deployment addresses this sidecar * currently hosts. Forwarded to the hub link, which announces them on @@ -146,6 +156,7 @@ export function createSidecarOrchestrator( mailInboundRouter, signalInboundRouter, drainInboundRouter, + sourcesInboundRouter, getWorkflowAddresses, pingIntervalMs, reconnectDelayMs, @@ -208,6 +219,7 @@ export function createSidecarOrchestrator( ...(mailInboundRouter !== undefined ? { mailInboundRouter } : {}), ...(signalInboundRouter !== undefined ? { signalInboundRouter } : {}), ...(drainInboundRouter !== undefined ? { drainInboundRouter } : {}), + ...(sourcesInboundRouter !== undefined ? { sourcesInboundRouter } : {}), ...(getWorkflowAddresses !== undefined ? { getWorkflowAddresses } : {}), ...(pingIntervalMs !== undefined ? { pingIntervalMs } : {}), ...(reconnectDelayMs !== undefined ? { reconnectDelayMs } : {}), diff --git a/packages/hub-agent/src/ws/hub-link.test.ts b/packages/hub-agent/src/ws/hub-link.test.ts index 40627e7f..d2907362 100644 --- a/packages/hub-agent/src/ws/hub-link.test.ts +++ b/packages/hub-agent/src/ws/hub-link.test.ts @@ -1014,6 +1014,184 @@ describe("sidecar↔hub integration", () => { } }); + const ROTATION_SOURCE = { + id: "primary", + provider: "anthropic", + baseURL: "https://api.anthropic.com", + apiKey: "sk-rotation", + model: "claude-rotation", + }; + + test("sourcesInboundRouter acks an inbound sources.update round-trip", async () => { + const transport = createInMemoryTransport(); + const sessions = createMockSessionManager(); + const deploymentAddress = "dep_sources-ack@integration.interchange"; + + const routed: { agentAddress: string }[] = []; + const sourcesInboundRouter = { + async tryRoute(frame: { agentAddress: string }): Promise { + routed.push({ agentAddress: frame.agentAddress }); + return true; + }, + }; + + const client = createHubLink({ + hubURL: `ws://localhost:${env.server.port}/ws`, + sidecarId: "sc-sources-ack", + token: "test-token", + transport, + sessions, + ...withTestDeployBindings(), + sourcesInboundRouter, + getWorkflowAddresses: () => [deploymentAddress], + }); + + client.connect(); + try { + await waitFor(() => + env.router.getRoutableAddresses().includes(deploymentAddress), + ); + // Resolves on the sidecar's session.ack. Without a reply this would + // await the full request timeout, so a prompt resolution is the proof + // the round-trip no longer hangs. + await env.router.sendSourcesUpdate( + deploymentAddress, + [ROTATION_SOURCE], + "primary", + ); + expect(routed).toHaveLength(1); + expect(routed[0]?.agentAddress).toBe(deploymentAddress); + } finally { + client.close(); + await waitFor( + () => !env.router.getConnectedSidecars().includes("sc-sources-ack"), + ); + } + }); + + test("an unrouted sources.update is answered with session.error", async () => { + const transport = createInMemoryTransport(); + const sessions = createMockSessionManager(); + const deploymentAddress = "dep_sources-unrouted@integration.interchange"; + + const sourcesInboundRouter = { + async tryRoute(): Promise { + return false; + }, + }; + + const client = createHubLink({ + hubURL: `ws://localhost:${env.server.port}/ws`, + sidecarId: "sc-sources-unrouted", + token: "test-token", + transport, + sessions, + ...withTestDeployBindings(), + sourcesInboundRouter, + getWorkflowAddresses: () => [deploymentAddress], + }); + + client.connect(); + try { + await waitFor(() => + env.router.getRoutableAddresses().includes(deploymentAddress), + ); + await expect( + env.router.sendSourcesUpdate( + deploymentAddress, + [ROTATION_SOURCE], + "primary", + ), + ).rejects.toThrow(/no deployment registered/); + } finally { + client.close(); + await waitFor( + () => + !env.router.getConnectedSidecars().includes("sc-sources-unrouted"), + ); + } + }); + + test("a rejected sources.update surfaces the reason as session.error", async () => { + const transport = createInMemoryTransport(); + const sessions = createMockSessionManager(); + const deploymentAddress = "dep_sources-reject@integration.interchange"; + + const sourcesInboundRouter = { + async tryRoute(): Promise { + throw new Error("supervisor is recycling"); + }, + }; + + const client = createHubLink({ + hubURL: `ws://localhost:${env.server.port}/ws`, + sidecarId: "sc-sources-reject", + token: "test-token", + transport, + sessions, + ...withTestDeployBindings(), + sourcesInboundRouter, + getWorkflowAddresses: () => [deploymentAddress], + }); + + client.connect(); + try { + await waitFor(() => + env.router.getRoutableAddresses().includes(deploymentAddress), + ); + await expect( + env.router.sendSourcesUpdate( + deploymentAddress, + [ROTATION_SOURCE], + "primary", + ), + ).rejects.toThrow(/supervisor is recycling/); + } finally { + client.close(); + await waitFor( + () => !env.router.getConnectedSidecars().includes("sc-sources-reject"), + ); + } + }); + + test("a sources.update with no router wired is answered with session.error", async () => { + const transport = createInMemoryTransport(); + const sessions = createMockSessionManager(); + const deploymentAddress = "dep_sources-norouter@integration.interchange"; + + const client = createHubLink({ + hubURL: `ws://localhost:${env.server.port}/ws`, + sidecarId: "sc-sources-norouter", + token: "test-token", + transport, + sessions, + ...withTestDeployBindings(), + // No sourcesInboundRouter: a request/ack frame must still be answered + // or the hub hangs on its request timeout. + getWorkflowAddresses: () => [deploymentAddress], + }); + + client.connect(); + try { + await waitFor(() => + env.router.getRoutableAddresses().includes(deploymentAddress), + ); + await expect( + env.router.sendSourcesUpdate( + deploymentAddress, + [ROTATION_SOURCE], + "primary", + ), + ).rejects.toThrow(/no sourcesInboundRouter/); + } finally { + client.close(); + await waitFor( + () => + !env.router.getConnectedSidecars().includes("sc-sources-norouter"), + ); + } + }); + // The hub-link's `pushWorkflowRunPack` retries the FIRST push to a // never-bootstrapped `(repoId, ref)` once on failure, absorbing the // hub-side `initRepo` CAS race. The retry guard is a Set keyed by diff --git a/packages/hub-agent/src/ws/hub-link.ts b/packages/hub-agent/src/ws/hub-link.ts index 177b4e6d..ef501ae6 100644 --- a/packages/hub-agent/src/ws/hub-link.ts +++ b/packages/hub-agent/src/ws/hub-link.ts @@ -24,6 +24,7 @@ import { type RepoId, type SignalDeliverFrame, type DrainDeliverFrame, + type SourcesUpdateFrame, type SyncRequestFrame, } from "@intx/types/sidecar"; import { createPackReceiver, createPackSender } from "@intx/pack-transport"; @@ -169,6 +170,28 @@ export interface DrainInboundRouter { tryRoute(frame: DrainDeliverFrame): Promise; } +/** + * Per-deployment-address sources-rotation registry the link consults on + * every inbound `sources.update` frame. Unlike signal/drain, `sources.update` + * is a REQUEST/ACK frame, so the link answers `session.ack` / `session.error` + * rather than logging and dropping -- a missing answer hangs the hub's + * request for its full timeout. + * + * The shape lives on hub-agent so the link does not import the sidecar + * host's wiring module, and so tests can substitute a stub. + */ +export interface SourcesInboundRouter { + /** + * Attempt to dispatch `frame` to the supervisor registered against + * `frame.agentAddress`. Resolves `true` when a handler accepted the + * rotation, `false` when no handler is registered (an unrouted address). + * Rejects when the handler is registered but the rotation is invalid or + * the supervisor's `deliverSources` throws; the link turns a rejection + * into a `session.error` carrying the reason. + */ + tryRoute(frame: SourcesUpdateFrame): Promise; +} + export type HubLinkConfig = { hubURL: string; sidecarId: string; @@ -219,6 +242,16 @@ export type HubLinkConfig = { * silent. */ drainInboundRouter?: DrainInboundRouter; + /** + * Optional inbound sources-rotation dispatcher. When present, the link + * routes every inbound `sources.update` frame through this router and + * answers the request/ack frame: `session.ack` when the router accepted + * the rotation, `session.error` when no deployment is registered, when + * the rotation is invalid, or when delivery throws. Absent means the + * link answers `session.error` for every rotation -- required because a + * request/ack frame with no reply hangs the hub's request. + */ + sourcesInboundRouter?: SourcesInboundRouter; /** * Returns the workflow-substrate deployment addresses this sidecar * currently hosts a live supervisor for. Called on every (re)connect to @@ -271,6 +304,7 @@ export function createHubLink(config: HubLinkConfig): HubLink { mailInboundRouter, signalInboundRouter, drainInboundRouter, + sourcesInboundRouter, getWorkflowAddresses = () => [], pingIntervalMs = DEFAULT_PING_INTERVAL_MS, reconnectDelayMs = DEFAULT_RECONNECT_DELAY_MS, @@ -707,6 +741,44 @@ export function createHubLink(config: HubLinkConfig): HubLink { } } + async function handleSourcesUpdate(frame: SourcesUpdateFrame): Promise { + // `sources.update` is request/ack (the hub awaits a reply within its + // request timeout), so every path answers `session.ack` or + // `session.error` -- unlike the fire-and-forget signal/drain frames + // that log and drop. A missing router still answers, or the hub hangs. + if (sourcesInboundRouter === undefined) { + send({ + type: "session.error", + requestId: frame.requestId, + error: "no sourcesInboundRouter is wired", + }); + return; + } + try { + const routed = await sourcesInboundRouter.tryRoute(frame); + if (routed) { + send({ type: "session.ack", requestId: frame.requestId }); + } else { + send({ + type: "session.error", + requestId: frame.requestId, + error: `no deployment registered for ${frame.agentAddress}`, + }); + } + } catch (err) { + // A registered address whose rotation was rejected: an invalid list + // (the router validates before dispatch) or the supervisor's + // `deliverSources` throwing (e.g. a recycling phase). The reason + // rides back verbatim so the hub sees why the rotation failed. + const msg = err instanceof Error ? err.message : String(err); + send({ + type: "session.error", + requestId: frame.requestId, + error: msg, + }); + } + } + async function pushWorkflowRunPack(opts: { agentAddress: string; repoId: RepoId; @@ -856,6 +928,9 @@ export function createHubLink(config: HubLinkConfig): HubLink { case "drain.deliver": await handleDrainDeliver(frame); break; + case "sources.update": + await handleSourcesUpdate(frame); + break; case "repo.pack.ack": handlePackAck(frame); break; diff --git a/packages/hub-sessions/src/hub-session-orchestrator.ts b/packages/hub-sessions/src/hub-session-orchestrator.ts index fcff263f..6349a8fb 100644 --- a/packages/hub-sessions/src/hub-session-orchestrator.ts +++ b/packages/hub-sessions/src/hub-session-orchestrator.ts @@ -140,9 +140,7 @@ export function createHubSessionOrchestrator( // A supervised deployment carries its grants and sources in the // deploy pack and refreshes them over the supervisor's IPC // credentials snapshot at spawn and recycle, so reconnect does not - // push them over the wire. The legacy grants.update / sources.update - // frames are no longer handled sidecar-side; pushing them here would - // hang on the sidecar's dropped frame and reject the reconnect. + // re-push them over the wire. const now = new Date(); if (instance.status !== "running") { diff --git a/packages/types/src/sidecar.ts b/packages/types/src/sidecar.ts index 302264b0..42c39327 100644 --- a/packages/types/src/sidecar.ts +++ b/packages/types/src/sidecar.ts @@ -395,10 +395,11 @@ export const GrantsUpdateFrame = type({ export type GrantsUpdateFrame = typeof GrantsUpdateFrame.infer; /** - * Push updated inference sources to a running agent. The sidecar hot-swaps - * the active source on the harness (selected by id from `defaultSource`) - * and re-persists the agent config. Responds with session.ack or - * session.error. + * Push an updated inference-source list to a running single-step + * deployment. The sidecar routes it to the deployment's supervisor, which + * delivers it to the warm agent and swaps its sources in place; element 0 + * of `sources` is the active source and must equal `defaultSource`. + * Responds with session.ack or session.error. */ export const SourcesUpdateFrame = type({ type: "'sources.update'", diff --git a/packages/workflow-host/src/index.ts b/packages/workflow-host/src/index.ts index abfe6b7c..b3968312 100644 --- a/packages/workflow-host/src/index.ts +++ b/packages/workflow-host/src/index.ts @@ -94,6 +94,7 @@ export { EventPayload, FrameEnvelope, IPC_CRYPTO, + SourcesUpdatedData, MacedEnvelope, OutboundAttachmentPayload, OutboundMessagePayload, diff --git a/packages/workflow-host/src/ipc/index.ts b/packages/workflow-host/src/ipc/index.ts index c030af34..ddc7bcf0 100644 --- a/packages/workflow-host/src/ipc/index.ts +++ b/packages/workflow-host/src/ipc/index.ts @@ -142,6 +142,7 @@ export { ControlPayload, OutboundAttachmentPayload, OutboundMessagePayload, + SourcesUpdatedData, createControlChannelSender, receiveControlChannel, type ControlChannelSender, From a47aef363a927254b2ade95f272acb6b072d701c Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Sun, 5 Jul 2026 12:20:49 -0500 Subject: [PATCH 061/101] Answer a malformed request frame instead of dropping it silently A structurally-malformed inbound request/ack control frame -- one that fails the top-level HubFrame parse -- was logged and dropped with no reply, so the hub's request hung to its full timeout. Recover the frame's correlation key, which usually survives a malformed nested field, and answer with the matching error: the requestId-correlated frames (sources.update, grants.update, session.abort) with a session.error, and the agentAddress-correlated frames (agent.deploy, agent.undeploy) with an agent.error. A frame with no recoverable key, or a fire-and-forget frame, stays logged and dropped. The recovery lives in the shared dispatch layer, ahead of the per-frame handlers, so it covers both control-frame families in one place. The chunked repo.pack streaming transfers are a separate request/ack path, not handled here. --- packages/hub-agent/src/ws/hub-link.test.ts | 117 +++++++++++++++++++++ packages/hub-agent/src/ws/hub-link.ts | 103 ++++++++++++++++++ 2 files changed, 220 insertions(+) diff --git a/packages/hub-agent/src/ws/hub-link.test.ts b/packages/hub-agent/src/ws/hub-link.test.ts index d2907362..ef8f7bc2 100644 --- a/packages/hub-agent/src/ws/hub-link.test.ts +++ b/packages/hub-agent/src/ws/hub-link.test.ts @@ -13,10 +13,12 @@ import { base64Encode, hexEncode } from "@intx/types"; import type { HarnessConfig } from "@intx/types/runtime"; import { + answerMalformedRequestFrame, createHubLink, type DeployRouter, type ReconnectScheduler, } from "./hub-link"; +import type { AgentErrorFrame, SessionErrorFrame } from "@intx/types/sidecar"; import type { AgentKeyStore } from "../agent-key-store"; import type { SessionManager } from "../session-manager"; @@ -1424,3 +1426,118 @@ describe("register frame on connect", () => { } }); }); + +describe("answerMalformedRequestFrame", () => { + test("answers a malformed sources.update with session.error carrying the requestId", () => { + const sent: (SessionErrorFrame | AgentErrorFrame)[] = []; + // A structurally-invalid sources list (empty source object) that failed + // the top-level parse but kept its type + requestId. + const answered = answerMalformedRequestFrame( + { + type: "sources.update", + requestId: "req-1", + agentAddress: "ins_x@example.com", + sources: [{}], + defaultSource: "x", + }, + "sources[0].id must be a string", + (frame) => sent.push(frame), + ); + expect(answered).toBe(true); + expect(sent).toHaveLength(1); + expect(sent[0]).toMatchObject({ + type: "session.error", + requestId: "req-1", + }); + expect(sent[0]?.error).toMatch(/malformed sources.update frame/); + }); + + test("answers a malformed agent.deploy with agent.error carrying the agentAddress", () => { + const sent: (SessionErrorFrame | AgentErrorFrame)[] = []; + const answered = answerMalformedRequestFrame( + { + type: "agent.deploy", + agentAddress: "ins_deploy@example.com", + agentId: "x", + }, + "config must be an object", + (frame) => sent.push(frame), + ); + expect(answered).toBe(true); + expect(sent).toHaveLength(1); + expect(sent[0]).toMatchObject({ + type: "agent.error", + agentAddress: "ins_deploy@example.com", + }); + expect(sent[0]?.error).toMatch(/malformed agent.deploy frame/); + }); + + test("drops the retired requestId-correlated frames the sidecar no longer dispatches", () => { + // grants.update / session.abort have hub senders but no sidecar + // dispatch handler (retired) and no production caller, so a malformed + // one is dropped just like a well-formed one -- the answerable set + // covers only frames the sidecar actually handles. + for (const frameType of ["grants.update", "session.abort"]) { + const sent: (SessionErrorFrame | AgentErrorFrame)[] = []; + const answered = answerMalformedRequestFrame( + { + type: frameType, + requestId: "req-2", + agentAddress: "ins_x@example.com", + }, + "payload must be valid", + (frame) => sent.push(frame), + ); + expect(answered).toBe(false); + expect(sent).toHaveLength(0); + } + }); + + test("answers a malformed agent.undeploy with agent.error", () => { + const sent: (SessionErrorFrame | AgentErrorFrame)[] = []; + const answered = answerMalformedRequestFrame( + { type: "agent.undeploy", agentAddress: "ins_undeploy@example.com" }, + "reason must be a string", + (frame) => sent.push(frame), + ); + expect(answered).toBe(true); + expect(sent[0]).toMatchObject({ + type: "agent.error", + agentAddress: "ins_undeploy@example.com", + }); + expect(sent[0]?.error).toMatch(/malformed agent.undeploy frame/); + }); + + test("does not answer a sources.update with no recoverable requestId", () => { + const sent: (SessionErrorFrame | AgentErrorFrame)[] = []; + const answered = answerMalformedRequestFrame( + { type: "sources.update", sources: [{}] }, + "bad", + (frame) => sent.push(frame), + ); + expect(answered).toBe(false); + expect(sent).toHaveLength(0); + }); + + test("does not answer a fire-and-forget frame even with a requestId present", () => { + const sent: (SessionErrorFrame | AgentErrorFrame)[] = []; + const answered = answerMalformedRequestFrame( + { type: "signal.deliver", agentAddress: "x", requestId: "r" }, + "bad", + (frame) => sent.push(frame), + ); + expect(answered).toBe(false); + expect(sent).toHaveLength(0); + }); + + test("does not answer a frame with no recognizable type", () => { + const sent: (SessionErrorFrame | AgentErrorFrame)[] = []; + const answered = answerMalformedRequestFrame( + { garbage: true }, + "bad", + (frame) => sent.push(frame), + ); + expect(answered).toBe(false); + expect(sent).toHaveLength(0); + }); +}); diff --git a/packages/hub-agent/src/ws/hub-link.ts b/packages/hub-agent/src/ws/hub-link.ts index ef501ae6..20c1e009 100644 --- a/packages/hub-agent/src/ws/hub-link.ts +++ b/packages/hub-agent/src/ws/hub-link.ts @@ -14,6 +14,8 @@ import { HubFrame, type SidecarFrame, type AgentDeployFrame, + type AgentErrorFrame, + type SessionErrorFrame, type AgentUndeployFrame, type ChallengeFrame, type ChallengeFailedFrame, @@ -46,6 +48,101 @@ export type SessionEventSink = ( const logger = getLogger(["interchange", "hub-agent", "ws"]); +/** + * Permissive envelope over a raw inbound frame that failed `HubFrame` + * validation. A malformed request/ack frame usually still carries an + * intact discriminator and correlation key -- the malformation is in a + * nested field -- so these top-level fields can be recovered to answer the + * requester. + */ +const MalformedRequestEnvelope = type({ + "type?": "string", + "requestId?": "string", + "agentAddress?": "string", +}); + +/** + * Inbound request/ack frames the sidecar dispatches that the hub + * correlates by `requestId`, whose failure reply is a `session.error`. + * Only `sources.update` qualifies. The hub also has `grants.update` / + * `session.abort` senders, but the sidecar has no dispatch handler for + * them (retired) and no production caller sends them, so a malformed one + * has no more of a requester to answer than a well-formed one -- both are + * dropped, keeping the answerable set aligned with what the sidecar + * actually handles. + */ +const SESSION_ERROR_REQUEST_TYPES: ReadonlySet = new Set([ + "sources.update", +]); + +/** + * Inbound request/ack frames the hub correlates by `agentAddress` and + * whose failure reply is an `agent.error` -- the frames the hub tracks in + * its per-address pending-deploy / pending-undeploy maps. + */ +const AGENT_ERROR_REQUEST_TYPES: ReadonlySet = new Set([ + "agent.deploy", + "agent.undeploy", +]); + +/** + * Answer a malformed inbound request/ack control frame with an error reply + * so the hub's request does not hang to its timeout. Two control-frame + * families answer through their correlation key: the `requestId`-correlated + * frame (sources.update) replies `session.error`; the + * `agentAddress`-correlated frames (agent.deploy, agent.undeploy) reply + * `agent.error`. The fire-and-forget frames + * (mail/signal/drain/...) have no requester waiting on a reply, so a + * malformed one is correctly left to be logged and dropped by the caller. + * + * The chunked `repo.pack` streaming transfers are also request/ack (keyed + * by `transferId`, rejected by `repo.pack.reject`) but are deliberately + * NOT handled here: a valid `repo.pack.reject` carries a structured + * `repoId` and a constrained reason that cannot be reliably synthesized + * from a frame malformed in those very fields, so recovering that path is + * separate work. + * + * Returns `true` when it answered; `false` when no correlation key is + * recoverable (an unknown/absent type, a fire-and-forget frame, or a + * request/ack frame whose key is itself missing) -- the caller then logs + * and drops, because there is nothing to answer. + */ +export function answerMalformedRequestFrame( + raw: unknown, + summary: string, + send: (frame: SessionErrorFrame | AgentErrorFrame) => void, +): boolean { + const envelope = MalformedRequestEnvelope(raw); + if (envelope instanceof type.errors) return false; + const frameType = envelope.type; + if (frameType === undefined) return false; + if ( + SESSION_ERROR_REQUEST_TYPES.has(frameType) && + envelope.requestId !== undefined && + envelope.requestId.length > 0 + ) { + send({ + type: "session.error", + requestId: envelope.requestId, + error: `malformed ${frameType} frame: ${summary}`, + }); + return true; + } + if ( + AGENT_ERROR_REQUEST_TYPES.has(frameType) && + envelope.agentAddress !== undefined && + envelope.agentAddress.length > 0 + ) { + send({ + type: "agent.error", + agentAddress: envelope.agentAddress, + error: `malformed ${frameType} frame: ${summary}`, + }); + return true; + } + return false; +} + const DEFAULT_PING_INTERVAL_MS = 30_000; const DEFAULT_RECONNECT_DELAY_MS = 3_000; @@ -862,6 +959,12 @@ export function createHubLink(config: HubLinkConfig): HubLink { } const validated = HubFrame(raw); if (validated instanceof type.errors) { + // A malformed request/ack frame must still be answered, or the hub's + // request hangs to its timeout. `sources.update` and `agent.deploy` + // usually keep an intact correlation key even when a nested field is + // malformed, so reply with the matching error frame; a fire-and-forget + // frame (or one with no recoverable key) is only logged and dropped. + answerMalformedRequestFrame(raw, validated.summary, send); logger.warn`Invalid hub frame: ${validated.summary}`; return; } From 3b2b10f1a50e8cf1c3a6deeddc84acb2b54580e0 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Sun, 5 Jul 2026 13:48:19 -0500 Subject: [PATCH 062/101] Reject a malformed pack frame instead of hanging its transfer The chunked repo.pack transfers (repo.pack.push, repo.pack.done) are the third request/ack family: a structurally-malformed one that fails the top-level parse was dropped with no reply, so the hub's pack transfer hung to its timeout -- the longest of any request frame. Recover the transferId, agentAddress, and repoId and answer with a repo.pack.reject whose reason is corrupt. When the repoId or the transferId is itself the malformed field, the frame stays dropped: a valid repo.pack.reject cannot be built without them. That residual case is a hub bug malforming the very fields used to address the reply, not a payload the receiver could act on. --- packages/hub-agent/src/ws/hub-link.test.ts | 104 ++++++++++++++++++--- packages/hub-agent/src/ws/hub-link.ts | 58 ++++++++++-- 2 files changed, 143 insertions(+), 19 deletions(-) diff --git a/packages/hub-agent/src/ws/hub-link.test.ts b/packages/hub-agent/src/ws/hub-link.test.ts index ef8f7bc2..c217de79 100644 --- a/packages/hub-agent/src/ws/hub-link.test.ts +++ b/packages/hub-agent/src/ws/hub-link.test.ts @@ -18,7 +18,11 @@ import { type DeployRouter, type ReconnectScheduler, } from "./hub-link"; -import type { AgentErrorFrame, SessionErrorFrame } from "@intx/types/sidecar"; +import type { + AgentErrorFrame, + PackRejectFrame, + SessionErrorFrame, +} from "@intx/types/sidecar"; import type { AgentKeyStore } from "../agent-key-store"; import type { SessionManager } from "../session-manager"; @@ -1429,7 +1433,7 @@ describe("register frame on connect", () => { describe("answerMalformedRequestFrame", () => { test("answers a malformed sources.update with session.error carrying the requestId", () => { - const sent: (SessionErrorFrame | AgentErrorFrame)[] = []; + const sent: (SessionErrorFrame | AgentErrorFrame | PackRejectFrame)[] = []; // A structurally-invalid sources list (empty source object) that failed // the top-level parse but kept its type + requestId. const answered = answerMalformedRequestFrame( @@ -1448,12 +1452,12 @@ describe("answerMalformedRequestFrame", () => { expect(sent[0]).toMatchObject({ type: "session.error", requestId: "req-1", + error: expect.stringMatching(/malformed sources.update frame/), }); - expect(sent[0]?.error).toMatch(/malformed sources.update frame/); }); test("answers a malformed agent.deploy with agent.error carrying the agentAddress", () => { - const sent: (SessionErrorFrame | AgentErrorFrame)[] = []; + const sent: (SessionErrorFrame | AgentErrorFrame | PackRejectFrame)[] = []; const answered = answerMalformedRequestFrame( { type: "agent.deploy", @@ -1468,8 +1472,8 @@ describe("answerMalformedRequestFrame", () => { expect(sent[0]).toMatchObject({ type: "agent.error", agentAddress: "ins_deploy@example.com", + error: expect.stringMatching(/malformed agent.deploy frame/), }); - expect(sent[0]?.error).toMatch(/malformed agent.deploy frame/); }); test("drops the retired requestId-correlated frames the sidecar no longer dispatches", () => { @@ -1478,7 +1482,8 @@ describe("answerMalformedRequestFrame", () => { // one is dropped just like a well-formed one -- the answerable set // covers only frames the sidecar actually handles. for (const frameType of ["grants.update", "session.abort"]) { - const sent: (SessionErrorFrame | AgentErrorFrame)[] = []; + const sent: (SessionErrorFrame | AgentErrorFrame | PackRejectFrame)[] = + []; const answered = answerMalformedRequestFrame( { type: frameType, @@ -1494,7 +1499,7 @@ describe("answerMalformedRequestFrame", () => { }); test("answers a malformed agent.undeploy with agent.error", () => { - const sent: (SessionErrorFrame | AgentErrorFrame)[] = []; + const sent: (SessionErrorFrame | AgentErrorFrame | PackRejectFrame)[] = []; const answered = answerMalformedRequestFrame( { type: "agent.undeploy", agentAddress: "ins_undeploy@example.com" }, "reason must be a string", @@ -1504,12 +1509,89 @@ describe("answerMalformedRequestFrame", () => { expect(sent[0]).toMatchObject({ type: "agent.error", agentAddress: "ins_undeploy@example.com", + error: expect.stringMatching(/malformed agent.undeploy frame/), + }); + }); + + test("answers a malformed repo.pack frame with repo.pack.reject on its transferId", () => { + for (const frameType of ["repo.pack.push", "repo.pack.done"]) { + const sent: (SessionErrorFrame | AgentErrorFrame | PackRejectFrame)[] = + []; + const answered = answerMalformedRequestFrame( + { + type: frameType, + agentAddress: "ins_pack@example.com", + repoId: { kind: "workflow-run", id: "dep-1" }, + transferId: "xfer-1", + seq: "not-a-number", + }, + "a nested field is malformed", + (frame) => sent.push(frame), + ); + expect(answered).toBe(true); + expect(sent[0]).toMatchObject({ + type: "repo.pack.reject", + transferId: "xfer-1", + reason: "corrupt", + }); + } + }); + + test("does not answer a repo.pack frame with no recoverable transferId", () => { + const sent: (SessionErrorFrame | AgentErrorFrame | PackRejectFrame)[] = []; + const answered = answerMalformedRequestFrame( + { + type: "repo.pack.push", + agentAddress: "ins_pack@example.com", + repoId: { kind: "workflow-run", id: "dep-1" }, + }, + "bad", + (frame) => sent.push(frame), + ); + expect(answered).toBe(false); + expect(sent).toHaveLength(0); + }); + + test("does not answer a repo.pack frame whose repoId is itself malformed", () => { + const sent: (SessionErrorFrame | AgentErrorFrame | PackRejectFrame)[] = []; + const answered = answerMalformedRequestFrame( + { + type: "repo.pack.push", + agentAddress: "ins_pack@example.com", + transferId: "xfer-3", + repoId: { kind: "not-a-real-kind" }, + }, + "bad", + (frame) => sent.push(frame), + ); + expect(answered).toBe(false); + expect(sent).toHaveLength(0); + }); + + test("recovers a non-pack frame that carries a malformed repoId-shaped field", () => { + // repoId is validated only inside the pack branch, so a requestId- or + // agentAddress-correlated frame that happens to carry a garbage + // repoId still recovers through its own correlation key. + const sent: (SessionErrorFrame | AgentErrorFrame | PackRejectFrame)[] = []; + const answered = answerMalformedRequestFrame( + { + type: "sources.update", + requestId: "req-9", + repoId: { kind: "not-a-real-kind" }, + sources: [{}], + }, + "sources invalid", + (frame) => sent.push(frame), + ); + expect(answered).toBe(true); + expect(sent[0]).toMatchObject({ + type: "session.error", + requestId: "req-9", }); - expect(sent[0]?.error).toMatch(/malformed agent.undeploy frame/); }); test("does not answer a sources.update with no recoverable requestId", () => { - const sent: (SessionErrorFrame | AgentErrorFrame)[] = []; + const sent: (SessionErrorFrame | AgentErrorFrame | PackRejectFrame)[] = []; const answered = answerMalformedRequestFrame( { type: "sources.update", sources: [{}] }, "bad", @@ -1520,7 +1602,7 @@ describe("answerMalformedRequestFrame", () => { }); test("does not answer a fire-and-forget frame even with a requestId present", () => { - const sent: (SessionErrorFrame | AgentErrorFrame)[] = []; + const sent: (SessionErrorFrame | AgentErrorFrame | PackRejectFrame)[] = []; const answered = answerMalformedRequestFrame( { type: "signal.deliver", agentAddress: "x", requestId: "r" }, "bad", @@ -1531,7 +1613,7 @@ describe("answerMalformedRequestFrame", () => { }); test("does not answer a frame with no recognizable type", () => { - const sent: (SessionErrorFrame | AgentErrorFrame)[] = []; + const sent: (SessionErrorFrame | AgentErrorFrame | PackRejectFrame)[] = []; const answered = answerMalformedRequestFrame( { garbage: true }, "bad", diff --git a/packages/hub-agent/src/ws/hub-link.ts b/packages/hub-agent/src/ws/hub-link.ts index 20c1e009..5aedac44 100644 --- a/packages/hub-agent/src/ws/hub-link.ts +++ b/packages/hub-agent/src/ws/hub-link.ts @@ -23,7 +23,7 @@ import { type PackDoneFrame, type PackAckFrame, type PackRejectFrame, - type RepoId, + RepoId, type SignalDeliverFrame, type DrainDeliverFrame, type SourcesUpdateFrame, @@ -59,6 +59,12 @@ const MalformedRequestEnvelope = type({ "type?": "string", "requestId?": "string", "agentAddress?": "string", + "transferId?": "string", + // `repoId` is carried as `unknown` and validated only inside the pack + // branch below. Validating it here would fail the whole envelope for a + // non-pack frame that happens to carry a malformed `repoId`-shaped field, + // sinking its recovery through its own correlation key. + "repoId?": "unknown", }); /** @@ -85,6 +91,17 @@ const AGENT_ERROR_REQUEST_TYPES: ReadonlySet = new Set([ "agent.undeploy", ]); +/** + * Inbound chunked-pack request frames the hub correlates by `transferId` + * and whose failure reply is a `repo.pack.reject`. The hub tracks these in + * its per-transfer pending map with the longest timeout of any request + * frame. + */ +const PACK_REJECT_REQUEST_TYPES: ReadonlySet = new Set([ + "repo.pack.push", + "repo.pack.done", +]); + /** * Answer a malformed inbound request/ack control frame with an error reply * so the hub's request does not hang to its timeout. Two control-frame @@ -95,12 +112,13 @@ const AGENT_ERROR_REQUEST_TYPES: ReadonlySet = new Set([ * (mail/signal/drain/...) have no requester waiting on a reply, so a * malformed one is correctly left to be logged and dropped by the caller. * - * The chunked `repo.pack` streaming transfers are also request/ack (keyed - * by `transferId`, rejected by `repo.pack.reject`) but are deliberately - * NOT handled here: a valid `repo.pack.reject` carries a structured - * `repoId` and a constrained reason that cannot be reliably synthesized - * from a frame malformed in those very fields, so recovering that path is - * separate work. + * The chunked `repo.pack` streaming transfers (repo.pack.push, + * repo.pack.done) are the third family: correlated by `transferId`, + * rejected by `repo.pack.reject`. A valid reject also carries the frame's + * `agentAddress` and structured `repoId`, so it is answerable only when + * all three survive the malformation; when `repoId` (or the transferId) is + * itself unrecoverable the frame is left to be logged and dropped, because + * a valid `repo.pack.reject` cannot be constructed without them. * * Returns `true` when it answered; `false` when no correlation key is * recoverable (an unknown/absent type, a fire-and-forget frame, or a @@ -110,7 +128,7 @@ const AGENT_ERROR_REQUEST_TYPES: ReadonlySet = new Set([ export function answerMalformedRequestFrame( raw: unknown, summary: string, - send: (frame: SessionErrorFrame | AgentErrorFrame) => void, + send: (frame: SessionErrorFrame | AgentErrorFrame | PackRejectFrame) => void, ): boolean { const envelope = MalformedRequestEnvelope(raw); if (envelope instanceof type.errors) return false; @@ -140,6 +158,30 @@ export function answerMalformedRequestFrame( }); return true; } + if ( + PACK_REJECT_REQUEST_TYPES.has(frameType) && + envelope.transferId !== undefined && + envelope.transferId.length > 0 && + envelope.agentAddress !== undefined && + envelope.agentAddress.length > 0 + ) { + // A valid repo.pack.reject carries the frame's structured repoId, so + // recover it here (kept out of the shared envelope to protect the other + // families). When the repoId is itself malformed there is no valid + // reject to build, so the frame is left to be dropped. The hub + // correlates the reject by transferId alone; "corrupt" is the reason + // for a frame that failed to parse. + const repoId = RepoId(envelope.repoId); + if (repoId instanceof type.errors) return false; + send({ + type: "repo.pack.reject", + agentAddress: envelope.agentAddress, + repoId, + transferId: envelope.transferId, + reason: "corrupt", + }); + return true; + } return false; } From b912e448a2dd44013eb03a806a5ca62bdcff9788 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Sun, 5 Jul 2026 16:19:39 -0500 Subject: [PATCH 063/101] Retire the orphaned session.abort and grants.update wire surface Retiring the in-process session runtime left the hub able to send the session.abort and grants.update control frames while the sidecar-side dispatch handlers were already gone and nothing called the senders -- dead hub-side surface with no live receiver. Live propagation of a mid-run grant change or a running-agent abort now has no wire path; a supervised replacement for each semantic is scoped separately. --- docs/ARCHITECTURE.md | 4 +- docs/AUTH.md | 4 +- docs/HARNESS_DESIGN.md | 13 ++++--- docs/IMPLEMENTATION.md | 15 ++++---- packages/hub-agent/src/ws/hub-link.test.ts | 11 +++--- packages/hub-agent/src/ws/hub-link.ts | 11 +++--- packages/hub-api/src/routes/agents.test.ts | 2 - packages/hub-api/src/routes/assets.test.ts | 2 - .../hub-api/src/routes/catalog-routes.test.ts | 2 - .../hub-api/src/routes/git-tokens.test.ts | 2 - packages/hub-api/src/routes/instances.test.ts | 6 --- packages/hub-api/src/routes/workflows.test.ts | 2 - .../hub-sessions/src/session-service.test.ts | 6 --- .../src/ws/sidecar-handler.test.ts | 38 ++++++++++++++++--- .../hub-sessions/src/ws/sidecar-handler.ts | 32 +--------------- packages/types/src/sidecar.ts | 31 +-------------- tests/hub-api/asset-routes.test.ts | 2 - 17 files changed, 62 insertions(+), 121 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 78714412..515be791 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -93,7 +93,7 @@ The reactor director selects which model to use for each inference call (per-cal Model provider selection follows the priority order established at launch. The highest-priority source is active; on a source-specific failure — an authentication failure, a protocol mismatch, or a transient network or timeout error the inference harness has already retried — the reactor fails over to the next source in the list. A rate limit is the exception: it waits out a short same-source backoff first, since it clears with time, and only then fails over. The error surfaces only once the list is exhausted. Source-invariant failures (a context-window overflow, a fatal request error, or an abort) do not trigger failover, since no other source would serve the call differently. Each new inference cycle restarts at the most-preferred source. v1 uses strict priority; equal-priority load-balancing and configurable selection strategies (weighted round-robin, latency-based) are a planned extension. -The model provider list is not fixed for the agent's lifetime. The control plane can push model provider updates to running agents — adding newly available model providers, removing revoked ones, or adjusting priorities — following the same pattern as dynamic grant updates. The harness applies updates without interrupting in-flight inference calls; the updated list takes effect on the next selection. +The model provider list is not fixed for the agent's lifetime. The control plane can push model provider updates to a running deployment — adding newly available model providers, removing revoked ones, or adjusting priorities. The sidecar routes the update to the deployment's supervisor, which swaps the running agent's sources in place. The harness applies updates without interrupting in-flight inference calls; the updated list takes effect on the next selection. The agent is unaware of which model provider is serving a given inference call. @@ -195,7 +195,7 @@ This three-source model enables important patterns: **Inherited capabilities** apply when agents create other agents. The parent agent can grant its child a subset of its own creator-sourced capabilities, establishing a delegation chain. Children cannot exceed their parent's authority, but they can carry creator-sourced capabilities that future invokers of the child would not have on their own. -**Grant revocation** is policy-driven with a default of fail-secure. If the creator's grants are revoked after agents have been launched with creator-sourced grants, running agents lose the affected grants immediately — the control plane pushes a grants update to the harness. Tenants can configure grace periods or notification-only behavior for specific grant types. Invoker-granted capabilities expire when the agent stops unless explicitly persisted. +**Grant revocation** is policy-driven with a default of fail-secure. If the creator's grants are revoked after agents have been launched with creator-sourced grants, running agents must lose the affected grants. Propagating a revocation to an already-running deployment is not currently implemented — the earlier grants-update wire mechanism has been retired, and its supervised replacement is designed separately; a change takes effect when the deployment next loads its grants. Tenants can configure grace periods or notification-only behavior for specific grant types. Invoker-granted capabilities expire when the agent stops unless explicitly persisted. Authorization grants are part of the agent's auditable state. The harness logs what was granted, by whom (tenant, creator, or invoker), when, and tracks all usage of delegated credentials. diff --git a/docs/AUTH.md b/docs/AUTH.md index 26d981d1..22e40f3f 100644 --- a/docs/AUTH.md +++ b/docs/AUTH.md @@ -203,11 +203,11 @@ Conditions are evaluated at runtime by the authorization engine. A grant with un Grant revocation is policy-driven with a default of fail-secure. -**Creator grant revocation**: If the creator's authority is revoked after agents have been launched with creator-sourced grants, running agents lose the affected grants. The control plane pushes a `grants.update` frame to the harness, which replaces its live materialized grant set. The harness authorizes each tool call against this set before the call executes, so a revoked capability is blocked at the next authorization check; a tool call already in flight is not interrupted. Tenants can configure grace periods or notification-only behavior for specific grant types. +**Creator grant revocation**: If the creator's authority is revoked after agents have been launched with creator-sourced grants, running agents must lose the affected grants. The harness authorizes each tool call against its live materialized grant set before the call executes, so a revoked capability is blocked once that set no longer contains it; a tool call already in flight is not interrupted. Propagating a revocation to an already-running deployment is not currently implemented — the earlier `grants.update` wire mechanism has been retired, and its supervised replacement is designed separately. Until then, the change takes effect when the deployment next loads its grants (the deploy pack at spawn and the supervisor's IPC credentials snapshot at recycle). Tenants can configure grace periods or notification-only behavior for specific grant types. **Invoker grant revocation**: Invoker-granted capabilities expire when the agent stops unless explicitly persisted. They are session-scoped by default. -**Tenant policy changes**: When tenant policies change (role modifications, system role updates), the control plane re-evaluates affected agents and pushes grant updates to their harnesses. +**Tenant policy changes**: When tenant policies change (role modifications, system role updates), the control plane re-evaluates affected agents. Propagating the resulting grant changes to already-running deployments shares the retired-mechanism gap described above; a change takes effect when each deployment next loads its grants. This parallels the credential revocation model described in CREDENTIALS.md — both follow the same fail-secure default with configurable tenant policies. diff --git a/docs/HARNESS_DESIGN.md b/docs/HARNESS_DESIGN.md index 31d7c236..2684d5c8 100644 --- a/docs/HARNESS_DESIGN.md +++ b/docs/HARNESS_DESIGN.md @@ -166,12 +166,13 @@ After self-restoration, the sidecar connects to the hub and proves ownership of 2. Hub generates a 32-byte random nonce per address and sends a `challenge` frame 3. Sidecar signs `nonce || agent_address` (concatenated bytes) with each agent's private key and sends a `challenge.response` frame 4. Hub verifies each signature against the stored public key for that address -5. Verified addresses are provisionally added to the routing table (required so the `grants.update` request/ack round-trip can reach the sidecar) -6. Hub sends `grants.update` with current grants for each verified address -7. Hub compares each agent's advertised deploy ref against its own. For agents whose ref is stale or absent, the hub creates and sends a fresh deploy pack (fire-and-forget, does not block reconnect completion) -8. On grant refresh success, the address remains in the routing table and queued messages are flushed -9. On failure (grant refresh rejected or timed out), the address is rolled back from the routing table — its queue is preserved for the next reconnect attempt and the sidecar receives `challenge.failed` -10. For addresses that fail cryptographic verification, hub sends `challenge.failed` with the address and reason +5. Verified addresses are provisionally added to the routing table so the hub's `agent.reconnected` reaction can run for each +6. Hub compares each agent's advertised deploy ref against its own. For agents whose ref is stale or absent, the hub creates and sends a fresh deploy pack (fire-and-forget, does not block reconnect completion) +7. On a successful reaction, the address remains in the routing table and queued messages are flushed +8. On failure (the reconnect reaction rejected by governance), the address is rolled back from the routing table — its queue is preserved for the next reconnect attempt and the sidecar receives `challenge.failed` +9. For addresses that fail cryptographic verification, hub sends `challenge.failed` with the address and reason + +A supervised deployment carries its grants in the deploy pack and refreshes them over the supervisor's IPC credentials snapshot at spawn and recycle, so reconnect no longer performs a grant-refresh round-trip over the wire. ### Reconnection Frames diff --git a/docs/IMPLEMENTATION.md b/docs/IMPLEMENTATION.md index 41234ed3..74a68563 100644 --- a/docs/IMPLEMENTATION.md +++ b/docs/IMPLEMENTATION.md @@ -237,9 +237,10 @@ The IMAP inbox is the source of truth for conversation history. Session channels **Prototype (hub-mediated sidecar transport):** 1. Sidecar reconnects to the hub and proves ownership of agent addresses via signed challenge (see HARNESS_DESIGN.md) -2. Hub refreshes grants via `grants.update` frame — the sidecar must have current grants before processing any messages -3. Hub flushes queued undelivered messages as `message.send` frames for verified agents -4. Sidecar loads agent context from isogit and resumes operation +2. Hub flushes queued undelivered messages as `message.send` frames for verified agents +3. Sidecar loads agent context from isogit and resumes operation + +A supervised deployment carries its grants in the deploy pack and refreshes them over the supervisor's IPC credentials snapshot at spawn and recycle, so reconnect does not refresh grants over the wire. In the prototype, the hub's database serves as the delivery queue for messages sent while the sidecar is disconnected. The sidecar's isogit repository is the source of truth for agent inference context. The hub maintains a sidecar-to-agent mapping so it knows which sidecar to route messages to for a given agent address. See HARNESS_DESIGN.md for the reconnection wire protocol. @@ -272,11 +273,9 @@ Undeploy is an acknowledged operation. The sidecar shuts the deployment's superv **Grant management:** -| Direction | Frame | Purpose | -| ------------- | --------------- | ---------------------------------------------------- | -| Hub → Sidecar | `grants.update` | Push updated grants to a running agent (request/ack) | +The live `grants.update` wire path has been retired. A supervised deployment receives its grants in the deploy pack and refreshes them over the supervisor's IPC credentials snapshot at spawn and recycle; there is no wire frame that pushes a grant change to an already-running deployment. Propagating a mid-run grant change (for example, a revocation) to a running deployment is not currently implemented. -On first connection (no existing agents), the sidecar sends a `register` frame. On reconnection (agents in data directory), it sends `reconnect`. After successful challenge verification, verified addresses are provisionally added to the routing table so the `grants.update` round-trip can reach the sidecar. If grant refresh fails, the address is rolled back from the routing table and its queued messages are preserved for the next reconnect attempt. +On first connection (no existing agents), the sidecar sends a `register` frame. On reconnection (agents in data directory), it sends `reconnect`. After successful challenge verification, verified addresses are provisionally added to the routing table so the hub's `agent.reconnected` reaction can run for each; an address whose reaction is rejected by governance is rolled back from the routing table and its queued messages are preserved for the next reconnect attempt. ### Debug and Telemetry Streams @@ -1173,7 +1172,7 @@ Pack transfers share the WebSocket with live session traffic. To prevent interfe - Pack chunks are interleaved with other frames at the WebSocket message level (messages are atomic; the receiver processes each independently) - The sender limits unacknowledged pack data to a configurable window before pausing -- Session-critical frames (`mail.inbound`, `message.send`, `session.abort`) are never delayed by pack transfers +- Session-critical frames (`mail.inbound`, `message.send`) are never delayed by pack transfers - The receiver buffers incoming chunks in memory (or a temp file for large transfers) and only unpacks atomically on `pack.done` ### Rejection Reasons diff --git a/packages/hub-agent/src/ws/hub-link.test.ts b/packages/hub-agent/src/ws/hub-link.test.ts index c217de79..fbd43cf3 100644 --- a/packages/hub-agent/src/ws/hub-link.test.ts +++ b/packages/hub-agent/src/ws/hub-link.test.ts @@ -1476,12 +1476,11 @@ describe("answerMalformedRequestFrame", () => { }); }); - test("drops the retired requestId-correlated frames the sidecar no longer dispatches", () => { - // grants.update / session.abort have hub senders but no sidecar - // dispatch handler (retired) and no production caller, so a malformed - // one is dropped just like a well-formed one -- the answerable set - // covers only frames the sidecar actually handles. - for (const frameType of ["grants.update", "session.abort"]) { + test("drops an unhandled request-shaped frame instead of answering it", () => { + // A request-shaped frame whose type is in none of the answerable sets + // (SESSION_ERROR / AGENT_ERROR / PACK_REJECT) has no requester to + // answer, so a malformed one is dropped rather than answered. + for (const frameType of ["some.unhandled.request", "another.unknown"]) { const sent: (SessionErrorFrame | AgentErrorFrame | PackRejectFrame)[] = []; const answered = answerMalformedRequestFrame( diff --git a/packages/hub-agent/src/ws/hub-link.ts b/packages/hub-agent/src/ws/hub-link.ts index 5aedac44..ee842ed5 100644 --- a/packages/hub-agent/src/ws/hub-link.ts +++ b/packages/hub-agent/src/ws/hub-link.ts @@ -70,12 +70,11 @@ const MalformedRequestEnvelope = type({ /** * Inbound request/ack frames the sidecar dispatches that the hub * correlates by `requestId`, whose failure reply is a `session.error`. - * Only `sources.update` qualifies. The hub also has `grants.update` / - * `session.abort` senders, but the sidecar has no dispatch handler for - * them (retired) and no production caller sends them, so a malformed one - * has no more of a requester to answer than a well-formed one -- both are - * dropped, keeping the answerable set aligned with what the sidecar - * actually handles. + * Only `sources.update` qualifies -- it is the sole frame answered with a + * `session.error`. Frames answered through the other correlation keys live + * in `AGENT_ERROR_REQUEST_TYPES` and `PACK_REJECT_REQUEST_TYPES`; a + * request-shaped frame in none of the three sets has no requester to + * answer and is dropped. */ const SESSION_ERROR_REQUEST_TYPES: ReadonlySet = new Set([ "sources.update", diff --git a/packages/hub-api/src/routes/agents.test.ts b/packages/hub-api/src/routes/agents.test.ts index e074941d..4c773432 100644 --- a/packages/hub-api/src/routes/agents.test.ts +++ b/packages/hub-api/src/routes/agents.test.ts @@ -174,8 +174,6 @@ function createMockSidecarRouter(): SidecarRouter { routeMail: () => notImpl("routeMail"), sendAgentDeploy: () => notImpl("sendAgentDeploy"), sendAgentUndeploy: () => notImpl("sendAgentUndeploy"), - sendSessionAbort: () => notImpl("sendSessionAbort"), - sendGrantsUpdate: () => notImpl("sendGrantsUpdate"), sendSourcesUpdate: () => notImpl("sendSourcesUpdate"), sendPack: () => notImpl("sendPack"), sendProvisionStep: () => notImpl("sendProvisionStep"), diff --git a/packages/hub-api/src/routes/assets.test.ts b/packages/hub-api/src/routes/assets.test.ts index 8874c448..901e1f7b 100644 --- a/packages/hub-api/src/routes/assets.test.ts +++ b/packages/hub-api/src/routes/assets.test.ts @@ -290,8 +290,6 @@ function createMockSidecarRouter(): SidecarRouter { routeMail: () => notImpl("routeMail"), sendAgentDeploy: () => notImpl("sendAgentDeploy"), sendAgentUndeploy: () => notImpl("sendAgentUndeploy"), - sendSessionAbort: () => notImpl("sendSessionAbort"), - sendGrantsUpdate: () => notImpl("sendGrantsUpdate"), sendSourcesUpdate: () => notImpl("sendSourcesUpdate"), sendPack: () => notImpl("sendPack"), sendProvisionStep: () => notImpl("sendProvisionStep"), diff --git a/packages/hub-api/src/routes/catalog-routes.test.ts b/packages/hub-api/src/routes/catalog-routes.test.ts index 2424cb6b..bd8557ce 100644 --- a/packages/hub-api/src/routes/catalog-routes.test.ts +++ b/packages/hub-api/src/routes/catalog-routes.test.ts @@ -188,8 +188,6 @@ function createMockSidecarRouter(): SidecarRouter { routeMail: () => notImpl("routeMail"), sendAgentDeploy: () => notImpl("sendAgentDeploy"), sendAgentUndeploy: () => notImpl("sendAgentUndeploy"), - sendSessionAbort: () => notImpl("sendSessionAbort"), - sendGrantsUpdate: () => notImpl("sendGrantsUpdate"), // Mutations fire a fire-and-forget source push; resolve so it is a no-op. sendSourcesUpdate: () => Promise.resolve(), sendPack: () => notImpl("sendPack"), diff --git a/packages/hub-api/src/routes/git-tokens.test.ts b/packages/hub-api/src/routes/git-tokens.test.ts index 8bc04a12..85445508 100644 --- a/packages/hub-api/src/routes/git-tokens.test.ts +++ b/packages/hub-api/src/routes/git-tokens.test.ts @@ -213,8 +213,6 @@ function createMockSidecarRouter(): SidecarRouter { routeMail: () => notImpl("routeMail"), sendAgentDeploy: () => notImpl("sendAgentDeploy"), sendAgentUndeploy: () => notImpl("sendAgentUndeploy"), - sendSessionAbort: () => notImpl("sendSessionAbort"), - sendGrantsUpdate: () => notImpl("sendGrantsUpdate"), sendSourcesUpdate: () => notImpl("sendSourcesUpdate"), sendPack: () => notImpl("sendPack"), sendProvisionStep: () => notImpl("sendProvisionStep"), diff --git a/packages/hub-api/src/routes/instances.test.ts b/packages/hub-api/src/routes/instances.test.ts index 1b32717f..b7e0a8ba 100644 --- a/packages/hub-api/src/routes/instances.test.ts +++ b/packages/hub-api/src/routes/instances.test.ts @@ -240,12 +240,6 @@ function createMockSidecarRouter( sendAgentUndeploy(_addr, _reason) { return notImpl("sendAgentUndeploy"); }, - sendSessionAbort(_addr, _reason) { - return notImpl("sendSessionAbort"); - }, - sendGrantsUpdate(_addr, _grants) { - return notImpl("sendGrantsUpdate"); - }, sendSourcesUpdate(_addr, _sources, _defaultSource) { return notImpl("sendSourcesUpdate"); }, diff --git a/packages/hub-api/src/routes/workflows.test.ts b/packages/hub-api/src/routes/workflows.test.ts index 31945ae4..95e4f040 100644 --- a/packages/hub-api/src/routes/workflows.test.ts +++ b/packages/hub-api/src/routes/workflows.test.ts @@ -196,8 +196,6 @@ function createMockSidecarRouter( }, sendAgentDeploy: () => notImpl("sendAgentDeploy"), sendAgentUndeploy: () => notImpl("sendAgentUndeploy"), - sendSessionAbort: () => notImpl("sendSessionAbort"), - sendGrantsUpdate: () => notImpl("sendGrantsUpdate"), sendSourcesUpdate: () => notImpl("sendSourcesUpdate"), sendPack: () => notImpl("sendPack"), sendProvisionStep: () => notImpl("sendProvisionStep"), diff --git a/packages/hub-sessions/src/session-service.test.ts b/packages/hub-sessions/src/session-service.test.ts index 53e75660..e5096b53 100644 --- a/packages/hub-sessions/src/session-service.test.ts +++ b/packages/hub-sessions/src/session-service.test.ts @@ -70,12 +70,6 @@ function createMockRouter(): SidecarRouter & { sendAgentUndeploy: track( "sendAgentUndeploy", ) as SidecarRouter["sendAgentUndeploy"], - sendSessionAbort: track( - "sendSessionAbort", - ) as SidecarRouter["sendSessionAbort"], - sendGrantsUpdate: track( - "sendGrantsUpdate", - ) as SidecarRouter["sendGrantsUpdate"], sendSourcesUpdate: track( "sendSourcesUpdate", ) as SidecarRouter["sendSourcesUpdate"], diff --git a/packages/hub-sessions/src/ws/sidecar-handler.test.ts b/packages/hub-sessions/src/ws/sidecar-handler.test.ts index fb456b9a..0705f925 100644 --- a/packages/hub-sessions/src/ws/sidecar-handler.test.ts +++ b/packages/hub-sessions/src/ws/sidecar-handler.test.ts @@ -43,6 +43,20 @@ describe("SidecarRouter", () => { const TEST_HUB_KEY = "a".repeat(64); + // A minimal valid source list for exercising the requestId-correlated + // sendRequest path (sendSourcesUpdate). The routing/reconnect invariants + // under test are independent of the source content. + const TEST_SOURCES = [ + { + id: "anthropic:claude-sonnet-4-20250514", + provider: "anthropic", + baseURL: "https://api.anthropic.com", + apiKey: "sk-test", + model: "claude-sonnet-4-20250514", + }, + ]; + const TEST_DEFAULT_SOURCE = "anthropic:claude-sonnet-4-20250514"; + beforeEach(() => { router = createSidecarRouter({ requestTimeoutMs: 500, @@ -1016,7 +1030,7 @@ describe("SidecarRouter", () => { await expect(promise).rejects.toThrow(/disconnected/); }); - test("abort preserves routing when address re-registered during await", async () => { + test("preserves routing when address re-registered during a request await", async () => { const ws1 = createMockWs(); router.handleOpen(ws1); router.handleMessage( @@ -1029,7 +1043,11 @@ describe("SidecarRouter", () => { }), ); - const promise = router.sendSessionAbort("agent@local", "user_disconnect"); + const promise = router.sendSourcesUpdate( + "agent@local", + TEST_SOURCES, + TEST_DEFAULT_SOURCE, + ); const frame = lastSent(ws1); const ws2 = createMockWs(); @@ -1053,7 +1071,7 @@ describe("SidecarRouter", () => { expect(router.getRoutableAddresses()).toContain("agent@local"); }); - test("closing stale sidecar after reconnect-during-abort does not evict address", async () => { + test("closing stale sidecar after reconnect-during-request does not evict address", async () => { const ws1 = createMockWs(); router.handleOpen(ws1); router.handleMessage( @@ -1066,7 +1084,11 @@ describe("SidecarRouter", () => { }), ); - const promise = router.sendSessionAbort("agent@local", "user_disconnect"); + const promise = router.sendSourcesUpdate( + "agent@local", + TEST_SOURCES, + TEST_DEFAULT_SOURCE, + ); const frame = lastSent(ws1); const ws2 = createMockWs(); @@ -2343,7 +2365,7 @@ describe("SidecarRouter", () => { expect(flushed[1].rawMessage).toBe("msg-2"); }); - test("sendSessionAbort rejects when agent is disconnected", async () => { + test("sendSourcesUpdate rejects when agent is disconnected", async () => { const ws = createMockWs(); router.handleOpen(ws); router.handleMessage( @@ -2358,7 +2380,11 @@ describe("SidecarRouter", () => { router.handleClose(ws); await expect( - router.sendSessionAbort("agent@local", "user_disconnect"), + router.sendSourcesUpdate( + "agent@local", + TEST_SOURCES, + TEST_DEFAULT_SOURCE, + ), ).rejects.toThrow("No sidecar connected"); }); }); diff --git a/packages/hub-sessions/src/ws/sidecar-handler.ts b/packages/hub-sessions/src/ws/sidecar-handler.ts index 6349c434..d03d2014 100644 --- a/packages/hub-sessions/src/ws/sidecar-handler.ts +++ b/packages/hub-sessions/src/ws/sidecar-handler.ts @@ -18,12 +18,10 @@ import { type RepoId, } from "@intx/types/sidecar"; import type { - AbortReason, ConnectorThreadState, HarnessConfig, InferenceSource, } from "@intx/types/runtime"; -import type { GrantRule } from "@intx/types/authz"; import { createSidecarEmitter, type SidecarEventEmitter, @@ -126,8 +124,6 @@ export type SidecarRouter = { workflow?: AgentDeployFrame["workflow"], ): Promise<{ publicKey: string }>; sendAgentUndeploy(agentAddress: string, reason: string): Promise; - sendSessionAbort(agentAddress: string, reason: AbortReason): Promise; - sendGrantsUpdate(agentAddress: string, grants: GrantRule[]): Promise; sendSourcesUpdate( agentAddress: string, sources: InferenceSource[], @@ -846,7 +842,7 @@ export function createSidecarRouter( // Add verified addresses to the routing table immediately so that // `agent.reconnected` subscribers can use sendRequest-based methods - // (e.g. sendGrantsUpdate). Addresses that fail governance are + // (e.g. sendSourcesUpdate). Addresses that fail governance are // rolled back from the routing table afterward. for (const addr of verified) { // If a different ws still owns this address (live takeover via @@ -1771,18 +1767,6 @@ export function createSidecarRouter( }); } - async function sendSessionAbort( - agentAddress: string, - reason: AbortReason, - ): Promise { - await sendRequest(agentAddress, (requestId) => ({ - type: "session.abort", - requestId, - agentAddress, - reason, - })); - } - function removeAgentAddress(ws: WsHandle, agentAddress: string): void { addressIndex.delete(agentAddress); const conn = connections.get(ws); @@ -1837,18 +1821,6 @@ export function createSidecarRouter( return connectorStates.get(agentAddress) ?? null; } - async function sendGrantsUpdate( - agentAddress: string, - grants: GrantRule[], - ): Promise { - await sendRequest(agentAddress, (requestId) => ({ - type: "grants.update", - requestId, - agentAddress, - grants, - })); - } - async function sendSourcesUpdate( agentAddress: string, sources: InferenceSource[], @@ -1937,8 +1909,6 @@ export function createSidecarRouter( routeMail, sendAgentDeploy, sendAgentUndeploy, - sendSessionAbort, - sendGrantsUpdate, sendSourcesUpdate, sendPack, bindStepRoute, diff --git a/packages/types/src/sidecar.ts b/packages/types/src/sidecar.ts index 42c39327..4580fa1f 100644 --- a/packages/types/src/sidecar.ts +++ b/packages/types/src/sidecar.ts @@ -9,13 +9,11 @@ import { type } from "arktype"; import { - AbortReason, ConnectorThreadState, HarnessConfig, InferenceEvent, InferenceSource, } from "./runtime"; -import { WireGrantRule } from "./grant-wire"; // --------------------------------------------------------------------------- // Sidecar → Hub @@ -151,7 +149,7 @@ export const PingFrame = type({ type: "'ping'" }); export type PingFrame = typeof PingFrame.infer; /** - * Acknowledges a request from the hub (session.abort, grants.update). + * Acknowledges a request from the hub (sources.update). */ export const SessionAckFrame = type({ type: "'session.ack'", @@ -362,18 +360,6 @@ export const ChallengeFailedFrame = type({ }); export type ChallengeFailedFrame = typeof ChallengeFailedFrame.infer; -/** - * Kill switch. Aborts a running agent immediately with the given reason. - * Responds with session.ack or session.error. - */ -export const SessionAbortFrame = type({ - type: "'session.abort'", - requestId: "string", - agentAddress: "string", - reason: AbortReason, -}); -export type SessionAbortFrame = typeof SessionAbortFrame.infer; - /** * Keepalive pong sent by the hub in response to a ping frame. * If the sidecar stops receiving pongs, it considers the hub dead. @@ -381,19 +367,6 @@ export type SessionAbortFrame = typeof SessionAbortFrame.infer; export const PongFrame = type({ type: "'pong'" }); export type PongFrame = typeof PongFrame.infer; -/** - * Push updated grants to a running agent. The sidecar replaces the agent's - * grant snapshot and re-persists the config. Responds with session.ack or - * session.error. - */ -export const GrantsUpdateFrame = type({ - type: "'grants.update'", - requestId: "string", - agentAddress: "string", - grants: WireGrantRule.array(), -}); -export type GrantsUpdateFrame = typeof GrantsUpdateFrame.infer; - /** * Push an updated inference-source list to a running single-step * deployment. The sidecar routes it to the deployment's supervisor, which @@ -732,8 +705,6 @@ export const HubFrame = MailInboundFrame.or(AgentDeployFrame) .or(ChallengeFrame) .or(ChallengeFailedFrame) .or(PongFrame) - .or(SessionAbortFrame) - .or(GrantsUpdateFrame) .or(SourcesUpdateFrame) .or(PackPushFrame) .or(PackDoneFrame) diff --git a/tests/hub-api/asset-routes.test.ts b/tests/hub-api/asset-routes.test.ts index 3a750e25..88a6ed4e 100644 --- a/tests/hub-api/asset-routes.test.ts +++ b/tests/hub-api/asset-routes.test.ts @@ -142,8 +142,6 @@ function createMockSidecarRouter(): SidecarRouter { routeMail: () => notImpl("routeMail"), sendAgentDeploy: () => notImpl("sendAgentDeploy"), sendAgentUndeploy: () => notImpl("sendAgentUndeploy"), - sendSessionAbort: () => notImpl("sendSessionAbort"), - sendGrantsUpdate: () => notImpl("sendGrantsUpdate"), sendSourcesUpdate: () => notImpl("sendSourcesUpdate"), sendPack: () => notImpl("sendPack"), sendProvisionStep: () => notImpl("sendProvisionStep"), From b81441f339a54aa60568244ee4a33ee2cc17fc0f Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Sun, 5 Jul 2026 16:56:17 -0500 Subject: [PATCH 064/101] Serialize the substrate-mirror entry points against overlap The awaited onRunBoundary mirror and the fire-and-forget onStateChanged mirror share the mirrored-count state with no serialization, so two overlapping runs could read the same boundary seq and emit two WAL entries at it -- and over-count the mirrored turns so a later mirror slices past an unmirrored turn and drops it from the log. Route both mirror and restore through one per-instance tail so every count-mutating run is serialized. Restore's connector-state restore can itself re-enter the mirror through onStateChanged, so establish the committed counts before that restore too. --- apps/sidecar/src/conversation-state.ts | 59 ++++- .../conversation-state-wal.test.ts | 220 ++++++++++++++++++ 2 files changed, 271 insertions(+), 8 deletions(-) diff --git a/apps/sidecar/src/conversation-state.ts b/apps/sidecar/src/conversation-state.ts index ce0d6529..0448c82d 100644 --- a/apps/sidecar/src/conversation-state.ts +++ b/apps/sidecar/src/conversation-state.ts @@ -347,6 +347,40 @@ export async function createDurableConversationStore( // checkpointBoundarySeq) and when to compact. let checkpointBoundarySeq = 0; + // Serialize the shared-counter critical section. Both `mirrorToSubstrate` + // and `restoreFromSubstrate` read and advance the three counts above, and + // either can re-enter the other: `connectorRouter.restore()` can fire + // `onStateChanged` synchronously, which enqueues a mirror. A single + // per-instance tail runs every such op one-at-a-time regardless of entry + // point, so two overlapping runs cannot read the same boundary seq and + // emit two WAL entries at it. Modeled on `runRepoOp` in the hub-agent + // session manager: the stored tail swallows rejections so one failed op + // does not poison the chain, while each caller still observes its own op's + // result (or rejection) through the returned promise. + // + // This serializes mirror-vs-mirror and mirror-vs-restore only. It does NOT + // address the reactor-vs-mirror peek-snapshot window documented on + // `runMirror` below (nothing must append to the reactor's turn array + // between its last writeTurns and the mirror's peekTurns) -- that is a + // different concurrency axis and is out of scope here. + let stateOpTail: Promise = Promise.resolve(); + function serializeStateOp(op: () => Promise): Promise { + const result = stateOpTail.then(op, op); + stateOpTail = result.then( + () => undefined, + () => undefined, + ); + return result; + } + + function restoreFromSubstrate(): Promise { + return serializeStateOp(runRestore); + } + + function mirrorToSubstrate(): Promise { + return serializeStateOp(runMirror); + } + function substrateAgentStateFsDir(): string { const repoDir = opts.substrate.getRepoDir(opts.workflowRunRepoId); return path.join( @@ -376,7 +410,7 @@ export async function createDurableConversationStore( return `${agentStatePrefix}${CHECKPOINT_META_FILE}`; } - async function restoreFromSubstrate(): Promise { + async function runRestore(): Promise { const reconstructed = await reconstructDurableConversation( substrateAgentStateFsDir(), opts.agentKey, @@ -397,6 +431,17 @@ export async function createDurableConversationStore( // so a future change-driven mirror carries the right base. await baseStorage.writeTurns(reconstructed.turns); baseStorage.setConnectorState(reconstructed.connectorState); + // Establish the committed counts BEFORE restoring the connector state. + // `connectorRouter.restore()` can fire `onStateChanged` synchronously + // (when the restored state differs from current), which enqueues a + // mirror. Serialization already chains that mirror behind this restore, + // but setting the counts first keeps them correct even if that ordering + // guarantee is ever weakened. The counts reflect the substrate state + // `reconstructed` was read from, which is durable independent of the + // local-store commit below. + mirroredBoundaryCount = reconstructed.boundaryCount; + mirroredTurnCount = reconstructed.totalTurns; + checkpointBoundarySeq = reconstructed.checkpointBoundarySeq; connectorRouter.restore(reconstructed.connectorState); await baseStorage.writeMetadata({ pendingOperations: reconstructed.pendingOperations, @@ -405,9 +450,6 @@ export async function createDurableConversationStore( await baseStorage.commit({ message: `restore conversation for ${opts.agentKey} from substrate`, }); - mirroredBoundaryCount = reconstructed.boundaryCount; - mirroredTurnCount = reconstructed.totalTurns; - checkpointBoundarySeq = reconstructed.checkpointBoundarySeq; return true; } @@ -501,7 +543,7 @@ export async function createDurableConversationStore( ); } - async function mirrorToSubstrate(): Promise { + async function runMirror(): Promise { // Slice the new turns from the reactor's in-memory array (retained // by the local single-writer store at the last writeTurns) instead // of re-reading and re-parsing the whole turns.jsonl every @@ -510,9 +552,10 @@ export async function createDurableConversationStore( // mutates the reactor's turn array between its last writeTurns and // this read. The mirror runs at onRunBoundary after send() settles, // and the reactor only appends inside a cycle (each ending in - // writeTurns), so peekTurns() equals the on-disk state here. A - // second, concurrent mirror trigger would have to preserve that - // ordering or slice from a snapshot. + // writeTurns), so peekTurns() equals the on-disk state here. + // Serializing the mirror entry points keeps two mirrors from + // overlapping this read, but the reactor must still not append + // between its writeTurns and this peek -- that axis is not serialized. const turns = baseStorage.peekTurns(); const metadata = await baseStorage.loadMetadata(); diff --git a/tests/workflow-deploy/conversation-state-wal.test.ts b/tests/workflow-deploy/conversation-state-wal.test.ts index dbfdf22f..9169cd02 100644 --- a/tests/workflow-deploy/conversation-state-wal.test.ts +++ b/tests/workflow-deploy/conversation-state-wal.test.ts @@ -26,6 +26,7 @@ import { generateKeyPair } from "@intx/crypto"; import { createSSHSignature } from "@intx/crypto"; import type { KeyPair, + ConnectorThreadState, ConversationTurn, PendingOperation, TokenUsage, @@ -555,4 +556,223 @@ describe("durable conversation store WAL + checkpoint (Phase D1)", () => { if (reconstructed === null) throw new Error("expected a reconstruction"); expect(reconstructed.turns).toEqual([userTurn("a"), userTurn("b")]); }); + + test("overlapping mirror runs serialize onto distinct boundaries and lose no turns", async () => { + // The dormant race this guards against: the awaited onRunBoundary mirror + // and the fire-and-forget onStateChanged mirror share the mirrored-count + // state with no serialization. Two overlapping runs both read the same + // boundary seq AND the same mirroredTurnCount, so they collide on one + // boundary and double-advance the turn count -- and a following mirror + // then slices past a genuinely new turn and drops it from the log. + // + // The barrier makes the interleave deterministic: the first WAL append is + // held until a would-be-concurrent second mirror has had time to reach + // its own append. Without serialization the second append runs on the + // stale counts; with it the second mirror does not start until the first + // advances them. + let releaseFirstAppend!: () => void; + const firstAppendHeld = new Promise((resolve) => { + releaseFirstAppend = resolve; + }); + let heldOnce = false; + const writeTreePreservingPrefix: RepoStore["writeTreePreservingPrefix"] = + async (principal, repoId, ref, args) => { + if (!heldOnce && args.preservePrefix.includes("/wal/")) { + heldOnce = true; + await firstAppendHeld; + } + return h.substrate.writeTreePreservingPrefix( + principal, + repoId, + ref, + args, + ); + }; + const wrappedSubstrate = new Proxy(h.substrate, { + get(target, prop, receiver): unknown { + if (prop === "writeTreePreservingPrefix") { + return writeTreePreservingPrefix; + } + return Reflect.get(target, prop, receiver); + }, + }); + + const store = await createDurableConversationStore({ + localStoreDir: localDir, + signer: h.signer, + substrate: wrappedSubstrate, + workflowRunRepoId: h.workflowRunRepoId, + workflowRunRef: WORKFLOW_RUN_REF, + principal: PRINCIPAL, + agentKey: AGENT_KEY, + }); + + // The local store holds [a, b]; fire two overlapping mirrors of it. + const ab = [userTurn("a"), userTurn("b")]; + await store.storage.writeTurns(ab); + await store.storage.writeMetadata({ + pendingOperations: [], + tokenUsage: tokenUsage(0), + }); + await store.storage.commit({ message: "ab" }); + + const both = Promise.all([ + store.mirrorToSubstrate(), + store.mirrorToSubstrate(), + ]); + // Let a would-be-concurrent second mirror reach its append before the + // first is released. Under serialization the second has not started, so + // only the first append is held here. + await new Promise((resolve) => setTimeout(resolve, 40)); + releaseFirstAppend(); + await both; + + // A following mirror carries a genuinely new turn. If the overlapping + // pair over-counted the mirrored turn count, this mirror slices past `c` + // and drops it from the durable log. + const abc = [...ab, userTurn("c")]; + await store.storage.writeTurns(abc); + await store.storage.writeMetadata({ + pendingOperations: [], + tokenUsage: tokenUsage(1), + }); + await store.storage.commit({ message: "abc" }); + await store.mirrorToSubstrate(); + + const reconstructed = await reconstructDurableConversation( + h.agentStateDir, + AGENT_KEY, + ); + if (reconstructed === null) throw new Error("expected a reconstruction"); + expect(reconstructed.turns).toEqual(abc); + }); + + test("a mirror re-entered by restore's connector-state change cannot clobber a concurrent boundary", async () => { + // Restore-reentrancy guard. restoreFromSubstrate calls + // connectorRouter.restore(), which fires onStateChanged synchronously when + // the restored connector state differs from the router's initial null -- + // and onStateChanged enqueues a (turnless) mirror. That reentrant mirror + // shares the mirrored-count state with restore and with the next real + // mirror. Unless restore runs on the same serialization tail (and + // establishes the counts before the connector restore), the reentrant + // append and a following real mirror can land on the SAME boundary seq, + // and the turnless one can overwrite the real turn. + // + // Seed two durable boundaries, the second carrying a non-null connector + // state -- exactly the future condition (a non-null connector state) under + // which this otherwise-dormant path activates. + const seed = await makeStore(h, localDir); + await seed.storage.writeTurns([userTurn("a")]); + await seed.storage.writeMetadata({ + pendingOperations: [], + tokenUsage: tokenUsage(0), + }); + await seed.storage.commit({ message: "a" }); + await seed.mirrorToSubstrate(); + + const connectorState: ConnectorThreadState = { + threadRoot: "", + lastMessageId: "", + replyTo: "user@example.com", + cc: [], + }; + seed.storage.setConnectorState(connectorState); + await seed.storage.writeTurns([userTurn("a"), userTurn("b")]); + await seed.storage.writeMetadata({ + pendingOperations: [], + tokenUsage: tokenUsage(1), + }); + await seed.storage.commit({ message: "b" }); + await seed.mirrorToSubstrate(); + + // A fresh store restores those two boundaries; a barrier holds the first + // WAL append (the reentrant turnless mirror) until a real `c` mirror is + // enqueued behind it. Every boundary seq the substrate writes is recorded + // (parsed from the append's commit message) so a same-seq clobber fails + // loud even if a future change masks the turn-loss symptom. + const seqsWritten: number[] = []; + let releaseFirstAppend!: () => void; + const firstAppendHeld = new Promise((resolve) => { + releaseFirstAppend = resolve; + }); + let heldOnce = false; + const writeTreePreservingPrefix: RepoStore["writeTreePreservingPrefix"] = + async (principal, repoId, ref, args) => { + const matched = /WAL boundary (\d+)/.exec(args.message); + if (matched !== null) seqsWritten.push(Number(matched[1])); + if (!heldOnce && args.preservePrefix.includes("/wal/")) { + heldOnce = true; + await firstAppendHeld; + } + return h.substrate.writeTreePreservingPrefix( + principal, + repoId, + ref, + args, + ); + }; + const wrappedSubstrate = new Proxy(h.substrate, { + get(target, prop, receiver): unknown { + if (prop === "writeTreePreservingPrefix") { + return writeTreePreservingPrefix; + } + return Reflect.get(target, prop, receiver); + }, + }); + + const freshLocalDir = path.join(h.baseDir, "reentry-local"); + const store = await createDurableConversationStore({ + localStoreDir: freshLocalDir, + signer: h.signer, + substrate: wrappedSubstrate, + workflowRunRepoId: h.workflowRunRepoId, + workflowRunRef: WORKFLOW_RUN_REF, + principal: PRINCIPAL, + agentKey: AGENT_KEY, + }); + + const restored = await store.restoreFromSubstrate(); + expect(restored).toBe(true); + // Wait until the reentrant mirror's append is parked, so it is + // unambiguously the first (held) WAL append. + for (let i = 0; i < 200 && !heldOnce; i += 1) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } + expect(heldOnce).toBe(true); + + // A real mirror carrying a genuinely new turn `c`, enqueued behind the + // parked reentrant mirror. + await store.storage.writeTurns([ + userTurn("a"), + userTurn("b"), + userTurn("c"), + ]); + await store.storage.writeMetadata({ + pendingOperations: [], + tokenUsage: tokenUsage(2), + }); + await store.storage.commit({ message: "c" }); + const cMirror = store.mirrorToSubstrate(); + + // Let a would-be-concurrent `c` mirror reach and pass its own append + // before the reentrant one is released. + await new Promise((resolve) => setTimeout(resolve, 40)); + releaseFirstAppend(); + await cMirror; + // The reentrant mirror is fire-and-forget; let it settle after release. + await new Promise((resolve) => setTimeout(resolve, 40)); + + const reconstructed = await reconstructDurableConversation( + h.agentStateDir, + AGENT_KEY, + ); + if (reconstructed === null) throw new Error("expected a reconstruction"); + expect(reconstructed.turns).toEqual([ + userTurn("a"), + userTurn("b"), + userTurn("c"), + ]); + // No two WAL appends targeted the same boundary seq. + expect(new Set(seqsWritten).size).toBe(seqsWritten.length); + }); }); From 76436061ddcf9bf7a4f2dd62504d98acd7794519 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Sun, 5 Jul 2026 18:14:21 -0500 Subject: [PATCH 065/101] Drop a stale tracker reference from the recovery-sweep comment A tracker id in a source comment points outside the repo and rots when the issue moves or closes; the comment already states the limitation it described. --- packages/workflow-host/src/supervisor/supervisor.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/workflow-host/src/supervisor/supervisor.ts b/packages/workflow-host/src/supervisor/supervisor.ts index 736f5800..2b5f2caf 100644 --- a/packages/workflow-host/src/supervisor/supervisor.ts +++ b/packages/workflow-host/src/supervisor/supervisor.ts @@ -1070,7 +1070,7 @@ export function createWorkflowSupervisor( // per-event form, which readers handle. There is no later trigger for // a run whose fold is interrupted here (e.g. by a crash before the // fold commits): the terminal signal fires once. A bounded recovery - // sweep is tracked as INTR-229; until then such a run stays + // sweep is not yet implemented; until then such a run stays // per-event. The fold commit carries no newly-added terminal event, // so it does not re-fire this terminal-write coupling. for (const { runId } of newlyTerminalRuns) { From 1af0ded26cc06fc643867ff36b7288bd81ac7c7a Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Sun, 5 Jul 2026 19:30:27 -0500 Subject: [PATCH 066/101] Build the child spawn env through one shared producer The recycle respawn built its child env in a second place that diverged from the spawn path: it omitted STEP_COUNT, which the child's spawn-time parser requires, so every recycle respawned a child that aborted at env parse before opening IPC and tore the whole deployment down. Route both the initial spawn and the recycle respawn through one buildChildSpawnEnv producer, keyed off a shared required-key list so the two paths cannot drift and a missing required key is a compile error rather than a production env-parse abort. --- .../workflow-host/src/child/env-bootstrap.ts | 23 +++++++ .../src/supervisor/recycle.test.ts | 46 +++++++++++++ .../workflow-host/src/supervisor/recycle.ts | 23 +++---- .../workflow-host/src/supervisor/spawn-env.ts | 65 +++++++++++++++++++ .../src/supervisor/supervisor.ts | 23 +++---- 5 files changed, 158 insertions(+), 22 deletions(-) create mode 100644 packages/workflow-host/src/supervisor/spawn-env.ts diff --git a/packages/workflow-host/src/child/env-bootstrap.ts b/packages/workflow-host/src/child/env-bootstrap.ts index 551fd462..560bf549 100644 --- a/packages/workflow-host/src/child/env-bootstrap.ts +++ b/packages/workflow-host/src/child/env-bootstrap.ts @@ -18,6 +18,29 @@ import { type } from "arktype"; import { hexDecode } from "@intx/types"; import { IPC_CRYPTO } from "../ipc/index"; +/** + * The required spawn-time env keys, named once so the supervisor-side + * producer (`buildChildSpawnEnv`) and this child-side parser share a + * single required-key contract. `WARM_KEEP` is optional and lives only in + * the shape below. The producer types its output against this list + * (`Record`), so omitting a listed key is a + * compile error. That this list stays in step with the validator shape + * below -- a hand-maintained arktype object -- is covered by the recycle + * env-contract regression test, which drives the real producer through + * this parser. This is the contract whose drift once omitted `STEP_COUNT` + * from the recycle env and broke every recycle. + */ +export const REQUIRED_SPAWN_ENV_KEYS = [ + "IPC_CHANNEL_ID", + "IPC_HMAC_KEY", + "HOST_PUBKEY", + "DEPLOYMENT_ID", + "DEFINITION_HASH", + "MAILBOX_ADDRESS", + "STEP_COUNT", +] as const; +export type RequiredSpawnEnvKey = (typeof REQUIRED_SPAWN_ENV_KEYS)[number]; + /** * Required env keys carried by the supervisor at spawn time. The * validator surface is intentionally narrow: every key documented at diff --git a/packages/workflow-host/src/supervisor/recycle.test.ts b/packages/workflow-host/src/supervisor/recycle.test.ts index 68bbce9a..2890b526 100644 --- a/packages/workflow-host/src/supervisor/recycle.test.ts +++ b/packages/workflow-host/src/supervisor/recycle.test.ts @@ -31,6 +31,7 @@ import { type WorkflowSupervisorBindings, } from "./index"; import { createRecyclePolicy, type RecyclePolicyBounds } from "./recycle"; +import { parseSpawnTimeEnv } from "../child/env-bootstrap"; import { defaultStepRepoId, STEP_GRANTS_PATH } from "./credentials"; import { createControlChannelSender, @@ -257,12 +258,15 @@ type FakeChild = { type SpawnTracker = { spawner: SubprocessSpawner; children: FakeChild[]; + spawnEnvs: Record[]; totalSpawns: number; }; function createSpawnTracker(opts: { sigtermExits?: boolean }): SpawnTracker { const children: FakeChild[] = []; + const spawnEnvs: Record[] = []; const spawner: SubprocessSpawner = ({ env }) => { + spawnEnvs.push(env); const supervisorToChild = createMemoryNdjsonStream(); const childToSupervisor = createMemoryNdjsonStream(); const eventChildToSupervisor = createMemoryFrameStream(); @@ -303,6 +307,7 @@ function createSpawnTracker(opts: { sigtermExits?: boolean }): SpawnTracker { return { spawner, children, + spawnEnvs, get totalSpawns() { return children.length; }, @@ -527,6 +532,47 @@ async function spawnSupervisor(opts: { } describe("supervisor recycle: operator-initiated", () => { + test("the recycle respawn env satisfies the child spawn-time env contract", async () => { + // Regression: the recycle respawn env must carry every required + // spawn-time key. A recycle env that omitted a required key (STEP_COUNT) + // made the respawned child abort in parseSpawnTimeEnv before opening + // IPC, so every recycle tore the deployment down. The fake spawner does + // not run parseSpawnTimeEnv, so this asserts the captured env against it + // directly -- the check the real child binary performs at boot. + const baseDir = await makeTempDir("recycle-env-"); + const ipcKeypair = await generateKeyPair(); + const mailBus = createMockMailBus(); + const tracker = createSpawnTracker({}); + const { supervisor } = await spawnSupervisor({ + baseDir, + tracker, + mailBus, + ipcKeypair, + }); + + const recyclePromise = supervisor.recycle({ reason: "env-contract" }); + while (tracker.children.length < 2) { + await new Promise((r) => setTimeout(r, 1)); + } + const secondChild = tracker.children[1]; + if (secondChild === undefined) { + throw new Error("second child missing after recycle spawn"); + } + await driveReady(secondChild, ipcKeypair); + await recyclePromise; + + expect(tracker.spawnEnvs).toHaveLength(2); + const spawnEnv = tracker.spawnEnvs[0]; + const respawnEnv = tracker.spawnEnvs[1]; + if (spawnEnv === undefined || respawnEnv === undefined) { + throw new Error("expected two captured spawn envs"); + } + // Both the initial spawn and the recycle respawn must satisfy the + // child's spawn-time env contract, with the same step count. + expect(parseSpawnTimeEnv(spawnEnv).stepCount).toBe(1); + expect(parseSpawnTimeEnv(respawnEnv).stepCount).toBe(1); + }); + test("drain mail sends, child is killed, fresh child spawns with a new channelId", async () => { const baseDir = await makeTempDir("recycle-op-"); const ipcKeypair = await generateKeyPair(); diff --git a/packages/workflow-host/src/supervisor/recycle.ts b/packages/workflow-host/src/supervisor/recycle.ts index f0ce145b..5f448844 100644 --- a/packages/workflow-host/src/supervisor/recycle.ts +++ b/packages/workflow-host/src/supervisor/recycle.ts @@ -83,7 +83,6 @@ import { getLogger } from "@intx/log"; import { generateKeyPair } from "@intx/crypto"; -import { hexEncode } from "@intx/types"; import { createControlChannelSender, @@ -101,6 +100,7 @@ import { type CredentialsSnapshot, } from "./credentials"; import type { SubprocessHandle, WorkflowSupervisorBindings } from "./types"; +import { buildChildSpawnEnv } from "./spawn-env"; import { DEFAULT_KILL_TIMEOUT_MS, killChildHandle } from "./child-termination"; const logger = getLogger(["workflow-host", "supervisor", "recycle"]); @@ -314,16 +314,17 @@ export async function triggerRecycle( const ipcKeypair = await ( ctx.bindings.ipcKeyPairFactory ?? generateKeyPair )(); - const env: Record = { - ...ctx.bindings.substrateEnv, - IPC_CHANNEL_ID: channelId, - IPC_HMAC_KEY: hexEncode(hmacKey), - HOST_PUBKEY: hexEncode(ipcKeypair.publicKey), - DEPLOYMENT_ID: ctx.bindings.deploymentId, - DEFINITION_HASH: ctx.definitionHash, - MAILBOX_ADDRESS: ctx.bindings.deploymentMailAddress, - WARM_KEEP: ctx.warmKeep ? "true" : "false", - }; + const env = buildChildSpawnEnv({ + substrateEnv: ctx.bindings.substrateEnv, + channelId, + hmacKey, + hostPublicKey: ipcKeypair.publicKey, + deploymentId: ctx.bindings.deploymentId, + deploymentMailAddress: ctx.bindings.deploymentMailAddress, + stepCount: ctx.bindings.stepCount, + definitionHash: ctx.definitionHash, + warmKeep: ctx.warmKeep, + }); const handle = ctx.bindings.subprocessSpawner({ binaryPath: ctx.bindings.binaryPath, diff --git a/packages/workflow-host/src/supervisor/spawn-env.ts b/packages/workflow-host/src/supervisor/spawn-env.ts new file mode 100644 index 00000000..12aeaad3 --- /dev/null +++ b/packages/workflow-host/src/supervisor/spawn-env.ts @@ -0,0 +1,65 @@ +// Supervisor-side producer of the workflow-process child's spawn-time env. +// +// Both the initial spawn and every recycle respawn route the env through +// this single builder, so the required-key contract the child-side parser +// (`parseSpawnTimeEnv`) enforces has exactly one producer. A divergence +// between the two paths -- the recycle env once omitted `STEP_COUNT` and +// broke every recycle -- is no longer expressible: the required keys are +// built as an exactly-typed record, so omitting one is a compile error. + +import { hexEncode } from "@intx/types"; + +import type { RequiredSpawnEnvKey } from "../child/env-bootstrap"; + +export interface ChildSpawnEnvParts { + /** + * The deployment's stable substrate env (DATA_DIR, the adapter manifest, + * the step inference sources, and so on). Layered UNDER the per-spawn + * anchors below so a required key can never be shadowed by a substrate + * value. + */ + substrateEnv: Record; + /** Supervisor-minted IPC channel id for this spawn. */ + channelId: string; + /** Shared HMAC key for the event channel, minted for this spawn. */ + hmacKey: Uint8Array; + /** Supervisor's Ed25519 public key for this spawn's control channel. */ + hostPublicKey: Uint8Array; + /** Deployment identity the supervisor manages. */ + deploymentId: string; + /** Mail address the deployment registered on the bus. */ + deploymentMailAddress: string; + /** Step count of the deployed workflow (`stepOrder.length`). */ + stepCount: number; + /** Content hash of the deployed workflow definition. */ + definitionHash: string; + /** Whether this deployment's agent is warm-kept across messages. */ + warmKeep: boolean; +} + +/** + * Build the spawn-time env for a workflow-process child. The single + * producer of what `parseSpawnTimeEnv` consumes; the initial spawn and + * every recycle respawn both call it, so neither can drift from the + * required-key contract. + */ +export function buildChildSpawnEnv( + parts: ChildSpawnEnvParts, +): Record { + // Exactly-typed so omitting a required key fails the type-check rather + // than surfacing as a child env-parse abort in production. + const required: Record = { + IPC_CHANNEL_ID: parts.channelId, + IPC_HMAC_KEY: hexEncode(parts.hmacKey), + HOST_PUBKEY: hexEncode(parts.hostPublicKey), + DEPLOYMENT_ID: parts.deploymentId, + DEFINITION_HASH: parts.definitionHash, + MAILBOX_ADDRESS: parts.deploymentMailAddress, + STEP_COUNT: String(parts.stepCount), + }; + return { + ...parts.substrateEnv, + ...required, + WARM_KEEP: parts.warmKeep ? "true" : "false", + }; +} diff --git a/packages/workflow-host/src/supervisor/supervisor.ts b/packages/workflow-host/src/supervisor/supervisor.ts index 2b5f2caf..6bac8f80 100644 --- a/packages/workflow-host/src/supervisor/supervisor.ts +++ b/packages/workflow-host/src/supervisor/supervisor.ts @@ -84,6 +84,7 @@ import { type CredentialsSnapshot, } from "./credentials"; import { commitCancelRequested } from "./cancel-signing"; +import { buildChildSpawnEnv } from "./spawn-env"; import { compactRunEvents } from "./run-event-compaction"; import { createDrainTimeoutAccumulator, @@ -1326,17 +1327,17 @@ export function createWorkflowSupervisor( const channelId = generateChannelId(); const hmacKey = generateHmacKey(); const ipcKeypair = await (bindings.ipcKeyPairFactory ?? generateKeyPair)(); - const env: Record = { - ...bindings.substrateEnv, - IPC_CHANNEL_ID: channelId, - IPC_HMAC_KEY: hexEncode(hmacKey), - HOST_PUBKEY: hexEncode(ipcKeypair.publicKey), - DEPLOYMENT_ID: bindings.deploymentId, - DEFINITION_HASH: opts.definitionHash, - MAILBOX_ADDRESS: bindings.deploymentMailAddress, - STEP_COUNT: String(bindings.stepCount), - WARM_KEEP: opts.warmKeep ? "true" : "false", - }; + const env = buildChildSpawnEnv({ + substrateEnv: bindings.substrateEnv, + channelId, + hmacKey, + hostPublicKey: ipcKeypair.publicKey, + deploymentId: bindings.deploymentId, + deploymentMailAddress: bindings.deploymentMailAddress, + stepCount: bindings.stepCount, + definitionHash: opts.definitionHash, + warmKeep: opts.warmKeep, + }); const handle = bindings.subprocessSpawner({ binaryPath: bindings.binaryPath, From 1001773dfc4087a922fcc9d45c6a172f60fb30d5 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Sun, 5 Jul 2026 19:50:42 -0500 Subject: [PATCH 067/101] Release the mail subscription when a spawn fails spawn registers the deployment mail address and subscribes a handler before the ready handshake. A handshake that times out, a child that exits mid-handshake, or a failed post-ready credentials push threw without releasing either, leaving a live subscription with no dispatch loop to service it -- and a redeploy would add a second one. Route each of those failure exits through shutdownInternal, which owns the starting-phase teardown, so the subscription and registration are released before the spawn rejects. --- .../src/supervisor/recycle.test.ts | 51 +++++++++++++++++++ .../src/supervisor/supervisor.ts | 42 ++++++++++----- 2 files changed, 81 insertions(+), 12 deletions(-) diff --git a/packages/workflow-host/src/supervisor/recycle.test.ts b/packages/workflow-host/src/supervisor/recycle.test.ts index 2890b526..af342cd7 100644 --- a/packages/workflow-host/src/supervisor/recycle.test.ts +++ b/packages/workflow-host/src/supervisor/recycle.test.ts @@ -531,6 +531,57 @@ async function spawnSupervisor(opts: { return { supervisor, spawnResult: result, firstSender }; } +describe("supervisor spawn: failure cleanup", () => { + test("a spawn that times out releases the mail subscription and address", async () => { + // Regression: spawn registers + subscribes the deployment mail address + // before the ready handshake. A handshake that times out (or the child + // exiting mid-handshake) must release that subscription and unregister + // the address; otherwise a live subscription lingers with no dispatch + // loop to service it and a redeploy adds a second one. + const baseDir = await makeTempDir("spawn-timeout-"); + const ipcKeypair = await generateKeyPair(); + const mailBus = createMockMailBus(); + // sigtermExits so the timeout path's kill settles the fake child. + const tracker = createSpawnTracker({ sigtermExits: true }); + await seedStepGrants( + baseDir, + defaultStepRepoId({ deploymentId: "deployment-x", stepId: "step-1" }), + [{ resource: "thing", action: "read" }], + ); + const bindings = await buildBindings({ + baseDir, + spawner: tracker.spawner, + mailBus, + ipcKeypair, + }); + const supervisor = createWorkflowSupervisor({ + ...bindings, + readyTimeoutMs: 20, + }); + + // Spawn but never drive the child's `ready` frame, so the handshake + // times out. + await expect( + supervisor.spawn({ + stepOrder: ["step-1"], + definitionHash: "def-hash-abc", + warmKeep: false, + onInferenceEvent: () => undefined, + }), + ).rejects.toThrow(/did not emit ready/); + + // The address must not remain registered, and the history must carry + // both the register and its matching unregister. + expect(mailBus.registered()).not.toContain("deployment-x@example.com"); + expect(mailBus.registrationHistory()).toContain( + "register:deployment-x@example.com", + ); + expect(mailBus.registrationHistory()).toContain( + "unregister:deployment-x@example.com", + ); + }); +}); + describe("supervisor recycle: operator-initiated", () => { test("the recycle respawn env satisfies the child spawn-time env contract", async () => { // Regression: the recycle respawn env must carry every required diff --git a/packages/workflow-host/src/supervisor/supervisor.ts b/packages/workflow-host/src/supervisor/supervisor.ts index 6bac8f80..8aa69e5f 100644 --- a/packages/workflow-host/src/supervisor/supervisor.ts +++ b/packages/workflow-host/src/supervisor/supervisor.ts @@ -1467,11 +1467,22 @@ export function createWorkflowSupervisor( clearTimer: readyClearTimer, logger, }); + // Release the mail subscription + address registration installed + // before the handshake. shutdownInternal owns the "starting"-phase + // teardown; its own kill against the already-killed handle is + // idempotent. Without this the failed spawn leaves a live + // subscription with no dispatch loop to service it. + await shutdownInternal({ + reason: `child did not emit ready within ${readyTimeoutMs}ms`, + }); throw new Error( `workflow-host supervisor: child did not emit ready within ${readyTimeoutMs}ms; killed`, ); } if (readyRace.kind === "failed") { + // The child exited during the handshake; release the subscription + // and registration the same way the timeout path does. + await shutdownInternal({ reason: "child exited before emitting ready" }); throw readyRace.err; } const readyInfo = readyRace.info; @@ -1485,19 +1496,26 @@ export function createWorkflowSupervisor( // same control channel `trigger.fire` uses, so the ordering // guarantee (`grants-updated` lands before `trigger.fire`) holds // for buffered and post-ready inbound mail alike. - await wired.wiring.controlSender.send({ - type: "grants-updated", - data: { - snapshot: { - steps: credentialsSnapshot.steps.map((s) => ({ - stepId: s.stepId, - address: s.address, - grants: [...s.grants], - contentHash: s.contentHash, - })), + try { + await wired.wiring.controlSender.send({ + type: "grants-updated", + data: { + snapshot: { + steps: credentialsSnapshot.steps.map((s) => ({ + stepId: s.stepId, + address: s.address, + grants: [...s.grants], + contentHash: s.contentHash, + })), + }, }, - }, - }); + }); + } catch (cause) { + // The child readied but died before or during the credentials push. + // Release the subscription and registration before failing the spawn. + await shutdownInternal({ reason: "grants-updated push to child failed" }); + throw cause; + } // Transition to running. The dispatch loop (started below) // picks up any pre-ready buffered mail through the FIFO inbox From 862e31473ca8ebcec4c861e4766f35ee5e348dac Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Sun, 5 Jul 2026 23:49:06 -0500 Subject: [PATCH 068/101] Carry a live source rotation across a recycle respawn The rotated inference-source list lived frozen in the deployment's substrateEnv, which every spawn spreads verbatim, so a recycle respawned the child on the deploy-time sources -- silently undoing a rotation done to fail away from a broken source. Recompute the source entry on every spawn and respawn through a new dynamicSpawnEnv binding the env builder pulls each time. The sidecar owns the table and its serialization; the single-step rotation handler revises it in place, so the respawn carries the rotation. deliverSources stays a flat live-only swap. --- apps/sidecar/src/workflow-host-wiring.test.ts | 94 ++++++++++++++++++- apps/sidecar/src/workflow-host-wiring.ts | 41 +++++++- .../src/supervisor/lifecycle-races.test.ts | 1 + .../supervisor/outbound-signed-send.test.ts | 1 + .../src/supervisor/recycle.test.ts | 60 ++++++++++++ .../workflow-host/src/supervisor/recycle.ts | 1 + .../workflow-host/src/supervisor/spawn-env.ts | 16 +++- .../src/supervisor/spawn-replay-fifo.test.ts | 1 + .../supervisor/stale-cohort-routing.test.ts | 1 + .../src/supervisor/substrate-write.test.ts | 1 + .../src/supervisor/supervisor.test.ts | 1 + .../src/supervisor/supervisor.ts | 1 + .../workflow-host/src/supervisor/types.ts | 11 +++ 13 files changed, 221 insertions(+), 9 deletions(-) diff --git a/apps/sidecar/src/workflow-host-wiring.test.ts b/apps/sidecar/src/workflow-host-wiring.test.ts index 18285632..aec0a382 100644 --- a/apps/sidecar/src/workflow-host-wiring.test.ts +++ b/apps/sidecar/src/workflow-host-wiring.test.ts @@ -86,6 +86,7 @@ describe("createSidecarWorkflowSupervisor", () => { deriveStepAddress: ({ deploymentId, stepId }) => `${deploymentId}-${stepId}@example.com`, substrateEnv: { DATA_DIR: "/tmp/wire" }, + dynamicSpawnEnv: () => ({}), subprocessSpawner: spawner, }); @@ -121,6 +122,7 @@ describe("createSidecarWorkflowSupervisor", () => { deriveStepAddress: ({ deploymentId, stepId }) => `${deploymentId}-${stepId}@example.com`, substrateEnv: {}, + dynamicSpawnEnv: () => ({}), subprocessSpawner: () => { throw new Error("spawner not invoked in this test"); }, @@ -1521,6 +1523,7 @@ describe("createSidecarDeployRouter multi-step branch", () => { env: Record; childToSupervisor: ReturnType; eventChildToSupervisor: ReturnType; + childSender?: ReturnType; }; const spawns: Spawn[] = []; const spawner: SubprocessSpawner = ({ env }) => { @@ -1546,7 +1549,10 @@ describe("createSidecarDeployRouter multi-step branch", () => { }; return handle; }; - async function driveReadyFor(index: number): Promise { + async function driveReadyFor( + index: number, + opts?: { sendRecycleRequest?: boolean }, + ): Promise { while (spawns.length <= index) { await new Promise((r) => setTimeout(r, 1)); } @@ -1569,6 +1575,9 @@ describe("createSidecarDeployRouter multi-step branch", () => { }, }, }); + // Retain the sender so a later recycle.request (recycleRequestFor) + // signs with the SAME keypair the supervisor pinned from this ready. + spawn.childSender = childSender; await childSender.send({ type: "ready", data: { @@ -1578,8 +1587,34 @@ describe("createSidecarDeployRouter multi-step branch", () => { ), }, }); + if (opts?.sendRecycleRequest === true) { + // Model the child asking to recycle; the supervisor's upstream pump + // consumes it and respawns. + await childSender.send({ + type: "recycle.request", + data: { reason: "sources-rotation-recycle" }, + }); + } } - return { spawner, driveReadyFor, spawnCount: () => spawns.length }; + return { + spawner, + driveReadyFor, + spawnCount: () => spawns.length, + envFor: (index: number): Record | undefined => + spawns[index]?.env, + async recycleRequestFor(index: number): Promise { + const sender = spawns[index]?.childSender; + if (sender === undefined) { + throw new Error( + `spawn ${String(index)} has no sender; drive its ready first`, + ); + } + await sender.send({ + type: "recycle.request", + data: { reason: "sources-rotation-recycle" }, + }); + }, + }; } function isRegistered( @@ -2013,6 +2048,61 @@ describe("createSidecarDeployRouter multi-step branch", () => { ).toBe(false); }); + test("a source rotation survives a recycle respawn", async () => { + // End-to-end guard for the rotation-survives-recycle fix: the single-step + // rotation handler mutates `currentSources`, `dynamicSpawnEnv` + // re-serializes it, and the recycle respawn's STEP_INFERENCE_SOURCES + // carries the ROTATED table -- not the frozen deploy-time one. + const sourcesRouter = createMultistepSourcesRouter(); + const spawner = makeReadyDrivingSpawner(10800); + const { router } = await buildMultistepFixture({ + spawner: spawner.spawner, + multistepSourcesRouter: sourcesRouter, + multistepSubstrateEnv: { + SIDECAR_DATA_DIR: await createTempBaseDir("sidecar-rot-survive-"), + }, + }); + + const addr = "ins_rotsurvive@example.com"; + const deployPromise = router.deploy(singleStepFrame(addr, "wf-rotsurvive")); + await spawner.driveReadyFor(0); + await deployPromise; + + // The initial spawn carries the deploy-time source table. + const initialEnv = spawner.envFor(0); + if (initialEnv === undefined) throw new Error("initial spawn env missing"); + expect( + JSON.parse(initialEnv[STEP_INFERENCE_SOURCES_ENV_KEY] ?? "null"), + ).toEqual({ "step-1": [makeInferenceSource("step-1")] }); + + // Rotate the single-step deployment's sources in place. + const rotated = makeInferenceSource("rotated"); + expect( + await sourcesRouter.tryRoute({ + type: "sources.update", + agentAddress: addr, + sources: [rotated], + defaultSource: "rotated", + }), + ).toBe(true); + + // The child asks to recycle; the supervisor respawns. + await spawner.recycleRequestFor(0); + while (spawner.spawnCount() < 2) { + await new Promise((r) => setTimeout(r, 1)); + } + await spawner.driveReadyFor(1); + + // The recycle respawn's sources are the ROTATED table, proving the + // rotation survived the recycle (before the fix it reverted to the + // deploy-time list frozen in substrateEnv). + const respawnEnv = spawner.envFor(1); + if (respawnEnv === undefined) throw new Error("respawn env missing"); + expect( + JSON.parse(respawnEnv[STEP_INFERENCE_SOURCES_ENV_KEY] ?? "null"), + ).toEqual({ "step-1": [rotated] }); + }); + test("two addresses whose deriveDeploymentId slugs collide are rejected at the second deploy", async () => { // deriveDeploymentId substitutes every disallowed character with // `-`, so two distinct addresses can collapse to the same slug. The slug diff --git a/apps/sidecar/src/workflow-host-wiring.ts b/apps/sidecar/src/workflow-host-wiring.ts index a787f6c2..fa367f5e 100644 --- a/apps/sidecar/src/workflow-host-wiring.ts +++ b/apps/sidecar/src/workflow-host-wiring.ts @@ -430,6 +430,12 @@ export type CreateSidecarWorkflowSupervisorOpts = { deriveStepRepoId?: DeriveStepRepoId; /** Substrate-config keys propagated to the child via spawn-time env. */ substrateEnv: Record; + /** + * Dynamic spawn-env fragment the supervisor recomputes on every spawn and + * recycle respawn (e.g. a live-rotated inference-source list). Its keys + * layer over `substrateEnv`. See the `dynamicSpawnEnv` supervisor binding. + */ + dynamicSpawnEnv: () => Record; /** * Override the subprocess spawner. Tests inject a deterministic * mock; production defaults to the `Bun.spawn`-backed @@ -1129,17 +1135,21 @@ export function createSidecarDeployRouter(deps: { // Per-deployment substrate-config keys the workflow-substrate-factory // validator requires. The boot edge's `multistepSubstrateEnv` carries // the boot-edge constants; the four workflow-definition / workflow-run - // identity keys are derived per-deploy here. `STEP_INFERENCE_SOURCES` - // threads each step's ordered inference-source failover chain into the - // child. + // identity keys are derived per-deploy here. const substrateEnv: Record = { ...multistepSubstrateEnv, WORKFLOW_DEFINITION_REPO_ID: spec.definition.id, WORKFLOW_DEFINITION_REF: "refs/heads/main", WORKFLOW_RUN_REPO_ID: deploymentId, WORKFLOW_RUN_REF: "refs/heads/main", - [STEP_INFERENCE_SOURCES_ENV_KEY]: JSON.stringify(spec.sources), }; + // Live-rotatable per-step inference sources. Seeded from the deploy + // spec, then revised in place by the single-step sources-rotation + // handler below. `STEP_INFERENCE_SOURCES` is NOT in the frozen + // `substrateEnv`: it is recomputed on every spawn and recycle respawn + // via `dynamicSpawnEnv`, so a rotation survives a recycle instead of + // reverting to the deploy-time list. + let currentSources = spec.sources; const wired = createSidecarWorkflowSupervisor({ transport: deps.transport, @@ -1156,6 +1166,13 @@ export function createSidecarDeployRouter(deps: { deriveStepAddress: stepStrategy.deriveStepAddress, deriveStepRepoId: stepStrategy.deriveStepRepoId, substrateEnv, + // Recomputed on every spawn AND recycle respawn. The rotation + // handler below revises `currentSources` in place, so a respawn + // re-serializes the current (possibly rotated) list rather than the + // frozen deploy-time value. + dynamicSpawnEnv: () => ({ + [STEP_INFERENCE_SOURCES_ENV_KEY]: JSON.stringify(currentSources), + }), subprocessSpawner: multistepSpawner, ...(deps.multistepBinaryPath !== undefined ? { binaryPath: deps.multistepBinaryPath } @@ -1296,9 +1313,24 @@ export function createSidecarDeployRouter(deps: { // so it registers no handler and `tryRoute` reports its address as // unrouted. if (warmKeep) { + // A single-step deployment's source table has exactly one entry, + // keyed by the head step. Derive that key once here (the layer that + // owns the single-key invariant); `deliverSources` stays flat and + // stepId-agnostic. + const rotationStepId = spec.definition.stepOrder[0]; + if (rotationStepId === undefined) { + throw new Error( + "single-step deploy has no step id for sources rotation", + ); + } deps.multistepSourcesRouter?.register( spec.agentAddress, async (args) => { + // Persist BEFORE the live-swap: a recycle mid-flight rejects + // `deliverSources`, but the respawn must still pick up the + // rotation. The wire boundary guarantees `args.sources[0]` is the + // default, which the recycle env form pins as the active source. + currentSources = { [rotationStepId]: args.sources }; await wired.supervisor.deliverSources({ sources: args.sources, defaultSource: args.defaultSource, @@ -1813,6 +1845,7 @@ export function createSidecarWorkflowSupervisor( subprocessSpawner: opts.subprocessSpawner ?? defaultSubprocessSpawner, binaryPath: opts.binaryPath ?? SIDECAR_WORKFLOW_CHILD_BINARY, substrateEnv: opts.substrateEnv, + dynamicSpawnEnv: opts.dynamicSpawnEnv, workflowRunRepoId: opts.workflowRunRepoId, workflowRunRef: opts.workflowRunRef, deploymentId: opts.deploymentId, diff --git a/packages/workflow-host/src/supervisor/lifecycle-races.test.ts b/packages/workflow-host/src/supervisor/lifecycle-races.test.ts index 7689461a..d08c85b1 100644 --- a/packages/workflow-host/src/supervisor/lifecycle-races.test.ts +++ b/packages/workflow-host/src/supervisor/lifecycle-races.test.ts @@ -343,6 +343,7 @@ async function buildBindings(opts: { subprocessSpawner: opts.spawner, binaryPath: "/fake/bin/workflow-child", substrateEnv: { DATA_DIR: opts.baseDir }, + dynamicSpawnEnv: () => ({}), workflowRunRepoId: { kind: "workflow-run", id: "deployment-x" }, workflowRunRef: "refs/heads/main", deploymentId: "deployment-x", diff --git a/packages/workflow-host/src/supervisor/outbound-signed-send.test.ts b/packages/workflow-host/src/supervisor/outbound-signed-send.test.ts index df1e75e5..5e8012e2 100644 --- a/packages/workflow-host/src/supervisor/outbound-signed-send.test.ts +++ b/packages/workflow-host/src/supervisor/outbound-signed-send.test.ts @@ -261,6 +261,7 @@ describe("supervisor-backed outbound signed send (Phase 4.3)", () => { }, binaryPath: "/fake/bin/workflow-child", substrateEnv: {}, + dynamicSpawnEnv: () => ({}), workflowRunRepoId: { kind: "workflow-run", id: "outbound-dep" }, workflowRunRef: "refs/heads/main", deploymentId: "outbound-dep", diff --git a/packages/workflow-host/src/supervisor/recycle.test.ts b/packages/workflow-host/src/supervisor/recycle.test.ts index af342cd7..6bf64d2a 100644 --- a/packages/workflow-host/src/supervisor/recycle.test.ts +++ b/packages/workflow-host/src/supervisor/recycle.test.ts @@ -473,6 +473,7 @@ async function buildBindings(opts: { subprocessSpawner: opts.spawner, binaryPath: "/fake/bin/workflow-child", substrateEnv: { DATA_DIR: opts.baseDir }, + dynamicSpawnEnv: () => ({}), workflowRunRepoId: { kind: "workflow-run", id: "deployment-x" }, workflowRunRef: "refs/heads/main", deploymentId: "deployment-x", @@ -582,6 +583,65 @@ describe("supervisor spawn: failure cleanup", () => { }); }); +describe("supervisor spawn: dynamic env", () => { + test("a recycle respawn re-pulls dynamicSpawnEnv so a host revision survives", async () => { + // Mechanism behind a source rotation surviving a recycle: the spawn-env + // builder pulls dynamicSpawnEnv() on every spawn AND respawn, so a value + // the host revised between the two reaches the respawned child instead + // of reverting to the frozen substrateEnv value. + const baseDir = await makeTempDir("recycle-dynenv-"); + const ipcKeypair = await generateKeyPair(); + const mailBus = createMockMailBus(); + const tracker = createSpawnTracker({}); + await seedStepGrants( + baseDir, + defaultStepRepoId({ deploymentId: "deployment-x", stepId: "step-1" }), + [{ resource: "thing", action: "read" }], + ); + let hostRevised = "before-rotation"; + const bindings = await buildBindings({ + baseDir, + spawner: tracker.spawner, + mailBus, + ipcKeypair, + }); + const supervisor = createWorkflowSupervisor({ + ...bindings, + dynamicSpawnEnv: () => ({ DYNAMIC_MARKER: hostRevised }), + }); + + const spawnPromise = supervisor.spawn({ + stepOrder: ["step-1"], + definitionHash: "def-hash-abc", + warmKeep: false, + onInferenceEvent: () => undefined, + }); + while (tracker.children.length === 0) { + await new Promise((r) => setTimeout(r, 1)); + } + const firstChild = tracker.children[0]; + if (firstChild === undefined) throw new Error("first child missing"); + await driveReady(firstChild, ipcKeypair); + await spawnPromise; + expect(tracker.spawnEnvs[0]?.DYNAMIC_MARKER).toBe("before-rotation"); + + // The host revises the dynamic value (models a live source rotation), + // then a recycle respawns the child. + hostRevised = "after-rotation"; + const recyclePromise = supervisor.recycle({ reason: "dynenv" }); + while (tracker.children.length < 2) { + await new Promise((r) => setTimeout(r, 1)); + } + const secondChild = tracker.children[1]; + if (secondChild === undefined) throw new Error("second child missing"); + await driveReady(secondChild, ipcKeypair); + await recyclePromise; + + // The respawn env carries the REVISED value -- the rotation survived. + expect(tracker.spawnEnvs[1]?.DYNAMIC_MARKER).toBe("after-rotation"); + }); +}); + describe("supervisor recycle: operator-initiated", () => { test("the recycle respawn env satisfies the child spawn-time env contract", async () => { // Regression: the recycle respawn env must carry every required diff --git a/packages/workflow-host/src/supervisor/recycle.ts b/packages/workflow-host/src/supervisor/recycle.ts index 5f448844..9d976479 100644 --- a/packages/workflow-host/src/supervisor/recycle.ts +++ b/packages/workflow-host/src/supervisor/recycle.ts @@ -316,6 +316,7 @@ export async function triggerRecycle( )(); const env = buildChildSpawnEnv({ substrateEnv: ctx.bindings.substrateEnv, + dynamicSpawnEnv: ctx.bindings.dynamicSpawnEnv, channelId, hmacKey, hostPublicKey: ipcKeypair.publicKey, diff --git a/packages/workflow-host/src/supervisor/spawn-env.ts b/packages/workflow-host/src/supervisor/spawn-env.ts index 12aeaad3..bb7db397 100644 --- a/packages/workflow-host/src/supervisor/spawn-env.ts +++ b/packages/workflow-host/src/supervisor/spawn-env.ts @@ -14,11 +14,18 @@ import type { RequiredSpawnEnvKey } from "../child/env-bootstrap"; export interface ChildSpawnEnvParts { /** * The deployment's stable substrate env (DATA_DIR, the adapter manifest, - * the step inference sources, and so on). Layered UNDER the per-spawn - * anchors below so a required key can never be shadowed by a substrate - * value. + * and so on), frozen for the deployment's lifetime. Layered UNDER the + * dynamic env fragment (so a host revision wins) and the per-spawn anchors + * (so a required key is never shadowed). */ substrateEnv: Record; + /** + * Host-supplied dynamic env fragment, recomputed for every spawn and + * respawn. Its keys layer OVER `substrateEnv` (so a value the host revised + * between spawns wins) and UNDER the required anchors. Returns `{}` when + * the host has no dynamic entries. + */ + dynamicSpawnEnv: () => Record; /** Supervisor-minted IPC channel id for this spawn. */ channelId: string; /** Shared HMAC key for the event channel, minted for this spawn. */ @@ -59,6 +66,9 @@ export function buildChildSpawnEnv( }; return { ...parts.substrateEnv, + // Host-revised entries win over the frozen substrate env; the required + // anchors below still win over everything. + ...parts.dynamicSpawnEnv(), ...required, WARM_KEEP: parts.warmKeep ? "true" : "false", }; diff --git a/packages/workflow-host/src/supervisor/spawn-replay-fifo.test.ts b/packages/workflow-host/src/supervisor/spawn-replay-fifo.test.ts index fcc185f6..c1b58fae 100644 --- a/packages/workflow-host/src/supervisor/spawn-replay-fifo.test.ts +++ b/packages/workflow-host/src/supervisor/spawn-replay-fifo.test.ts @@ -545,6 +545,7 @@ async function boot(opts: { prefix: string }): Promise< subprocessSpawner: spawner, binaryPath: "/fake/bin/workflow-child", substrateEnv: { DATA_DIR: baseDir }, + dynamicSpawnEnv: () => ({}), workflowRunRepoId, workflowRunRef: "refs/heads/main", deploymentId: "deployment-x", diff --git a/packages/workflow-host/src/supervisor/stale-cohort-routing.test.ts b/packages/workflow-host/src/supervisor/stale-cohort-routing.test.ts index e6e4132d..2632bbca 100644 --- a/packages/workflow-host/src/supervisor/stale-cohort-routing.test.ts +++ b/packages/workflow-host/src/supervisor/stale-cohort-routing.test.ts @@ -442,6 +442,7 @@ describe("H-S2 stale-cohort routing pinch-point", () => { subprocessSpawner: tracker.spawner, binaryPath: "/fake/bin", substrateEnv: { DATA_DIR: baseDir }, + dynamicSpawnEnv: () => ({}), workflowRunRepoId: { kind: "workflow-run", id: "deployment-x" }, workflowRunRef: "refs/heads/main", deploymentId: "deployment-x", diff --git a/packages/workflow-host/src/supervisor/substrate-write.test.ts b/packages/workflow-host/src/supervisor/substrate-write.test.ts index bb2555b1..887ad3ab 100644 --- a/packages/workflow-host/src/supervisor/substrate-write.test.ts +++ b/packages/workflow-host/src/supervisor/substrate-write.test.ts @@ -517,6 +517,7 @@ async function bootSupervisor(opts: { subprocessSpawner: spawner, binaryPath: "/fake/bin/workflow-child", substrateEnv: { DATA_DIR: baseDir }, + dynamicSpawnEnv: () => ({}), workflowRunRepoId: { kind: "workflow-run", id: "deployment-x" }, workflowRunRef: "refs/heads/main", deploymentId: "deployment-x", diff --git a/packages/workflow-host/src/supervisor/supervisor.test.ts b/packages/workflow-host/src/supervisor/supervisor.test.ts index c6113642..db1b0842 100644 --- a/packages/workflow-host/src/supervisor/supervisor.test.ts +++ b/packages/workflow-host/src/supervisor/supervisor.test.ts @@ -504,6 +504,7 @@ async function buildBindings(opts: { subprocessSpawner: opts.spawner, binaryPath: "/fake/bin/workflow-child", substrateEnv: { DATA_DIR: opts.baseDir }, + dynamicSpawnEnv: () => ({}), workflowRunRepoId: { kind: "workflow-run", id: "deployment-x" }, workflowRunRef: "refs/heads/main", deploymentId: "deployment-x", diff --git a/packages/workflow-host/src/supervisor/supervisor.ts b/packages/workflow-host/src/supervisor/supervisor.ts index 8aa69e5f..ef4f43b5 100644 --- a/packages/workflow-host/src/supervisor/supervisor.ts +++ b/packages/workflow-host/src/supervisor/supervisor.ts @@ -1329,6 +1329,7 @@ export function createWorkflowSupervisor( const ipcKeypair = await (bindings.ipcKeyPairFactory ?? generateKeyPair)(); const env = buildChildSpawnEnv({ substrateEnv: bindings.substrateEnv, + dynamicSpawnEnv: bindings.dynamicSpawnEnv, channelId, hmacKey, hostPublicKey: ipcKeypair.publicKey, diff --git a/packages/workflow-host/src/supervisor/types.ts b/packages/workflow-host/src/supervisor/types.ts index 164783fb..f16c93d4 100644 --- a/packages/workflow-host/src/supervisor/types.ts +++ b/packages/workflow-host/src/supervisor/types.ts @@ -281,6 +281,17 @@ export interface WorkflowSupervisorBindings { * construction logic owns the shape. */ substrateEnv: Record; + /** + * Per-spawn dynamic substrate-env entries the host recomputes for every + * spawn AND every recycle respawn. Distinct from `substrateEnv`, which is + * frozen for the deployment's lifetime: this callback lets a value the + * host revised between spawns (a live inference-source rotation) reach the + * respawned child. Invoked by the spawn-env builder on each spawn/respawn; + * its keys layer over `substrateEnv` and under the IPC anchors. Like + * `substrateEnv`, the supervisor does not inspect the returned keys -- the + * host owns their shape. A host with no dynamic entries returns `{}`. + */ + dynamicSpawnEnv: () => Record; /** * Workflow-run repo identity for the deployment. The supervisor * commits its own CancelRequested / drain events here. From 6e2d68392611f7f42d00bb41a324f39aba386194 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Mon, 6 Jul 2026 00:14:26 -0500 Subject: [PATCH 069/101] Keep a failing deploy cleanup from masking the real error The soft-failed-deploy catch awaited deleteWorkflowDeploymentRecord before rethrowing. A rejecting delete replaced the real deploy error and skipped releaseSlug, so the failure surfaced as a cleanup error and the slug leaked. Wrap the record delete in its own try/catch: a rejection is logged at error (the orphaned record is durable state the next boot re-drives) while the original cause still propagates and the slug is still released. --- apps/sidecar/src/workflow-host-wiring.ts | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/apps/sidecar/src/workflow-host-wiring.ts b/apps/sidecar/src/workflow-host-wiring.ts index fa367f5e..b7281a7b 100644 --- a/apps/sidecar/src/workflow-host-wiring.ts +++ b/apps/sidecar/src/workflow-host-wiring.ts @@ -1588,8 +1588,19 @@ export function createSidecarDeployRouter(deps: { } catch (cause) { // Soft failure (this process survived, the deploy threw): drop the // record and release the slug so the failed deploy is neither restored - // nor leaks its slug. - await deleteWorkflowDeploymentRecord(dataDir, deploymentId); + // nor leaks its slug. The record delete must not mask the real deploy + // error or skip releasing the slug: a rejecting delete is logged (the + // orphaned record is a durable-state leak the next boot scan re-drives) + // but `cause` is still what propagates and the slug is still released. + try { + await deleteWorkflowDeploymentRecord(dataDir, deploymentId); + } catch (cleanupError) { + const message = + cleanupError instanceof Error + ? cleanupError.message + : String(cleanupError); + logger.error`deploy cleanup: deleteWorkflowDeploymentRecord failed for ${deploymentId}: ${message}`; + } releaseSlug(deploymentId, frame.agentAddress); throw cause; } finally { From b5f69b329d6c35bf0208708fdd69a33b750c3d87 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Mon, 6 Jul 2026 00:28:48 -0500 Subject: [PATCH 070/101] Persist a source rotation so it survives a sidecar restart A single-step deployment's source rotation lived only in the deploy closure's in-memory currentSources, which the recycle path re-pulls but which a full sidecar restart loses -- the boot scan reseeds spec.sources from the deployment record, and that record still held the deploy-time sources. Write the rotated table into the deployment record on each rotation, through the shared buildDeploymentRecord builder, before committing the rotation in memory so a failed write leaves no partial state. Restore already reconstructs spec.sources from record.sources, so the respawned child picks up the rotation after a restart. --- apps/sidecar/src/workflow-host-wiring.test.ts | 111 ++++++++++++++++++ apps/sidecar/src/workflow-host-wiring.ts | 59 ++++++++-- 2 files changed, 157 insertions(+), 13 deletions(-) diff --git a/apps/sidecar/src/workflow-host-wiring.test.ts b/apps/sidecar/src/workflow-host-wiring.test.ts index aec0a382..1683e1d1 100644 --- a/apps/sidecar/src/workflow-host-wiring.test.ts +++ b/apps/sidecar/src/workflow-host-wiring.test.ts @@ -2103,6 +2103,117 @@ describe("createSidecarDeployRouter multi-step branch", () => { ).toEqual({ "step-1": [rotated] }); }); + test("a source rotation survives a full sidecar restart", async () => { + // Restart-durability: a rotation is persisted into the deployment record, + // so a fresh sidecar process (a restore over the same data dir) respawns + // the deployment on the ROTATED sources, not the deploy-time ones. + const dataDir = await createTempBaseDir("sidecar-rot-restart-"); + const addr = "ins_rotrestart@example.com"; + + // First process: deploy a single-step deployment and rotate its sources. + const sourcesRouter = createMultistepSourcesRouter(); + const first = makeReadyDrivingSpawner(10900); + const { router: routerA } = await buildMultistepFixture({ + spawner: first.spawner, + multistepSourcesRouter: sourcesRouter, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + const deployPromise = routerA.deploy( + singleStepFrame(addr, "wf-rotrestart"), + ); + await first.driveReadyFor(0); + await deployPromise; + + const rotated = makeInferenceSource("rotated"); + expect( + await sourcesRouter.tryRoute({ + type: "sources.update", + agentAddress: addr, + sources: [rotated], + defaultSource: "rotated", + }), + ).toBe(true); + + // Second process (simulated restart): a fresh router over the SAME data + // dir restores the deployment from its durable record. + const second = makeReadyDrivingSpawner(11000); + const { router: routerB } = await buildMultistepFixture({ + spawner: second.spawner, + transport: createInMemoryTransport(), + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + const restorePromise = routerB.restoreWorkflowDeployments(); + await second.driveReadyFor(0); + await restorePromise; + + // The restored spawn carries the ROTATED sources, read back from the + // durable record -- not the deploy-time list. + const restoredEnv = second.envFor(0); + if (restoredEnv === undefined) { + throw new Error("restored spawn env missing"); + } + expect( + JSON.parse(restoredEnv[STEP_INFERENCE_SOURCES_ENV_KEY] ?? "null"), + ).toEqual({ "step-1": [rotated] }); + }); + + test("a rotation whose persist fails leaves no partial effect", async () => { + // Atomicity: if the durable write rejects, the rotation takes NO effect. + // currentSources stays on the deploy-time table (so a recycle respawn is + // unrotated) and deliverSources is never reached. The write is ordered + // before the in-memory commit precisely so a failure cannot leave the + // three views (record, currentSources, live child) divergent. + const dataDir = await createTempBaseDir("sidecar-rot-failatomic-"); + const addr = "ins_rotfailatomic@example.com"; + const sourcesRouter = createMultistepSourcesRouter(); + const spawner = makeReadyDrivingSpawner(11100); + const { router } = await buildMultistepFixture({ + spawner: spawner.spawner, + multistepSourcesRouter: sourcesRouter, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + + const deployPromise = router.deploy(singleStepFrame(addr, "wf-rotfail")); + await spawner.driveReadyFor(0); + await deployPromise; + + // Replace the deployment record file with a directory so the rotation's + // writeWorkflowDeploymentRecord rejects (EISDIR) -- a deterministic, + // root-immune write fault (a chmod guard would be bypassed under root). + const recordFile = path.join( + dataDir, + "workflow-runs", + deriveDeploymentId(addr), + "deployment.json", + ); + await fs.rm(recordFile); + await fs.mkdir(recordFile); + + // The persist rejects, so the route rejects and nothing after the write + // runs. + await expect( + sourcesRouter.tryRoute({ + type: "sources.update", + agentAddress: addr, + sources: [makeInferenceSource("rotated")], + defaultSource: "rotated", + }), + ).rejects.toThrow(); + + // currentSources is untouched: a recycle respawn carries the DEPLOY-TIME + // sources, proving the failed rotation left no partial in-memory effect. + await spawner.recycleRequestFor(0); + while (spawner.spawnCount() < 2) { + await new Promise((r) => setTimeout(r, 1)); + } + await spawner.driveReadyFor(1); + const respawnEnv = spawner.envFor(1); + if (respawnEnv === undefined) throw new Error("respawn env missing"); + expect( + JSON.parse(respawnEnv[STEP_INFERENCE_SOURCES_ENV_KEY] ?? "null"), + ).toEqual({ "step-1": [makeInferenceSource("step-1")] }); + }); + test("two addresses whose deriveDeploymentId slugs collide are rejected at the second deploy", async () => { // deriveDeploymentId substitutes every disallowed character with // `-`, so two distinct addresses can collapse to the same slug. The slug diff --git a/apps/sidecar/src/workflow-host-wiring.ts b/apps/sidecar/src/workflow-host-wiring.ts index b7281a7b..e923ce28 100644 --- a/apps/sidecar/src/workflow-host-wiring.ts +++ b/apps/sidecar/src/workflow-host-wiring.ts @@ -1074,6 +1074,29 @@ export function createSidecarDeployRouter(deps: { hubPublicKey: string | undefined; } + /** + * Build the durable deployment record from a spec and a source table. The + * table is a parameter (not `spec.sources`) so the deploy path writes the + * deploy-time sources while the rotation handler writes the live-rotated + * ones -- both through one shape, so a rotation persists the same record a + * boot-time restore reseeds from. + */ + function buildDeploymentRecord( + spec: WorkflowDeploySpec, + sources: WorkflowDeploymentRecord["sources"], + ): WorkflowDeploymentRecord { + return { + version: 1, + agentAddress: spec.agentAddress, + definitionId: spec.definition.id, + sources, + ...(spec.sessionId !== undefined ? { sessionId: spec.sessionId } : {}), + ...(spec.hubPublicKey !== undefined + ? { hubPublicKey: spec.hubPublicKey } + : {}), + }; + } + /** * The single owner of the workflow-deployment spawn sequence: construct * the supervisor, register the single-step agent's outbound key + head @@ -1326,11 +1349,30 @@ export function createSidecarDeployRouter(deps: { deps.multistepSourcesRouter?.register( spec.agentAddress, async (args) => { - // Persist BEFORE the live-swap: a recycle mid-flight rejects - // `deliverSources`, but the respawn must still pick up the + // Persist the rotation durably BEFORE committing it in memory or + // pushing it live, so a failed write leaves no partial effect: + // currentSources, the record, and the child all stay on the + // prior sources and the throw is truthful. Persistence lets the + // rotation survive a full sidecar restart, not just a recycle -- + // the boot scan reseeds spec.sources from record.sources. + // Overwrites the deploy-time record in place. Skipped when no + // data dir was wired (a test router that never persists), + // matching the restore guard. + const rotated = { [rotationStepId]: args.sources }; + if (stepStateDataDir !== undefined) { + await writeWorkflowDeploymentRecord( + stepStateDataDir, + deploymentId, + buildDeploymentRecord(spec, rotated), + ); + } + // Commit in memory BEFORE the live-swap, and synchronously so no + // recycle can interleave before deliverSources: a recycle + // mid-flight rejects `deliverSources`, but the respawn re-pulls + // currentSources through dynamicSpawnEnv and must see the // rotation. The wire boundary guarantees `args.sources[0]` is the // default, which the recycle env form pins as the active source. - currentSources = { [rotationStepId]: args.sources }; + currentSources = rotated; await wired.supervisor.deliverSources({ sources: args.sources, defaultSource: args.defaultSource, @@ -1537,16 +1579,7 @@ export function createSidecarDeployRouter(deps: { ? frame.hubPublicKey : undefined, }; - const record: WorkflowDeploymentRecord = { - version: 1, - agentAddress: spec.agentAddress, - definitionId: spec.definition.id, - sources: spec.sources, - ...(spec.sessionId !== undefined ? { sessionId: spec.sessionId } : {}), - ...(spec.hubPublicKey !== undefined - ? { hubPublicKey: spec.hubPublicKey } - : {}), - }; + const record = buildDeploymentRecord(spec, spec.sources); claimSlug(deploymentId, frame.agentAddress); // Hold the single-flight reservation across the async body below and clear From 41019c08a45afa76ccf339f0f2ad3bf1319fb8c9 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Mon, 6 Jul 2026 00:42:44 -0500 Subject: [PATCH 071/101] Re-apply the live sources after a warm agent's first build A sources-updated frame processed during the warm agent's first (async, off-control-loop) build hit the still-empty warm cache as a no-op applySources, while the in-flight build had already captured the prior sources -- so the rotation was lost for the warm agent's life. After storing the freshly built agent, re-apply the live source table. The entry now exists, so applySources reaches it, and any rotation that landed during the build takes effect. --- .../sidecar/src/workflow-substrate-factory.ts | 1 + .../src/adapters/step-invoker.test.ts | 66 +++++++++++++++++++ .../src/adapters/step-invoker.ts | 23 ++++++- 3 files changed, 89 insertions(+), 1 deletion(-) diff --git a/apps/sidecar/src/workflow-substrate-factory.ts b/apps/sidecar/src/workflow-substrate-factory.ts index 941d7560..5ab72ec7 100644 --- a/apps/sidecar/src/workflow-substrate-factory.ts +++ b/apps/sidecar/src/workflow-substrate-factory.ts @@ -1146,6 +1146,7 @@ export function createSidecarSubstrateFactory( buildEnv: (buildReq) => buildStepEnv(buildReq, sourcesRef), agentFactory: stepAgentFactory, onEvent, + sourcesRef, ...(warmCache !== undefined ? { warmCache } : {}), ...(onRunBoundary !== undefined ? { onRunBoundary } : {}), })(req); diff --git a/packages/workflow-host/src/adapters/step-invoker.test.ts b/packages/workflow-host/src/adapters/step-invoker.test.ts index aea86737..f0e179ce 100644 --- a/packages/workflow-host/src/adapters/step-invoker.test.ts +++ b/packages/workflow-host/src/adapters/step-invoker.test.ts @@ -255,6 +255,72 @@ function buildRequest(opts: { }; } +describe("workflow-host StepInvoker adapter - warm source rotation", () => { + test("a rotation during the first warm build reaches the built agent", async () => { + // Finding: a sources rotation that lands during the (async) first warm + // build hits the still-empty cache as a no-op applySources, while the + // in-flight build already captured the prior sources. After the built + // agent is stored, the adapter re-applies the live table so the rotation + // is not lost for the warm agent's life. + const stub = buildStreamingStubAgent([]); + const setSourcesCalls: { + sources: InferenceSource[]; + defaultSource: string; + }[] = []; + stub.agent.setSources = (sources, defaultSource) => { + setSourcesCalls.push({ sources, defaultSource }); + }; + const workflowAuthorize: WorkflowAuthorizeFn = async () => ({ + effect: "allow", + matchingGrants: [], + resolvedBy: null, + }); + const warmCache = createWarmAgentCache(); + const rotated: InferenceSource = { ...STUB_SOURCE, id: "rotated" }; + const sourcesRef = { + current: { "step-1": [STUB_SOURCE] } as Record, + }; + + let releaseBuild!: () => void; + const buildGate = new Promise((resolve) => { + releaseBuild = resolve; + }); + let buildStarted = false; + + const invoker = createWorkflowStepInvoker({ + workflowAuthorize, + buildEnv: async () => stubBuildEnv(), + agentFactory: async () => { + buildStarted = true; + await buildGate; + return stub.agent; + }, + warmCache, + sourcesRef, + }); + + const invokePromise = invoker(buildRequest({ input: { goal: "ping" } })); + while (!buildStarted) { + await new Promise((r) => setTimeout(r, 1)); + } + // A rotation lands DURING the build: the run-loop updates the ref and + // calls applySources -- a no-op here because the cache is still empty. + sourcesRef.current = { "step-1": [rotated] }; + warmCache.applySources([rotated], "rotated"); + expect(setSourcesCalls).toHaveLength(0); + + releaseBuild(); + await invokePromise; + + // The re-apply after store applied the ROTATED table to the built agent. + expect(setSourcesCalls).toEqual([ + { sources: [rotated], defaultSource: "rotated" }, + ]); + + await warmCache.evictAll("test cleanup"); + }); +}); + describe("workflow-host StepInvoker adapter - happy path", () => { test("delivers synthesized message, captures reply, returns output shape", async () => { const stub = buildStubAgent(); diff --git a/packages/workflow-host/src/adapters/step-invoker.ts b/packages/workflow-host/src/adapters/step-invoker.ts index 146a9f44..e85ed524 100644 --- a/packages/workflow-host/src/adapters/step-invoker.ts +++ b/packages/workflow-host/src/adapters/step-invoker.ts @@ -60,7 +60,7 @@ import { type BaseEnv, } from "@intx/agent"; import { getLogger } from "@intx/log"; -import type { InferenceEvent } from "@intx/types/runtime"; +import type { InferenceEvent, InferenceSource } from "@intx/types/runtime"; import type { AuthorizeContext, StepInvokeRequest, @@ -152,6 +152,16 @@ export interface WorkflowStepInvokerOpts { * agent is never warm-kept. */ warmCache?: WarmAgentCache; + /** + * Live per-step inference-source table the run-loop mutates in place on a + * rotation. Supplied only on the warm path: after building and storing the + * warm agent, the adapter re-applies the current table so a rotation that + * landed during the (async) first build -- which the empty-cache + * `applySources` no-op could not reach, and which the in-flight build had + * already captured the prior sources for -- is not lost for the warm + * agent's life. + */ + sourcesRef?: { current: Record }; /** * Run-boundary hook for the warm path (design §3c durability). When * supplied, the adapter awaits it in the warm path's `finally` -- once @@ -301,6 +311,17 @@ async function invokeWarmStep( if (sink !== null) sink(event); }); warmCache.store(key, agent, eventSinkRef, eventForward); + // Re-apply the live source table to the just-built agent. A rotation + // that arrived during the (async) build hit the still-empty cache as a + // no-op `applySources` while the build had already captured the prior + // sources; now that the entry exists, this applies any such rotation so + // it is not lost for the warm agent's life. No-op when the table is + // unchanged. The wire boundary guarantees element 0 is the default. + const live = opts.sourcesRef?.current[key]; + const head = live?.[0]; + if (live !== undefined && head !== undefined) { + warmCache.applySources(live, head.id); + } } if (opts.onEvent !== undefined) { From 9bbd96081282ac71988ddb09ded7827d51457a63 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Mon, 6 Jul 2026 01:13:30 -0500 Subject: [PATCH 072/101] Route every startup-phase throw through the teardown owner A spawn that threw after the OS child was running but before the ready handshake -- during credentials assembly, mail registration, or the handshake itself -- propagated without killing the child or releasing the mail subscription and address registration. The child was orphaned and, past the registration point, a live subscription was left with no dispatch loop to service it. Wrap the startup path from the "starting" seam through the successful return in a single try whose catch runs shutdownInternal, the one owner of starting/running teardown. The per-site releases the ready-race branches carried are folded into it. The one throw site that runs before any state record owns the handle -- wireChild -- kills the handle directly, since shutdownInternal early-returns on the "idle" phase. The teardown's kill rejects the ready promise; attach a benign handler at wire time so that rejection is never unhandled when a startup throw tears down before the handshake fold would observe it. --- .../src/supervisor/supervisor.test.ts | 104 ++++ .../src/supervisor/supervisor.ts | 490 ++++++++++-------- 2 files changed, 365 insertions(+), 229 deletions(-) diff --git a/packages/workflow-host/src/supervisor/supervisor.test.ts b/packages/workflow-host/src/supervisor/supervisor.test.ts index db1b0842..1bad6c8b 100644 --- a/packages/workflow-host/src/supervisor/supervisor.test.ts +++ b/packages/workflow-host/src/supervisor/supervisor.test.ts @@ -857,6 +857,110 @@ describe("createWorkflowSupervisor", () => { expect(readyDeadline.cancelled).toBe(true); }); + // A spawn that throws AFTER the OS child is running but BEFORE the + // supervisor reaches the ready handshake must not orphan the child or + // leave the mail address registered. `shutdownInternal` owns that + // teardown once the state record enters "starting"; the spawn body + // routes every post-seam throw through it. + async function makePreRegistrationFailureHarness(opts: { + failSubscribe?: boolean; + failDeriveStepAddress?: boolean; + }) { + const baseDir = await makeTempDir("supervisor-spawn-leak-"); + await seedStepGrants( + baseDir, + defaultStepRepoId({ deploymentId: "deployment-x", stepId: "step-1" }), + [{ resource: "thing", action: "read" }], + ); + const supervisorIpcKeyPair = await generateKeyPair(); + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const eventChildToSupervisor = createMemoryFrameStream(); + let resolveExit: ((code: number) => void) | undefined; + const exited = new Promise((resolve) => { + resolveExit = resolve; + }); + const killSignals: string[] = []; + const spawner: SubprocessSpawner = () => ({ + pid: 4321, + controlWriter: supervisorToChild.writer, + controlReader: childToSupervisor.reader, + eventReader: eventChildToSupervisor.reader, + kill: (signal) => { + killSignals.push( + typeof signal === "string" ? signal : String(signal ?? ""), + ); + childToSupervisor.close(); + eventChildToSupervisor.close(); + resolveExit?.(0); + }, + exited, + }); + const mailBus = createMockMailBus(); + const bindingsMailBus: MailBusBindings = { + ...mailBus, + subscribeMailForAddress: + opts.failSubscribe === true + ? () => { + throw new Error("injected subscribe failure"); + } + : mailBus.subscribeMailForAddress, + }; + const baseBindings = await buildBindings({ + baseDir, + spawner, + signSpy: () => ({ sig: new Uint8Array(64), principalKind: "supervisor" }), + mailBus: bindingsMailBus, + }); + const bindings: WorkflowSupervisorBindings = { + ...baseBindings, + ipcKeyPairFactory: () => Promise.resolve(supervisorIpcKeyPair), + ...(opts.failDeriveStepAddress === true + ? { + deriveStepAddress: () => { + throw new Error("injected deriveStepAddress failure"); + }, + } + : {}), + }; + return { + supervisor: createWorkflowSupervisor(bindings), + killSignals, + registered: mailBus.registered, + }; + } + + const preRegistrationSpawnOpts = { + stepOrder: ["step-1"], + definitionHash: "def-hash-abc", + warmKeep: false, + onInferenceEvent: () => { + /* unused in the pre-registration failure tests */ + }, + }; + + test("a spawn whose mail subscription throws kills the child and releases the address", async () => { + const h = await makePreRegistrationFailureHarness({ failSubscribe: true }); + await expect(h.supervisor.spawn(preRegistrationSpawnOpts)).rejects.toThrow( + "injected subscribe failure", + ); + // The address was registered just before subscribe threw; the + // teardown must unregister it so no orphaned registration survives. + expect(h.registered()).toHaveLength(0); + expect(h.killSignals.length).toBeGreaterThan(0); + }); + + test("a spawn whose credentials assembly throws kills the child", async () => { + const h = await makePreRegistrationFailureHarness({ + failDeriveStepAddress: true, + }); + await expect(h.supervisor.spawn(preRegistrationSpawnOpts)).rejects.toThrow( + "injected deriveStepAddress failure", + ); + expect(h.registered()).toHaveLength(0); + expect(h.killSignals.length).toBeGreaterThan(0); + }); + test("drain() forwards the `drain` control frame and arms a drainTimeout accumulator per in-flight run", async () => { const baseDir = await makeTempDir("supervisor-drain-arm-"); await seedStepGrants( diff --git a/packages/workflow-host/src/supervisor/supervisor.ts b/packages/workflow-host/src/supervisor/supervisor.ts index ef4f43b5..3db74ba1 100644 --- a/packages/workflow-host/src/supervisor/supervisor.ts +++ b/packages/workflow-host/src/supervisor/supervisor.ts @@ -1345,12 +1345,38 @@ export function createWorkflowSupervisor( env, }); - const wired = await wireChild({ - channelId, - hmacKey, - ipcKeypair, - handle, - onInferenceEvent: opts.onInferenceEvent, + let wired: Awaited>; + try { + wired = await wireChild({ + channelId, + hmacKey, + ipcKeypair, + handle, + onInferenceEvent: opts.onInferenceEvent, + }); + } catch (cause) { + // wireChild threw before any state record owns the handle, so + // shutdownInternal -- which reaches the handle through the + // active-state record -- would early-return on the "idle" phase + // without killing it. Kill the freshly-spawned child directly to + // avoid orphaning the OS process. + await killChildHandle(handle, DEFAULT_KILL_TIMEOUT_MS, { + setTimer: readySetTimer, + clearTimer: readyClearTimer, + logger, + }); + throw cause; + } + + // The ready handshake below folds `wired.readyPromise` into an + // outcome value, handling its rejection. But a startup teardown that + // fires BEFORE the handshake -- a throw during credentials assembly + // or mail registration -- kills the child, and that kill rejects + // `readyPromise` (the control channel ends). Attach a benign handler + // now so the rejection is never unhandled on that path; the + // handshake's own fold still observes the outcome when it runs. + void wired.readyPromise.catch(() => { + /* handled by the ready-handshake fold when the handshake runs */ }); // Cohort abort controller covers terminal-event watcher @@ -1377,127 +1403,131 @@ export function createWorkflowSupervisor( replayDone: null, }; - const credentialsSnapshot = await assembleCredentialsSnapshot({ - repoStore: bindings.repoStore, - principal: bindings.readPrincipal, - stepOrder: opts.stepOrder, - deploymentId: bindings.deploymentId, - deriveStepAddress: bindings.deriveStepAddress, - ...(bindings.deriveStepRepoId !== undefined - ? { deriveStepRepoId: bindings.deriveStepRepoId } - : {}), - }); - state.credentialsSnapshot = credentialsSnapshot; - - // Replay any orphaned `processing/` entries back to `inbox/` - // BEFORE the dispatch loop's first dequeue. A crash mid-dispatch - // in a prior supervisor incarnation can leave an entry in - // `processing/` with no owner; the FIFO contract requires the - // entry move back to `inbox/` so the next dispatch picks it up - // in its original arrival position. The replay runs off the - // spawn critical path (the substrate write may roundtrip through - // the pack-pushing wrap and a slow hub), but `runDispatchLoop` - // takes the promise as an argument and awaits it before its - // first `dequeueToProcessing` so a fresh inbound mail that lands - // during the replay window cannot ship ahead of the orphan once - // the replay completes. - const replayDone = inboxPrimitives - .replayProcessingToInbox( - bindings.repoStore, - inboxWritePrincipal, - bindings.workflowRunRepoId, - bindings.deploymentMailAddress, - ) - .then(() => { - wakeDispatch(); - }) - .catch((cause) => { - // Documented best-effort: a failed replay leaves orphaned - // `processing/` entries parked and the dispatch loop will - // then ship newly-enqueued mail ahead of them, violating - // the FIFO contract described in the comment above. - // Tightening this to a fatal `onChildCrash` was attempted - // but caused spurious crashes in the integration suite - // where the first spawn legitimately has no - // `processing/` directory to replay; resolving that - // requires either a no-op-on-missing variant of - // `replayProcessingToInbox` or a dispatch-loop periodic - // sweep that picks up parked orphans. Left as logged - // best-effort until that lands. - const message = cause instanceof Error ? cause.message : String(cause); - logger.warn`replayProcessingToInbox on spawn failed: ${message}`; - }); - // Hold the replay promise on the active-state record so - // `shutdownInternal` awaits its settlement before tearing the - // bindings down. A shutdown that lands while the replay is in - // flight would otherwise leave the substrate write pending past - // the supervisor's exit. - state.replayDone = replayDone; - - bindings.mailBus.registerAddress(bindings.deploymentMailAddress); - const mailUnsubscribe = bindings.mailBus.subscribeMailForAddress( - bindings.deploymentMailAddress, - onMailMessage, - ); - state.mailUnsubscribe = mailUnsubscribe; - - // Bound the `ready` handshake. `wired.readyPromise` resolves on `ready` - // and rejects when the control channel ends (the child exited); a child - // that neither readies nor exits would block here forever. Fold all three - // outcomes into values so the single `readyClearTimer` below runs on every - // path -- ready, child-exit failure, and timeout -- before we act on the - // result. A `Promise.race` that could reject would skip the clear on the - // child-exit path and leak an armed deadline that keeps the event loop - // alive for up to `readyTimeoutMs`. The deadline is resolve-only, so it - // contributes no rejection of its own. Kill on timeout uses the - // SIGTERM->SIGKILL escalation because a wedged child may ignore SIGTERM; - // SIGKILL guarantees `exited` settles. - const readyOutcome = wired.readyPromise.then( - (info) => ({ kind: "ready" as const, info }), - (err: unknown) => ({ kind: "failed" as const, err }), - ); - const readyDeadline = waitDeadline(readySetTimer, readyTimeoutMs); - const readyRace = await Promise.race([ - readyOutcome, - readyDeadline.promise.then(() => ({ kind: "timeout" as const })), - ]); - readyClearTimer(readyDeadline.handle); - if (readyRace.kind === "timeout") { - await killChildHandle(wired.wiring.handle, DEFAULT_KILL_TIMEOUT_MS, { - setTimer: readySetTimer, - clearTimer: readyClearTimer, - logger, - }); - // Release the mail subscription + address registration installed - // before the handshake. shutdownInternal owns the "starting"-phase - // teardown; its own kill against the already-killed handle is - // idempotent. Without this the failed spawn leaves a live - // subscription with no dispatch loop to service it. - await shutdownInternal({ - reason: `child did not emit ready within ${readyTimeoutMs}ms`, + // Everything from here to the successful `return` runs with the state + // record in "starting" (then "running"). A throw at any of these + // steps -- credentials assembly, mail registration, the ready + // handshake, the credentials push, the dispatch-loop start -- routes + // through shutdownInternal, the single owner of starting/running + // teardown: it kills the handle and releases the mail subscription + // and address registration installed below. + try { + const credentialsSnapshot = await assembleCredentialsSnapshot({ + repoStore: bindings.repoStore, + principal: bindings.readPrincipal, + stepOrder: opts.stepOrder, + deploymentId: bindings.deploymentId, + deriveStepAddress: bindings.deriveStepAddress, + ...(bindings.deriveStepRepoId !== undefined + ? { deriveStepRepoId: bindings.deriveStepRepoId } + : {}), }); - throw new Error( - `workflow-host supervisor: child did not emit ready within ${readyTimeoutMs}ms; killed`, + state.credentialsSnapshot = credentialsSnapshot; + + // Replay any orphaned `processing/` entries back to `inbox/` + // BEFORE the dispatch loop's first dequeue. A crash mid-dispatch + // in a prior supervisor incarnation can leave an entry in + // `processing/` with no owner; the FIFO contract requires the + // entry move back to `inbox/` so the next dispatch picks it up + // in its original arrival position. The replay runs off the + // spawn critical path (the substrate write may roundtrip through + // the pack-pushing wrap and a slow hub), but `runDispatchLoop` + // takes the promise as an argument and awaits it before its + // first `dequeueToProcessing` so a fresh inbound mail that lands + // during the replay window cannot ship ahead of the orphan once + // the replay completes. + const replayDone = inboxPrimitives + .replayProcessingToInbox( + bindings.repoStore, + inboxWritePrincipal, + bindings.workflowRunRepoId, + bindings.deploymentMailAddress, + ) + .then(() => { + wakeDispatch(); + }) + .catch((cause) => { + // Documented best-effort: a failed replay leaves orphaned + // `processing/` entries parked and the dispatch loop will + // then ship newly-enqueued mail ahead of them, violating + // the FIFO contract described in the comment above. + // Tightening this to a fatal `onChildCrash` was attempted + // but caused spurious crashes in the integration suite + // where the first spawn legitimately has no + // `processing/` directory to replay; resolving that + // requires either a no-op-on-missing variant of + // `replayProcessingToInbox` or a dispatch-loop periodic + // sweep that picks up parked orphans. Left as logged + // best-effort until that lands. + const message = + cause instanceof Error ? cause.message : String(cause); + logger.warn`replayProcessingToInbox on spawn failed: ${message}`; + }); + // Hold the replay promise on the active-state record so + // `shutdownInternal` awaits its settlement before tearing the + // bindings down. A shutdown that lands while the replay is in + // flight would otherwise leave the substrate write pending past + // the supervisor's exit. + state.replayDone = replayDone; + + bindings.mailBus.registerAddress(bindings.deploymentMailAddress); + const mailUnsubscribe = bindings.mailBus.subscribeMailForAddress( + bindings.deploymentMailAddress, + onMailMessage, ); - } - if (readyRace.kind === "failed") { - // The child exited during the handshake; release the subscription - // and registration the same way the timeout path does. - await shutdownInternal({ reason: "child exited before emitting ready" }); - throw readyRace.err; - } - const readyInfo = readyRace.info; - - // Push the assembled credentialsSnapshot to the child before the - // mail buffer drains. Without this, the child's - // `createCredentialsBackedAuthorize` closure observes a null - // snapshot ref on the first authorize call and throws "no - // credentialsSnapshot active"; the run's first step fails before - // the runtime body can commit `StepCompleted`. The send rides the - // same control channel `trigger.fire` uses, so the ordering - // guarantee (`grants-updated` lands before `trigger.fire`) holds - // for buffered and post-ready inbound mail alike. - try { + state.mailUnsubscribe = mailUnsubscribe; + + // Bound the `ready` handshake. `wired.readyPromise` resolves on `ready` + // and rejects when the control channel ends (the child exited); a child + // that neither readies nor exits would block here forever. Fold all three + // outcomes into values so the single `readyClearTimer` below runs on every + // path -- ready, child-exit failure, and timeout -- before we act on the + // result. A `Promise.race` that could reject would skip the clear on the + // child-exit path and leak an armed deadline that keeps the event loop + // alive for up to `readyTimeoutMs`. The deadline is resolve-only, so it + // contributes no rejection of its own. Kill on timeout uses the + // SIGTERM->SIGKILL escalation because a wedged child may ignore SIGTERM; + // SIGKILL guarantees `exited` settles. + const readyOutcome = wired.readyPromise.then( + (info) => ({ kind: "ready" as const, info }), + (err: unknown) => ({ kind: "failed" as const, err }), + ); + const readyDeadline = waitDeadline(readySetTimer, readyTimeoutMs); + const readyRace = await Promise.race([ + readyOutcome, + readyDeadline.promise.then(() => ({ kind: "timeout" as const })), + ]); + readyClearTimer(readyDeadline.handle); + if (readyRace.kind === "timeout") { + await killChildHandle(wired.wiring.handle, DEFAULT_KILL_TIMEOUT_MS, { + setTimer: readySetTimer, + clearTimer: readyClearTimer, + logger, + }); + // The SIGTERM->SIGKILL escalation above is deliberate: a wedged + // child may ignore the plain kill shutdownInternal issues. The + // outer catch then runs shutdownInternal for the "starting"-phase + // teardown (subscription + address release); its kill against the + // already-killed handle is idempotent. + throw new Error( + `workflow-host supervisor: child did not emit ready within ${readyTimeoutMs}ms; killed`, + ); + } + if (readyRace.kind === "failed") { + // The child exited during the handshake; the outer catch releases + // the subscription and registration via shutdownInternal. + throw readyRace.err; + } + const readyInfo = readyRace.info; + + // Push the assembled credentialsSnapshot to the child before the + // mail buffer drains. Without this, the child's + // `createCredentialsBackedAuthorize` closure observes a null + // snapshot ref on the first authorize call and throws "no + // credentialsSnapshot active"; the run's first step fails before + // the runtime body can commit `StepCompleted`. The send rides the + // same control channel `trigger.fire` uses, so the ordering + // guarantee (`grants-updated` lands before `trigger.fire`) holds + // for buffered and post-ready inbound mail alike. await wired.wiring.controlSender.send({ type: "grants-updated", data: { @@ -1511,117 +1541,119 @@ export function createWorkflowSupervisor( }, }, }); - } catch (cause) { - // The child readied but died before or during the credentials push. - // Release the subscription and registration before failing the spawn. - await shutdownInternal({ reason: "grants-updated push to child failed" }); - throw cause; - } - // Transition to running. The dispatch loop (started below) - // picks up any pre-ready buffered mail through the FIFO inbox - // queue rather than through an in-memory buffer; arrival order - // is preserved by the envelope's `receivedAt` prefix on the - // inbox filename. - const startingPhaseCohortAbort = state.terminalCohortAbort; - if (startingPhaseCohortAbort === null) { - throw new Error( - "supervisor: terminalCohortAbort missing after spawn handshake", + // Transition to running. The dispatch loop (started below) + // picks up any pre-ready buffered mail through the FIFO inbox + // queue rather than through an in-memory buffer; arrival order + // is preserved by the envelope's `receivedAt` prefix on the + // inbox filename. + const startingPhaseCohortAbort = state.terminalCohortAbort; + if (startingPhaseCohortAbort === null) { + throw new Error( + "supervisor: terminalCohortAbort missing after spawn handshake", + ); + } + const startingPhaseBroadcaster = state.terminalBroadcaster; + const dispatchLoop = runDispatchLoop( + wired.wiring.controlSender, + startingPhaseCohortAbort, + startingPhaseBroadcaster, + replayDone, ); - } - const startingPhaseBroadcaster = state.terminalBroadcaster; - const dispatchLoop = runDispatchLoop( - wired.wiring.controlSender, - startingPhaseCohortAbort, - startingPhaseBroadcaster, - replayDone, - ); - // Surface dispatch-loop failures via the logger; the loop's own - // catch already swallows per-iteration faults, but a structural - // failure (e.g. the cohort abort handler itself throws) lands - // here. - void dispatchLoop.catch((cause) => { - const message = cause instanceof Error ? cause.message : String(cause); - logger.error`dispatch loop terminated with error: ${message}`; - }); - state = { - phase: "running", - handle, - controlSender: wired.wiring.controlSender, - channelId, - eventPump: wired.wiring.eventPump, - onInferenceEvent: opts.onInferenceEvent, - mailUnsubscribe, - credentialsSnapshot, - terminalCohortAbort: startingPhaseCohortAbort, - terminalBroadcaster: startingPhaseBroadcaster, - dispatchLoop, - replayDone, - }; - // Kick the dispatch loop in case mail landed in the inbox - // before the loop's first `await dispatchWake`. A wake against a - // freshly-minted promise is a no-op; the dispatch loop's first - // dequeue happens unconditionally. - wakeDispatch(); + // Surface dispatch-loop failures via the logger; the loop's own + // catch already swallows per-iteration faults, but a structural + // failure (e.g. the cohort abort handler itself throws) lands + // here. + void dispatchLoop.catch((cause) => { + const message = cause instanceof Error ? cause.message : String(cause); + logger.error`dispatch loop terminated with error: ${message}`; + }); + state = { + phase: "running", + handle, + controlSender: wired.wiring.controlSender, + channelId, + eventPump: wired.wiring.eventPump, + onInferenceEvent: opts.onInferenceEvent, + mailUnsubscribe, + credentialsSnapshot, + terminalCohortAbort: startingPhaseCohortAbort, + terminalBroadcaster: startingPhaseBroadcaster, + dispatchLoop, + replayDone, + }; + // Kick the dispatch loop in case mail landed in the inbox + // before the loop's first `await dispatchWake`. A wake against a + // freshly-minted promise is a no-op; the dispatch loop's first + // dequeue happens unconditionally. + wakeDispatch(); - // Cache the spawn context for the recycle path. The recycle path - // reuses the same stepOrder/definitionHash/onInferenceEvent on - // every respawn -- those are the strict-orthogonality anchors - // with redeploy, and the supervisor never mutates them. - const now = bindings.recyclePolicyNow ?? defaultNow; - spawnContext = { - stepOrder: opts.stepOrder, - definitionHash: opts.definitionHash, - warmKeep: opts.warmKeep, - onInferenceEvent: opts.onInferenceEvent, - spawnedAt: now(), - }; + // Cache the spawn context for the recycle path. The recycle path + // reuses the same stepOrder/definitionHash/onInferenceEvent on + // every respawn -- those are the strict-orthogonality anchors + // with redeploy, and the supervisor never mutates them. + const now = bindings.recyclePolicyNow ?? defaultNow; + spawnContext = { + stepOrder: opts.stepOrder, + definitionHash: opts.definitionHash, + warmKeep: opts.warmKeep, + onInferenceEvent: opts.onInferenceEvent, + spawnedAt: now(), + }; - // Start the upstream control pump so the supervisor sees the - // child's `recycle.request` (and any future upstream variant) as - // it arrives. The pump exits when the iterator ends, which - // happens when the child closes its end of the control channel - // -- either on shutdown or on recycle's `kill` step. The pump - // closes over the cohort's broadcaster captured at pump-start - // time so a `terminal.event` frame the iterator dequeues after a - // recycle has minted a new cohort routes to THIS cohort's (now - // disposed) broadcaster, not the successor's. - void pumpUpstreamControl( - wired.controlIncoming, - startingPhaseBroadcaster, - ).catch((cause) => { - const message = cause instanceof Error ? cause.message : String(cause); - logger.error`upstream control pump failed: ${message}`; - }); + // Start the upstream control pump so the supervisor sees the + // child's `recycle.request` (and any future upstream variant) as + // it arrives. The pump exits when the iterator ends, which + // happens when the child closes its end of the control channel + // -- either on shutdown or on recycle's `kill` step. The pump + // closes over the cohort's broadcaster captured at pump-start + // time so a `terminal.event` frame the iterator dequeues after a + // recycle has minted a new cohort routes to THIS cohort's (now + // disposed) broadcaster, not the successor's. + void pumpUpstreamControl( + wired.controlIncoming, + startingPhaseBroadcaster, + ).catch((cause) => { + const message = cause instanceof Error ? cause.message : String(cause); + logger.error`upstream control pump failed: ${message}`; + }); - // Arm the recycle policy. The policy is a no-op when all bounds - // are `undefined`; bounds resolution lives inside `createRecyclePolicy`. - if (bindings.recyclePolicy !== undefined) { - const setTimer = bindings.recyclePolicySetTimer ?? defaultSetTimer; - const clearTimer = bindings.recyclePolicyClearTimer ?? defaultClearTimer; - recyclePolicy = createRecyclePolicy({ - bounds: bindings.recyclePolicy, - now, - spawnedAt: spawnContext.spawnedAt, - ...(bindings.readRssBytes !== undefined - ? { readRssBytes: bindings.readRssBytes } - : {}), - ...(bindings.readGrantsAgeMs !== undefined - ? { readGrantsAgeMs: bindings.readGrantsAgeMs } - : {}), - setTimer, - clearTimer, - trigger: async (reason) => { - await recycle({ reason, origin: "policy" }); - }, + // Arm the recycle policy. The policy is a no-op when all bounds + // are `undefined`; bounds resolution lives inside `createRecyclePolicy`. + if (bindings.recyclePolicy !== undefined) { + const setTimer = bindings.recyclePolicySetTimer ?? defaultSetTimer; + const clearTimer = + bindings.recyclePolicyClearTimer ?? defaultClearTimer; + recyclePolicy = createRecyclePolicy({ + bounds: bindings.recyclePolicy, + now, + spawnedAt: spawnContext.spawnedAt, + ...(bindings.readRssBytes !== undefined + ? { readRssBytes: bindings.readRssBytes } + : {}), + ...(bindings.readGrantsAgeMs !== undefined + ? { readGrantsAgeMs: bindings.readGrantsAgeMs } + : {}), + setTimer, + clearTimer, + trigger: async (reason) => { + await recycle({ reason, origin: "policy" }); + }, + }); + } + + return { + pid: readyInfo.childPid, + channelId, + credentialsSnapshot, + }; + } catch (cause) { + const message = cause instanceof Error ? cause.message : String(cause); + await shutdownInternal({ + reason: `spawn failed during startup: ${message}`, }); + throw cause; } - - return { - pid: readyInfo.childPid, - channelId, - credentialsSnapshot, - }; } /** From 3748ac26cefaae1319d7f2225372c302ae838a07 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Mon, 6 Jul 2026 01:53:21 -0500 Subject: [PATCH 073/101] Refresh comments the trivial-path removal left stale The in-process trivial deploy path is gone, but comments across the sidecar, hub-sessions, and type-contract layers still described it: a "trivial" deploy branch that supposedly never registered signal or drain handlers, a `recordRunEvent` writer that no longer exists, an AgentDeployFrame "third shape" the router now rejects, and several contrasts against a legacy in-process path a reader can no longer find. Rewrite each to match the current model: the deploy router registers signal and drain handlers for every deployment and the sources handler only for a single-step warm one; the supervisor's conversation-state writes reach the substrate store; a frame carrying neither `workflow` nor `provisionStep` is rejected with no in-process fall-through. --- apps/sidecar/src/index.ts | 10 ++--- apps/sidecar/src/workflow-host-wiring.test.ts | 11 +++--- apps/sidecar/src/workflow-run-pack-client.ts | 22 +++++------ packages/hub-api/src/routes/instances.ts | 6 +-- .../hub-sessions/src/session-service.test.ts | 2 +- packages/hub-sessions/src/session-service.ts | 39 +++++++++---------- .../hub-sessions/src/ws/sidecar-handler.ts | 6 +-- packages/types/src/sidecar.ts | 19 +++++---- .../src/capability-approval.ts | 5 --- packages/workflow-deploy/src/index.ts | 4 +- .../src/supervisor/credentials.ts | 13 ++++--- 11 files changed, 65 insertions(+), 72 deletions(-) diff --git a/apps/sidecar/src/index.ts b/apps/sidecar/src/index.ts index fd1466ed..3c0ba1c6 100644 --- a/apps/sidecar/src/index.ts +++ b/apps/sidecar/src/index.ts @@ -202,11 +202,11 @@ const SIDECAR_SIGNING_DIR = path.join(dataDir, ".sidecar-signing"); const sidecarSigningKey = await loadOrMintSidecarKeypair(SIDECAR_SIGNING_DIR); // Construct the substrate-backed RepoStore at the boot edge. The -// supervisor consumes this through the deploy router; the trivial -// branch's `recordRunEvent` reaches `writeTreePreservingPrefix` on -// this store. The boot-edge facade below wraps the store so a -// successful workflow-run write fires the pack push hook before its -// Promise resolves. +// supervisor consumes this through the deploy router; the supervisor's +// conversation-state writes reach `writeTreePreservingPrefix` on this +// store. The boot-edge facade below wraps the store so a successful +// workflow-run write fires the pack push hook before its Promise +// resolves. const agentRepoStore = createAgentRepoStore({ dataDir, signingKey: sidecarSigningKey, diff --git a/apps/sidecar/src/workflow-host-wiring.test.ts b/apps/sidecar/src/workflow-host-wiring.test.ts index 1683e1d1..3cc278ae 100644 --- a/apps/sidecar/src/workflow-host-wiring.test.ts +++ b/apps/sidecar/src/workflow-host-wiring.test.ts @@ -398,11 +398,12 @@ function makeMultistepFrame(args: MultistepDeployArgs): AgentDeployFrame { agentAddress: args.agentAddress ?? "multi@example.com", agentId: "multi-agent", hubPublicKey: "hub-pk", - // The wire-side HarnessConfig has many required fields. The router - // never inspects `config` on the multi-step branch (only the - // trivial branch hands it to provisionAgent), so an opaque - // placeholder satisfies the surface contract for these tests. - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the multi-step branch does not read config + // The wire-side HarnessConfig has many required fields. On the + // workflow deploy path the router reads only `config.sessionId` + // and `config.grants`, both of which tolerate the empty + // placeholder (they resolve to `undefined`), so an opaque `{}` + // satisfies the surface contract for these tests. + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the workflow path reads only config.sessionId/config.grants, which tolerate undefined config: {} as AgentDeployFrame["config"], workflow: { definition: args.definition, diff --git a/apps/sidecar/src/workflow-run-pack-client.ts b/apps/sidecar/src/workflow-run-pack-client.ts index 7216d435..8278fe77 100644 --- a/apps/sidecar/src/workflow-run-pack-client.ts +++ b/apps/sidecar/src/workflow-run-pack-client.ts @@ -85,7 +85,7 @@ export function createWorkflowRunPackClient( /** * Mapping registry the boot-edge substrate facade consults to resolve - * `repoId.id` (the workflow-run deploymentId, which the trivial branch + * `repoId.id` (the workflow-run deploymentId, which the deploy router * derives by slugging the agent's mail address) back into the * agentAddress carried on every outbound pack frame. Populated by the * deploy router as each `agent.deploy` frame lands. @@ -179,11 +179,11 @@ export type MultistepSignalHandler = (args: { /** * Per-deployment-address signal handler registry the sidecar hub-link - * consults on every inbound `signal.deliver` frame. The trivial deploy - * path never registers a handler. The multi-step deploy router + * consults on every inbound `signal.deliver` frame. The deploy router * registers a handler against the deployment's mail address after - * `wired.supervisor.spawn` succeeds; the handler dispatches the signal - * into the supervisor's `deliverSignal`. + * `wired.supervisor.spawn` succeeds, for single-step and multi-step + * deployments alike; the handler dispatches the signal into the + * supervisor's `deliverSignal`. * * The registry lives at the sidecar's host layer (not inside the * workflow-host library) for the same boundary reason as @@ -244,11 +244,11 @@ export type MultistepDrainHandler = (args: { /** * Per-deployment-address drain handler registry the sidecar hub-link - * consults on every inbound `drain.deliver` frame. The trivial deploy - * path never registers a handler. The multi-step deploy router + * consults on every inbound `drain.deliver` frame. The deploy router * registers a handler against the deployment's mail address after - * `wired.supervisor.spawn` succeeds; the handler dispatches into the - * supervisor's `drain`. + * `wired.supervisor.spawn` succeeds, for single-step and multi-step + * deployments alike; the handler dispatches into the supervisor's + * `drain`. * * The registry lives at the sidecar's host layer (not inside the * workflow-host library) for the same boundary reason as @@ -304,8 +304,8 @@ export type MultistepSourcesHandler = (args: { /** * Per-deployment-address sources-rotation handler registry. Only a * single-step warm deployment registers a handler (after - * `wired.supervisor.spawn` succeeds); the trivial and multi-step paths - * never do, so `tryRoute` resolves a rotation only for a registered + * `wired.supervisor.spawn` succeeds); a multi-step deployment never + * does, so `tryRoute` resolves a rotation only for a registered * single-step address and returns `false` for any other. * * The registry lives at the sidecar's host layer for the same boundary diff --git a/packages/hub-api/src/routes/instances.ts b/packages/hub-api/src/routes/instances.ts index 0c32972e..bdf0cbec 100644 --- a/packages/hub-api/src/routes/instances.ts +++ b/packages/hub-api/src/routes/instances.ts @@ -513,9 +513,9 @@ export function createInstanceRoutes({ try { // Deploy the instance as a single-step workflow at the head: it runs - // as a supervised workflow-process child, not the legacy trivial - // in-process path. The real `agentId` (row.id) is passed so the child - // resolves the instance's skills and pinned tool packages. The + // as a supervised workflow-process child. The real `agentId` (row.id) + // is passed so the child resolves the instance's skills and pinned + // tool packages. The // returned head public key is surfaced separately via the sidecar's // `agent.deploy.ack`, so the route discards it here. await sessionService.deployInstanceAtHead({ diff --git a/packages/hub-sessions/src/session-service.test.ts b/packages/hub-sessions/src/session-service.test.ts index e5096b53..623efafd 100644 --- a/packages/hub-sessions/src/session-service.test.ts +++ b/packages/hub-sessions/src/session-service.test.ts @@ -1507,7 +1507,7 @@ describe("deployWorkflowDefinition", () => { // Multi-step branch: each step is provisioned via a per-step // sendAgentDeploy without a workflow field, then a single // deployment-level agent.deploy frame carries the workflow - // projection. The trivial branch never emits a workflow field at all. + // projection. const workflowFrames = sentWorkflows.filter( (w): w is NonNullable => w !== undefined, ); diff --git a/packages/hub-sessions/src/session-service.ts b/packages/hub-sessions/src/session-service.ts index 60bc76f9..4fe82ad7 100644 --- a/packages/hub-sessions/src/session-service.ts +++ b/packages/hub-sessions/src/session-service.ts @@ -118,9 +118,9 @@ export type SessionService = { * wrapping the harness as a one-step workflow and routing it through the * deploy core with the instance's real identity. Replaces `launchSession` * as the production instance-deploy entry point: the instance runs as a - * supervised workflow-process child rather than the legacy trivial - * in-process path. Writes no `workflow_deployment` row. Returns the head's - * agent-key ack (the key the head signs its reconnect challenges with). + * supervised workflow-process child. Writes no `workflow_deployment` + * row. Returns the head's agent-key ack (the key the head signs its + * reconnect challenges with). */ deployInstanceAtHead(params: { agentAddress: string; @@ -454,9 +454,9 @@ export function bridgeOrchestratorDeployContent( * Wire the workflow-deploy orchestrator's `sendMultiStepDeploy` * dependency against `SidecarRouter.sendAgentDeploy`. The router * accepts an optional `workflow` projection on the deploy frame; the - * sidecar's deploy router uses field presence to discriminate the - * multi-step branch from the trivial branch. The supervisor public key - * returned by the sidecar's `agent.deploy.ack` is threaded back as the + * sidecar's deploy router uses field presence to route the frame to + * the workflow deploy path. The supervisor public key returned by the + * sidecar's `agent.deploy.ack` is threaded back as the * `MultiStepDeployResult.publicKey`. * * Exported so the co-located caller-site test can assert that the @@ -548,17 +548,16 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { /** * Stage a deploy on the sidecar: resolve assets and tool packages, write * the deploy tree, provision the agent, and deliver the deploy + asset - * packs (Phases 0-2b). Does NOT start the warm harness -- callers that - * want one (the legacy agent-deploy path) invoke `startWarmSession` - * afterward. Phase 1's provision has three shapes: + * packs (Phases 0-2b). Phase 1's provision has two shapes: * - `workflowFrame` set: the single-step head hand-off fires the * deployment `agent.deploy` frame that spawns the workflow-process * child. Returns the supervisor public key. * - `stageOnly` set: a multi-step per-step stage binds a transient route * for the step address, fires a no-spawn provision frame (init repo + - * record hub key), and unbinds the route once the packs land. No warm - * harness, no child. - * - neither: the legacy plain provision frame (warm harness). + * record hub key), and unbinds the route once the packs land. No + * child. + * A call with neither is rejected -- the legacy warm-harness path + * is gone. */ async function executeLaunchPhases(params: { agentAddress: string; @@ -573,7 +572,7 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { * source pins (the sidecar initializes the head repo on receipt and * spawns the workflow-process child) instead of the plain provision * frame. The returned supervisor public key comes from that frame's - * ack; the caller skips `startWarmSession` for a workflow deploy. + * ack. * * Mutually exclusive with `stageOnly`. */ @@ -767,8 +766,7 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { // on receipt (so the Phase 2 pack has a repo to apply into) and spawns // the workflow-process child. A stage-only per-step deploy sends a // no-spawn provision frame: the sidecar inits the step's agent-state - // repo and records the hub key, but spawns nothing. The plain-provision - // frame stays for the legacy agent-deploy passthrough. Firing the frame + // repo and records the hub key, but spawns nothing. Firing the frame // before the Phase 2 pack is the ordering barrier -- the repo must // exist before the pack applies. A workflow frame's ack surfaces the // supervisor public key to the caller. @@ -875,11 +873,10 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { * Deploy a one-step workflow once at the head. Reuses the full * launch-phase machinery (deploy-tree write, pack, asset fan-out) via * `executeLaunchPhases`, swapping the Phase 1 provision frame for the - * workflow frame; it never calls `startWarmSession`, so no warm harness - * starts. The workflow frame makes the sidecar initialize the head repo - * and spawn the workflow-process child; the follow-up pack lands the - * head's deploy - * tree. Returns the supervisor's principal public key from the frame's + * workflow frame. The workflow frame makes the sidecar initialize the + * head repo and spawn the workflow-process child; the follow-up pack + * lands the head's deploy tree. Returns the supervisor's principal + * public key from the frame's * ack. A workflow-frame launch always yields a deploy-ack key; its * absence is a wiring bug, not a tolerable case. */ @@ -993,7 +990,7 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { * the harness as a one-step workflow (the same wrap `launchSession` uses) and * route it through `deploySingleStepAtHead` with the instance's REAL identity * -- so the head address IS the instance address and the deploy runs as a - * supervised workflow-process child, not the legacy trivial in-process path. + * supervised workflow-process child. * * Unlike the orchestrator's `runSingleStepAtHead`, this calls * `deploySingleStepAtHead` directly with the route's real `agentId` diff --git a/packages/hub-sessions/src/ws/sidecar-handler.ts b/packages/hub-sessions/src/ws/sidecar-handler.ts index d03d2014..c4555855 100644 --- a/packages/hub-sessions/src/ws/sidecar-handler.ts +++ b/packages/hub-sessions/src/ws/sidecar-handler.ts @@ -109,9 +109,9 @@ export type SidecarRouter = { * Send an `agent.deploy` frame to the sidecar. When `workflow` is * supplied, the frame carries the multi-step deploy projection * (workflow definition plus per-step source pins); the sidecar's - * deploy router uses the field's presence to discriminate the - * multi-step branch from the trivial branch. Absent on every legacy - * agent-deploy. + * deploy router routes it to the workflow deploy path. The sole + * caller supplies `workflow` on every deploy; per-step provisioning + * uses `sendProvisionStep`. * * The returned promise resolves with the supervisor's principal * public key (hex-encoded Ed25519) carried on `agent.deploy.ack`. diff --git a/packages/types/src/sidecar.ts b/packages/types/src/sidecar.ts index 4580fa1f..5e59faf9 100644 --- a/packages/types/src/sidecar.ts +++ b/packages/types/src/sidecar.ts @@ -249,11 +249,10 @@ export const DrainDeliverFrame = type({ export type DrainDeliverFrame = typeof DrainDeliverFrame.infer; /** - * Workflow projection carried on an `agent.deploy` frame for the - * multi-step branch. Presence of the field at the deploy router is the - * discriminator between the trivial-launch path (single-step, - * `config`-driven) and the multi-step path (workflow-process spawn, - * per-step source pins). + * Workflow projection carried on an `agent.deploy` frame. Its presence + * at the deploy router routes the frame to the workflow deploy path -- + * single- or multi-step, both of which spawn the workflow-process child + * -- as opposed to a per-step provision frame. * * `definition` is the wire projection of `WorkflowDefinition` from * `@intx/workflow`. The arktype validator enforces the structural @@ -301,10 +300,10 @@ export const AgentDeployWorkflow = type({ export type AgentDeployWorkflow = typeof AgentDeployWorkflow.infer; /** - * Deploy an agent to this sidecar. The sidecar initializes a harness from - * the config. + * Deploy an agent to this sidecar. The sidecar spawns a supervised + * workflow-process child to host the deployment. * - * The deploy router discriminates three shapes by field presence without + * The deploy router discriminates two shapes by field presence without * consulting `config`: * - `workflow` set: a workflow deployment (single-step head or multi-step) * that spawns the supervised workflow-process child. @@ -313,8 +312,8 @@ export type AgentDeployWorkflow = typeof AgentDeployWorkflow.infer; * records the hub key so the follow-up deploy pack applies and verifies, * but spawns nothing. The deployment-level `workflow` frame (sent once * after every step is provisioned) spawns the child. - * - neither: the legacy trivial in-process harness deploy. - * `workflow` and `provisionStep` are mutually exclusive. + * A frame carrying neither is rejected -- there is no in-process + * fall-through. `workflow` and `provisionStep` are mutually exclusive. */ export const AgentDeployFrame = type({ type: "'agent.deploy'", diff --git a/packages/workflow-deploy/src/capability-approval.ts b/packages/workflow-deploy/src/capability-approval.ts index ca408118..64c0d26a 100644 --- a/packages/workflow-deploy/src/capability-approval.ts +++ b/packages/workflow-deploy/src/capability-approval.ts @@ -12,11 +12,6 @@ // - A non-empty `unresolvedDirectors` field on the walk result is // itself a deploy-time failure; the gate surfaces it through // `ApprovalDecision` and the orchestrator aborts. -// - The trivial-workflow uniformity bridge: operators populating the -// approval set from the legacy per-agent grant store carry through -// the previously-approved grants so the operator does not see a -// fresh approval prompt for grants that were already approved on the -// legacy agent-deploy path. import type { CapabilityWalkResult } from "./capability-walk"; diff --git a/packages/workflow-deploy/src/index.ts b/packages/workflow-deploy/src/index.ts index 6bb9f570..951b31a7 100644 --- a/packages/workflow-deploy/src/index.ts +++ b/packages/workflow-deploy/src/index.ts @@ -8,8 +8,8 @@ // - approval gate: consumes the walk's output plus an operator- // supplied `ApprovalSet` and yields a per-step pending delta. // - orchestrator: validates the workflow, runs the walk + approval -// gate, writes the workflow repo, and branches on the trivial-vs- -// multi-step dichotomy for per-agent launches. +// gate, writes the workflow repo, and branches on the single-step- +// vs-multi-step dichotomy for per-agent launches. export { walkCapabilities, diff --git a/packages/workflow-host/src/supervisor/credentials.ts b/packages/workflow-host/src/supervisor/credentials.ts index c249404b..1c6343ef 100644 --- a/packages/workflow-host/src/supervisor/credentials.ts +++ b/packages/workflow-host/src/supervisor/credentials.ts @@ -126,10 +126,11 @@ export type AssembleCredentialsSnapshotOpts = { /** * Default mapping from `(deploymentId, stepId)` to the agent-state - * repo's identity. Multi-step deployments use `- - * ` to keep each step's grants isolated to its own repo; - * trivial deployments collapse to the same convention with the - * sole step's id appended. + * repo id: `-`, isolating each step's grants in + * its own repo. Applied to a one-step `stepOrder` it yields a single + * such repo. The single-step launched-agent deploy overrides this + * default (see `DeriveStepRepoId`) to reuse the legacy agent-state + * repo. */ export function defaultStepRepoId(args: { deploymentId: string; @@ -150,8 +151,8 @@ export function defaultStepRepoId(args: { * through the git object database. * * A missing grants file is treated as "no grants" (empty array) so - * the trivial path with no operator-supplied grants does not crash; - * a malformed file does crash, because the file's presence implies + * a step whose repo carries no grants file does not crash; a + * malformed file does crash, because the file's presence implies * the deploy orchestrator intended a snapshot and a structural * failure is a programming bug at the boundary. */ From ec685bacd4dab79c4818f8ea56d119c29dde9f23 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Mon, 6 Jul 2026 06:59:58 -0500 Subject: [PATCH 074/101] Rewrite the harness design doc for the deployment-record model The sidecar no longer persists a per-agent `agent.json` HarnessConfig and self-restores an in-process harness from it. A deployment now runs as a supervised workflow-process child, and its restart record is a per-deployment `deployment.json` under `workflow-runs//` that the boot scan re-drives through the same spawn path a live deploy uses. Rewrite the deploy/undeploy, directory-layout, self-restoration, and security-model sections to describe the deployment-record model. The per-agent key custody, challenge/response, and authority model are unchanged and left as-is. --- docs/HARNESS_DESIGN.md | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/docs/HARNESS_DESIGN.md b/docs/HARNESS_DESIGN.md index 2684d5c8..a8834f79 100644 --- a/docs/HARNESS_DESIGN.md +++ b/docs/HARNESS_DESIGN.md @@ -79,11 +79,11 @@ All communication between hub and sidecar is over a single persistent WebSocket | `agent.deploy.ack` | `agentAddress`, `publicKey` | Agent deployed, here is its public key | | `agent.error` | `agentAddress`, `error` | Deployment failed | -When the hub sends `agent.deploy`, the sidecar generates a key pair (if new) or loads the existing one, persists the `HarnessConfig` to `agent.json`, initializes the harness, and responds with `agent.deploy.ack` including the agent's hex-encoded public key. The hub stores this public key for reconnect verification. +When the hub sends `agent.deploy`, the sidecar spawns a supervised **workflow-process child** to host the deployment and responds with `agent.deploy.ack`. For a single-step (launched-agent) deployment the sidecar generates the agent's key pair (if new) or loads the existing one, records the hub's public key for later deploy-pack verification, initializes the head's on-disk deploy-tree repository (the narrow `initRepo`, not the retired `provisionAgent`), and acks the agent's hex-encoded public key — the hub stores it in `agent_instance.publicKey` for reconnect verification. A multi-step deployment has no head agent identity: its per-step addresses are workflow-derived, so the ack instead carries the sidecar's supervisor principal key, which the hub discards. Before the child is spawned, the inputs a restart cannot otherwise recover — the per-step inference sources, the session id, and (single-step only) the hub public key — are written to a per-deployment `deployment.json` record. -When the hub sends `agent.undeploy`, the sidecar stops the harness, unregisters the agent from the transport, and clears the persisted config. +When the hub sends `agent.undeploy`, the sidecar shuts the deployment's supervisor down (killing the workflow-process child and releasing its IPC pipes and event-channel handle), unregisters the deployment address from the transport and from the mail/signal/drain routers, reclaims the deployment's per-step scratch, and deletes the `deployment.json` record so a later boot does not re-spawn a torn-down deployment. The agent's key pair and its durable agent-state / conversation repositories are left in place so a redeploy on the same address resumes them. -Credentials travel in the `agent.deploy` frame's `config.providers` array. There is no separate credential push endpoint. +Credentials travel in the `agent.deploy` frame's inference **sources** — `config.sources`, and the per-step `workflow.sources` failover chains — where each `InferenceSource` carries its own API key. There is no separate credential push endpoint. ## Per-Agent Key Pairs @@ -95,21 +95,28 @@ Directory layout under `SIDECAR_DATA_DIR`: ``` SIDECAR_DATA_DIR/ - agent-name_at_tenant_interchange_network/ - .git/ # isogit repository (context, audit records) - agent.json # persisted HarnessConfig for session restore + / # per-agent key custody + head deploy-tree repo + .git/ # isogit repository (deploy tree, context, audit records) keys/ - id_ed25519 # agent private key (raw 32 bytes) - id_ed25519.pub # agent public key (raw 32 bytes) + id_ed25519 # agent private key (raw 32 bytes, mode 0600) + id_ed25519.pub # agent public key (raw 32 bytes) + workflow-runs/ + / # workflow-run substrate for one deployment + deployment.json # per-deployment restore record (mode 0600); see below + workflow-step-state/ + / # ephemeral per-step scratch, reclaimed on undeploy + agent-conversation-state/ # durable per-agent conversation, survives undeploy ``` -The `agent.json` file stores the full harness configuration (system prompt, model, providers, session ID) so that the sidecar can restore agent sessions on restart without re-receiving the config from the hub. +The per-agent key directory is keyed by the sanitized agent address; the workflow subtrees are keyed by the derived deployment id. + +The `deployment.json` record stores only what a restart cannot otherwise recover: the deployment's `agentAddress`, the `definitionId` naming its workflow definition on disk, each step's ordered inference-**sources** failover chain (`sources`), the optional inference `sessionId`, and — for a single-step deployment — the `hubPublicKey`. A `version` field guards the schema so a stale record can be rejected rather than parsed blindly. The record deliberately does **not** duplicate the workflow definition (kept on disk under its `definitionId` and re-read at restore) or the step grants (kept in each step's agent-state repo). Because each source embeds its API key, the record is written owner-only (mode 0600). The directory name is the agent address with `@` replaced by `_at_` and non-alphanumeric characters (except `-` and `_`) replaced by `_`. ## Agent Deployment vs User Sessions -The sidecar manages agents, not user sessions. When the hub deploys an agent to a sidecar, the sidecar starts a harness for that agent. The harness runs continuously, receiving messages from any source — other agents, users, system signals. User sessions are a hub-side concept: the hub tracks which users are interacting with which agents and routes user messages to the agent's address accordingly, but the sidecar does not know or care about individual user sessions. +The sidecar manages agents, not user sessions. When the hub deploys an agent to a sidecar, the sidecar spawns a supervised **workflow-process child** for that deployment. The child runs continuously, receiving messages from any source — other agents, users, system signals — and builds the agent harness inside its own process. User sessions are a hub-side concept: the hub tracks which users are interacting with which agents and routes user messages to the agent's address accordingly, but the sidecar does not know or care about individual user sessions. The hub maintains a sidecar-to-agent mapping in its database. This mapping determines where to route messages for a given agent address. When a sidecar disconnects, the hub knows which agents are affected and queues messages for them until the sidecar reconnects. @@ -156,7 +163,7 @@ On reconnection (agents successfully restored from `SIDECAR_DATA_DIR`), the side ### Self-Restoration -When the WebSocket connection to the hub opens, the sidecar scans `SIDECAR_DATA_DIR` for agent directories containing both a key pair and an `agent.json` config. For each valid directory, the sidecar restores the harness from the persisted config. The `register` or `reconnect` frame is held until restoration completes. This restoration happens entirely on the sidecar side — the hub is not involved in session restoration. +At boot, **before** opening the WebSocket connection to the hub, the sidecar scans `SIDECAR_DATA_DIR/workflow-runs/` for `deployment.json` records (`scanWorkflowDeploymentRecords`). Each record is validated at the trust boundary, its stored `agentAddress` is cross-checked against its directory name, its workflow definition is re-read and re-validated off disk with the same gates a fresh deploy applies, and its pinned inference sources are re-admitted against the providers this sidecar can build; a record that fails any check is logged and skipped so one bad deployment cannot strand the rest. Each surviving record is restored through the **same** spawn path a live deploy uses — a supervised workflow-process child, with the single-step head's key pair and recorded hub key re-established. Restoration completes before the `register` or `reconnect` frame is sent, and happens entirely on the sidecar side; the hub is not involved in restoration. ### Challenge/Response Verification @@ -210,7 +217,7 @@ The sidecar's isogit repository is the source of truth for agent inference conte ## Security Model -Credentials travel in the `agent.deploy` frame's `config.providers` array and are held in memory by the harness. They are never persisted to disk (the `agent.json` file contains the full config including provider entries with API keys — this is a known limitation of the prototype that should be addressed before production use). +Credentials travel in the `agent.deploy` frame's inference `sources` (each `InferenceSource` carries its own API key) and are held in memory by the running deployment. They are also persisted to disk in the deployment's `deployment.json` record — the `sources` field embeds those API keys — so the sidecar can restore a deployment on restart without re-receiving them from the hub. The record is written owner-only (mode 0600), but storing provider API keys on the sidecar's disk at all is a known limitation of the prototype that should be addressed before production use. ## Key Rotation From a9ffa6d0408d179b3f0c1fd8ba37bc33c2668b48 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Mon, 6 Jul 2026 12:11:07 -0500 Subject: [PATCH 075/101] Write the deployment record atomically and durably The deployment record is the sole restore source for a deployment's inference sources and hub key, and a source rotation overwrites it in place. A bare truncating write could leave a torn record on a crash mid-rotation, and the boot scan then skips the deployment entirely -- losing it rather than restoring a stale source list. Route the write through a temp-file-plus-rename helper that fsyncs before the rename, so a reader only ever observes a complete record across both process death and power loss. --- apps/sidecar/src/atomic-write.test.ts | 102 ++++++++++++++++++ apps/sidecar/src/atomic-write.ts | 79 ++++++++++++++ .../src/workflow-deployment-record.test.ts | 19 ++++ .../sidecar/src/workflow-deployment-record.ts | 17 +-- 4 files changed, 211 insertions(+), 6 deletions(-) create mode 100644 apps/sidecar/src/atomic-write.test.ts create mode 100644 apps/sidecar/src/atomic-write.ts diff --git a/apps/sidecar/src/atomic-write.test.ts b/apps/sidecar/src/atomic-write.test.ts new file mode 100644 index 00000000..7b81961a --- /dev/null +++ b/apps/sidecar/src/atomic-write.test.ts @@ -0,0 +1,102 @@ +import { describe, test, expect } from "bun:test"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import { writeFileAtomicDurable } from "./atomic-write"; + +async function makeDir(): Promise { + return fs.mkdtemp(path.join(os.tmpdir(), "atomic-write-")); +} + +async function listNames(dir: string): Promise { + return (await fs.readdir(dir)).sort(); +} + +describe("writeFileAtomicDurable", () => { + test("round-trips contents to the target path", async () => { + const dir = await makeDir(); + const file = path.join(dir, "record.json"); + + await writeFileAtomicDurable(file, '{"a":1}', { mode: 0o600 }); + expect(await fs.readFile(file, "utf8")).toBe('{"a":1}'); + + await fs.rm(dir, { recursive: true, force: true }); + }); + + test("applies mode on the created file", async () => { + const dir = await makeDir(); + const file = path.join(dir, "record.json"); + + await writeFileAtomicDurable(file, "x", { mode: 0o600 }); + const stat = await fs.stat(file); + expect(stat.mode & 0o777).toBe(0o600); + + await fs.rm(dir, { recursive: true, force: true }); + }); + + test("re-applies mode when overwriting an existing file", async () => { + const dir = await makeDir(); + const file = path.join(dir, "record.json"); + + // Seed a pre-existing file with a laxer mode. A plain in-place + // overwrite keeps the original mode; the temp+rename path creates a + // fresh file every write, so the requested mode actually lands. + await fs.writeFile(file, "old", { mode: 0o644 }); + await writeFileAtomicDurable(file, "new", { mode: 0o600 }); + + expect(await fs.readFile(file, "utf8")).toBe("new"); + const stat = await fs.stat(file); + expect(stat.mode & 0o777).toBe(0o600); + + await fs.rm(dir, { recursive: true, force: true }); + }); + + test("leaves no temp orphan after a successful write", async () => { + const dir = await makeDir(); + const file = path.join(dir, "record.json"); + + await writeFileAtomicDurable(file, "x", { mode: 0o600 }); + expect(await listNames(dir)).toEqual(["record.json"]); + + await fs.rm(dir, { recursive: true, force: true }); + }); + + test("a write that fails before staging leaves the prior file intact", async () => { + const dir = await makeDir(); + const file = path.join(dir, "record.json"); + await fs.writeFile(file, "old", { mode: 0o600 }); + + // Deny writes to the directory so the temp-file creation fails + // outright. The prior complete record must survive untouched -- an + // interrupted rotation must never corrupt the sole restore source. + await fs.chmod(dir, 0o500); + try { + await expect( + writeFileAtomicDurable(file, "new", { mode: 0o600 }), + ).rejects.toThrow(); + expect(await fs.readFile(file, "utf8")).toBe("old"); + expect(await listNames(dir)).toEqual(["record.json"]); + } finally { + await fs.chmod(dir, 0o700); + await fs.rm(dir, { recursive: true, force: true }); + } + }); + + test("a write that fails at the rename unlinks its staged temp", async () => { + const dir = await makeDir(); + const file = path.join(dir, "record.json"); + + // Occupy the target path with a directory so the rename fails after + // the temp file has already been staged and fsynced. This exercises + // the error-path unlink: the staged temp must be removed rather than + // stranded as an orphan. + await fs.mkdir(file); + await expect( + writeFileAtomicDurable(file, "new", { mode: 0o600 }), + ).rejects.toThrow(); + expect(await listNames(dir)).toEqual(["record.json"]); + + await fs.rm(dir, { recursive: true, force: true }); + }); +}); diff --git a/apps/sidecar/src/atomic-write.ts b/apps/sidecar/src/atomic-write.ts new file mode 100644 index 00000000..48b116e3 --- /dev/null +++ b/apps/sidecar/src/atomic-write.ts @@ -0,0 +1,79 @@ +// Atomic, durable file replacement for the sidecar's non-rebuildable +// on-disk records. Distinct from the cache's rebuildable temp+rename +// (no fsync, a lost write just forces a re-fetch) and from +// `fsyncWriteFile`'s in-place fsync write (no atomicity, a torn write +// leaves a half-file): this is the tier for a sole restore source that +// must survive both a process kill and a power loss without ever +// exposing a torn record. + +import { open, rename, unlink } from "node:fs/promises"; +import { dirname } from "node:path"; + +import { getLogger } from "@intx/log"; +import { hexEncode } from "@intx/types"; + +const logger = getLogger(["interchange", "sidecar", "atomic-write"]); + +export interface AtomicWriteOptions { + /** Permission mode applied when the temp file is created. */ + mode: number; +} + +/** + * Replace `path` with `contents` atomically and durably. The bytes land + * in a fresh per-write temp file that is fsynced and then `rename`d over + * `path`; because rename is atomic within a directory, a reader only + * ever observes the prior complete file or the new complete file, never + * a torn one. The fsync before the rename is what extends that + * guarantee past process death to OS crash / power loss: without it, the + * ext4 delayed-allocation window can surface the renamed path as a + * zero-length file after a power loss. + * + * The parent directory is fsynced after the rename so the new link is + * itself durable, but a filesystem that rejects directory fsync + * (FAT/exFAT, some network mounts) only degrades durability -- the file + * is already renamed and fsynced -- so that failure is logged, not + * thrown. + * + * `mode` is applied on the temp file's creation, so it takes effect on + * every write. A plain in-place overwrite of an existing file would + * silently keep the original file's mode instead. + * + * The temp file follows `createTarballCache`'s `.tmp..` + * naming convention for consistency across the sidecar's staged writes. + */ +export async function writeFileAtomicDurable( + path: string, + contents: string, + options: AtomicWriteOptions, +): Promise { + const tmp = `${path}.tmp.${String(process.pid)}.${hexEncode(crypto.getRandomValues(new Uint8Array(8)))}`; + try { + const handle = await open(tmp, "w", options.mode); + try { + await handle.writeFile(contents); + await handle.sync(); + } finally { + await handle.close(); + } + await rename(tmp, path); + } catch (cause) { + // The write failed and is about to rethrow; unlink the temp so a + // failed write leaves no orphan. Best-effort: the temp may never + // have been created, and a second failure here must not mask the + // original cause. + await unlink(tmp).catch(() => undefined); + throw cause; + } + + try { + const dirHandle = await open(dirname(path), "r"); + try { + await dirHandle.sync(); + } finally { + await dirHandle.close(); + } + } catch (err) { + logger.warn`parent-dir fsync failed for ${path}; durability is degraded but the file is renamed and fsynced — ${err instanceof Error ? err.message : String(err)}`; + } +} diff --git a/apps/sidecar/src/workflow-deployment-record.test.ts b/apps/sidecar/src/workflow-deployment-record.test.ts index db321dc1..8b3c8426 100644 --- a/apps/sidecar/src/workflow-deployment-record.test.ts +++ b/apps/sidecar/src/workflow-deployment-record.test.ts @@ -114,6 +114,25 @@ describe("workflow deployment record store", () => { await fs.rm(dataDir, { recursive: true, force: true }); }); + test("overwriting a record leaves the new one and no temp orphan", async () => { + const dataDir = await makeDataDir(); + const deploymentId = "rotated-1"; + + // A source rotation overwrites the existing record in place. The + // atomic write must replace it cleanly, leaving only the record and + // no `.tmp` staging file behind. + await writeWorkflowDeploymentRecord(dataDir, deploymentId, SINGLE_STEP); + await writeWorkflowDeploymentRecord(dataDir, deploymentId, MULTI_STEP); + + const dir = path.join(dataDir, "workflow-runs", deploymentId); + expect(await fs.readdir(dir)).toEqual(["deployment.json"]); + + const scanned = await scanWorkflowDeploymentRecords(dataDir); + expect(scanned.map((s) => s.record)).toEqual([MULTI_STEP]); + + await fs.rm(dataDir, { recursive: true, force: true }); + }); + test("delete removes the record and is a no-op when absent", async () => { const dataDir = await makeDataDir(); const deploymentId = "gone-1"; diff --git a/apps/sidecar/src/workflow-deployment-record.ts b/apps/sidecar/src/workflow-deployment-record.ts index 10a2485b..01aaaf14 100644 --- a/apps/sidecar/src/workflow-deployment-record.ts +++ b/apps/sidecar/src/workflow-deployment-record.ts @@ -15,7 +15,7 @@ // `definitionId`, and each step's grants live in its agent-state repo, so // neither is duplicated here. -import { mkdir, readdir, readFile, rm, writeFile } from "node:fs/promises"; +import { mkdir, readdir, readFile, rm } from "node:fs/promises"; import { dirname, join as pathJoin } from "node:path"; import { type } from "arktype"; @@ -23,6 +23,8 @@ import { type } from "arktype"; import { getLogger } from "@intx/log"; import { InferenceSource } from "@intx/types/runtime"; +import { writeFileAtomicDurable } from "./atomic-write"; + const logger = getLogger([ "interchange", "sidecar", @@ -74,11 +76,14 @@ export async function writeWorkflowDeploymentRecord( ): Promise { const path = recordPath(dataDir, deploymentId); await mkdir(dirname(path), { recursive: true }); - // Owner-only (0o600): the record embeds each source's `apiKey`, so it must - // not be world-readable on a shared host. This matches the private-key - // writes elsewhere on the sidecar. - await writeFile(path, JSON.stringify(record, null, 2), { - encoding: "utf8", + // Atomic + durable: this is the sole restore source for the + // deployment's `sources`/`hubPublicKey`, and a rotation overwrites the + // existing record in place, so an interrupted write must never expose a + // torn record the boot scan would then skip. Owner-only (0o600): the + // record embeds each source's `apiKey`, so it must not be world-readable + // on a shared host, matching the private-key writes elsewhere on the + // sidecar. + await writeFileAtomicDurable(path, JSON.stringify(record, null, 2), { mode: 0o600, }); } From bb97c38d324400a863279812275ab4d40a9c75a4 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Mon, 6 Jul 2026 13:25:40 -0500 Subject: [PATCH 076/101] Bound the recycle respawn ready handshake with a timeout The recycle path awaited the respawned child's ready handshake with no deadline, so a child that spawned but never emitted ready parked the supervisor in the recycling phase forever with no automatic recovery -- inbound mail queued against a dispatch loop that never started, and only an external undeploy could free it. The spawn path already bounds the same handshake; mirror it here: race the ready outcome against a resolve-only deadline and, on timeout or a handshake failure, reap the new child and throw so the supervisor's recycle-failure teardown lands a clean stopped state the operator can see and redeploy. The new child is reaped in the recycle path itself because it is never installed on supervisor state before the throw, so the failure teardown (which reaps the prior cohort) would otherwise leak it. The credentials-snapshot read sits outside this deadline by design; a wedged substrate is bounded at its own layer, not by this handshake timer. --- .../src/supervisor/child-termination.ts | 10 ++ .../src/supervisor/recycle.test.ts | 130 ++++++++++++++++++ .../workflow-host/src/supervisor/recycle.ts | 83 ++++++++++- .../src/supervisor/supervisor.ts | 16 +-- 4 files changed, 224 insertions(+), 15 deletions(-) diff --git a/packages/workflow-host/src/supervisor/child-termination.ts b/packages/workflow-host/src/supervisor/child-termination.ts index 90aeef90..13e1f81c 100644 --- a/packages/workflow-host/src/supervisor/child-termination.ts +++ b/packages/workflow-host/src/supervisor/child-termination.ts @@ -20,6 +20,16 @@ import type { SubprocessHandle } from "./types"; */ export const DEFAULT_KILL_TIMEOUT_MS = 5_000; +/** + * Default deadline for a child's `ready` handshake -- the window a freshly + * spawned child has to emit `ready` before the supervisor kills it and + * treats the spawn (or recycle respawn) as failed. Used when a caller + * supplies no override. Shared here, alongside the kill default, so both + * the spawn path and the recycle path bound the handshake identically + * without either re-declaring the constant. + */ +export const DEFAULT_READY_TIMEOUT_MS = 30_000; + /** * Injected dependencies for `killChildHandle`. `setTimer`/`clearTimer` * default to the real `setTimeout`/`clearTimeout` when omitted so tests diff --git a/packages/workflow-host/src/supervisor/recycle.test.ts b/packages/workflow-host/src/supervisor/recycle.test.ts index 6bf64d2a..2671006f 100644 --- a/packages/workflow-host/src/supervisor/recycle.test.ts +++ b/packages/workflow-host/src/supervisor/recycle.test.ts @@ -793,6 +793,136 @@ describe("supervisor recycle: failure after the cohort handoff", () => { }); }); +describe("supervisor recycle: respawn handshake bound", () => { + test("a respawn that never emits ready times out, reaps the new child, and reaches a clean stopped state", async () => { + const baseDir = await makeTempDir("recycle-ready-timeout-"); + const ipcKeypair = await generateKeyPair(); + const mailBus = createMockMailBus(); + // sigtermExits:false so the reap escalates SIGTERM -> SIGKILL: a wedged + // respawn is exactly the child that ignores SIGTERM, and the reap must + // still guarantee it dies. + const tracker = createSpawnTracker({ sigtermExits: false }); + await seedStepGrants( + baseDir, + defaultStepRepoId({ deploymentId: "deployment-x", stepId: "step-1" }), + [{ resource: "thing", action: "read" }], + ); + const bindings = await buildBindings({ + baseDir, + spawner: tracker.spawner, + mailBus, + ipcKeypair, + }); + const supervisor = createWorkflowSupervisor({ + ...bindings, + // Collapse the recycle path's ready deadline and kill-escalation + // deadline to the next macrotask so the wedged-respawn path resolves + // without real-time waits. The child never emits ready, so timeout + // always wins the handshake race. The spawn path keeps the default + // real timer, so the first child readies normally. + recyclePolicySetTimer: (cb) => setTimeout(cb, 0), + recyclePolicyClearTimer: () => undefined, + }); + const spawnPromise = supervisor.spawn({ + stepOrder: ["step-1"], + definitionHash: "def-hash-abc", + warmKeep: false, + onInferenceEvent: () => undefined, + }); + while (tracker.children.length === 0) { + await new Promise((r) => setTimeout(r, 1)); + } + const first = tracker.children[0]; + if (first === undefined) throw new Error("first child missing"); + await driveReady(first, ipcKeypair); + await spawnPromise; + + // Recycle spawns a second child but never drives its ready frame, so + // the bounded handshake times out. + let caught: unknown; + try { + await supervisor.recycle({ reason: "ready-timeout-test" }); + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(Error); + expect(caught instanceof Error && caught.message).toMatch( + /did not emit ready/, + ); + + // The wedged respawn was reaped through the full SIGTERM -> SIGKILL + // ladder rather than left running as an orphan. + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + expect(second.killSignals).toEqual(["SIGTERM", "SIGKILL"]); + + // The recycle failure routed through shutdownInternal to `stopped`, so + // a follow-up shutdown is a no-op. + await supervisor.shutdown(); + }); + + test("a respawn whose control channel ends before ready rejects and reaps the new child", async () => { + const baseDir = await makeTempDir("recycle-ready-failed-"); + const ipcKeypair = await generateKeyPair(); + const mailBus = createMockMailBus(); + // sigtermExits:true so the reap's SIGTERM settles the fake child; this + // path proves the child is killed even though the control channel end + // does not itself terminate the process. + const tracker = createSpawnTracker({ sigtermExits: true }); + await seedStepGrants( + baseDir, + defaultStepRepoId({ deploymentId: "deployment-x", stepId: "step-1" }), + [{ resource: "thing", action: "read" }], + ); + const bindings = await buildBindings({ + baseDir, + spawner: tracker.spawner, + mailBus, + ipcKeypair, + }); + // Default (real, 30s) ready timer: the deadline must NOT fire, so the + // control-channel-end failure wins the handshake race deterministically. + const supervisor = createWorkflowSupervisor(bindings); + const spawnPromise = supervisor.spawn({ + stepOrder: ["step-1"], + definitionHash: "def-hash-abc", + warmKeep: false, + onInferenceEvent: () => undefined, + }); + while (tracker.children.length === 0) { + await new Promise((r) => setTimeout(r, 1)); + } + const first = tracker.children[0]; + if (first === undefined) throw new Error("first child missing"); + await driveReady(first, ipcKeypair); + await spawnPromise; + + const recyclePromise = supervisor.recycle({ reason: "ready-failed-test" }); + while (tracker.children.length < 2) { + await new Promise((r) => setTimeout(r, 1)); + } + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + // End the new child's control channel before it emits ready: the + // handshake fails rather than hangs. + second.childToSupervisor.close(); + + let caught: unknown; + try { + await recyclePromise; + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(Error); + expect(caught instanceof Error && caught.message).toMatch( + /control channel ended before child emitted ready/, + ); + expect(second.killSignals).toContain("SIGTERM"); + + await supervisor.shutdown(); + }); +}); + describe("supervisor recycle: deliverSignal phase guard", () => { test("deliverSignal during the recycling window rejects rather than writing to the dying cohort's controlSender", async () => { // M2: during `recycling`, the supervisor's `state.controlSender` diff --git a/packages/workflow-host/src/supervisor/recycle.ts b/packages/workflow-host/src/supervisor/recycle.ts index 9d976479..049125a8 100644 --- a/packages/workflow-host/src/supervisor/recycle.ts +++ b/packages/workflow-host/src/supervisor/recycle.ts @@ -101,7 +101,14 @@ import { } from "./credentials"; import type { SubprocessHandle, WorkflowSupervisorBindings } from "./types"; import { buildChildSpawnEnv } from "./spawn-env"; -import { DEFAULT_KILL_TIMEOUT_MS, killChildHandle } from "./child-termination"; +import { + DEFAULT_KILL_TIMEOUT_MS, + DEFAULT_READY_TIMEOUT_MS, + defaultClearTimer, + defaultSetTimer, + killChildHandle, + waitDeadline, +} from "./child-termination"; const logger = getLogger(["workflow-host", "supervisor", "recycle"]); @@ -225,6 +232,14 @@ export interface RecycleContext { * `DEFAULT_KILL_TIMEOUT_MS`. */ readonly killTimeoutMs?: number; + /** + * Deadline (ms) for the respawned child's `ready` handshake, matching + * the bound the spawn path applies. The supervisor resolves the + * effective value at its edge (`bindings.readyTimeoutMs ?? + * DEFAULT_READY_TIMEOUT_MS`) and passes it through; the `??` fallback + * here only fires for a direct test caller. + */ + readonly readyTimeoutMs?: number; /** * Optional drain deadline (ms) used in step 1. The supervisor's own * drainTimeout accumulator escalates separately; this deadline is @@ -346,6 +361,14 @@ export async function triggerRecycle( }); const readyPromise = waitForReady(controlIncoming); + // Attach a benign handler at creation, before the fold below consumes + // the rejection: `readyPromise` is created here but not raced until + // after `assembleCredentialsSnapshot` awaits. A child that exits during + // that window rejects `readyPromise` with no handler yet attached -- an + // unhandled rejection across the await boundary. Mirrors the spawn + // path's identical guard. Attaching `.catch` here and `.then` at the + // race is fine; both observe the same settled value. + void readyPromise.catch(() => undefined); const eventIter = receiveEventChannel({ hmacKey, @@ -372,8 +395,62 @@ export async function triggerRecycle( }); // Steps 4 + 5: self-discover + resume. These run inside the child - // before it emits `ready`; the supervisor waits. - const readyInfo = await readyPromise; + // before it emits `ready`; the supervisor waits, but bounded -- a child + // that neither readies nor exits must not park the recycle (and thus + // the supervisor) in `recycling` forever. Bound the handshake exactly + // as the spawn path bounds its own: fold ready/failed into values, race + // a resolve-only deadline, clear the timer on every path. + // + // NOTE: `assembleCredentialsSnapshot` above is a substrate read that + // sits OUTSIDE this deadline; a wedged substrate is a supervisor-side + // fault bounded at its own layer, not by overloading this handshake + // timer. The respawn is deadline-bounded on the child handshake, not on + // that read. + const readyTimeoutMs = ctx.readyTimeoutMs ?? DEFAULT_READY_TIMEOUT_MS; + const setTimer = ctx.setTimer ?? defaultSetTimer; + const clearTimer = ctx.clearTimer ?? defaultClearTimer; + const readyOutcome = readyPromise.then( + (info) => ({ kind: "ready" as const, info }), + (err: unknown) => ({ kind: "failed" as const, err }), + ); + const readyDeadline = waitDeadline(setTimer, readyTimeoutMs); + const readyRace = await Promise.race([ + readyOutcome, + readyDeadline.promise.then(() => ({ kind: "timeout" as const })), + ]); + clearTimer(readyDeadline.handle); + if (readyRace.kind !== "ready") { + // The new child was never installed on `state`, so the supervisor's + // recycle-failure teardown (which reaps the PRIOR cohort) would leak + // it. Reap it here. Kill FIRST, then finalize the pumps: process + // death drives EOF on both channels, which unparks `waitForReady`'s + // in-flight `iter.next()` (letting `controlIncoming.return` complete) + // and ends `pumpEvents`. Awaiting either finalizer before the kill + // would hang behind the still-open channels. `killChildHandle` on an + // already-dead handle is a cheap no-op, so the `failed` path (a + // control-channel end does not guarantee the process died, since the + // event channel is separate) is reaped too, not just the timeout. + await killChildHandle(handle, killTimeoutMs, { + logger, + setTimer, + clearTimer, + }); + void eventPump.catch((cause: unknown) => { + const message = cause instanceof Error ? cause.message : String(cause); + logger.warn`recycle: reaped child eventPump failed after handshake failure: ${message}`; + }); + void controlIncoming.return(undefined).catch((cause: unknown) => { + const message = cause instanceof Error ? cause.message : String(cause); + logger.warn`recycle: reaped child controlIncoming.return failed after handshake failure: ${message}`; + }); + if (readyRace.kind === "timeout") { + throw new Error( + `workflow-host supervisor recycle: child did not emit ready within ${String(readyTimeoutMs)}ms; killed`, + ); + } + throw readyRace.err; + } + const readyInfo = readyRace.info; logger.info`recycle ${opts.origin}: child ready (pid=${String(readyInfo.childPid)}, newChannelId=${channelId})`; const newWiring: ChildWiring = { diff --git a/packages/workflow-host/src/supervisor/supervisor.ts b/packages/workflow-host/src/supervisor/supervisor.ts index 3db74ba1..3db0ef26 100644 --- a/packages/workflow-host/src/supervisor/supervisor.ts +++ b/packages/workflow-host/src/supervisor/supervisor.ts @@ -116,6 +116,7 @@ import { } from "./terminal-broadcaster"; import { DEFAULT_KILL_TIMEOUT_MS, + DEFAULT_READY_TIMEOUT_MS, defaultClearTimer, defaultSetTimer, killChildHandle, @@ -124,18 +125,6 @@ import { const logger = getLogger(["workflow-host", "supervisor"]); -/** - * Default bound on the child's spawn-time `ready` handshake. A spawned - * child that neither emits `ready` nor exits would otherwise block - * `spawn` forever; on expiry the supervisor kills the child and rejects - * the spawn. Generous enough for a real child to boot, open its IPC - * channels, and sign `ready` under load; short enough that a wedged child - * does not stall the spawn (and, on the sidecar, boot-time restore) - * indefinitely. Operator-overridable via `WorkflowSupervisorBindings. - * readyTimeoutMs`. - */ -const DEFAULT_READY_TIMEOUT_MS = 30_000; - /** * Default watchdog timeout for the supervisor's * `synchronouslyDispatchTerminalWrite`. The handler holds the @@ -2369,6 +2358,9 @@ export function createWorkflowSupervisor( wakeDispatch(); }, onCrash: onChildCrash, + // Edge-resolved once at the supervisor factory; recycle bounds + // the respawn handshake with the same value the spawn path uses. + readyTimeoutMs, ...(bindings.recyclePolicySetTimer !== undefined ? { setTimer: bindings.recyclePolicySetTimer } : {}), From 26644b1987edba72c763d08f8d601a0fad628ffa Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Mon, 6 Jul 2026 14:15:49 -0500 Subject: [PATCH 077/101] Swap the rotation sources before persisting to survive a recycle The single-step source-rotation handler persisted the deployment record before committing the new sources to the in-memory respawn hint. A recycle that interleaved the persist await read the stale hint through dynamicSpawnEnv and stood the child up on the prior sources, while the durable record had already moved on -- the running child contradicted durable intent, and a restart would "correct" it to the sources the child should already have had. Swap the in-memory hint synchronously before the persist so an interleaving recycle respawns on the same sources being persisted. The durable write still precedes the live deliverSources swap, and the hint rolls back on a failed persist so the hint and the record stay in agreement in the common failure case. The only residual disagreement is a child briefly ahead of durable truth on a failed persist that a recycle interleaved, which the next recycle heals -- the benign direction. The record writer is injected so a test can block or fail the persist at the interleave point. --- apps/sidecar/src/workflow-host-wiring.test.ts | 188 +++++++++++++++++- apps/sidecar/src/workflow-host-wiring.ts | 75 +++++-- docs/HARNESS_DESIGN.md | 2 + 3 files changed, 236 insertions(+), 29 deletions(-) diff --git a/apps/sidecar/src/workflow-host-wiring.test.ts b/apps/sidecar/src/workflow-host-wiring.test.ts index 3cc278ae..65144f49 100644 --- a/apps/sidecar/src/workflow-host-wiring.test.ts +++ b/apps/sidecar/src/workflow-host-wiring.test.ts @@ -33,6 +33,7 @@ import { type MultistepSourcesRouter, } from "./workflow-run-pack-client"; import { + scanWorkflowDeploymentRecords, writeWorkflowDeploymentRecord, type WorkflowDeploymentRecord, } from "./workflow-deployment-record"; @@ -553,6 +554,12 @@ describe("createSidecarDeployRouter multi-step branch", () => { * omitted a fresh keypair is minted per call as before. */ headKeyPair?: Awaited>; + /** + * Injectable deployment-record writer. The rotation-interleave tests + * pass a blockable/failing stub so a recycle can be driven into the + * rotation's persist window; omitted, the router uses the real writer. + */ + writeWorkflowDeploymentRecord?: typeof writeWorkflowDeploymentRecord; }) { const transport = opts.transport ?? createInMemoryTransport(); const keyPair = await generateKeyPair(); @@ -629,6 +636,11 @@ describe("createSidecarDeployRouter multi-step branch", () => { ...(opts.readyTimeoutMs !== undefined ? { readyTimeoutMs: opts.readyTimeoutMs } : {}), + ...(opts.writeWorkflowDeploymentRecord !== undefined + ? { + writeWorkflowDeploymentRecord: opts.writeWorkflowDeploymentRecord, + } + : {}), }); return { router, @@ -2159,11 +2171,13 @@ describe("createSidecarDeployRouter multi-step branch", () => { }); test("a rotation whose persist fails leaves no partial effect", async () => { - // Atomicity: if the durable write rejects, the rotation takes NO effect. - // currentSources stays on the deploy-time table (so a recycle respawn is - // unrotated) and deliverSources is never reached. The write is ordered - // before the in-memory commit precisely so a failure cannot leave the - // three views (record, currentSources, live child) divergent. + // Atomicity: if the durable write rejects, the rotation takes NO net + // effect. The handler swaps currentSources synchronously, then rolls it + // back on a failed persist, so once the handler throws currentSources is + // on the deploy-time table (a recycle respawn AFTER the failure is + // unrotated) and deliverSources is never reached. Rolling the in-memory + // hint back keeps currentSources and the record in agreement in this + // no-interleaved-recycle failure case. const dataDir = await createTempBaseDir("sidecar-rot-failatomic-"); const addr = "ins_rotfailatomic@example.com"; const sourcesRouter = createMultistepSourcesRouter(); @@ -2201,8 +2215,74 @@ describe("createSidecarDeployRouter multi-step branch", () => { }), ).rejects.toThrow(); - // currentSources is untouched: a recycle respawn carries the DEPLOY-TIME - // sources, proving the failed rotation left no partial in-memory effect. + // currentSources was rolled back: a recycle respawn AFTER the failed + // rotation carries the DEPLOY-TIME sources, proving the failed rotation + // left no net in-memory effect. + await spawner.recycleRequestFor(0); + while (spawner.spawnCount() < 2) { + await new Promise((r) => setTimeout(r, 1)); + } + await spawner.driveReadyFor(1); + const respawnEnv = spawner.envFor(1); + if (respawnEnv === undefined) throw new Error("respawn env missing"); + expect( + JSON.parse(respawnEnv[STEP_INFERENCE_SOURCES_ENV_KEY] ?? "null"), + ).toEqual({ "step-1": [makeInferenceSource("step-1")] }); + }); + + test("a recycle interleaving the rotation persist respawns on the rotated sources", async () => { + // Interleave guard: the rotation swaps currentSources synchronously + // BEFORE the durable persist, so a recycle that lands inside the persist + // window respawns the child on the ROTATED sources -- consistent with + // what is being persisted -- rather than the stale deploy-time table. + const dataDir = await createTempBaseDir("sidecar-rot-interleave-ok-"); + const addr = "ins_rotinterok@example.com"; + const sourcesRouter = createMultistepSourcesRouter(); + const spawner = makeReadyDrivingSpawner(11200); + + // A persist that blocks on a test-controlled gate once armed, so a recycle + // can be driven into the rotation's persist window. The deploy's own + // persist (before the gate is armed) passes straight through to the real + // writer. + let gate: Promise | null = null; + let release: () => void = () => undefined; + const persist: typeof writeWorkflowDeploymentRecord = async ( + d, + id, + rec, + ) => { + if (gate !== null) await gate; + await writeWorkflowDeploymentRecord(d, id, rec); + }; + + const { router } = await buildMultistepFixture({ + spawner: spawner.spawner, + multistepSourcesRouter: sourcesRouter, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + writeWorkflowDeploymentRecord: persist, + }); + + const deployPromise = router.deploy(singleStepFrame(addr, "wf-rotinterok")); + await spawner.driveReadyFor(0); + await deployPromise; + + // Arm the gate so the rotation's persist blocks mid-window. + gate = new Promise((resolve) => { + release = resolve; + }); + + // Start the rotation but do not await it: the handler swaps currentSources + // synchronously, then parks on the blocked persist before deliverSources. + const rotated = makeInferenceSource("rotated"); + const rotatePromise = sourcesRouter.tryRoute({ + type: "sources.update", + agentAddress: addr, + sources: [rotated], + defaultSource: "rotated", + }); + + // Drive a recycle while the persist is blocked. The respawn env must carry + // the ROTATED sources, because the synchronous swap already ran. await spawner.recycleRequestFor(0); while (spawner.spawnCount() < 2) { await new Promise((r) => setTimeout(r, 1)); @@ -2212,6 +2292,100 @@ describe("createSidecarDeployRouter multi-step branch", () => { if (respawnEnv === undefined) throw new Error("respawn env missing"); expect( JSON.parse(respawnEnv[STEP_INFERENCE_SOURCES_ENV_KEY] ?? "null"), + ).toEqual({ "step-1": [rotated] }); + + // Release the persist; the rotation completes without throwing and the + // durable record converges on the rotated sources. + release(); + await rotatePromise; + const scanned = await scanWorkflowDeploymentRecords(dataDir); + const record = scanned.find( + (s) => s.deploymentId === deriveDeploymentId(addr), + ); + expect(record?.record.sources).toEqual({ "step-1": [rotated] }); + }); + + test("a recycle interleaving a failed rotation persist respawns on new sources then rolls back", async () => { + // The benign residual, pinned: a persist that fails WHILE a recycle + // interleaves leaves the just-respawned child transiently ahead on the + // rotated sources (the intended, self-healing direction), while + // currentSources rolls back to the deploy-time table so the NEXT recycle + // reverts the child to durable truth. + const dataDir = await createTempBaseDir("sidecar-rot-interleave-fail-"); + const addr = "ins_rotinterfail@example.com"; + const sourcesRouter = createMultistepSourcesRouter(); + const spawner = makeReadyDrivingSpawner(11300); + + let gate: Promise | null = null; + let release: () => void = () => undefined; + const persist: typeof writeWorkflowDeploymentRecord = async ( + d, + id, + rec, + ) => { + if (gate !== null) { + await gate; + throw new Error("rotation persist boom"); + } + await writeWorkflowDeploymentRecord(d, id, rec); + }; + + const { router } = await buildMultistepFixture({ + spawner: spawner.spawner, + multistepSourcesRouter: sourcesRouter, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + writeWorkflowDeploymentRecord: persist, + }); + + const deployPromise = router.deploy( + singleStepFrame(addr, "wf-rotinterfail"), + ); + await spawner.driveReadyFor(0); + await deployPromise; + + gate = new Promise((resolve) => { + release = resolve; + }); + + const rotated = makeInferenceSource("rotated"); + const rotatePromise = sourcesRouter.tryRoute({ + type: "sources.update", + agentAddress: addr, + sources: [rotated], + defaultSource: "rotated", + }); + + // A recycle interleaves the about-to-fail persist: the respawn still + // carries the ROTATED sources, because the swap already ran. + await spawner.recycleRequestFor(0); + while (spawner.spawnCount() < 2) { + await new Promise((r) => setTimeout(r, 1)); + } + await spawner.driveReadyFor(1); + const respawnEnv = spawner.envFor(1); + if (respawnEnv === undefined) throw new Error("respawn env missing"); + expect( + JSON.parse(respawnEnv[STEP_INFERENCE_SOURCES_ENV_KEY] ?? "null"), + ).toEqual({ "step-1": [rotated] }); + + // Release the persist so it rejects; the rotation rolls currentSources + // back and rethrows. + release(); + await expect(rotatePromise).rejects.toThrow(/rotation persist boom/); + + // A SECOND recycle now respawns on the ROLLED-BACK deploy-time sources, + // healing the transient down to durable truth. + await spawner.recycleRequestFor(1); + while (spawner.spawnCount() < 3) { + await new Promise((r) => setTimeout(r, 1)); + } + await spawner.driveReadyFor(2); + const secondRespawnEnv = spawner.envFor(2); + if (secondRespawnEnv === undefined) { + throw new Error("second respawn env missing"); + } + expect( + JSON.parse(secondRespawnEnv[STEP_INFERENCE_SOURCES_ENV_KEY] ?? "null"), ).toEqual({ "step-1": [makeInferenceSource("step-1")] }); }); diff --git a/apps/sidecar/src/workflow-host-wiring.ts b/apps/sidecar/src/workflow-host-wiring.ts index e923ce28..e333f14a 100644 --- a/apps/sidecar/src/workflow-host-wiring.ts +++ b/apps/sidecar/src/workflow-host-wiring.ts @@ -885,6 +885,14 @@ export function createSidecarDeployRouter(deps: { * the deploy or boot-time restore. */ readyTimeoutMs?: number; + /** + * Deployment-record writer, injectable so a test can block or fail the + * persist at a controlled point -- the natural seam for exercising a + * recycle that interleaves the source-rotation persist window. Defaults + * to the real `writeWorkflowDeploymentRecord`; production never overrides + * it. + */ + writeWorkflowDeploymentRecord?: typeof writeWorkflowDeploymentRecord; }): SidecarDeployRouter { // Validate the signing seed at construction so a malformed key fails // sidecar boot rather than the first multi-step deploy, where the @@ -914,6 +922,8 @@ export function createSidecarDeployRouter(deps: { // no child ever rooted scratch and the undeploy reclaim is correctly // skipped. const stepStateDataDir = multistepSubstrateEnv.SIDECAR_DATA_DIR; + const persistDeploymentRecord = + deps.writeWorkflowDeploymentRecord ?? writeWorkflowDeploymentRecord; const multistepSpawner = deps.multistepSubprocessSpawner ?? defaultSubprocessSpawner; const multistepDeriveStepAddress: DeriveStepAddress = @@ -1349,30 +1359,51 @@ export function createSidecarDeployRouter(deps: { deps.multistepSourcesRouter?.register( spec.agentAddress, async (args) => { - // Persist the rotation durably BEFORE committing it in memory or - // pushing it live, so a failed write leaves no partial effect: - // currentSources, the record, and the child all stay on the - // prior sources and the throw is truthful. Persistence lets the - // rotation survive a full sidecar restart, not just a recycle -- - // the boot scan reseeds spec.sources from record.sources. - // Overwrites the deploy-time record in place. Skipped when no - // data dir was wired (a test router that never persists), - // matching the restore guard. const rotated = { [rotationStepId]: args.sources }; + // Swap `currentSources` synchronously BEFORE the durable persist. + // `currentSources` is the process-local respawn hint the + // supervisor reads synchronously through `dynamicSpawnEnv`, so a + // recycle that interleaves the persist `await` must respawn the + // child on the SAME sources being persisted, not the stale prior + // table. The obvious inverse -- persist first, then swap -- is + // rejected: it leaves the child on the OLD sources during the + // persist window while the record has already moved to NEW, so a + // recycle there respawns stale and a restart would "correct" it, + // i.e. the running child contradicts durable intent. Swapping + // first makes the only residual disagreement child-ahead-of- + // durable on a failed persist, which the next recycle heals down + // to the rolled-back durable truth -- the benign direction. The + // wire boundary guarantees `args.sources[0]` is the default, + // which the recycle env form pins as the active source. + const prevSources = currentSources; + currentSources = rotated; + // The durable write still precedes the LIVE swap + // (`deliverSources`), preserving persist-before-externally-visible + // for state that outlives the process; only the process-local + // respawn hint moves ahead. On a failed persist, roll the hint + // back so `currentSources` and the record stay in agreement in the + // common (no interleaved recycle) failure case -- the invariant + // restart consistency depends on. Persistence lets the rotation + // survive a full sidecar restart, not just a recycle: the boot + // scan reseeds spec.sources from record.sources. Overwrites the + // deploy-time record in place. Skipped when no data dir was wired + // (a test router that never persists), matching the restore guard. if (stepStateDataDir !== undefined) { - await writeWorkflowDeploymentRecord( - stepStateDataDir, - deploymentId, - buildDeploymentRecord(spec, rotated), - ); + try { + await persistDeploymentRecord( + stepStateDataDir, + deploymentId, + buildDeploymentRecord(spec, rotated), + ); + } catch (cause) { + // Restoring unconditionally is safe because rotations are + // single-flight per deployment: the hub awaits each + // sources.update ack before sending the next, so prevSources + // cannot clobber a concurrently-committed later rotation. + currentSources = prevSources; + throw cause; + } } - // Commit in memory BEFORE the live-swap, and synchronously so no - // recycle can interleave before deliverSources: a recycle - // mid-flight rejects `deliverSources`, but the respawn re-pulls - // currentSources through dynamicSpawnEnv and must see the - // rotation. The wire boundary guarantees `args.sources[0]` is the - // default, which the recycle env form pins as the active source. - currentSources = rotated; await wired.supervisor.deliverSources({ sources: args.sources, defaultSource: args.defaultSource, @@ -1594,7 +1625,7 @@ export function createSidecarDeployRouter(deps: { // leaves a record the boot scan re-drives (an idempotent re-spawn; the // child's in-flight-run discovery resumes any run). A soft-failed deploy // deletes it below, so only a crash-interrupted deploy leaves one. - await writeWorkflowDeploymentRecord(dataDir, deploymentId, record); + await persistDeploymentRecord(dataDir, deploymentId, record); // Materialize the deploy-only durable state the spawned child and the // supervisor read from disk: the workflow definition (`workflow.json`) diff --git a/docs/HARNESS_DESIGN.md b/docs/HARNESS_DESIGN.md index a8834f79..1162c619 100644 --- a/docs/HARNESS_DESIGN.md +++ b/docs/HARNESS_DESIGN.md @@ -112,6 +112,8 @@ The per-agent key directory is keyed by the sanitized agent address; the workflo The `deployment.json` record stores only what a restart cannot otherwise recover: the deployment's `agentAddress`, the `definitionId` naming its workflow definition on disk, each step's ordered inference-**sources** failover chain (`sources`), the optional inference `sessionId`, and — for a single-step deployment — the `hubPublicKey`. A `version` field guards the schema so a stale record can be rejected rather than parsed blindly. The record deliberately does **not** duplicate the workflow definition (kept on disk under its `definitionId` and re-read at restore) or the step grants (kept in each step's agent-state repo). Because each source embeds its API key, the record is written owner-only (mode 0600). +A live source rotation for a single-step deployment overwrites this record's `sources` before it takes effect. Persistence is what makes a rotation durable: a rotation whose write fails is not durable, and the deployment falls back to the last durably-recorded source list on the next recycle or restart. + The directory name is the agent address with `@` replaced by `_at_` and non-alphanumeric characters (except `-` and `_`) replaced by `_`. ## Agent Deployment vs User Sessions From c487da4d22972d6b3930b75b992062a4b9f64bd3 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Mon, 6 Jul 2026 15:06:39 -0500 Subject: [PATCH 078/101] Round-trip run events faithfully in the child run tests The child run tests injected a substrate whose write path discarded the staged tree, so the run event log the runtime committed never landed on disk and the workflow-run repo-store adapter read it back as an empty log. A completed run then surfaced a terminalStatus with no matching terminal event in its result events. The runtime, the adapter, and the emitter all tolerated the empty read, so the infidelity stayed invisible. Persist the events the way the real substrate does: run the merge callback against the prior entries under the preserved prefix, then write the merged subtree under the repo dir so the adapter's disk read round-trips. A completed run's emitted terminal frame now carries the real committed seq, which the warm-path test asserts is non-zero so a regression to a discarding substrate is caught. --- .../workflow-host/src/child/run-child.test.ts | 59 +++++++++++++++++-- 1 file changed, 55 insertions(+), 4 deletions(-) diff --git a/packages/workflow-host/src/child/run-child.test.ts b/packages/workflow-host/src/child/run-child.test.ts index 266d1785..e0b4447f 100644 --- a/packages/workflow-host/src/child/run-child.test.ts +++ b/packages/workflow-host/src/child/run-child.test.ts @@ -159,12 +159,58 @@ function createMemoryFrameStream() { }; } +// Read every file directly under the `runs//events/` prefix into the +// full-path-keyed map the merge callback expects (`appendBatchEvents` +// slices the prefix off each key to recover the seq). +async function readPrefixEntries( + repoDir: string, + prefix: string, +): Promise> { + const entries = new Map(); + const prefixDir = path.join(repoDir, prefix); + let names: string[]; + try { + names = await fs.readdir(prefixDir); + } catch (cause) { + // No prior entries under the prefix yet: the first append starts the + // subtree. + if (cause instanceof Error && "code" in cause && cause.code === "ENOENT") { + return entries; + } + throw cause; + } + for (const name of names) { + const full = path.join(prefixDir, name); + if (!(await fs.stat(full)).isFile()) continue; + entries.set(`${prefix}${name}`, await fs.readFile(full)); + } + return entries; +} + function createStubRepoStore(baseDir: string): RepoStore { const stub: Partial = { getRepoDir(repoId: RepoId): string { return path.join(baseDir, repoId.kind, repoId.id); }, - async writeTreePreservingPrefix(_principal, _repoId, _ref, _args) { + async writeTreePreservingPrefix(_principal, repoId, _ref, args) { + // Persist the run event log the way the real substrate does so the + // `createWorkflowRunRepoStore` adapter's disk read round-trips: run + // the merge callback against the prior entries under the preserved + // prefix, then replace that subtree with the merged result (files + // outside the prefix are untouched). Discarding the write here would + // make a completed run's `result.events` read back empty. + const repoDir = path.join(baseDir, repoId.kind, repoId.id); + const existing = await readPrefixEntries(repoDir, args.preservePrefix); + const merged = await args.merge(existing); + await fs.rm(path.join(repoDir, args.preservePrefix), { + recursive: true, + force: true, + }); + for (const [relPath, content] of Object.entries(merged)) { + const full = path.join(repoDir, relPath); + await fs.mkdir(path.dirname(full), { recursive: true }); + await fs.writeFile(full, content); + } return { commitSha: "deadbeefcafef00d", newlyTerminalRuns: [] }; }, }; @@ -719,14 +765,19 @@ describe("runWorkflowChild", () => { data: { runId: "run-1", messageId: "msg-1", receivedAt: 1 }, }); - let sawTerminal = false; + let terminalSeq: number | null = null; for await (const payload of recvIter) { if (payload.type === "terminal.event" && payload.data.runId === "run-1") { - sawTerminal = true; + terminalSeq = payload.data.seq; break; } } - expect(sawTerminal).toBe(true); + // The frame's seq is sourced from the run's committed terminal event, so + // a real (non-zero) seq proves the child test substrate faithfully + // persisted and read back the run's event log rather than round-tripping + // an empty one. + expect(terminalSeq).not.toBeNull(); + expect(terminalSeq).toBeGreaterThan(0); // The run reached terminal AND the completion continuation ran (it // emitted the terminal.event we just observed). On the cold path the // same continuation would have called cleanupRunStorage by now; the From 4cedbe8fa542cda114c915504f6e96b5bc994b83 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Mon, 6 Jul 2026 15:13:36 -0500 Subject: [PATCH 079/101] Throw on a missing or mismatched terminal event in the child The child's terminal-event emitter coerced a missing terminal event to seq 0 and an absent RunFailed message to an empty string, and derived the frame kind from the run's terminal status rather than the committed event. A run whose committed log lacked a terminal event, or carried one whose kind disagreed with the status, emitted a frame that did not mirror any on-disk audit entry -- desyncing the supervisor from the durable log that resume reads, so it would settle a run the log still shows in-flight. Source every field from the committed terminal event, and throw when it is absent or its kind disagrees with the terminal status. No frame keeps the supervisor and the durable log agreeing the run is unsettled, so the next recycle or restart resumes it, and a genuine producer bug surfaces loudly through the caller's completion continuation rather than as silent audit corruption. This mirrors the supervisor's own terminal-event synthesis. --- .../workflow-host/src/child/run-child.test.ts | 113 ++++++++++++++++++ packages/workflow-host/src/child/run-child.ts | 86 ++++++++----- 2 files changed, 166 insertions(+), 33 deletions(-) diff --git a/packages/workflow-host/src/child/run-child.test.ts b/packages/workflow-host/src/child/run-child.test.ts index e0b4447f..8568b79c 100644 --- a/packages/workflow-host/src/child/run-child.test.ts +++ b/packages/workflow-host/src/child/run-child.test.ts @@ -43,12 +43,16 @@ import { type ChildStepInvoker, type RunWorkflowChildBindings, } from "./index"; +import { emitTerminalEvent } from "./run-child"; +import type { RunResult } from "@intx/workflow"; import { createControlChannelSender, generateChannelId, generateHmacKey, receiveControlChannel, receiveEventChannel, + type ControlChannelSender, + type ControlPayload, type FrameReader, type FrameWriter, type NdjsonReader, @@ -1904,3 +1908,112 @@ describe("event channel writer wiring", () => { expect(crashes).toHaveLength(0); }); }); + +describe("emitTerminalEvent", () => { + function capturingSender(): { + sender: ControlChannelSender; + sent: ControlPayload[]; + } { + const sent: ControlPayload[] = []; + const sender: ControlChannelSender = { + seq: 0, + send: (payload) => { + sent.push(payload); + return Promise.resolve(); + }, + }; + return { sender, sent }; + } + + test("mirrors a RunCompleted terminal event, sourcing seq and at from it", async () => { + const { sender, sent } = capturingSender(); + const result: RunResult = { + runId: "run-ok", + terminalStatus: "completed", + outputs: {}, + events: [{ kind: "RunCompleted", seq: 5, at: "2026-01-01T00:00:05Z" }], + }; + await emitTerminalEvent(sender, result); + expect(sent).toEqual([ + { + type: "terminal.event", + data: { + runId: "run-ok", + seq: 5, + kind: "RunCompleted", + at: "2026-01-01T00:00:05Z", + }, + }, + ]); + }); + + test("mirrors a RunFailed terminal event, sourcing the error message from it", async () => { + const { sender, sent } = capturingSender(); + const result: RunResult = { + runId: "run-boom", + terminalStatus: "failed", + outputs: {}, + events: [ + { + kind: "RunFailed", + seq: 7, + at: "2026-01-01T00:00:07Z", + error: { message: "step blew up" }, + }, + ], + }; + await emitTerminalEvent(sender, result); + expect(sent).toEqual([ + { + type: "terminal.event", + data: { + runId: "run-boom", + seq: 7, + kind: "RunFailed", + at: "2026-01-01T00:00:07Z", + error: { message: "step blew up" }, + }, + }, + ]); + }); + + test("throws when the committed log carries no terminal event", () => { + const { sender, sent } = capturingSender(); + const result: RunResult = { + runId: "run-nolog", + terminalStatus: "completed", + outputs: {}, + events: [], + }; + // The runtime commits the terminal event last, so its absence is a + // producer bug. Throw rather than emit a seq-0 frame that would desync + // the supervisor from the durable log. + expect(() => emitTerminalEvent(sender, result)).toThrow( + /carries no terminal event/, + ); + expect(sent).toEqual([]); + }); + + test("throws when the terminal event kind disagrees with the terminal status", () => { + const { sender, sent } = capturingSender(); + const result: RunResult = { + runId: "run-mismatch", + terminalStatus: "completed", + outputs: {}, + events: [ + { + kind: "RunFailed", + seq: 9, + at: "2026-01-01T00:00:09Z", + error: { message: "actually failed" }, + }, + ], + }; + // Emitting a RunCompleted frame carrying the RunFailed's seq would claim + // completion while pointing at a failure's audit entry. + expect(() => emitTerminalEvent(sender, result)).toThrow( + /committed terminal event is RunFailed/, + ); + expect(sent).toEqual([]); + }); +}); diff --git a/packages/workflow-host/src/child/run-child.ts b/packages/workflow-host/src/child/run-child.ts index 372049c8..419772cc 100644 --- a/packages/workflow-host/src/child/run-child.ts +++ b/packages/workflow-host/src/child/run-child.ts @@ -1069,29 +1069,34 @@ function buildRuntimeEnv(args: { * loop and any armed drainTimeout accumulator subscribed for the * runId. * - * The mapping is total: every `terminalStatus` the runtime body - * surfaces corresponds to exactly one `kind` in the wire union. The - * `error.message` on `RunFailed` is taken from the last - * `RunFailed`/`StepFailed` event the runtime emitted; when the log - * does not carry one the supervisor's downstream consumers see an - * empty message rather than a thrown error (the wire shape requires - * the field). + * The frame mirrors the run's committed terminal event: every field -- + * `kind`, `seq`, `at`, and (for `RunFailed`) `error.message` -- is + * sourced from that event, which is why the frame's `seq` matches the + * on-disk audit-log entry. `terminalStatus` is only the cross-check: the + * found event's `kind` must agree with it. A missing terminal event, or + * one whose kind disagrees, is a runtime producer bug (the runtime + * commits the terminal event last), and emitting a frame anyway would + * desync the supervisor from the durable log that `discoverInFlightRuns` + * reads on resume -- the supervisor would settle a run the on-disk log + * still shows in-flight. So this throws instead: no frame keeps the + * supervisor and the durable log agreeing that the run is unsettled, and + * the next recycle/restart resumes it. The throw propagates to the + * caller's `complete` continuation, which logs it. * - * Errors flowing out of `upstreamSender.send` are logged but not - * rethrown -- the supervisor's dispatch loop is the authoritative - * settler for the dispatch entry through its cohort abort signal, so a - * lost terminal frame surfaces structurally as a wedged dispatch - * rather than a silent lifecycle failure. + * Errors flowing out of `upstreamSender.send` are a different case -- + * a transport send failure, logged but not rethrown. The supervisor's + * dispatch loop is the authoritative settler through its cohort abort + * signal, so a lost frame surfaces structurally as a wedged dispatch + * rather than a silent lifecycle failure. The invariant throws above run + * before the send so that catch never swallows them. */ -function emitTerminalEvent( +export function emitTerminalEvent( upstreamSender: ControlChannelSender, result: RunResult, ): Promise { - const at = new Date().toISOString(); - // Recover the terminal event blob from the committed event log so - // the wire frame's seq matches the on-disk audit-log entry. The - // runtime body commits the terminal event last; walking from the - // end finds it in one step without rebuilding the state machine. + // Recover the terminal event from the committed event log. The runtime + // body commits the terminal event last; walking from the end finds it in + // one step without rebuilding the state machine. let terminalEvent: (typeof result.events)[number] | null = null; for (let i = result.events.length - 1; i >= 0; i -= 1) { const candidate = result.events[i]; @@ -1105,34 +1110,49 @@ function emitTerminalEvent( break; } } - const seq = terminalEvent?.seq ?? 0; - const eventAt = terminalEvent?.at ?? at; + if (terminalEvent === null) { + throw new Error( + `emitTerminalEvent: run ${result.runId} terminated as ${result.terminalStatus} but its committed event log carries no terminal event (the runtime commits it last; this is a producer bug)`, + ); + } + const expectedKind = + result.terminalStatus === "completed" + ? "RunCompleted" + : result.terminalStatus === "cancelled" + ? "RunCancelled" + : "RunFailed"; + if (terminalEvent.kind !== expectedKind) { + throw new Error( + `emitTerminalEvent: run ${result.runId} terminated as ${result.terminalStatus} but its committed terminal event is ${terminalEvent.kind}`, + ); + } + // The RunFailed-missing-error.message case the supervisor's + // `synthesizeTerminalEvent` guards is unreachable here: `result.events` + // is typed `WorkflowEvent[]`, and `RunFailed.error.message` is a + // non-optional `string`, so a RunFailed reached here always carries one. + // The supervisor needs that guard because it parses untrusted JSON. let payload: Extract["data"]; - if (result.terminalStatus === "completed") { + if (terminalEvent.kind === "RunCompleted") { payload = { runId: result.runId, - seq, + seq: terminalEvent.seq, kind: "RunCompleted", - at: eventAt, + at: terminalEvent.at, }; - } else if (result.terminalStatus === "cancelled") { + } else if (terminalEvent.kind === "RunCancelled") { payload = { runId: result.runId, - seq, + seq: terminalEvent.seq, kind: "RunCancelled", - at: eventAt, + at: terminalEvent.at, }; } else { - const message = - terminalEvent !== null && terminalEvent.kind === "RunFailed" - ? terminalEvent.error.message - : ""; payload = { runId: result.runId, - seq, + seq: terminalEvent.seq, kind: "RunFailed", - at: eventAt, - error: { message }, + at: terminalEvent.at, + error: { message: terminalEvent.error.message }, }; } return upstreamSender From 898ab23965111da96b93662012f2560c2c419a18 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Mon, 6 Jul 2026 16:30:31 -0500 Subject: [PATCH 080/101] Store the deployment address and public key on its row The reconnect ownership challenge looks a deployment's public key up by address, but workflow-deployment addresses had nowhere on the hub to hold that key -- only launched agents' agent_instance rows carried one. Add an address column and a nullable public_key column to the workflow_deployment projection row, with a unique index on address so a double-insert fails loud. The deploy path records the address at insert; the public key stays null until the sidecar's deploy-ack persists the minted deployment key. No behavior change yet: nothing reads the new columns. This is the storage the deploy-ack persistence and the reconnect challenge build on. --- .../db/migrations/0035_violet_giant_man.sql | 3 + .../db/migrations/meta/0035_snapshot.json | 3225 +++++++++++++++++ packages/db/migrations/meta/_journal.json | 7 + .../db/src/schema/workflow-deployments.ts | 25 +- packages/hub-sessions/src/session-service.ts | 3 + 5 files changed, 3261 insertions(+), 2 deletions(-) create mode 100644 packages/db/migrations/0035_violet_giant_man.sql create mode 100644 packages/db/migrations/meta/0035_snapshot.json diff --git a/packages/db/migrations/0035_violet_giant_man.sql b/packages/db/migrations/0035_violet_giant_man.sql new file mode 100644 index 00000000..4d615f9e --- /dev/null +++ b/packages/db/migrations/0035_violet_giant_man.sql @@ -0,0 +1,3 @@ +ALTER TABLE "workflow_deployment" ADD COLUMN "address" text NOT NULL;--> statement-breakpoint +ALTER TABLE "workflow_deployment" ADD COLUMN "public_key" text;--> statement-breakpoint +CREATE UNIQUE INDEX "workflow_deployment_address_idx" ON "workflow_deployment" USING btree ("address"); \ No newline at end of file diff --git a/packages/db/migrations/meta/0035_snapshot.json b/packages/db/migrations/meta/0035_snapshot.json new file mode 100644 index 00000000..21845ce4 --- /dev/null +++ b/packages/db/migrations/meta/0035_snapshot.json @@ -0,0 +1,3225 @@ +{ + "id": "ddb550f1-b5b9-4094-941b-6d0ad963d01a", + "prevId": "b997db7b-976b-4511-8780-9052911edf5c", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.account": { + "name": "account", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": ["token"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": ["email"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification": { + "name": "verification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.federation_trust": { + "name": "federation_trust", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_tenant_id": { + "name": "target_tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "direction": { + "name": "direction", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "federation_trust_tenant_id_tenant_id_fk": { + "name": "federation_trust_tenant_id_tenant_id_fk", + "tableFrom": "federation_trust", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "federation_trust_target_tenant_id_tenant_id_fk": { + "name": "federation_trust_target_tenant_id_tenant_id_fk", + "tableFrom": "federation_trust", + "tableTo": "tenant", + "columnsFrom": ["target_tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "federation_trust_tenant_id_target_tenant_id_unique": { + "name": "federation_trust_tenant_id_target_tenant_id_unique", + "nullsNotDistinct": false, + "columns": ["tenant_id", "target_tenant_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tenant": { + "name": "tenant", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "domain": { + "name": "domain", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_id": { + "name": "parent_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "tenant_parent_id_tenant_id_fk": { + "name": "tenant_parent_id_tenant_id_fk", + "tableFrom": "tenant", + "tableTo": "tenant", + "columnsFrom": ["parent_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "tenant_slug_unique": { + "name": "tenant_slug_unique", + "nullsNotDistinct": false, + "columns": ["slug"] + }, + "tenant_domain_unique": { + "name": "tenant_domain_unique", + "nullsNotDistinct": false, + "columns": ["domain"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.principal": { + "name": "principal", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ref_id": { + "name": "ref_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "principal_tenant_id_tenant_id_fk": { + "name": "principal_tenant_id_tenant_id_fk", + "tableFrom": "principal", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "principal_tenant_id_kind_ref_id_unique": { + "name": "principal_tenant_id_kind_ref_id_unique", + "nullsNotDistinct": false, + "columns": ["tenant_id", "kind", "ref_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agent_role": { + "name": "agent_role", + "schema": "", + "columns": { + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role_id": { + "name": "role_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "agent_role_agent_id_agent_id_fk": { + "name": "agent_role_agent_id_agent_id_fk", + "tableFrom": "agent_role", + "tableTo": "agent", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_role_role_id_role_id_fk": { + "name": "agent_role_role_id_role_id_fk", + "tableFrom": "agent_role", + "tableTo": "role", + "columnsFrom": ["role_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "agent_role_agent_id_role_id_pk": { + "name": "agent_role_agent_id_role_id_pk", + "columns": ["agent_id", "role_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.principal_role": { + "name": "principal_role", + "schema": "", + "columns": { + "principal_id": { + "name": "principal_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role_id": { + "name": "role_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "principal_role_principal_id_principal_id_fk": { + "name": "principal_role_principal_id_principal_id_fk", + "tableFrom": "principal_role", + "tableTo": "principal", + "columnsFrom": ["principal_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "principal_role_role_id_role_id_fk": { + "name": "principal_role_role_id_role_id_fk", + "tableFrom": "principal_role", + "tableTo": "role", + "columnsFrom": ["role_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "principal_role_principal_id_role_id_pk": { + "name": "principal_role_principal_id_role_id_pk", + "columns": ["principal_id", "role_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.role": { + "name": "role", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_system": { + "name": "is_system", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "role_tenant_id_tenant_id_fk": { + "name": "role_tenant_id_tenant_id_fk", + "tableFrom": "role", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.grant": { + "name": "grant", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role_id": { + "name": "role_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "principal_id": { + "name": "principal_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resource": { + "name": "resource", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "effect": { + "name": "effect", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "conditions": { + "name": "conditions", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "origin": { + "name": "origin", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "grant_tenant_id_tenant_id_fk": { + "name": "grant_tenant_id_tenant_id_fk", + "tableFrom": "grant", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "grant_role_id_role_id_fk": { + "name": "grant_role_id_role_id_fk", + "tableFrom": "grant", + "tableTo": "role", + "columnsFrom": ["role_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "grant_principal_id_principal_id_fk": { + "name": "grant_principal_id_principal_id_fk", + "tableFrom": "grant", + "tableTo": "principal", + "columnsFrom": ["principal_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agent": { + "name": "agent", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "creator_principal_id": { + "name": "creator_principal_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "system_prompt": { + "name": "system_prompt", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "context_config": { + "name": "context_config", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "initial_state": { + "name": "initial_state", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "model_config": { + "name": "model_config", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "capabilities": { + "name": "capabilities", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "credential_requirements": { + "name": "credential_requirements", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "model_requirements": { + "name": "model_requirements", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "tool_packages": { + "name": "tool_packages", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "grant_requirements": { + "name": "grant_requirements", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "current_version": { + "name": "current_version", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'1'" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'deployed'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "agent_tenant_id_tenant_id_fk": { + "name": "agent_tenant_id_tenant_id_fk", + "tableFrom": "agent", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_creator_principal_id_principal_id_fk": { + "name": "agent_creator_principal_id_principal_id_fk", + "tableFrom": "agent", + "tableTo": "principal", + "columnsFrom": ["creator_principal_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agent_version": { + "name": "agent_version", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "agent_version_agent_id_agent_id_fk": { + "name": "agent_version_agent_id_agent_id_fk", + "tableFrom": "agent_version", + "tableTo": "agent", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.provider": { + "name": "provider", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "plugin": { + "name": "plugin", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "authorization_url": { + "name": "authorization_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "token_url": { + "name": "token_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_info_url": { + "name": "user_info_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "provider_tenant_id_tenant_id_fk": { + "name": "provider_tenant_id_tenant_id_fk", + "tableFrom": "provider", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "provider_tenant_name": { + "name": "provider_tenant_name", + "nullsNotDistinct": false, + "columns": ["tenant_id", "name"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client": { + "name": "oauth_client", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_secret": { + "name": "client_secret", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "redirect_uris": { + "name": "redirect_uris", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "default_scopes": { + "name": "default_scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "oauth_client_tenant_id_tenant_id_fk": { + "name": "oauth_client_tenant_id_tenant_id_fk", + "tableFrom": "oauth_client", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_client_provider_id_provider_id_fk": { + "name": "oauth_client_provider_id_provider_id_fk", + "tableFrom": "oauth_client", + "tableTo": "provider", + "columnsFrom": ["provider_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_client_tenant_provider": { + "name": "oauth_client_tenant_provider", + "nullsNotDistinct": false, + "columns": ["tenant_id", "provider_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.credential": { + "name": "credential", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "principal_id": { + "name": "principal_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "oauth_client_id": { + "name": "oauth_client_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "secret": { + "name": "secret", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "refresh_secret": { + "name": "refresh_secret", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "credential_tenant_id_tenant_id_fk": { + "name": "credential_tenant_id_tenant_id_fk", + "tableFrom": "credential", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_principal_id_principal_id_fk": { + "name": "credential_principal_id_principal_id_fk", + "tableFrom": "credential", + "tableTo": "principal", + "columnsFrom": ["principal_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "credential_provider_id_provider_id_fk": { + "name": "credential_provider_id_provider_id_fk", + "tableFrom": "credential", + "tableTo": "provider", + "columnsFrom": ["provider_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_oauth_client_id_oauth_client_id_fk": { + "name": "credential_oauth_client_id_oauth_client_id_fk", + "tableFrom": "credential", + "tableTo": "oauth_client", + "columnsFrom": ["oauth_client_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "credential_tenant_name": { + "name": "credential_tenant_name", + "nullsNotDistinct": false, + "columns": ["tenant_id", "name"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.asset": { + "name": "asset", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "creator_principal_id": { + "name": "creator_principal_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "asset_tenant_id_tenant_id_fk": { + "name": "asset_tenant_id_tenant_id_fk", + "tableFrom": "asset", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "asset_creator_principal_id_principal_id_fk": { + "name": "asset_creator_principal_id_principal_id_fk", + "tableFrom": "asset", + "tableTo": "principal", + "columnsFrom": ["creator_principal_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "asset_tenant_kind_name": { + "name": "asset_tenant_kind_name", + "nullsNotDistinct": false, + "columns": ["tenant_id", "kind", "name"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agent_asset": { + "name": "agent_asset", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "asset_id": { + "name": "asset_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ref": { + "name": "ref", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_mode": { + "name": "access_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'read-only'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "agent_asset_agent_id_agent_id_fk": { + "name": "agent_asset_agent_id_agent_id_fk", + "tableFrom": "agent_asset", + "tableTo": "agent", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_asset_asset_id_asset_id_fk": { + "name": "agent_asset_asset_id_asset_id_fk", + "tableFrom": "agent_asset", + "tableTo": "asset", + "columnsFrom": ["asset_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "agent_asset_agent_asset": { + "name": "agent_asset_agent_asset", + "nullsNotDistinct": false, + "columns": ["agent_id", "asset_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.transaction": { + "name": "transaction", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "wallet_id": { + "name": "wallet_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "direction": { + "name": "direction", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "amount": { + "name": "amount", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "currency": { + "name": "currency", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_id": { + "name": "recipient_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sender_id": { + "name": "sender_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "request_id": { + "name": "request_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "transaction_wallet_id_wallet_id_fk": { + "name": "transaction_wallet_id_wallet_id_fk", + "tableFrom": "transaction", + "tableTo": "wallet", + "columnsFrom": ["wallet_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "transaction_agent_id_agent_id_fk": { + "name": "transaction_agent_id_agent_id_fk", + "tableFrom": "transaction", + "tableTo": "agent", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.wallet": { + "name": "wallet", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "backend_type": { + "name": "backend_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "currency": { + "name": "currency", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "balance": { + "name": "balance", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "wallet_tenant_id_tenant_id_fk": { + "name": "wallet_tenant_id_tenant_id_fk", + "tableFrom": "wallet", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.offering": { + "name": "offering", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "pricing": { + "name": "pricing", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "schema": { + "name": "schema", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "offering_agent_id_agent_id_fk": { + "name": "offering_agent_id_agent_id_fk", + "tableFrom": "offering", + "tableTo": "agent", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "offering_tenant_id_tenant_id_fk": { + "name": "offering_tenant_id_tenant_id_fk", + "tableFrom": "offering", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.model": { + "name": "model", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "canonical_name": { + "name": "canonical_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "disabled": { + "name": "disabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "model_tenant_id_tenant_id_fk": { + "name": "model_tenant_id_tenant_id_fk", + "tableFrom": "model", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "model_tenant_canonical_name": { + "name": "model_tenant_canonical_name", + "nullsNotDistinct": false, + "columns": ["tenant_id", "canonical_name"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.model_offering": { + "name": "model_offering", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "model_id": { + "name": "model_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "deployment_tags": { + "name": "deployment_tags", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "capabilities": { + "name": "capabilities", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "disabled": { + "name": "disabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "model_offering_tenant_id_tenant_id_fk": { + "name": "model_offering_tenant_id_tenant_id_fk", + "tableFrom": "model_offering", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "model_offering_model_id_model_id_fk": { + "name": "model_offering_model_id_model_id_fk", + "tableFrom": "model_offering", + "tableTo": "model", + "columnsFrom": ["model_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "model_offering_provider_id_model_provider_id_fk": { + "name": "model_offering_provider_id_model_provider_id_fk", + "tableFrom": "model_offering", + "tableTo": "model_provider", + "columnsFrom": ["provider_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "model_offering_tenant_model_provider": { + "name": "model_offering_tenant_model_provider", + "nullsNotDistinct": false, + "columns": ["tenant_id", "model_id", "provider_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.model_pricing": { + "name": "model_pricing", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "offering_id": { + "name": "offering_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "currency": { + "name": "currency", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "input_token_price": { + "name": "input_token_price", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "output_token_price": { + "name": "output_token_price", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cache_read_token_price": { + "name": "cache_read_token_price", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cache_write_token_price": { + "name": "cache_write_token_price", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "thinking_token_price": { + "name": "thinking_token_price", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "per_request_fee": { + "name": "per_request_fee", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "per_image_fee": { + "name": "per_image_fee", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "per_audio_fee": { + "name": "per_audio_fee", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "effective_from": { + "name": "effective_from", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "model_pricing_tenant_id_tenant_id_fk": { + "name": "model_pricing_tenant_id_tenant_id_fk", + "tableFrom": "model_pricing", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "model_pricing_offering_id_model_offering_id_fk": { + "name": "model_pricing_offering_id_model_offering_id_fk", + "tableFrom": "model_pricing", + "tableTo": "model_offering", + "columnsFrom": ["offering_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "model_pricing_offering_currency_effective_from": { + "name": "model_pricing_offering_currency_effective_from", + "nullsNotDistinct": false, + "columns": ["offering_id", "currency", "effective_from"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.model_provider": { + "name": "model_provider", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "plugin": { + "name": "plugin", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "base_url": { + "name": "base_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "credential_id": { + "name": "credential_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "wallet_id": { + "name": "wallet_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "disabled": { + "name": "disabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "model_provider_tenant_id_tenant_id_fk": { + "name": "model_provider_tenant_id_tenant_id_fk", + "tableFrom": "model_provider", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "model_provider_credential_id_credential_id_fk": { + "name": "model_provider_credential_id_credential_id_fk", + "tableFrom": "model_provider", + "tableTo": "credential", + "columnsFrom": ["credential_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + }, + "model_provider_wallet_id_wallet_id_fk": { + "name": "model_provider_wallet_id_wallet_id_fk", + "tableFrom": "model_provider", + "tableTo": "wallet", + "columnsFrom": ["wallet_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "model_provider_tenant_name": { + "name": "model_provider_tenant_name", + "nullsNotDistinct": false, + "columns": ["tenant_id", "name"] + } + }, + "policies": {}, + "checkConstraints": { + "model_provider_auth_xor": { + "name": "model_provider_auth_xor", + "value": "(\"model_provider\".\"credential_id\" is not null) <> (\"model_provider\".\"wallet_id\" is not null)" + } + }, + "isRLSEnabled": false + }, + "public.sidecar": { + "name": "sidecar", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'online'" + }, + "last_heartbeat": { + "name": "last_heartbeat", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agent_session": { + "name": "agent_session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "principal_id": { + "name": "principal_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "ended_at": { + "name": "ended_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "agent_session_tenant_id_tenant_id_fk": { + "name": "agent_session_tenant_id_tenant_id_fk", + "tableFrom": "agent_session", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_session_agent_id_agent_id_fk": { + "name": "agent_session_agent_id_agent_id_fk", + "tableFrom": "agent_session", + "tableTo": "agent", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_session_principal_id_principal_id_fk": { + "name": "agent_session_principal_id_principal_id_fk", + "tableFrom": "agent_session", + "tableTo": "principal", + "columnsFrom": ["principal_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agent_instance": { + "name": "agent_instance", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "principal_id": { + "name": "principal_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version_id": { + "name": "version_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'deployed'" + }, + "sidecar_id": { + "name": "sidecar_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "public_key": { + "name": "public_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "kernel_id": { + "name": "kernel_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "model_preferences": { + "name": "model_preferences", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "ended_at": { + "name": "ended_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "agent_instance_agent_id_agent_id_fk": { + "name": "agent_instance_agent_id_agent_id_fk", + "tableFrom": "agent_instance", + "tableTo": "agent", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_instance_tenant_id_tenant_id_fk": { + "name": "agent_instance_tenant_id_tenant_id_fk", + "tableFrom": "agent_instance", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_instance_principal_id_principal_id_fk": { + "name": "agent_instance_principal_id_principal_id_fk", + "tableFrom": "agent_instance", + "tableTo": "principal", + "columnsFrom": ["principal_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "agent_instance_version_id_agent_version_id_fk": { + "name": "agent_instance_version_id_agent_version_id_fk", + "tableFrom": "agent_instance", + "tableTo": "agent_version", + "columnsFrom": ["version_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "agent_instance_session_id_agent_session_id_fk": { + "name": "agent_instance_session_id_agent_session_id_fk", + "tableFrom": "agent_instance", + "tableTo": "agent_session", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "agent_instance_sidecar_id_sidecar_id_fk": { + "name": "agent_instance_sidecar_id_sidecar_id_fk", + "tableFrom": "agent_instance", + "tableTo": "sidecar", + "columnsFrom": ["sidecar_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "agent_instance_address_unique": { + "name": "agent_instance_address_unique", + "nullsNotDistinct": false, + "columns": ["address"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session_asset": { + "name": "session_asset", + "schema": "", + "columns": { + "instance_id": { + "name": "instance_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_asset_id": { + "name": "agent_asset_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "mount_path": { + "name": "mount_path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "asset_pack_sha": { + "name": "asset_pack_sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_commit_sha": { + "name": "source_commit_sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "materialized_at": { + "name": "materialized_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "session_asset_pack_sha_idx": { + "name": "session_asset_pack_sha_idx", + "columns": [ + { + "expression": "asset_pack_sha", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_asset_instance_id_agent_instance_id_fk": { + "name": "session_asset_instance_id_agent_instance_id_fk", + "tableFrom": "session_asset", + "tableTo": "agent_instance", + "columnsFrom": ["instance_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "session_asset_agent_asset_id_agent_asset_id_fk": { + "name": "session_asset_agent_asset_id_agent_asset_id_fk", + "tableFrom": "session_asset", + "tableTo": "agent_asset", + "columnsFrom": ["agent_asset_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "session_asset_instance_id_mount_path_pk": { + "name": "session_asset_instance_id_mount_path_pk", + "columns": ["instance_id", "mount_path"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.inference_turn": { + "name": "inference_turn", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "instance_id": { + "name": "instance_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'running'" + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ended_at": { + "name": "ended_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "inference_turn_instance_id_started_at_idx": { + "name": "inference_turn_instance_id_started_at_idx", + "columns": [ + { + "expression": "instance_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "inference_turn_session_id_agent_session_id_fk": { + "name": "inference_turn_session_id_agent_session_id_fk", + "tableFrom": "inference_turn", + "tableTo": "agent_session", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "inference_turn_instance_id_agent_instance_id_fk": { + "name": "inference_turn_instance_id_agent_instance_id_fk", + "tableFrom": "inference_turn", + "tableTo": "agent_instance", + "columnsFrom": ["instance_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "inference_turn_tenant_id_tenant_id_fk": { + "name": "inference_turn_tenant_id_tenant_id_fk", + "tableFrom": "inference_turn", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session_mail": { + "name": "session_mail", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "instance_id": { + "name": "instance_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "direction": { + "name": "direction", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "raw": { + "name": "raw", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "session_mail_instance_id_created_at_idx": { + "name": "session_mail_instance_id_created_at_idx", + "columns": [ + { + "expression": "instance_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_mail_session_id_agent_session_id_fk": { + "name": "session_mail_session_id_agent_session_id_fk", + "tableFrom": "session_mail", + "tableTo": "agent_session", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "session_mail_instance_id_agent_instance_id_fk": { + "name": "session_mail_instance_id_agent_instance_id_fk", + "tableFrom": "session_mail", + "tableTo": "agent_instance", + "columnsFrom": ["instance_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "session_mail_tenant_id_tenant_id_fk": { + "name": "session_mail_tenant_id_tenant_id_fk", + "tableFrom": "session_mail", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.turn_part": { + "name": "turn_part", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "turn_id": { + "name": "turn_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "ordinal": { + "name": "ordinal", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "turn_part_turn_id_inference_turn_id_fk": { + "name": "turn_part_turn_id_inference_turn_id_fk", + "tableFrom": "turn_part", + "tableTo": "inference_turn", + "columnsFrom": ["turn_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "turn_part_session_id_agent_session_id_fk": { + "name": "turn_part_session_id_agent_session_id_fk", + "tableFrom": "turn_part", + "tableTo": "agent_session", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.git_token": { + "name": "git_token", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "principal_id": { + "name": "principal_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_hash_sha256": { + "name": "token_hash_sha256", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "resource": { + "name": "resource", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ref_pattern": { + "name": "ref_pattern", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actions": { + "name": "actions", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "git_token_user_id_name_active_idx": { + "name": "git_token_user_id_name_active_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"git_token\".\"revoked_at\" is null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "git_token_tenant_id_tenant_id_fk": { + "name": "git_token_tenant_id_tenant_id_fk", + "tableFrom": "git_token", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "git_token_user_id_user_id_fk": { + "name": "git_token_user_id_user_id_fk", + "tableFrom": "git_token", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "git_token_principal_id_principal_id_fk": { + "name": "git_token_principal_id_principal_id_fk", + "tableFrom": "git_token", + "tableTo": "principal", + "columnsFrom": ["principal_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "git_token_token_hash_sha256_unique": { + "name": "git_token_token_hash_sha256_unique", + "nullsNotDistinct": false, + "columns": ["token_hash_sha256"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_deployment": { + "name": "workflow_deployment", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "definition_asset_id": { + "name": "definition_asset_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_key": { + "name": "public_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'deployed'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_deployment_tenant_idx": { + "name": "workflow_deployment_tenant_idx", + "columns": [ + { + "expression": "tenant_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_deployment_address_idx": { + "name": "workflow_deployment_address_idx", + "columns": [ + { + "expression": "address", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_deployment_tenant_id_tenant_id_fk": { + "name": "workflow_deployment_tenant_id_tenant_id_fk", + "tableFrom": "workflow_deployment", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_deployment_definition_asset_id_asset_id_fk": { + "name": "workflow_deployment_definition_asset_id_asset_id_fk", + "tableFrom": "workflow_deployment", + "tableTo": "asset", + "columnsFrom": ["definition_asset_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/packages/db/migrations/meta/_journal.json b/packages/db/migrations/meta/_journal.json index daaaf1f2..c7a4aca2 100644 --- a/packages/db/migrations/meta/_journal.json +++ b/packages/db/migrations/meta/_journal.json @@ -246,6 +246,13 @@ "when": 1782164862412, "tag": "0034_sleepy_songbird", "breakpoints": true + }, + { + "idx": 35, + "version": "7", + "when": 1783373028980, + "tag": "0035_violet_giant_man", + "breakpoints": true } ] } diff --git a/packages/db/src/schema/workflow-deployments.ts b/packages/db/src/schema/workflow-deployments.ts index 5f6b58e7..ac5612cf 100644 --- a/packages/db/src/schema/workflow-deployments.ts +++ b/packages/db/src/schema/workflow-deployments.ts @@ -1,4 +1,10 @@ -import { index, pgTable, text, timestamp } from "drizzle-orm/pg-core"; +import { + index, + pgTable, + text, + timestamp, + uniqueIndex, +} from "drizzle-orm/pg-core"; import { asset } from "./assets"; import { tenant } from "./tenants"; @@ -25,11 +31,26 @@ export const workflowDeployment = pgTable( definitionAssetId: text("definition_asset_id") .notNull() .references(() => asset.id, { onDelete: "cascade" }), + // The deployment's routable address (`ins_@`), stored rather + // than re-derived at read time so the reconnect ownership challenge can + // look up the deployment's public key by address, symmetrically with the + // `agent_instance` path. Unique so a double-insert fails loud. + address: text("address").notNull(), + // The Ed25519 public key the sidecar minted for this deployment address, + // persisted at deploy-ack. Nullable by design: the row is written at + // deploy-start and the key arrives at ack, so a not-yet-acked (or + // pre-migration) deployment reads `null` and its reconnect challenge + // fails closed -- the address stays unrouted rather than routing without + // ownership proof. + publicKey: text("public_key"), status: text("status") .$type() .notNull() .default("deployed"), createdAt: timestamp("created_at").notNull().defaultNow(), }, - (t) => [index("workflow_deployment_tenant_idx").on(t.tenantId, t.createdAt)], + (t) => [ + index("workflow_deployment_tenant_idx").on(t.tenantId, t.createdAt), + uniqueIndex("workflow_deployment_address_idx").on(t.address), + ], ); diff --git a/packages/hub-sessions/src/session-service.ts b/packages/hub-sessions/src/session-service.ts index 4fe82ad7..16a71c81 100644 --- a/packages/hub-sessions/src/session-service.ts +++ b/packages/hub-sessions/src/session-service.ts @@ -1122,6 +1122,9 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { id: deploymentId, tenantId, definitionAssetId, + address: deriveDeploymentAddress({ deploymentId, deploymentDomain }), + // publicKey is left null here; the sidecar's deploy-ack persists the + // deployment's minted key once the child has spawned. status: "deployed", createdAt: now, }); From 42dae6115f4902c81f7a6f2042667264dce88ba6 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Mon, 6 Jul 2026 16:58:05 -0500 Subject: [PATCH 081/101] Persist and resolve the deployment public key hub-side Every deployment already mints an Ed25519 key for its address and signs reconnect challenges with it, but only single-step launched agents acked that key; a multi-step deployment acked the supervisor principal key, which the hub discarded and which does not match what the key store signs a challenge with. So a workflow-deployment address had no key the hub could verify a reconnect challenge against. Ack the deployment address's own key for every deployment, and persist it where the reconnect challenge reads it: a launched agent on its agent_instance row, a workflow-derived deployment on its workflow_deployment row (the deploy-ack handler no longer no-ops for those addresses). lookupPublicKey now routes by address space -- workflow-derived to the deployment row, launched to the instance row -- rather than one table, and filters the deployment row to a live deployment so a torn-down one's key cannot satisfy a challenge. A null or absent key returns null, which fails the challenge closed. No behavior change yet: nothing sends a reconnect frame, so no challenge runs. This is the key storage and lookup the reconnect flip builds on. --- apps/sidecar/src/workflow-host-wiring.test.ts | 22 +-- apps/sidecar/src/workflow-host-wiring.ts | 43 ++---- docs/HARNESS_DESIGN.md | 2 +- .../hub-sessions/src/hub-session-lookups.ts | 39 +++++- .../src/hub-session-orchestrator.test.ts | 11 +- .../src/hub-session-orchestrator.ts | 33 +++-- tests/db/deployment-key-lookup.test.ts | 130 ++++++++++++++++++ 7 files changed, 219 insertions(+), 61 deletions(-) create mode 100644 tests/db/deployment-key-lookup.test.ts diff --git a/apps/sidecar/src/workflow-host-wiring.test.ts b/apps/sidecar/src/workflow-host-wiring.test.ts index 65144f49..ec917564 100644 --- a/apps/sidecar/src/workflow-host-wiring.test.ts +++ b/apps/sidecar/src/workflow-host-wiring.test.ts @@ -651,7 +651,7 @@ describe("createSidecarDeployRouter multi-step branch", () => { }; } - test("validates the projection, constructs SpawnOpts from the frame, drives spawn, and surfaces the supervisor's principal pubkey", async () => { + test("validates the projection, constructs SpawnOpts from the frame, drives spawn, and acks the deployment address's public key", async () => { const supervisorIpcKeyPair = await generateKeyPair(); const childIpcKeyPair = await generateKeyPair(); const supervisorToChild = createMemoryNdjsonStream(); @@ -682,8 +682,13 @@ describe("createSidecarDeployRouter multi-step branch", () => { }; const multiDataDir = await createTempBaseDir("sidecar-multi-data-"); - const { router, keyPair } = await buildMultistepFixture({ + // The deployment address's own key -- what `loadOrGenerateKey` mints and + // `signChallenge` signs reconnect challenges with. Pin it so the ack + // assertion below is deterministic. + const deploymentKeyPair = await generateKeyPair(); + const { router } = await buildMultistepFixture({ spawner, + headKeyPair: deploymentKeyPair, multistepBinaryPath: "/fake/bin/multistep-workflow-child", multistepSubstrateEnv: { SIDECAR_DATA_DIR: multiDataDir, @@ -752,14 +757,13 @@ describe("createSidecarDeployRouter multi-step branch", () => { }); const result = await deployPromise; - // The publicKey is the sidecar's principal public key (hex). The - // router derives it from the signing seed; the resulting hex is a - // 64-character lowercase string. + // Every deployment -- single- or multi-step -- acks the deployment + // address's own public key, the one `signChallenge` signs reconnect + // challenges with, so the hub can verify ownership on reconnect. A + // multi-step deployment previously acked the supervisor principal key, + // which the hub discarded. The hex is a 64-character lowercase string. expect(result.publicKey).toMatch(/^[0-9a-f]{64}$/); - - // Sanity check: re-deriving the public key from the keypair lines - // up with the router's returned value. - expect(result.publicKey).toBe(hexEncode(keyPair.publicKey)); + expect(result.publicKey).toBe(hexEncode(deploymentKeyPair.publicKey)); // Teardown: kill the child so the spawn-time pumps unwind. // Use unused supervisorToChild to silence the linter. diff --git a/apps/sidecar/src/workflow-host-wiring.ts b/apps/sidecar/src/workflow-host-wiring.ts index e333f14a..1a36e86d 100644 --- a/apps/sidecar/src/workflow-host-wiring.ts +++ b/apps/sidecar/src/workflow-host-wiring.ts @@ -1242,25 +1242,20 @@ export function createSidecarDeployRouter(deps: { ); agentTransportRegistered = true; - // The public key the deploy ack surfaces to the hub. For a single-step - // head it is the AGENT key, set inside the block below; a genuine - // multi-step deployment has no head agent identity and falls back to the - // supervisor principal key at the return. (The registered deployment - // keypair above is used purely for outbound signing; a multi-step - // deployment address is workflow-derived, incurs no reconnect challenge, - // and so records no `agent_instance.publicKey` -- carrying it on the ack - // would be data written nowhere and read nowhere.) - let headAgentPublicKey: string | undefined; + // The public key the deploy ack surfaces to the hub is the deployment + // address's own Ed25519 key -- the one `loadOrGenerateKey` minted above, + // which `AgentKeyStore.signChallenge(spec.agentAddress)` also signs + // reconnect challenges with. EVERY deployment acks it, single- and + // multi-step alike, so the hub can verify the reconnect ownership + // challenge for both: a single-step head records it into + // `agent_instance.publicKey`; a workflow-derived deployment records it on + // its `workflow_deployment` row. A multi-step deployment previously acked + // the supervisor principal key -- which the hub discarded and which does + // NOT match what `signChallenge` signs with -- so its address could be + // re-claimed on reconnect without proof; carrying the deployment key + // closes that. + const deploymentPublicKey = hexEncode(keyPair.publicKey); if (spec.definition.stepOrder.length === 1) { - // A single-step head IS an agent identity: it signs its own outbound - // mail AND its reconnect challenges with this agent key (via the key - // store's signChallenge). The hub records the ack's key into - // `agent_instance.publicKey` (for a rerouted instance head, which has - // an instance row and is not workflow-derived) and verifies the - // reconnect challenge against it, so the ack MUST carry the agent key, - // not the supervisor principal key -- otherwise verification fails. - headAgentPublicKey = hexEncode(keyPair.publicKey); - // A single-step workflow stages its deploy tree at the head (the // lone step IS the head). Initialize the head's on-disk deploy-tree // repo (idempotent) so the hub's deploy-pack push has a repo to @@ -1413,16 +1408,6 @@ export function createSidecarDeployRouter(deps: { } routersRegistered = true; - // Resolve the ack public key BEFORE registering the deployment - // address so an (unreachable, deterministic) derivation failure - // unwinds the spawn without having touched the boot-edge - // `DeploymentAddressRegistry`. A single-step head acks its agent key - // (captured above); a multi-step deployment acks the supervisor - // principal key its workflow-run events are signed with. - const publicKey = - headAgentPublicKey ?? - (await derivePrincipalPublicKeyHex(deps.signingKeySeed)); - // Register the deployment-address mapping last so a failure in any // earlier step leaves the boot-edge `DeploymentAddressRegistry` // untouched. Nothing fallible runs after it, so the finally unwind @@ -1433,7 +1418,7 @@ export function createSidecarDeployRouter(deps: { }); succeeded = true; - return { publicKey }; + return { publicKey: deploymentPublicKey }; } finally { if (!succeeded) { // Unwind in reverse registration order so each step undoes state diff --git a/docs/HARNESS_DESIGN.md b/docs/HARNESS_DESIGN.md index 1162c619..6be03ba1 100644 --- a/docs/HARNESS_DESIGN.md +++ b/docs/HARNESS_DESIGN.md @@ -79,7 +79,7 @@ All communication between hub and sidecar is over a single persistent WebSocket | `agent.deploy.ack` | `agentAddress`, `publicKey` | Agent deployed, here is its public key | | `agent.error` | `agentAddress`, `error` | Deployment failed | -When the hub sends `agent.deploy`, the sidecar spawns a supervised **workflow-process child** to host the deployment and responds with `agent.deploy.ack`. For a single-step (launched-agent) deployment the sidecar generates the agent's key pair (if new) or loads the existing one, records the hub's public key for later deploy-pack verification, initializes the head's on-disk deploy-tree repository (the narrow `initRepo`, not the retired `provisionAgent`), and acks the agent's hex-encoded public key — the hub stores it in `agent_instance.publicKey` for reconnect verification. A multi-step deployment has no head agent identity: its per-step addresses are workflow-derived, so the ack instead carries the sidecar's supervisor principal key, which the hub discards. Before the child is spawned, the inputs a restart cannot otherwise recover — the per-step inference sources, the session id, and (single-step only) the hub public key — are written to a per-deployment `deployment.json` record. +When the hub sends `agent.deploy`, the sidecar spawns a supervised **workflow-process child** to host the deployment and responds with `agent.deploy.ack`. For a single-step (launched-agent) deployment the sidecar generates the agent's key pair (if new) or loads the existing one, records the hub's public key for later deploy-pack verification, initializes the head's on-disk deploy-tree repository (the narrow `initRepo`, not the retired `provisionAgent`), and acks the agent's hex-encoded public key — the hub stores it in `agent_instance.publicKey` for reconnect verification. A multi-step deployment's address is workflow-derived and has no `agent_instance` row, so the ack carries the deployment address's own key — the same Ed25519 key the sidecar signs reconnect challenges with — and the hub stores it on the deployment's `workflow_deployment` row (keyed by address), mirroring the launched-agent path so both are reconnect-verifiable. Before the child is spawned, the inputs a restart cannot otherwise recover — the per-step inference sources, the session id, and (single-step only) the hub public key — are written to a per-deployment `deployment.json` record. When the hub sends `agent.undeploy`, the sidecar shuts the deployment's supervisor down (killing the workflow-process child and releasing its IPC pipes and event-channel handle), unregisters the deployment address from the transport and from the mail/signal/drain routers, reclaims the deployment's per-step scratch, and deletes the `deployment.json` record so a later boot does not re-spawn a torn-down deployment. The agent's key pair and its durable agent-state / conversation repositories are left in place so a redeploy on the same address resumes them. diff --git a/packages/hub-sessions/src/hub-session-lookups.ts b/packages/hub-sessions/src/hub-session-lookups.ts index e87f0297..acfb9f8c 100644 --- a/packages/hub-sessions/src/hub-session-lookups.ts +++ b/packages/hub-sessions/src/hub-session-lookups.ts @@ -8,9 +8,14 @@ import { eq, and, isNull } from "drizzle-orm"; import type { DB } from "@intx/db"; -import { agentInstance, sessionMail } from "@intx/db/schema"; +import { + agentInstance, + sessionMail, + workflowDeployment, +} from "@intx/db/schema"; import { getLogger } from "@intx/log"; import { parseAgentAddress } from "@intx/types"; +import { isWorkflowDerivedAddress } from "@intx/workflow-deploy"; import type { AgentRepoStore } from "./agent-repo"; import { generateId } from "@intx/hub-common"; @@ -30,6 +35,33 @@ export function createHubSessionLookups( return { async lookupPublicKey(agentAddress) { + // Route by address space, not a blind two-table fallback: a + // workflow-derived address's key lives on its workflow_deployment + // row, a launched agent's on its agent_instance row, and the two + // spaces are disjoint. Routing (rather than falling back) means a + // launched agent that is missing its instance row returns null and + // fails its challenge visibly, instead of silently resolving against + // the wrong table. + if (isWorkflowDerivedAddress(agentAddress)) { + // Filter to a live ("deployed") deployment so a torn-down + // deployment's key can no longer satisfy a challenge. A null + // publicKey (deployed but not yet acked, or pre-migration) or an + // absent row returns null -- the challenge fails closed and the + // address stays unrouted rather than routing without ownership + // proof. + const row = await db + .select({ publicKey: workflowDeployment.publicKey }) + .from(workflowDeployment) + .where( + and( + eq(workflowDeployment.address, agentAddress), + eq(workflowDeployment.status, "deployed"), + ), + ) + .limit(1) + .then((rows) => rows[0]); + return row?.publicKey ?? null; + } const row = await db .select({ publicKey: agentInstance.publicKey }) .from(agentInstance) @@ -41,10 +73,7 @@ export function createHubSessionLookups( ) .limit(1) .then((rows) => rows[0]); - if (!row) { - return null; - } - return row.publicKey; + return row?.publicKey ?? null; }, async lookupDeployRef(agentAddress) { diff --git a/packages/hub-sessions/src/hub-session-orchestrator.test.ts b/packages/hub-sessions/src/hub-session-orchestrator.test.ts index 8ab2d01a..3538bfdb 100644 --- a/packages/hub-sessions/src/hub-session-orchestrator.test.ts +++ b/packages/hub-sessions/src/hub-session-orchestrator.test.ts @@ -343,13 +343,18 @@ describe("createHubSessionOrchestrator", () => { expect(harness.updates[0]?.set).toEqual({ publicKey: "deadbeef" }); }); - test("no-ops for a derived workflow-step address with no instance row", async () => { + test("persists the public key for a workflow-derived deployment address", async () => { harness = setup({ instance: undefined }); await harness.events.emitAndAwait("agent.deploy.ack", { - agentAddress: "ins_dep_abc-step1@workflow.interchange", + agentAddress: "ins_dep_abc@workflow.interchange", publicKey: "deadbeef", }); - expect(harness.updates).toHaveLength(0); + // A workflow-derived deployment address has no agent_instance row; its + // key is persisted on the workflow_deployment projection row (keyed by + // address) so the reconnect challenge can verify it. Previously this + // no-oped, which is what left these addresses un-verifiable. + expect(harness.updates).toHaveLength(1); + expect(harness.updates[0]?.set).toEqual({ publicKey: "deadbeef" }); }); test("throws when a launched-agent instance address has no row", async () => { diff --git a/packages/hub-sessions/src/hub-session-orchestrator.ts b/packages/hub-sessions/src/hub-session-orchestrator.ts index 6349a8fb..eadfb5df 100644 --- a/packages/hub-sessions/src/hub-session-orchestrator.ts +++ b/packages/hub-sessions/src/hub-session-orchestrator.ts @@ -13,7 +13,7 @@ import { eq } from "drizzle-orm"; import { type } from "arktype"; import type { DB } from "@intx/db"; -import { agentInstance } from "@intx/db/schema"; +import { agentInstance, workflowDeployment } from "@intx/db/schema"; import { parseMailToEmail } from "@intx/mime"; import { parseInferenceEvent } from "@intx/types/runtime"; import { getLogger } from "@intx/log"; @@ -100,21 +100,26 @@ export function createHubSessionOrchestrator( unsubscribers.push( events.on("agent.deploy.ack", async ({ agentAddress, publicKey }) => { + // Workflow-derived addresses (the deployment-level + // `ins_@` and the per-step + // `ins_-@`) have no agent_instance row; + // their public key lives on the workflow_deployment projection row, + // keyed by address. Persist it there so the reconnect ownership + // challenge can verify the deployment address. Only the + // deployment-level address has a row, so a stray per-step ack updates + // nothing. (This was previously a no-op, which is what left + // workflow-deployment addresses un-verifiable on reconnect.) + if (isWorkflowDerivedAddress(agentAddress)) { + await db + .update(workflowDeployment) + .set({ publicKey }) + .where(eq(workflowDeployment.address, agentAddress)); + return; + } + // A launched agent has an agent_instance row; a missing one is a bug + // to surface, not to drop silently. const instance = await findInstance(db, agentAddress); if (instance === undefined) { - // Workflow agents are provisioned at derived addresses — the - // deployment-level `ins_@` and the per-step - // `ins_-@` — with no agent_instance - // row, and nothing reads back their public key. The only reader - // of `agentInstance.publicKey` is the instance-keyed reconnect - // challenge, which workflow-derived addresses never take. So a - // missing row for a workflow-derived address is a correct no-op. - // A missing row for any other address is a launched agent whose - // instance should exist; surface that rather than silently - // dropping it. - if (isWorkflowDerivedAddress(agentAddress)) { - return; - } throw new Error( `No active instance found for deploy ack on address "${agentAddress}"`, ); diff --git a/tests/db/deployment-key-lookup.test.ts b/tests/db/deployment-key-lookup.test.ts new file mode 100644 index 00000000..37e4961f --- /dev/null +++ b/tests/db/deployment-key-lookup.test.ts @@ -0,0 +1,130 @@ +import { + afterAll, + beforeAll, + beforeEach, + describe, + expect, + test, +} from "bun:test"; + +import { + createHubSessionLookups, + type AgentRepoStore, +} from "@intx/hub-sessions"; +import { workflowDeployment } from "@intx/db/schema"; +import { + createTestDb, + harnessDbEnvAvailable, + type TestDb, +} from "@intx/test-harness/db-harness"; +import { seedAsset, seedTenants } from "@intx/test-harness/seed"; + +// The reconnect ownership challenge verifies a deployment address against a +// public key resolved by `lookupPublicKey`. These tests pin the workflow- +// deployment side of that lookup: keyed by address, gated on a live +// ("deployed") deployment, fail-closed on a missing/null key, and routed by +// address space so a launched-agent address never resolves against the +// deployment table. +describe.skipIf(!harnessDbEnvAvailable())( + "lookupPublicKey deployment-key routing (real DB)", + () => { + let h: TestDb; + + beforeAll(async () => { + h = await createTestDb(); + }); + + afterAll(async () => { + await h.close(); + }); + + beforeEach(async () => { + await h.reset(); + }); + + // lookupPublicKey never touches the repo store, so a throwing stub keeps + // the AgentRepoStore surface satisfied without a real on-disk store. + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub; lookupPublicKey does not touch the repo store + const stubRepoStore = new Proxy( + {}, + { + get() { + throw new Error("agentRepoStore is not used by lookupPublicKey"); + }, + }, + ) as AgentRepoStore; + + function lookupPublicKey(address: string): Promise { + return createHubSessionLookups({ + db: h.db, + agentRepoStore: stubRepoStore, + }).lookupPublicKey(address); + } + + async function seedDeployment(opts: { + address: string; + publicKey: string | null; + status: "deployed" | "error"; + }): Promise { + await seedTenants(h.db, [{ id: "t1" }]); + await seedAsset(h.db, { + id: "asset1", + tenantId: "t1", + kind: "workflow", + name: "wf", + }); + await h.db.insert(workflowDeployment).values({ + id: "dep1", + tenantId: "t1", + definitionAssetId: "asset1", + address: opts.address, + publicKey: opts.publicKey, + status: opts.status, + }); + } + + test("resolves a deployed deployment's key by address", async () => { + await seedDeployment({ + address: "ins_dep_abc@wf.example", + publicKey: "pk1", + status: "deployed", + }); + expect(await lookupPublicKey("ins_dep_abc@wf.example")).toBe("pk1"); + }); + + test("returns null when the deployment has not yet acked a key", async () => { + await seedDeployment({ + address: "ins_dep_abc@wf.example", + publicKey: null, + status: "deployed", + }); + expect(await lookupPublicKey("ins_dep_abc@wf.example")).toBeNull(); + }); + + test("returns null for a non-deployed (torn-down) deployment", async () => { + await seedDeployment({ + address: "ins_dep_abc@wf.example", + publicKey: "pk1", + status: "error", + }); + expect(await lookupPublicKey("ins_dep_abc@wf.example")).toBeNull(); + }); + + test("returns null for an unknown deployment address", async () => { + expect(await lookupPublicKey("ins_dep_missing@wf.example")).toBeNull(); + }); + + test("routes a launched-agent address to agent_instance, never the deployment table", async () => { + // A workflow_deployment row must not answer for a launched-agent + // (ins_...) address. The two address spaces are disjoint and the lookup + // routes by discriminator, so an absent agent_instance row returns null + // rather than leaking a deployment key. + await seedDeployment({ + address: "ins_dep_abc@wf.example", + publicKey: "pk1", + status: "deployed", + }); + expect(await lookupPublicKey("ins_launched@wf.example")).toBeNull(); + }); + }, +); From 9539468d544e8ff2099ead70395b129c9140d237 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Mon, 6 Jul 2026 22:30:12 -0500 Subject: [PATCH 082/101] Challenge workflow-deployment addresses on reconnect Workflow-substrate deployment addresses (ins_dep_...) re-registered through a keyless `workflowAddresses` frame field that routed them without any ownership proof. A sidecar holding a valid token could name another deployment's address on reconnect and capture its mail. Launched-agent addresses never had this gap: they prove ownership via Ed25519 challenge/response on reconnect. Close it by routing deployment addresses through that same challenged path: - Remove the `workflowAddresses` field from the register and reconnect frames. The sidecar announces deployments in the reconnect frame's `agentAddresses`, and connect() sends an empty first-connect register followed by the challenged reconnect. - The hub challenges every reconnect address uniformly and routes it only after a valid signature. A workflow-derived address is tracked on the connection's workflow set and skips the agent_instance-only reconnect reaction, but enters the routing table only post-challenge. - lookupPublicKey resolves a deployment key from workflow_deployment, gated on a live `deployed` status, and fails closed on a missing key, mirroring the launched-agent path. - When a new connection's verified reconnect reclaims a workflow address, evict it from the superseded connection's owned set, so that connection's later close does not cancel the new owner's in-flight pack transfer. Rewrite the tests that encoded the keyless model to assert the ownership invariant: challenged happy-path, wrong-key rejection, fail-closed on a missing key, close eviction, reclaim-only-via-its-own- challenge, and superseded-reclaim not cancelling the new owner's transfer. Wire the challenge round-trip into the hub-link integration harness and replace the invalid deployment fixtures (dep_..., and hyphenated ins_dep-...) with real ins_dep_ addresses; those fixtures were rejected by parseAgentAddress and never exercised the workflow-derived branch at all. The register frame's routing path is not modified here. --- docs/IMPLEMENTATION.md | 8 +- .../src/ws/hub-link-mail-router.test.ts | 37 +- packages/hub-agent/src/ws/hub-link.test.ts | 122 +++-- packages/hub-agent/src/ws/hub-link.ts | 54 ++- .../src/ws/sidecar-handler.test.ts | 432 +++++++++++------- .../hub-sessions/src/ws/sidecar-handler.ts | 116 +++-- packages/types/src/sidecar.ts | 18 +- 7 files changed, 505 insertions(+), 282 deletions(-) diff --git a/docs/IMPLEMENTATION.md b/docs/IMPLEMENTATION.md index 74a68563..5b82e790 100644 --- a/docs/IMPLEMENTATION.md +++ b/docs/IMPLEMENTATION.md @@ -1242,11 +1242,11 @@ If the WebSocket disconnects mid-transfer, no git state is corrupted: the sideca ### Reconnect Sequencing -On reconnect the sidecar restores its workflow deployments from local disk (see the deployment restore path) and re-announces them so the hub restores their routes: +On reconnect the sidecar restores its workflow deployments from local disk (see the deployment restore path) and re-announces them so the hub restores their routes. A workflow-deployment address is **not** keyless: it carries the deployment's own Ed25519 key (minted at deploy, acked to the hub), so it proves ownership through the same challenge/response a launched agent does. -1. Sidecar sends a single `register` frame carrying its live workflow-deployment addresses (`workflowAddresses`) -2. Hub re-registers those addresses for routing. Workflow-deployment addresses are hub-minted and keyless, so they re-register without the challenge/response flow a per-agent key would require -3. For a deployment whose deploy ref the hub tracks as behind, the hub initiates a pack transfer and waits for `pack.ack` +1. Sidecar sends an empty `register` frame to establish the connection, then a `reconnect` frame carrying its live workflow-deployment addresses (in `agentAddresses`) +2. For each address the hub resolves the stored deployment key (`workflow_deployment.publicKey`, gated on a live `deployed` status), issues a random nonce, and routes the address only after the sidecar returns a valid signature over `nonce ‖ address`. An address with no live key fails closed and stays unrouted — a token-holding sidecar cannot reclaim a deployment's route without the deployment's key +3. The hub does not run its deploy-ref freshness catch-up for a workflow-deployment address — that pack-transfer path is launched-agent only (`isWorkflowDerivedAddress` short-circuits it). A deployment's in-flight run state is reconstructed sidecar-locally at restore, not re-fetched from the hub The sidecar verifies every inbound deploy pack's commit signature against the hub key it recorded at deploy time before applying it, so pack content is never applied on an unverified signature. diff --git a/packages/hub-agent/src/ws/hub-link-mail-router.test.ts b/packages/hub-agent/src/ws/hub-link-mail-router.test.ts index 93259c5a..31d62346 100644 --- a/packages/hub-agent/src/ws/hub-link-mail-router.test.ts +++ b/packages/hub-agent/src/ws/hub-link-mail-router.test.ts @@ -21,13 +21,13 @@ import { type WsHandle, } from "@intx/hub-sessions"; import { createInMemoryTransport } from "@intx/mail-memory"; -import { base64Encode } from "@intx/types"; +import { base64Encode, hexEncode } from "@intx/types"; import type { HarnessConfig, InboundMessage, KeyPair, } from "@intx/types/runtime"; -import { signEd25519, verifySSHSignature } from "@intx/crypto"; +import { generateKeyPair, signEd25519, verifySSHSignature } from "@intx/crypto"; import { hexDecode } from "@intx/types"; import { createHubLink, type DeployRouter } from "./hub-link"; @@ -162,12 +162,20 @@ const VALID_MESSAGE = new TextEncoder().encode( type TestEnv = { server: ReturnType; router: SidecarRouter; + // address -> hex-encoded Ed25519 public key, backing the hub's fail-closed + // `lookupPublicKey`. A workflow deployment routes only after signing the + // hub's reconnect nonce with the key registered here. + deploymentKeys: Map; }; function startTestServer(): TestEnv { + const deploymentKeys = new Map(); const router = createSidecarRouter({ requestTimeoutMs: 5000, hubPublicKey: "a".repeat(64), + lookups: { + lookupPublicKey: async (address) => deploymentKeys.get(address) ?? null, + }, }); const app = new Hono(); @@ -205,7 +213,7 @@ function startTestServer(): TestEnv { port: 0, }); - return { server, router }; + return { server, router, deploymentKeys }; } const env = startTestServer(); @@ -214,11 +222,27 @@ afterAll(async () => { await env.server.stop(true); }); +/** + * Wire a workflow deployment for the challenged reconnect path: mint a + * keypair, register it in the sidecar keyStore (so `signChallenge` answers the + * hub nonce) and in the hub's `deploymentKeys` lookup (so the hub challenges + * and verifies). The deployment address then routes once the reconnect + * challenge round-trips. + */ +async function provisionDeploymentKey( + keyStore: ReturnType, + address: string, +): Promise { + const kp = await generateKeyPair(); + keyStore.registerKey(address, kp); + env.deploymentKeys.set(address, hexEncode(kp.publicKey)); +} + describe("hub-link mail.inbound throwing router", () => { test("a throwing mailInboundRouter does not wedge subsequent frames", async () => { const transport = createInMemoryTransport(); const sessions = createMockSessionManager(); - const deploymentAddress = "dep_wedge-1@integration.interchange"; + const deploymentAddress = "ins_dep_wedge1@integration.interchange"; sessions.addresses.push(deploymentAddress); let calls = 0; @@ -234,14 +258,17 @@ describe("hub-link mail.inbound throwing router", () => { }, }; + const bindings = withTestDeployBindings(); + await provisionDeploymentKey(bindings.keyStore, deploymentAddress); const client = createHubLink({ hubURL: `ws://localhost:${env.server.port}/ws`, sidecarId: "sc-mail-wedge", token: "test-token", transport, sessions, - ...withTestDeployBindings(), + ...bindings, mailInboundRouter, + getWorkflowAddresses: () => [deploymentAddress], }); client.connect(); diff --git a/packages/hub-agent/src/ws/hub-link.test.ts b/packages/hub-agent/src/ws/hub-link.test.ts index fbd43cf3..5e192598 100644 --- a/packages/hub-agent/src/ws/hub-link.test.ts +++ b/packages/hub-agent/src/ws/hub-link.test.ts @@ -8,7 +8,7 @@ import { type WsHandle, } from "@intx/hub-sessions"; import { createInMemoryTransport } from "@intx/mail-memory"; -import { signEd25519, verifySSHSignature } from "@intx/crypto"; +import { generateKeyPair, signEd25519, verifySSHSignature } from "@intx/crypto"; import { base64Encode, hexEncode } from "@intx/types"; import type { HarnessConfig } from "@intx/types/runtime"; @@ -183,15 +183,25 @@ type TestEnv = { router: SidecarRouter; agentEvents: { addr: string; sid: string; event: unknown }[]; outboundMail: { rawMessage: string; recipients: string[] }[]; + // address -> hex-encoded Ed25519 public key, backing the hub's + // `lookupPublicKey`. A workflow deployment announced through the + // challenged reconnect frame routes only after signing the hub's nonce + // with the key registered here; tests populate it via + // `provisionDeploymentKey`. + deploymentKeys: Map; }; function startTestServer(): TestEnv { const agentEvents: TestEnv["agentEvents"] = []; const outboundMail: TestEnv["outboundMail"] = []; + const deploymentKeys = new Map(); const router = createSidecarRouter({ requestTimeoutMs: 5000, hubPublicKey: "a".repeat(64), + lookups: { + lookupPublicKey: async (address) => deploymentKeys.get(address) ?? null, + }, }); router.events.on("agent.event", ({ agentAddress, sessionId, event }) => { agentEvents.push({ addr: agentAddress, sid: sessionId, event }); @@ -238,7 +248,7 @@ function startTestServer(): TestEnv { port: 0, }); - return { server, router, agentEvents, outboundMail }; + return { server, router, agentEvents, outboundMail, deploymentKeys }; } // --------------------------------------------------------------------------- @@ -251,6 +261,23 @@ afterAll(async () => { await env.server.stop(true); }); +/** + * Wire a workflow deployment for the challenged reconnect path: mint an + * Ed25519 keypair, register it in the sidecar's keyStore (so `signChallenge` + * can answer the hub's nonce) and in the hub's `deploymentKeys` lookup (so the + * hub issues a challenge and verifies the signature). After this, the + * deployment address named in `getWorkflowAddresses` routes once the + * reconnect challenge round-trips -- the same proof a launched agent makes. + */ +async function provisionDeploymentKey( + keyStore: ReturnType, + address: string, +): Promise { + const kp = await generateKeyPair(); + keyStore.registerKey(address, kp); + env.deploymentKeys.set(address, hexEncode(kp.publicKey)); +} + describe("sidecar↔hub integration", () => { test("sidecar registers with hub on connect", async () => { const transport = createInMemoryTransport(); @@ -403,33 +430,32 @@ describe("sidecar↔hub integration", () => { test("disconnect cleans up routing table", async () => { const transport = createInMemoryTransport(); const sessions = createMockSessionManager(); + const deploymentAddress = "ins_dep_disc1@integration.interchange"; + const bindings = withTestDeployBindings(); + await provisionDeploymentKey(bindings.keyStore, deploymentAddress); const client = createHubLink({ hubURL: `ws://localhost:${env.server.port}/ws`, sidecarId: "sc-disconnect", token: "test-token", - transport, sessions, - ...withTestDeployBindings(), - getWorkflowAddresses: () => ["tracked@test.interchange"], + ...bindings, + getWorkflowAddresses: () => [deploymentAddress], }); client.connect(); + // Routability lags the connection: it lands only after the reconnect + // challenge round-trips, so wait on the routable address directly. await waitFor(() => - env.router.getConnectedSidecars().includes("sc-disconnect"), - ); - expect(env.router.getRoutableAddresses()).toContain( - "tracked@test.interchange", + env.router.getRoutableAddresses().includes(deploymentAddress), ); client.close(); await waitFor( () => !env.router.getConnectedSidecars().includes("sc-disconnect"), ); - expect(env.router.getRoutableAddresses()).not.toContain( - "tracked@test.interchange", - ); + expect(env.router.getRoutableAddresses()).not.toContain(deploymentAddress); }); test("repo.pack.reject sent when applyDeployPack throws signature_invalid", async () => { @@ -922,7 +948,7 @@ describe("sidecar↔hub integration", () => { // mail.inbound for it goes through the link's switch case, which // must consult mailInboundRouter first and -- on a `true` return // -- skip transport.deliver and sessions.commitInboundMail. - const deploymentAddress = "dep_multistep-1@integration.interchange"; + const deploymentAddress = "ins_dep_mail1@integration.interchange"; const routed: { address: string; bytes: Uint8Array }[] = []; const mailInboundRouter = { @@ -932,13 +958,15 @@ describe("sidecar↔hub integration", () => { }, }; + const bindings = withTestDeployBindings(); + await provisionDeploymentKey(bindings.keyStore, deploymentAddress); const client = createHubLink({ hubURL: `ws://localhost:${env.server.port}/ws`, sidecarId: "sc-multistep-mail", token: "test-token", transport, sessions, - ...withTestDeployBindings(), + ...bindings, mailInboundRouter, getWorkflowAddresses: () => [deploymentAddress], }); @@ -969,7 +997,7 @@ describe("sidecar↔hub integration", () => { test("drainInboundRouter dispatches an inbound drain.deliver frame", async () => { const transport = createInMemoryTransport(); const sessions = createMockSessionManager(); - const deploymentAddress = "dep_drain-1@integration.interchange"; + const deploymentAddress = "ins_dep_drain1@integration.interchange"; const routed: { agentAddress: string; deadlineMs: number }[] = []; const drainInboundRouter = { @@ -985,13 +1013,15 @@ describe("sidecar↔hub integration", () => { }, }; + const bindings = withTestDeployBindings(); + await provisionDeploymentKey(bindings.keyStore, deploymentAddress); const client = createHubLink({ hubURL: `ws://localhost:${env.server.port}/ws`, sidecarId: "sc-drain-router", token: "test-token", transport, sessions, - ...withTestDeployBindings(), + ...bindings, drainInboundRouter, getWorkflowAddresses: () => [deploymentAddress], }); @@ -1031,7 +1061,7 @@ describe("sidecar↔hub integration", () => { test("sourcesInboundRouter acks an inbound sources.update round-trip", async () => { const transport = createInMemoryTransport(); const sessions = createMockSessionManager(); - const deploymentAddress = "dep_sources-ack@integration.interchange"; + const deploymentAddress = "ins_dep_srcack@integration.interchange"; const routed: { agentAddress: string }[] = []; const sourcesInboundRouter = { @@ -1041,13 +1071,15 @@ describe("sidecar↔hub integration", () => { }, }; + const bindings = withTestDeployBindings(); + await provisionDeploymentKey(bindings.keyStore, deploymentAddress); const client = createHubLink({ hubURL: `ws://localhost:${env.server.port}/ws`, sidecarId: "sc-sources-ack", token: "test-token", transport, sessions, - ...withTestDeployBindings(), + ...bindings, sourcesInboundRouter, getWorkflowAddresses: () => [deploymentAddress], }); @@ -1078,7 +1110,7 @@ describe("sidecar↔hub integration", () => { test("an unrouted sources.update is answered with session.error", async () => { const transport = createInMemoryTransport(); const sessions = createMockSessionManager(); - const deploymentAddress = "dep_sources-unrouted@integration.interchange"; + const deploymentAddress = "ins_dep_srcunrouted@integration.interchange"; const sourcesInboundRouter = { async tryRoute(): Promise { @@ -1086,13 +1118,15 @@ describe("sidecar↔hub integration", () => { }, }; + const bindings = withTestDeployBindings(); + await provisionDeploymentKey(bindings.keyStore, deploymentAddress); const client = createHubLink({ hubURL: `ws://localhost:${env.server.port}/ws`, sidecarId: "sc-sources-unrouted", token: "test-token", transport, sessions, - ...withTestDeployBindings(), + ...bindings, sourcesInboundRouter, getWorkflowAddresses: () => [deploymentAddress], }); @@ -1121,7 +1155,7 @@ describe("sidecar↔hub integration", () => { test("a rejected sources.update surfaces the reason as session.error", async () => { const transport = createInMemoryTransport(); const sessions = createMockSessionManager(); - const deploymentAddress = "dep_sources-reject@integration.interchange"; + const deploymentAddress = "ins_dep_srcreject@integration.interchange"; const sourcesInboundRouter = { async tryRoute(): Promise { @@ -1129,13 +1163,15 @@ describe("sidecar↔hub integration", () => { }, }; + const bindings = withTestDeployBindings(); + await provisionDeploymentKey(bindings.keyStore, deploymentAddress); const client = createHubLink({ hubURL: `ws://localhost:${env.server.port}/ws`, sidecarId: "sc-sources-reject", token: "test-token", transport, sessions, - ...withTestDeployBindings(), + ...bindings, sourcesInboundRouter, getWorkflowAddresses: () => [deploymentAddress], }); @@ -1163,15 +1199,17 @@ describe("sidecar↔hub integration", () => { test("a sources.update with no router wired is answered with session.error", async () => { const transport = createInMemoryTransport(); const sessions = createMockSessionManager(); - const deploymentAddress = "dep_sources-norouter@integration.interchange"; + const deploymentAddress = "ins_dep_srcnorouter@integration.interchange"; + const bindings = withTestDeployBindings(); + await provisionDeploymentKey(bindings.keyStore, deploymentAddress); const client = createHubLink({ hubURL: `ws://localhost:${env.server.port}/ws`, sidecarId: "sc-sources-norouter", token: "test-token", transport, sessions, - ...withTestDeployBindings(), + ...bindings, // No sourcesInboundRouter: a request/ack frame must still be answered // or the hub hangs on its request timeout. getWorkflowAddresses: () => [deploymentAddress], @@ -1371,8 +1409,8 @@ describe("sidecar↔hub integration", () => { }); }); -describe("register frame on connect", () => { - test("ships a single register carrying the sidecar's workflow addresses", async () => { +describe("register + reconnect frames on connect", () => { + test("ships an empty register then a challenged reconnect carrying the workflow addresses", async () => { const frames: string[] = []; const app = new Hono(); app.get( @@ -1389,7 +1427,7 @@ describe("register frame on connect", () => { const transport = createInMemoryTransport(); const sessions = createMockSessionManager(); - const workflowAddresses = ["ins_dep-1@integration.interchange"]; + const workflowAddresses = ["ins_dep_reg1@integration.interchange"]; const client = createHubLink({ hubURL: `ws://localhost:${server.port}/ws`, @@ -1403,27 +1441,27 @@ describe("register frame on connect", () => { client.connect(); try { - await waitFor(() => - frames - .map((s) => JSON.parse(s)) - .some((f: { type: string }) => f.type === "register"), - ); + await waitFor(() => { + const types = frames.map((s) => JSON.parse(s).type); + return types.includes("register") && types.includes("reconnect"); + }); const parsed = frames.map((s) => JSON.parse(s)); const registerFrames = parsed.filter( (f: { type: string }) => f.type === "register", ); - // Exactly one register: the in-process session runtime is retired, so - // there is no empty-register-then-reconnect dance. + const reconnectFrames = parsed.filter( + (f: { type: string }) => f.type === "reconnect", + ); + // First-connect register carries no addresses and no workflow field: + // a workflow deployment is announced only through the challenged + // reconnect, never as a keyless register route. expect(registerFrames).toHaveLength(1); - // Session addresses are always empty now; the sidecar's workflow - // deployments ride the same register frame so the hub re-registers - // their keyless routes without a challenge. expect(registerFrames[0].agentAddresses).toEqual([]); - expect(registerFrames[0].workflowAddresses).toEqual(workflowAddresses); - // No reconnect frame: disk-restored sessions no longer exist. - expect(parsed.some((f: { type: string }) => f.type === "reconnect")).toBe( - false, - ); + expect(registerFrames[0].workflowAddresses).toBeUndefined(); + // The reconnect frame announces the workflow addresses in + // agentAddresses, so each proves ownership via the Ed25519 challenge. + expect(reconnectFrames).toHaveLength(1); + expect(reconnectFrames[0].agentAddresses).toEqual(workflowAddresses); } finally { client.close(); await server.stop(true); diff --git a/packages/hub-agent/src/ws/hub-link.ts b/packages/hub-agent/src/ws/hub-link.ts index ee842ed5..aff82b5f 100644 --- a/packages/hub-agent/src/ws/hub-link.ts +++ b/packages/hub-agent/src/ws/hub-link.ts @@ -393,9 +393,11 @@ export type HubLinkConfig = { /** * Returns the workflow-substrate deployment addresses this sidecar * currently hosts a live supervisor for. Called on every (re)connect to - * announce them to the hub for routing: they are hub-minted with no - * per-address key, so they re-register WITHOUT the challenge flow, and - * without this announcement the hub drops their route on a WS reconnect. + * announce them to the hub for routing through the CHALLENGED reconnect + * frame: each deployment carries its own Ed25519 key (minted at deploy, + * acked to the hub), so it proves ownership via challenge/response exactly + * like a launched agent -- there is no keyless routing shortcut. Without + * this announcement the hub drops the deployment's route on a WS reconnect. * Defaults to none when omitted (tests / deployments with no workflow * substrate). */ @@ -1117,22 +1119,48 @@ export function createHubLink(config: HubLinkConfig): HubLink { packSender.cancelAll("Connection lost"); // Announce this sidecar to the hub for routing. The hub learns of a - // sidecar only from a register frame, and `connections` is the map - // `sendAgentDeploy` consults to route a deploy. Session addresses are - // always empty -- the in-process session runtime is retired -- but the - // frame still establishes the sidecar in that map. Workflow deployments - // restored at boot (before this connect) re-register here WITHOUT a - // challenge: they are hub-minted and keyless, so the hub would otherwise - // drop their route on a WS reconnect. - const workflowAddresses = getWorkflowAddresses(); + // sidecar only from a register frame; `connections` is the map + // `sendAgentDeploy` consults to route a deploy. This first-connect + // register carries no addresses -- it only establishes the sidecar in + // that map. Restored deployments are announced through the CHALLENGED + // reconnect frame below. send({ type: "register", sidecarId, token, - agentAddresses: sessions.getAddresses(), - ...(workflowAddresses.length > 0 ? { workflowAddresses } : {}), + agentAddresses: [], }); flush(); + + // Re-announce every deployment restored at boot through the reconnect + // frame so the hub proves ownership of each address (Ed25519 + // challenge/response, signed by the deployment's own key via + // `signChallenge`) before it routes mail. Routing a restored address + // through `register`/`workflowAddresses` -- unchallenged -- would let a + // rogue sidecar holding a valid token reclaim a victim's address. + // Restore runs before `connect()`, so `getWorkflowAddresses()` is + // already populated; the only async work is reading each address's + // deploy ref for the hub's deploy-pack freshness check. + const restoredAddresses = getWorkflowAddresses(); + if (restoredAddresses.length > 0) { + void (async () => { + const deployRefs: Record = {}; + for (const address of restoredAddresses) { + const ref = await sessions.getDeployRef(address); + if (ref !== null) { + deployRefs[address] = ref; + } + } + send({ + type: "reconnect", + sidecarId, + token, + agentAddresses: restoredAddresses, + ...(Object.keys(deployRefs).length > 0 ? { deployRefs } : {}), + }); + flush(); + })(); + } }); ws.addEventListener("message", (event) => { diff --git a/packages/hub-sessions/src/ws/sidecar-handler.test.ts b/packages/hub-sessions/src/ws/sidecar-handler.test.ts index 0705f925..8a48ea3a 100644 --- a/packages/hub-sessions/src/ws/sidecar-handler.test.ts +++ b/packages/hub-sessions/src/ws/sidecar-handler.test.ts @@ -185,121 +185,150 @@ describe("SidecarRouter", () => { }); }); - describe("workflow-address re-registration", () => { - test("register routes workflow addresses without a challenge", () => { - // The restart scenario: a sidecar hosting only workflow deployments - // reconnects and announces them via `workflowAddresses`. They must - // become routable directly -- they are hub-minted with no per-address - // key to challenge. Before this, such a sidecar sent no terminal frame - // carrying them, so the hub never re-learned the route. - const ws = createMockWs(); - router.handleOpen(ws); - router.handleMessage( - ws, - JSON.stringify({ - type: "register", - sidecarId: "sc-1", - token: "tok", - agentAddresses: [], - workflowAddresses: ["ins_dep_abc@local"], - }), - ); + describe("workflow-address reconnect (challenged, at parity with launched agents)", () => { + // A workflow-substrate deployment address (ins_dep_...) proves ownership + // through the SAME Ed25519 challenge as a launched agent: the hub resolves + // the deployment's public key, issues a nonce, and routes the address only + // after a valid signature. These tests pin that parity -- there is no + // keyless register-field shortcut for a workflow address anymore, so a + // token-holding sidecar cannot reclaim a deployment's route without the + // deployment's own key. + const WF_ADDR = "ins_dep_abc@local"; + + // Router whose key lookup resolves WF_ADDR to `publicKeyHex` (a live + // deployment) or to null (unknown / torn-down), mirroring the + // launched-agent reconnect tests. + function workflowReconnectRouter(publicKeyHex: string | null) { + return createSidecarRouter({ + requestTimeoutMs: 5000, + hubPublicKey: TEST_HUB_KEY, + lookups: { + lookupPublicKey: async (addr) => + addr === WF_ADDR ? publicKeyHex : null, + }, + }); + } - expect(router.getRoutableAddresses()).toEqual(["ins_dep_abc@local"]); - expect(router.routeMail("ins_dep_abc@local", "dGVzdA==")).toBe(true); - }); + function findFrame(ws: ReturnType, type: string) { + return ws.sent.map((s) => JSON.parse(s)).find((f) => f.type === type); + } - test("reconnect preserves a workflow address not in the challenged list", () => { - // A reconnect lists its workflow addresses only in `workflowAddresses`, - // never in the challenged `agentAddresses`. The internal empty-register - // clears the connection's addresses; the reconnect re-adds the workflow - // set, so it stays routable across the reconnect. - const ws = createMockWs(); - router.handleOpen(ws); - router.handleMessage( + async function reconnect( + r: ReturnType, + ws: ReturnType, + ) { + r.handleOpen(ws); + r.handleMessage( ws, JSON.stringify({ - type: "register", + type: "reconnect", sidecarId: "sc-1", token: "tok", - agentAddresses: [], - workflowAddresses: ["ins_dep_abc@local"], + agentAddresses: [WF_ADDR], }), ); - expect(router.routeMail("ins_dep_abc@local", "dGVzdA==")).toBe(true); + await new Promise((res) => setTimeout(res, 50)); + } - router.handleMessage( + async function respondToChallenge( + r: ReturnType, + ws: ReturnType, + privateKey: Uint8Array, + ) { + const { address, nonce } = findFrame(ws, "challenge").challenges[0]; + r.handleMessage( ws, JSON.stringify({ - type: "reconnect", - sidecarId: "sc-1", - token: "tok", - agentAddresses: [], - workflowAddresses: ["ins_dep_abc@local"], + type: "challenge.response", + responses: [ + { + address, + signature: await signChallenge(nonce, address, privateKey), + }, + ], }), ); + await new Promise((res) => setTimeout(res, 50)); + } + + test("routes a workflow address only after a passed challenge", async () => { + const kp = await generateKeyPair(); + const r = workflowReconnectRouter(hexEncode(kp.publicKey)); + const ws = createMockWs(); + await reconnect(r, ws); + + // The reconnect frame alone earns nothing: the address is not routable + // until the challenge is answered. + expect(r.getRoutableAddresses()).not.toContain(WF_ADDR); + const challenge = findFrame(ws, "challenge"); + expect(challenge).toBeDefined(); + expect(challenge.challenges[0].address).toBe(WF_ADDR); + + await respondToChallenge(r, ws, kp.privateKey); + + expect(r.getRoutableAddresses()).toContain(WF_ADDR); + expect(r.routeMail(WF_ADDR, "dGVzdA==")).toBe(true); + }); - expect(router.routeMail("ins_dep_abc@local", "dGVzdA==")).toBe(true); + test("rejects a workflow address signed with the wrong key (no hijack)", async () => { + const kp = await generateKeyPair(); + const wrongKp = await generateKeyPair(); + const r = workflowReconnectRouter(hexEncode(kp.publicKey)); + const ws = createMockWs(); + await reconnect(r, ws); + + await respondToChallenge(r, ws, wrongKp.privateKey); + + expect(r.getRoutableAddresses()).not.toContain(WF_ADDR); + expect(r.routeMail(WF_ADDR, "dGVzdA==")).toBe(false); + expect(findFrame(ws, "challenge.failed").address).toBe(WF_ADDR); }); - test("disconnect removes workflow addresses from the routing table", () => { - // Guards the leak: without removing the connection's workflow addresses - // on close, a stale `addr -> dead ws` entry would survive in the routing - // table and the next sidecar could not cleanly reclaim it. + test("fails closed when the deployment has no live key", async () => { + // lookupPublicKey resolves null -- an unknown or torn-down deployment. + // No challenge is issued and the address never routes. + const r = workflowReconnectRouter(null); const ws = createMockWs(); - router.handleOpen(ws); - router.handleMessage( - ws, - JSON.stringify({ - type: "register", - sidecarId: "sc-1", - token: "tok", - agentAddresses: [], - workflowAddresses: ["ins_dep_abc@local"], - }), - ); - expect(router.routeMail("ins_dep_abc@local", "dGVzdA==")).toBe(true); + await reconnect(r, ws); - router.handleClose(ws); + expect(r.getRoutableAddresses()).not.toContain(WF_ADDR); + expect(findFrame(ws, "challenge")).toBeUndefined(); + expect(findFrame(ws, "challenge.failed").address).toBe(WF_ADDR); + }); - expect(router.getRoutableAddresses()).toEqual([]); - expect(router.routeMail("ins_dep_abc@local", "dGVzdA==")).toBe(false); + test("disconnect removes a verified workflow address from routing", async () => { + const kp = await generateKeyPair(); + const r = workflowReconnectRouter(hexEncode(kp.publicKey)); + const ws = createMockWs(); + await reconnect(r, ws); + await respondToChallenge(r, ws, kp.privateKey); + expect(r.routeMail(WF_ADDR, "dGVzdA==")).toBe(true); + + r.handleClose(ws); + + expect(r.getRoutableAddresses()).toEqual([]); + expect(r.routeMail(WF_ADDR, "dGVzdA==")).toBe(false); }); - test("a fresh ws reclaims a workflow address after an abrupt disconnect", () => { - // Restart without a clean close: the old ws still holds the route. A - // fresh ws re-announcing the same workflow address takes over routing, - // and the stale ws's later close must not clobber the new owner (the - // ownership guard in handleClose). + test("a fresh ws reclaims a workflow address only by passing its own challenge", async () => { + const kp = await generateKeyPair(); + const r = workflowReconnectRouter(hexEncode(kp.publicKey)); + const oldWs = createMockWs(); - router.handleOpen(oldWs); - router.handleMessage( - oldWs, - JSON.stringify({ - type: "register", - sidecarId: "sc-1", - token: "tok", - agentAddresses: [], - workflowAddresses: ["ins_dep_abc@local"], - }), - ); + await reconnect(r, oldWs); + await respondToChallenge(r, oldWs, kp.privateKey); + expect(r.routeMail(WF_ADDR, "dGVzdA==")).toBe(true); const newWs = createMockWs(); - router.handleOpen(newWs); - router.handleMessage( - newWs, - JSON.stringify({ - type: "register", - sidecarId: "sc-1", - token: "tok", - agentAddresses: [], - workflowAddresses: ["ins_dep_abc@local"], - }), - ); + await reconnect(r, newWs); + await respondToChallenge(r, newWs, kp.privateKey); - router.handleClose(oldWs); + // The stale ws's later close must not clobber the new owner (the + // ownership guard in handleClose). + r.handleClose(oldWs); - expect(router.routeMail("ins_dep_abc@local", "dGVzdA==")).toBe(true); + expect(r.getRoutableAddresses()).toContain(WF_ADDR); + expect(r.routeMail(WF_ADDR, "dGVzdA==")).toBe(true); }); }); @@ -355,46 +384,63 @@ describe("SidecarRouter", () => { }); test("a reconnect after unbind does not resurrect the transient route", async () => { - // A real reconnect only runs when `lookupPublicKey` is configured - // (otherwise `handleReconnect` bails before re-registering anything). - // Use a dedicated router with the lookup so the reconnect actually - // executes and this exercises the re-registration path. + // The deployment address is re-announced on reconnect and proves + // ownership through the challenge; the transient per-step route + // (bind/unbind) is never persisted into that announcement, so a + // reconnect must not bring it back. + const kp = await generateKeyPair(); const reconnectRouter = createSidecarRouter({ - requestTimeoutMs: 500, + requestTimeoutMs: 5000, hubPublicKey: TEST_HUB_KEY, - lookups: { lookupPublicKey: async () => null }, + lookups: { + lookupPublicKey: async (addr) => + addr === DEPLOYMENT_ADDR ? hexEncode(kp.publicKey) : null, + }, }); const ws = createMockWs(); reconnectRouter.handleOpen(ws); - // The deployment address is a workflow-substrate address (no challenge). - reconnectRouter.handleMessage( - ws, - JSON.stringify({ - type: "register", - sidecarId: "sc-1", - token: "tok", - agentAddresses: [], - workflowAddresses: [DEPLOYMENT_ADDR], - }), - ); + + // Bring the deployment address up through the challenged reconnect path. + async function reconnectDeployment() { + reconnectRouter.handleMessage( + ws, + JSON.stringify({ + type: "reconnect", + sidecarId: "sc-1", + token: "tok", + agentAddresses: [DEPLOYMENT_ADDR], + }), + ); + await new Promise((r) => setTimeout(r, 50)); + const challenges = ws.sent + .map((s) => JSON.parse(s)) + .filter((f) => f.type === "challenge"); + const { address, nonce } = + challenges[challenges.length - 1].challenges[0]; + reconnectRouter.handleMessage( + ws, + JSON.stringify({ + type: "challenge.response", + responses: [ + { + address, + signature: await signChallenge(nonce, address, kp.privateKey), + }, + ], + }), + ); + await new Promise((r) => setTimeout(r, 50)); + } + + await reconnectDeployment(); + expect(reconnectRouter.getRoutableAddresses()).toContain(DEPLOYMENT_ADDR); + reconnectRouter.bindStepRoute(STEP_ADDR); reconnectRouter.unbindStepRoute(STEP_ADDR); - // The sidecar reconnects announcing only the deployment's workflow - // address -- never the transient per-step address, which was never - // persisted into the reconnect announcement. - reconnectRouter.handleMessage( - ws, - JSON.stringify({ - type: "reconnect", - sidecarId: "sc-1", - token: "tok", - agentAddresses: [], - workflowAddresses: [DEPLOYMENT_ADDR], - }), - ); - // Let the async reconnect re-registration settle. - await new Promise((r) => setTimeout(r, 0)); + // The sidecar reconnects announcing only the deployment address -- never + // the transient per-step address. + await reconnectDeployment(); // The reconnect actually ran (it did not bail and close the socket). expect(ws.closed).toBe(false); @@ -2831,27 +2877,19 @@ describe("SidecarRouter", () => { expect(ack.repoId).toEqual(repoId); }); - test("a workflow deployment announced via workflowAddresses can still push workflow-run packs", async () => { - // The reconnect/restart shape: the deployment address is announced via - // `workflowAddresses`, not `agentAddresses`. Pack-push authorization must - // still recognize it as owned by this connection -- otherwise the hub's - // workflow-run observation mirror silently stops updating after a - // reconnect (mail routing resumes, but pack push would reject the - // address as unrouted). + test("a workflow deployment owned by its connection can push workflow-run packs", async () => { + // Pack-push authorization keys on `connOwnsAddress`, which unions the + // connection's launched-agent and workflow-substrate address sets. A + // workflow deployment address owned by the connection must authorize a + // workflow-run pack -- otherwise the hub's workflow-run observation + // mirror silently stops updating (mail routing resumes, but pack push + // rejects the address as unrouted). The challenged-reconnect ownership + // path itself is covered by the "workflow-address reconnect" block; here + // the address is registered directly to keep the test on pack routing. const { router: r, calls } = buildPackRouter(); const ws = createMockWs(); const addr = "ins_dep_wfr@local"; - r.handleOpen(ws); - r.handleMessage( - ws, - JSON.stringify({ - type: "register", - sidecarId: "sc-wfr", - token: "tok", - agentAddresses: [], - workflowAddresses: [addr], - }), - ); + registerAddr(r, ws, "sc-wfr", addr); const repoId: RepoId = { kind: "workflow-run", id: "dep-wfr-2" }; pushPack(r, ws, { @@ -2885,31 +2923,113 @@ describe("SidecarRouter", () => { const transferId = "t-reclaim"; const oldWs = createMockWs(); - r.handleOpen(oldWs); - r.handleMessage( - oldWs, - JSON.stringify({ - type: "register", - sidecarId: "sc", - token: "tok", - agentAddresses: [], - workflowAddresses: [addr], - }), - ); + registerAddr(r, oldWs, "sc", addr); const newWs = createMockWs(); - r.handleOpen(newWs); + registerAddr(r, newWs, "sc", addr); + + // The new owner starts (but does not finish) a workflow-run transfer. + for (const chunk of chunkPack(pack)) { + r.handleMessage( + newWs, + JSON.stringify({ + type: "repo.pack.push", + agentAddress: addr, + repoId, + transferId, + seq: chunk.seq, + data: chunk.data, + }), + ); + } + + // The superseded connection closes mid-transfer. + r.handleClose(oldWs); + + // The new owner completes the transfer; it must not have been cancelled. r.handleMessage( newWs, JSON.stringify({ - type: "register", - sidecarId: "sc", - token: "tok", - agentAddresses: [], - workflowAddresses: [addr], + type: "repo.pack.done", + agentAddress: addr, + repoId, + transferId, + ref: "refs/heads/events", + commitSha: "a".repeat(40), }), ); + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(calls.length).toBe(1); + expect(calls[0]?.method).toBe("receiveWorkflowRunPack"); + }); + + test("a workflow address reclaimed via challenged reconnect does not cancel the new owner's transfer", async () => { + // The register-reclaim variant above covers the handleRegister ghost + // cleanup. This covers the CHALLENGED RECONNECT reclaim path, where the + // deployment lives on the connection's workflow set: the verified reclaim + // must evict the address from the superseded connection, or that + // connection's close runs cancelByAgent and kills the new owner's + // in-flight transfer. + const kp = await generateKeyPair(); + const calls: { method: string }[] = []; + const r = createSidecarRouter({ + requestTimeoutMs: 5000, + hubPublicKey: TEST_HUB_KEY, + lookups: { + lookupPublicKey: async () => hexEncode(kp.publicKey), + async receiveWorkflowRunPack() { + calls.push({ method: "receiveWorkflowRunPack" }); + return { accepted: true }; + }, + }, + }); + const addr = "ins_dep_reclaim_rc@local"; + const repoId: RepoId = { kind: "workflow-run", id: "dep-reclaim-rc" }; + const pack = new Uint8Array([4, 5, 6, 7]); + const transferId = "t-reclaim-rc"; + + async function reconnectVerify(ws: ReturnType) { + r.handleOpen(ws); + r.handleMessage( + ws, + JSON.stringify({ + type: "reconnect", + sidecarId: "sc", + token: "tok", + agentAddresses: [addr], + }), + ); + await new Promise((resolve) => setTimeout(resolve, 50)); + const challenge = ws.sent + .map((s) => JSON.parse(s)) + .find((f: { type: string }) => f.type === "challenge"); + r.handleMessage( + ws, + JSON.stringify({ + type: "challenge.response", + responses: [ + { + address: addr, + signature: await signChallenge( + challenge.challenges[0].nonce, + addr, + kp.privateKey, + ), + }, + ], + }), + ); + await new Promise((resolve) => setTimeout(resolve, 50)); + } + + const oldWs = createMockWs(); + await reconnectVerify(oldWs); + + const newWs = createMockWs(); + await reconnectVerify(newWs); + // The new owner starts (but does not finish) a workflow-run transfer. for (const chunk of chunkPack(pack)) { r.handleMessage( diff --git a/packages/hub-sessions/src/ws/sidecar-handler.ts b/packages/hub-sessions/src/ws/sidecar-handler.ts index c4555855..cc4eb491 100644 --- a/packages/hub-sessions/src/ws/sidecar-handler.ts +++ b/packages/hub-sessions/src/ws/sidecar-handler.ts @@ -8,6 +8,7 @@ import { getLogger } from "@intx/log"; import { verifyEd25519 } from "@intx/crypto"; import { chunkPack, createPackReceiver } from "@intx/pack-transport"; import { hexDecode, hexEncode } from "@intx/types"; +import { isWorkflowDerivedAddress } from "@intx/workflow-deploy"; import { type } from "arktype"; import { SidecarFrame, @@ -412,13 +413,7 @@ export function createSidecarRouter( switch (frame.type) { case "register": - handleRegister( - ws, - frame.sidecarId, - frame.token, - frame.agentAddresses, - frame.workflowAddresses ?? [], - ); + handleRegister(ws, frame.sidecarId, frame.token, frame.agentAddresses); break; case "reconnect": void handleReconnect( @@ -426,7 +421,6 @@ export function createSidecarRouter( frame.sidecarId, frame.token, frame.agentAddresses, - frame.workflowAddresses ?? [], frame.deployRefs ?? {}, ); break; @@ -535,7 +529,6 @@ export function createSidecarRouter( sidecarId: string, token: string, agentAddresses: string[], - workflowAddresses: string[] = [], ): void { if (validateToken !== undefined && !validateToken(sidecarId, token)) { logger.warn`Rejected registration from sidecar ${sidecarId}: invalid token`; @@ -562,9 +555,11 @@ export function createSidecarRouter( } const addrSet = new Set(agentAddresses); - // The frame carries the COMPLETE current live workflow-address set, so - // this replaces (not merges) the connection's workflow routing. - const workflowSet = new Set(workflowAddresses); + // The connection's workflow-address set starts empty and is populated + // only from VERIFIED reconnect addresses (post-challenge) and from + // `bindStepRoute`'s transient per-step routes -- never from an + // unauthenticated frame field. It backs `handleClose` reclamation. + const workflowSet = new Set(); // Clean up ghost entries from other connections that previously // owned addresses this sidecar is now claiming. @@ -593,24 +588,6 @@ export function createSidecarRouter( } } - // Same ghost cleanup for reclaimed workflow-substrate addresses: evict the - // address from the prior owner's set. Without this, when the superseded - // connection later closes (the abrupt-restart overlap window), its - // `handleClose` teardown would iterate the stale address and cancel THIS - // connection's live in-flight pack transfer / abandon its collector for - // the deployment it just took over. `cancelByAgent` is keyed by address, - // not by connection, so the stale close would hit the new owner. The - // session loop above evicts the reclaimed address the same way. - for (const addr of workflowSet) { - const prevWs = addressIndex.get(addr); - if (prevWs !== undefined && prevWs !== ws) { - const prevConn = connections.get(prevWs); - if (prevConn !== undefined) { - prevConn.workflowAddresses.delete(addr); - } - } - } - const conn: SidecarConnection = { sidecarId, agentAddresses: addrSet, @@ -635,17 +612,15 @@ export function createSidecarRouter( disconnectedAgents.delete(addr); } } - // Re-register workflow-substrate addresses for routing directly. They are - // hub-minted with no per-address key (no `agent_instance` row), so there - // is no challenge to run -- the same way they first entered `addressIndex` - // at deploy time via `sendAgentDeploy`. A later-connecting ws claiming the - // same address overwrites the pointer here; the prior owner's `handleClose` - // then no-ops on it via its ownership guard. - for (const addr of workflowSet) { - addressIndex.set(addr, ws); - } + // Register does NOT route any workflow-substrate address: a restored + // deployment address is announced through the challenged reconnect frame + // and enters `addressIndex` only after its ownership challenge passes. + // Routing an address here from an unauthenticated register frame is the + // exact bypass that let a token-holding sidecar reclaim a victim's + // address, so this handler writes only the (empty, first-connect) session + // set above. - logger.info`Sidecar ${sidecarId} registered with ${String(agentAddresses.length)} agents and ${String(workflowSet.size)} workflow deployments`; + logger.info`Sidecar ${sidecarId} registered with ${String(agentAddresses.length)} agents`; } async function handleReconnect( @@ -653,7 +628,6 @@ export function createSidecarRouter( sidecarId: string, token: string, agentAddresses: string[], - workflowAddresses: string[] = [], deployRefs: Record = {}, ): Promise { if (validateToken !== undefined && !validateToken(sidecarId, token)) { @@ -686,10 +660,11 @@ export function createSidecarRouter( // freshly-deployed agent from routing. const previouslyOwned = new Set(connections.get(ws)?.agentAddresses); - // Register the sidecar connection immediately (with no session addresses, - // but WITH the workflow-substrate addresses -- those need no challenge) - // so it can receive frames while the session challenge is pending. - handleRegister(ws, sidecarId, token, [], workflowAddresses); + // Register the sidecar connection immediately (with no addresses) so it + // can receive frames while the ownership challenge is pending. Every + // reconnect address -- session and workflow-derived alike -- enters + // routing only through the verified path below, never unchallenged here. + handleRegister(ws, sidecarId, token, []); const conn = connections.get(ws); if (conn === undefined) return; @@ -852,8 +827,31 @@ export function createSidecarRouter( const prevWs = addressIndex.get(addr); if (prevWs !== undefined && prevWs !== ws) { connectorStates.delete(addr); + // Evict the reclaimed address from the superseded connection's owned + // set. handleClose's cancelByAgent sweep iterates a connection's owned + // union WITHOUT an ownership guard, so if the stale connection still + // listed this address it would cancel THIS connection's in-flight pack + // transfer for it when it finally closes. Delete from both sets: a + // workflow-derived address lives on the workflow set, a launched agent + // on the session set, and delete is a no-op for the absent one. + const prevConn = connections.get(prevWs); + if (prevConn !== undefined) { + prevConn.workflowAddresses.delete(addr); + prevConn.agentAddresses.delete(addr); + } + } + // Track the address on the set that matches its lifecycle so + // handleClose reclaims it correctly: a workflow-derived deployment + // address goes on the workflow set (no disconnect queue -- its + // in-flight state is reconstructed sidecar-locally on the next + // reconnect), a launched agent on the session set (queued for + // reconnect). The routing pointer is the same either way; only now + // it is written behind a passed challenge. + if (isWorkflowDerivedAddress(addr)) { + conn.workflowAddresses.add(addr); + } else { + conn.agentAddresses.add(addr); } - conn.agentAddresses.add(addr); addressIndex.set(addr, ws); } @@ -861,6 +859,17 @@ export function createSidecarRouter( const failed: string[] = []; for (const addr of verified) { + // The `agent.reconnected` reaction is session lifecycle -- instance + // status flip, event-collector restore -- owned by the agent_instance + // concept. A workflow-derived deployment address has no agent_instance + // row, so the reaction's `requireInstance` would throw and roll the + // just-verified address back out of routing. It needs routing + queue + // flush only, which the passed challenge has now made safe; skip the + // session reaction for it. + if (isWorkflowDerivedAddress(addr)) { + ready.push(addr); + continue; + } if (events.listenerCount("agent.reconnected") === 0) { ready.push(addr); continue; @@ -883,7 +892,10 @@ export function createSidecarRouter( return; } - // Roll back failed addresses from the routing table. + // Roll back failed addresses from the routing table. Only the session set + // is touched: a workflow-derived address can never be in `failed` -- it + // early-`continue`s to `ready` above, before the reaction that populates + // `failed` -- so it is never on the workflow set at this point. for (const addr of failed) { conn.agentAddresses.delete(addr); addressIndex.delete(addr); @@ -901,6 +913,11 @@ export function createSidecarRouter( const checkDeployRef = lookups.lookupDeployRef; if (checkDeployRef !== undefined) { for (const addr of ready) { + // Deploy-pack freshness (createDeployPack / parseAgentId) is an + // agent-repo path scoped to launched agents; it is not validated for + // workflow-derived deployment addresses, and the current register + // path never ran it for them. Skip it here to match that. + if (isWorkflowDerivedAddress(addr)) continue; void (async () => { try { const hubRef = await checkDeployRef(addr); @@ -1101,8 +1118,9 @@ export function createSidecarRouter( // owned union so a reconnected workflow deployment's transfer is // cancelled too; the deduped set avoids a double-cancel for an address // that is in both sets. A reclaimed address is not present here -- the - // ghost cleanup in handleRegister evicts it from this (superseded) - // connection -- so a stale close does not cancel the new owner's work. + // verified reconnect path that took it over evicts it from this + // (superseded) connection's owned set -- so a stale close does not cancel + // the new owner's work. const owned = ownedAddresses(conn); for (const addr of owned) { agentStatePackReceiver.cancelByAgent(addr); diff --git a/packages/types/src/sidecar.ts b/packages/types/src/sidecar.ts index 5e59faf9..deeef353 100644 --- a/packages/types/src/sidecar.ts +++ b/packages/types/src/sidecar.ts @@ -29,29 +29,21 @@ export const RegisterFrame = type({ sidecarId: "string", token: "string", agentAddresses: "string[]", - // Live workflow-substrate deployment addresses (ins_dep_...) this sidecar - // currently hosts. Unlike `agentAddresses`, these are hub-minted and - // carry no per-address key, so the hub re-registers them for routing - // directly (no challenge) -- the same way they were first registered at - // deploy time. Absent on sidecars/paths that host none. - "workflowAddresses?": "string[]", }); export type RegisterFrame = typeof RegisterFrame.infer; /** - * Sent on connect when the sidecar has existing agent repositories from a - * previous run. Lists the agent addresses it can serve, triggering the - * challenge/response verification flow. + * Sent on connect when the sidecar has agent repositories or deployments + * from a previous run. Lists the addresses it can serve, triggering the + * challenge/response ownership-verification flow for every one of them -- + * launched agents and workflow deployments alike, so both are proven, not + * routed on trust. */ export const ReconnectFrame = type({ type: "'reconnect'", sidecarId: "string", token: "string", agentAddresses: "string[]", - // See `RegisterFrame.workflowAddresses`. Carried on the reconnect frame too - // so a sidecar restoring both single-agent sessions and workflow - // deployments re-registers both in one connect. - "workflowAddresses?": "string[]", "deployRefs?": "Record", }); export type ReconnectFrame = typeof ReconnectFrame.infer; From ba061bfb5e7c3ae7426c48f9e023db093e883753 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Mon, 6 Jul 2026 23:13:39 -0500 Subject: [PATCH 083/101] Gate register-frame routing on key existence `handleRegister` routed every address a register frame named on the sidecar token alone, with no per-address ownership proof -- the same token-only routing the reconnect challenge exists to prevent, reachable through the sibling register frame. A token-holding sidecar could name another deployment's (or agent's) keyed address in a register frame and capture its future mail; the ghost-cleanup even evicted the true owner first, so a refused-but-partially-applied claim could also strand the victim. Gate routing on key existence: a register frame routes an address only when `lookupPublicKey` returns null for it -- a genuine keyless first-deploy, matching the documented token-bounded first-deploy trust model. An address that already has a stored key is refused and must prove ownership through the challenged reconnect path. The keyless-only set is computed up front, before the ghost-cleanup and every routing mutation, so a refused (keyed) address touches nothing: no eviction of a live owner, hence no downgrade from hijack to a denial of service on the victim. When the key lookup is not configured, or the lookup throws (a transient DB failure), the handler fails closed -- routes nothing and logs an error -- rather than routing unverified or letting the rejection float out of the void-dispatched handler and crash the hub. `handleRegister` becomes async to await the lookup, mirroring `handleReconnect`. --- docs/IMPLEMENTATION.md | 2 + .../src/ws/sidecar-handler.test.ts | 426 +++++++++++++++--- .../hub-sessions/src/ws/sidecar-handler.ts | 78 +++- 3 files changed, 440 insertions(+), 66 deletions(-) diff --git a/docs/IMPLEMENTATION.md b/docs/IMPLEMENTATION.md index 5b82e790..b24c5d2a 100644 --- a/docs/IMPLEMENTATION.md +++ b/docs/IMPLEMENTATION.md @@ -1252,6 +1252,8 @@ The sidecar verifies every inbound deploy pack's commit signature against the hu On first deploy (no prior key exists), the sidecar is authenticated by its registration token but cannot prove agent key ownership (the key does not exist yet). The hub sends `agent.deploy` to provision the agent, and the sidecar generates the key and returns it in `agent.deploy.ack`. The registration token and the authenticated WebSocket channel bound the trust for first-deploy; challenge/response protects all subsequent interactions. +The hub enforces that boundary structurally in `handleRegister`: a `register` frame routes an address only when `lookupPublicKey` returns null for it — a genuine keyless first-deploy. An address that already has a stored key is refused and must re-enter routing through the challenged reconnect, so a token-holding sidecar cannot reclaim a keyed address's route (its own or a victim's) on token auth alone. The check runs before any routing mutation, so a refused address never even evicts its current owner. If the key lookup is not configured, register fails closed (routes nothing and logs an error) rather than routing unverified. + ### State Push Policy The sidecar pushes state to the hub based on configurable policy: diff --git a/packages/hub-sessions/src/ws/sidecar-handler.test.ts b/packages/hub-sessions/src/ws/sidecar-handler.test.ts index 8a48ea3a..fe7d38d0 100644 --- a/packages/hub-sessions/src/ws/sidecar-handler.test.ts +++ b/packages/hub-sessions/src/ws/sidecar-handler.test.ts @@ -1,4 +1,5 @@ import { describe, test, expect, beforeEach } from "bun:test"; +import { configureSync, getConfig, resetSync } from "@intx/log"; import { generateKeyPair, signEd25519 } from "@intx/crypto"; import { hexDecode, hexEncode, parseAgentAddress } from "@intx/types"; import { chunkPack } from "@intx/pack-transport"; @@ -38,6 +39,14 @@ function lastSent(ws: ReturnType) { return JSON.parse(last); } +// Let the async register key-existence gate settle. A `register` frame now +// routes on a microtask (handleRegister awaits `lookupPublicKey` per address), +// so a test that reads the routing table right after sending one must await +// this first. +async function tick(): Promise { + await new Promise((resolve) => setTimeout(resolve, 0)); +} + describe("SidecarRouter", () => { let router: ReturnType; @@ -61,11 +70,17 @@ describe("SidecarRouter", () => { router = createSidecarRouter({ requestTimeoutMs: 500, hubPublicKey: TEST_HUB_KEY, + // Always-null lookup: no address has a stored key, so the register + // key-existence gate classifies every registered address as a keyless + // first-deploy and routes it. This makes the dependency explicit for + // the pure-routing tests (the gate is fail-closed without it) while + // preserving their "register routes the address" behavior. + lookups: { lookupPublicKey: async () => null }, }); }); describe("registration", () => { - test("register frame populates routing table", () => { + test("register frame populates routing table", async () => { const ws = createMockWs(); router.handleOpen(ws); router.handleMessage( @@ -77,6 +92,7 @@ describe("SidecarRouter", () => { agentAddresses: ["agent-a@local", "agent-b@local"], }), ); + await tick(); expect(router.getConnectedSidecars()).toEqual(["sc-1"]); expect(router.getRoutableAddresses().sort()).toEqual([ @@ -85,7 +101,7 @@ describe("SidecarRouter", () => { ]); }); - test("re-registration updates addresses", () => { + test("re-registration updates addresses", async () => { const ws = createMockWs(); router.handleOpen(ws); router.handleMessage( @@ -97,6 +113,7 @@ describe("SidecarRouter", () => { agentAddresses: ["agent-a@local"], }), ); + await tick(); router.handleMessage( ws, @@ -107,6 +124,7 @@ describe("SidecarRouter", () => { agentAddresses: ["agent-c@local"], }), ); + await tick(); expect(router.getRoutableAddresses()).toEqual(["agent-c@local"]); }); @@ -149,7 +167,7 @@ describe("SidecarRouter", () => { expect(router.getConnectedSidecars()).toEqual([]); }); - test("re-registration by another sidecar cleans ghost from old connection", () => { + test("re-registration by another sidecar cleans ghost from old connection", async () => { const ws1 = createMockWs(); router.handleOpen(ws1); router.handleMessage( @@ -161,6 +179,7 @@ describe("SidecarRouter", () => { agentAddresses: ["agent@local", "other@local"], }), ); + await tick(); const ws2 = createMockWs(); router.handleOpen(ws2); @@ -173,6 +192,7 @@ describe("SidecarRouter", () => { agentAddresses: ["agent@local"], }), ); + await tick(); // ws2 now owns agent@local. Closing ws1 should only remove // other@local (which ws1 still owns), not agent@local. @@ -185,6 +205,239 @@ describe("SidecarRouter", () => { }); }); + describe("register key-existence gate", () => { + // A register frame is token-authenticated but proves no per-address + // ownership, so it may route only a KEYLESS first-deploy address. An + // address that already has a stored key must prove ownership through the + // challenged reconnect path; register must neither route it to the caller + // nor disturb its existing owner. + const KEYED = "victim@local"; + + function gatedRouter(publicKeyHex: string | null) { + return createSidecarRouter({ + requestTimeoutMs: 5000, + hubPublicKey: TEST_HUB_KEY, + lookups: { + lookupPublicKey: async (addr) => + addr === KEYED ? publicKeyHex : null, + }, + }); + } + + // Bring KEYED up as owned by `ws` through the challenged reconnect path. + async function reconnectVerify( + r: ReturnType, + ws: ReturnType, + privateKey: Uint8Array, + ) { + r.handleOpen(ws); + r.handleMessage( + ws, + JSON.stringify({ + type: "reconnect", + sidecarId: "owner", + token: "tok", + agentAddresses: [KEYED], + }), + ); + await new Promise((res) => setTimeout(res, 50)); + const challenge = ws.sent + .map((s) => JSON.parse(s)) + .find((f: { type: string }) => f.type === "challenge"); + r.handleMessage( + ws, + JSON.stringify({ + type: "challenge.response", + responses: [ + { + address: KEYED, + signature: await signChallenge( + challenge.challenges[0].nonce, + KEYED, + privateKey, + ), + }, + ], + }), + ); + await new Promise((res) => setTimeout(res, 50)); + } + + test("routes a keyless first-deploy address", async () => { + const r = gatedRouter(null); + const ws = createMockWs(); + r.handleOpen(ws); + r.handleMessage( + ws, + JSON.stringify({ + type: "register", + sidecarId: "sc-1", + token: "tok", + agentAddresses: ["fresh@local"], + }), + ); + await tick(); + + expect(r.getRoutableAddresses()).toContain("fresh@local"); + expect(r.routeMail("fresh@local", "dGVzdA==")).toBe(true); + }); + + test("refuses a keyed address and leaves its owner's route untouched", async () => { + const kp = await generateKeyPair(); + const r = gatedRouter(hexEncode(kp.publicKey)); + + // The true owner establishes KEYED via the challenged reconnect. + const ownerWs = createMockWs(); + await reconnectVerify(r, ownerWs, kp.privateKey); + expect(r.getRoutableAddresses()).toContain(KEYED); + + // A rogue sidecar with a valid token names KEYED in a register frame. + const rogueWs = createMockWs(); + r.handleOpen(rogueWs); + r.handleMessage( + rogueWs, + JSON.stringify({ + type: "register", + sidecarId: "rogue", + token: "tok", + agentAddresses: [KEYED], + }), + ); + await tick(); + + // KEYED is still routed to the owner, not the rogue: the gate refused to + // route it AND -- because it runs before the ghost-cleanup -- never + // evicted the owner. No hijack, and no downgrade to a denial of service. + expect(r.getRoutableAddresses()).toContain(KEYED); + ownerWs.sent.length = 0; + rogueWs.sent.length = 0; + expect(r.routeMail(KEYED, "dGVzdA==")).toBe(true); + expect(ownerWs.sent).toHaveLength(1); + expect(rogueWs.sent).toHaveLength(0); + }); + + test("fails closed and surfaces an error when no lookup is configured", async () => { + const captured: { level: string; message: string }[] = []; + const savedConfig = getConfig(); + configureSync({ + reset: true, + sinks: { + capture: (record) => { + const message = Array.isArray(record.message) + ? record.message + .map((part) => + typeof part === "string" ? part : JSON.stringify(part), + ) + .join("") + : String(record.message); + captured.push({ level: record.level, message }); + }, + }, + loggers: [{ category: [], lowestLevel: "debug", sinks: ["capture"] }], + }); + try { + // No lookups configured: the gate cannot distinguish a keyed address + // from a first-deploy, so it must route nothing rather than permit. + const r = createSidecarRouter({ + requestTimeoutMs: 500, + hubPublicKey: TEST_HUB_KEY, + }); + const ws = createMockWs(); + r.handleOpen(ws); + r.handleMessage( + ws, + JSON.stringify({ + type: "register", + sidecarId: "sc-1", + token: "tok", + agentAddresses: ["x@local"], + }), + ); + await tick(); + + expect(r.getRoutableAddresses()).toEqual([]); + expect(r.routeMail("x@local", "dGVzdA==")).toBe(false); + // The missing dependency surfaces as an error, not a silent permit. + expect( + captured.some( + (l) => + l.level === "error" && + l.message.includes("lookupPublicKey is not configured"), + ), + ).toBe(true); + } finally { + if (savedConfig) { + configureSync({ reset: true, ...savedConfig }); + } else { + resetSync(); + } + } + }); + + test("fails closed on a key-lookup error instead of crashing", async () => { + // A rejecting lookup (e.g. a transient DB failure) must be caught and + // surfaced, not floated out of the void-dispatched handler as an + // unhandled rejection that could take down the hub. Fail closed: route + // nothing and log an error. (The test completing rather than hanging on + // an unhandled rejection is itself part of the assertion.) + const captured: { level: string; message: string }[] = []; + const savedConfig = getConfig(); + configureSync({ + reset: true, + sinks: { + capture: (record) => { + const message = Array.isArray(record.message) + ? record.message + .map((part) => + typeof part === "string" ? part : JSON.stringify(part), + ) + .join("") + : String(record.message); + captured.push({ level: record.level, message }); + }, + }, + loggers: [{ category: [], lowestLevel: "debug", sinks: ["capture"] }], + }); + try { + const r = createSidecarRouter({ + requestTimeoutMs: 500, + hubPublicKey: TEST_HUB_KEY, + lookups: { + lookupPublicKey: async () => { + throw new Error("db unavailable"); + }, + }, + }); + const ws = createMockWs(); + r.handleOpen(ws); + r.handleMessage( + ws, + JSON.stringify({ + type: "register", + sidecarId: "sc-1", + token: "tok", + agentAddresses: ["x@local"], + }), + ); + await tick(); + + expect(r.getRoutableAddresses()).toEqual([]); + expect( + captured.some( + (l) => + l.level === "error" && l.message.includes("Key lookup failed"), + ), + ).toBe(true); + } finally { + if (savedConfig) { + configureSync({ reset: true, ...savedConfig }); + } else { + resetSync(); + } + } + }); + }); + describe("workflow-address reconnect (challenged, at parity with launched agents)", () => { // A workflow-substrate deployment address (ins_dep_...) proves ownership // through the SAME Ed25519 challenge as a launched agent: the hub resolves @@ -337,7 +590,9 @@ describe("SidecarRouter", () => { const STEP_ADDR = "ins_dep_multi-step1@local"; const ZERO_SHA = "0".repeat(40); - function registerDeploymentSidecar(): ReturnType { + async function registerDeploymentSidecar(): Promise< + ReturnType + > { const ws = createMockWs(); router.handleOpen(ws); router.handleMessage( @@ -349,11 +604,14 @@ describe("SidecarRouter", () => { agentAddresses: [DEPLOYMENT_ADDR], }), ); + // Register routing is async (the key-existence gate awaits the lookup); + // settle it before assertions read the routing table. + await new Promise((res) => setTimeout(res, 0)); return ws; } test("bind makes a step address routable; unbind removes it", async () => { - registerDeploymentSidecar(); + await registerDeploymentSidecar(); // An unbound step address has no route: `sendPack` rejects before // touching the wire. @@ -454,14 +712,14 @@ describe("SidecarRouter", () => { expect(() => router.bindStepRoute(STEP_ADDR)).toThrow(/No sidecar/); }); - test("unbind is a no-op for an address that was never bound", () => { - registerDeploymentSidecar(); + test("unbind is a no-op for an address that was never bound", async () => { + await registerDeploymentSidecar(); expect(() => router.unbindStepRoute(STEP_ADDR)).not.toThrow(); expect(router.getRoutableAddresses()).not.toContain(STEP_ADDR); }); - test("handleClose reclaims a still-bound step route", () => { - const ws = registerDeploymentSidecar(); + test("handleClose reclaims a still-bound step route", async () => { + const ws = await registerDeploymentSidecar(); router.bindStepRoute(STEP_ADDR); expect(router.getRoutableAddresses()).toContain(STEP_ADDR); @@ -473,7 +731,7 @@ describe("SidecarRouter", () => { }); describe("mail routing", () => { - test("routes mail between two sidecars", () => { + test("routes mail between two sidecars", async () => { const ws1 = createMockWs(); const ws2 = createMockWs(); router.handleOpen(ws1); @@ -497,6 +755,7 @@ describe("SidecarRouter", () => { agentAddresses: ["receiver@local"], }), ); + await tick(); router.handleMessage( ws1, @@ -517,7 +776,7 @@ describe("SidecarRouter", () => { expect(router.routeMail("nobody@local", "dGVzdA==")).toBe(false); }); - test("routeMail returns true for routable address", () => { + test("routeMail returns true for routable address", async () => { const ws = createMockWs(); router.handleOpen(ws); router.handleMessage( @@ -529,6 +788,7 @@ describe("SidecarRouter", () => { agentAddresses: ["agent@local"], }), ); + await tick(); expect(router.routeMail("agent@local", "dGVzdA==")).toBe(true); const delivered = lastSent(ws); @@ -537,7 +797,9 @@ describe("SidecarRouter", () => { test("unroutable mail emits mail.outbound.undelivered", () => { const outbound: { rawMessage: string; recipients: string[] }[] = []; - const router = createSidecarRouter({}); + const router = createSidecarRouter({ + lookups: { lookupPublicKey: async () => null }, + }); router.events.on("mail.outbound.undelivered", (event) => { outbound.push({ rawMessage: event.rawMessage, @@ -578,6 +840,7 @@ describe("SidecarRouter", () => { const persisted: { id: string; address: string }[] = []; const router = createSidecarRouter({ lookups: { + lookupPublicKey: async () => null, persistMail: async ({ senderAddress, recipients }) => { // Mirrors the fixed persistMail: always create the // outbound record for the sender, and only create @@ -656,6 +919,7 @@ describe("SidecarRouter", () => { const persisted: { id: string; address: string }[] = []; const router = createSidecarRouter({ lookups: { + lookupPublicKey: async () => null, persistMail: async ({ senderAddress, recipients }) => { return [ { @@ -946,6 +1210,7 @@ describe("SidecarRouter", () => { agentAddresses: ["agent@local"], }), ); + await tick(); const promise = router.sendAgentUndeploy("agent@local", "session_ended"); const frame = lastSent(ws); @@ -1026,6 +1291,7 @@ describe("SidecarRouter", () => { agentAddresses: ["undeploy-dc@local"], }), ); + await tick(); // Start an undeploy but disconnect before the ack arrives. const promise = router.sendAgentUndeploy("undeploy-dc@local", "teardown"); @@ -1088,6 +1354,7 @@ describe("SidecarRouter", () => { agentAddresses: ["agent@local"], }), ); + await tick(); const promise = router.sendSourcesUpdate( "agent@local", @@ -1129,6 +1396,7 @@ describe("SidecarRouter", () => { agentAddresses: ["agent@local"], }), ); + await tick(); const promise = router.sendSourcesUpdate( "agent@local", @@ -1164,7 +1432,9 @@ describe("SidecarRouter", () => { describe("agent events", () => { test("agent.event frames are forwarded to subscribers", () => { const events: { addr: string; sid: string; event: unknown }[] = []; - const router = createSidecarRouter({}); + const router = createSidecarRouter({ + lookups: { lookupPublicKey: async () => null }, + }); router.events.on("agent.event", ({ agentAddress, sessionId, event }) => { events.push({ addr: agentAddress, sid: sessionId, event }); }); @@ -1202,7 +1472,9 @@ describe("SidecarRouter", () => { test("agent.event frames are emitted on router.events", () => { const seen: { addr: string; sid: string }[] = []; - const router = createSidecarRouter({}); + const router = createSidecarRouter({ + lookups: { lookupPublicKey: async () => null }, + }); router.events.on("agent.event", ({ agentAddress, sessionId }) => { seen.push({ addr: agentAddress, sid: sessionId }); }); @@ -1231,8 +1503,10 @@ describe("SidecarRouter", () => { expect(seen).toEqual([{ addr: "agent@local", sid: "sess-1" }]); }); - test("sidecar.disconnect is emitted on router.events", () => { - const router = createSidecarRouter({}); + test("sidecar.disconnect is emitted on router.events", async () => { + const router = createSidecarRouter({ + lookups: { lookupPublicKey: async () => null }, + }); const seen: string[][] = []; router.events.on("sidecar.disconnect", ({ ownedAddresses }) => { seen.push(ownedAddresses); @@ -1249,13 +1523,16 @@ describe("SidecarRouter", () => { agentAddresses: ["agent@local"], }), ); + await tick(); router.handleClose(ws); expect(seen).toEqual([["agent@local"]]); }); - test("connector.state.changed populates the cache and is readable via getConnectorState", () => { - const router = createSidecarRouter({}); + test("connector.state.changed populates the cache and is readable via getConnectorState", async () => { + const router = createSidecarRouter({ + lookups: { lookupPublicKey: async () => null }, + }); const ws = createMockWs(); router.handleOpen(ws); router.handleMessage( @@ -1267,6 +1544,7 @@ describe("SidecarRouter", () => { agentAddresses: ["agent@local"], }), ); + await tick(); // Before any state frame, the cache is absent → null. expect(router.getConnectorState("agent@local")).toBeNull(); @@ -1301,8 +1579,10 @@ describe("SidecarRouter", () => { expect(router.getConnectorState("agent@local")).toBeNull(); }); - test("connector.state.changed is emitted on router.events", () => { - const router = createSidecarRouter({}); + test("connector.state.changed is emitted on router.events", async () => { + const router = createSidecarRouter({ + lookups: { lookupPublicKey: async () => null }, + }); const seen: { addr: string; state: unknown }[] = []; router.events.on( "connector.state.changed", @@ -1322,6 +1602,7 @@ describe("SidecarRouter", () => { agentAddresses: ["agent@local"], }), ); + await tick(); const state = { threadRoot: "", @@ -1342,8 +1623,10 @@ describe("SidecarRouter", () => { expect(seen).toEqual([{ addr: "agent@local", state }]); }); - test("live takeover via register evicts the prior owner's cached connector state", () => { - const router = createSidecarRouter({}); + test("live takeover via register evicts the prior owner's cached connector state", async () => { + const router = createSidecarRouter({ + lookups: { lookupPublicKey: async () => null }, + }); // First sidecar registers and reports connector state. const ws1 = createMockWs(); @@ -1357,6 +1640,7 @@ describe("SidecarRouter", () => { agentAddresses: ["agent@local"], }), ); + await tick(); router.handleMessage( ws1, JSON.stringify({ @@ -1387,12 +1671,15 @@ describe("SidecarRouter", () => { agentAddresses: ["agent@local"], }), ); + await tick(); expect(router.getConnectorState("agent@local")).toBeNull(); }); - test("disconnect clears cached connector state for affected agents", () => { - const router = createSidecarRouter({}); + test("disconnect clears cached connector state for affected agents", async () => { + const router = createSidecarRouter({ + lookups: { lookupPublicKey: async () => null }, + }); const ws = createMockWs(); router.handleOpen(ws); router.handleMessage( @@ -1404,6 +1691,7 @@ describe("SidecarRouter", () => { agentAddresses: ["agent@local"], }), ); + await tick(); router.handleMessage( ws, @@ -2258,6 +2546,46 @@ describe("SidecarRouter", () => { }); describe("disconnect message queuing", () => { + // Establish an initial route for a KEYED address through the challenged + // reconnect path. A keyed address cannot be routed by a plain register + // (the key-existence gate rejects it), so the disconnect-queue tests -- + // whose agents have a stored key so the reconnect challenge can verify -- + // bring the address up the same way production does. + async function connectAgentViaChallenge( + r: ReturnType, + ws: ReturnType, + addr: string, + privateKey: Uint8Array, + ) { + r.handleOpen(ws); + r.handleMessage( + ws, + JSON.stringify({ + type: "reconnect", + sidecarId: "sc-1", + token: "tok", + agentAddresses: [addr], + }), + ); + await new Promise((res) => setTimeout(res, 50)); + const challengeFrame = ws.sent + .map((s) => JSON.parse(s)) + .find((f: { type: string }) => f.type === "challenge"); + const responses = await Promise.all( + challengeFrame.challenges.map( + async (c: { address: string; nonce: string }) => ({ + address: c.address, + signature: await signChallenge(c.nonce, c.address, privateKey), + }), + ), + ); + r.handleMessage( + ws, + JSON.stringify({ type: "challenge.response", responses }), + ); + await new Promise((res) => setTimeout(res, 50)); + } + test("mail queued during disconnect is flushed on reconnect", async () => { const kp = await generateKeyPair(); const router = createSidecarRouter({ @@ -2269,18 +2597,10 @@ describe("SidecarRouter", () => { }, }); - // Initial connection with one agent. + // Initial connection with one agent, via the challenged reconnect path + // (a keyed address cannot be routed by a plain register). const ws1 = createMockWs(); - router.handleOpen(ws1); - router.handleMessage( - ws1, - JSON.stringify({ - type: "register", - sidecarId: "sc-1", - token: "tok", - agentAddresses: ["agent@local"], - }), - ); + await connectAgentViaChallenge(router, ws1, "agent@local", kp.privateKey); // Disconnect — creates a queue entry. router.handleClose(ws1); @@ -2349,16 +2669,7 @@ describe("SidecarRouter", () => { }); const ws = createMockWs(); - router.handleOpen(ws); - router.handleMessage( - ws, - JSON.stringify({ - type: "register", - sidecarId: "sc-1", - token: "tok", - agentAddresses: ["agent@local"], - }), - ); + await connectAgentViaChallenge(router, ws, "agent@local", kp.privateKey); router.handleClose(ws); // Queue 3 messages with max size 2 — oldest should be evicted. @@ -2461,6 +2772,7 @@ describe("SidecarRouter", () => { const router = createSidecarRouter({ requestTimeoutMs: 500, pingTimeoutMs: 100, + lookups: { lookupPublicKey: async () => null }, }); const ws = createMockWs(); @@ -2757,6 +3069,10 @@ describe("SidecarRouter", () => { requestTimeoutMs: 500, hubPublicKey: TEST_HUB_KEY, lookups: { + // Keyless lookup: the pack tests register the deployment address as a + // first-deploy so the key-existence gate routes it (the gate is + // fail-closed without a lookup configured). + lookupPublicKey: async () => null, async receiveAgentStatePack(repoId, pack, ref, commitSha) { calls.push({ method: "receiveAgentStatePack", @@ -2782,7 +3098,7 @@ describe("SidecarRouter", () => { return { router: packRouter, calls }; } - function registerAddr( + async function registerAddr( r: ReturnType, ws: ReturnType, sidecarId: string, @@ -2798,6 +3114,9 @@ describe("SidecarRouter", () => { agentAddresses: [addr], }), ); + // Register routing is async (the key-existence gate awaits the lookup); + // settle it before assertions read the routing table. + await new Promise((res) => setTimeout(res, 0)); } function pushPack( @@ -2842,7 +3161,7 @@ describe("SidecarRouter", () => { const { router: r, calls } = buildPackRouter(); const ws = createMockWs(); const addr = "agent-wfr@local"; - registerAddr(r, ws, "sc-wfr", addr); + await registerAddr(r, ws, "sc-wfr", addr); const transferId = "t-wfr-1"; const repoId: RepoId = { kind: "workflow-run", id: "dep-wfr-1" }; @@ -2889,7 +3208,7 @@ describe("SidecarRouter", () => { const { router: r, calls } = buildPackRouter(); const ws = createMockWs(); const addr = "ins_dep_wfr@local"; - registerAddr(r, ws, "sc-wfr", addr); + await registerAddr(r, ws, "sc-wfr", addr); const repoId: RepoId = { kind: "workflow-run", id: "dep-wfr-2" }; pushPack(r, ws, { @@ -2923,10 +3242,10 @@ describe("SidecarRouter", () => { const transferId = "t-reclaim"; const oldWs = createMockWs(); - registerAddr(r, oldWs, "sc", addr); + await registerAddr(r, oldWs, "sc", addr); const newWs = createMockWs(); - registerAddr(r, newWs, "sc", addr); + await registerAddr(r, newWs, "sc", addr); // The new owner starts (but does not finish) a workflow-run transfer. for (const chunk of chunkPack(pack)) { @@ -3071,7 +3390,7 @@ describe("SidecarRouter", () => { const { router: r, calls } = buildPackRouter(); const ws = createMockWs(); const addr = "agent-mix@local"; - registerAddr(r, ws, "sc-mix", addr); + await registerAddr(r, ws, "sc-mix", addr); // Reuse the same transferId across kinds. The two receivers' // in-flight state must be independent, so this must NOT collide. @@ -3165,7 +3484,7 @@ describe("SidecarRouter", () => { }); const ws = createMockWs(); const addr = "agent-wfr-rej@local"; - registerAddr(r, ws, "sc-wfr-rej", addr); + await registerAddr(r, ws, "sc-wfr-rej", addr); const transferId = "t-wfr-rej"; const repoId: RepoId = { kind: "workflow-run", id: "dep-wfr-rej" }; @@ -3189,7 +3508,7 @@ describe("SidecarRouter", () => { }); describe("sendDrain", () => { - test("ships a drain.deliver frame to the sidecar holding the deployment", () => { + test("ships a drain.deliver frame to the sidecar holding the deployment", async () => { const ws = createMockWs(); router.handleOpen(ws); router.handleMessage( @@ -3201,6 +3520,7 @@ describe("SidecarRouter", () => { agentAddresses: ["dep@integration.interchange"], }), ); + await tick(); ws.sent.length = 0; router.sendDrain({ diff --git a/packages/hub-sessions/src/ws/sidecar-handler.ts b/packages/hub-sessions/src/ws/sidecar-handler.ts index cc4eb491..47ce9f27 100644 --- a/packages/hub-sessions/src/ws/sidecar-handler.ts +++ b/packages/hub-sessions/src/ws/sidecar-handler.ts @@ -413,7 +413,12 @@ export function createSidecarRouter( switch (frame.type) { case "register": - handleRegister(ws, frame.sidecarId, frame.token, frame.agentAddresses); + void handleRegister( + ws, + frame.sidecarId, + frame.token, + frame.agentAddresses, + ); break; case "reconnect": void handleReconnect( @@ -524,18 +529,56 @@ export function createSidecarRouter( } } - function handleRegister( + async function handleRegister( ws: WsHandle, sidecarId: string, token: string, agentAddresses: string[], - ): void { + ): Promise { if (validateToken !== undefined && !validateToken(sidecarId, token)) { logger.warn`Rejected registration from sidecar ${sidecarId}: invalid token`; ws.close(); return; } + // Key-existence gate. A register frame is token-authenticated but carries + // no per-address ownership proof, so it may route an address ONLY if that + // address has no stored key yet -- a genuine keyless first-deploy (the + // token-bounded first-deploy trust model). An address that already has a + // key must prove ownership through the challenged reconnect path; routing + // it here on token auth alone is the register-frame sibling of the + // reconnect hijack. The keyless-only set is computed up front, BEFORE the + // ghost-cleanup and every routing mutation below, so a rejected address + // touches nothing: no eviction of a live owner, hence no downgrade from + // hijack to denial-of-service on the victim. + const lookupKey = lookups.lookupPublicKey; + const routableAddresses: string[] = []; + for (const addr of agentAddresses) { + if (lookupKey === undefined) { + // Fail closed: without the ownership lookup a keyed address cannot be + // told apart from a first-deploy, so route nothing and surface the + // misconfiguration. Empty first-connect registers never reach here. + logger.error`Cannot gate register routing for ${addr}: lookupPublicKey is not configured; refusing to route (challenged reconnect required)`; + continue; + } + let existingKey: string | null; + try { + existingKey = await lookupKey(addr); + } catch (err) { + // Fail closed on a lookup error (e.g. a transient DB failure): route + // nothing for this address and surface the failure, rather than let + // the rejection float out of this void-dispatched handler and take + // down the hub. + logger.error`Key lookup failed for ${addr} during register: ${err instanceof Error ? err.message : String(err)}; failing closed (challenged reconnect required)`; + continue; + } + if (existingKey !== null) { + logger.warn`Refusing to route ${addr} via register: address already has a stored key; ownership must be proven via challenged reconnect`; + continue; + } + routableAddresses.push(addr); + } + // If this same ws is re-registering, drop its addressIndex entries so // the new register/reconnect rebuilds the owned set (handleReconnect // and the loop below re-add the addresses that remain owned). Do not @@ -544,6 +587,15 @@ export function createSidecarRouter( // current. A genuinely restarting harness opens a new ws (so existing // is undefined) and its stale state is cleared by the cross-ws ghost // loop below and by handleClose. + // + // An address this ws already proved via challenged reconnect (so it now + // carries a stored key) is dropped here and NOT re-added, because the gate + // above refuses a keyed address on register. A client that re-registers a + // keyed address on a live connection therefore self-evicts that route + // until its next reconnect. That is the intended contract -- keyed + // addresses route only via challenged reconnect, the honest client sends + // an empty register, and the harm is self-inflicted (a register frame can + // only name this connection's own routes, not another connection's). const existing = connections.get(ws); if (existing !== undefined) { for (const addr of existing.agentAddresses) { @@ -554,7 +606,7 @@ export function createSidecarRouter( } } - const addrSet = new Set(agentAddresses); + const addrSet = new Set(routableAddresses); // The connection's workflow-address set starts empty and is populated // only from VERIFIED reconnect addresses (post-challenge) and from // `bindStepRoute`'s transient per-step routes -- never from an @@ -612,15 +664,14 @@ export function createSidecarRouter( disconnectedAgents.delete(addr); } } - // Register does NOT route any workflow-substrate address: a restored - // deployment address is announced through the challenged reconnect frame - // and enters `addressIndex` only after its ownership challenge passes. - // Routing an address here from an unauthenticated register frame is the - // exact bypass that let a token-holding sidecar reclaim a victim's - // address, so this handler writes only the (empty, first-connect) session - // set above. + // Only keyless first-deploy addresses reach `addressIndex` here; an + // address that already has a stored key was filtered out by the gate + // above and must re-enter routing through the challenged reconnect path. + // Because the gate runs before the ghost-cleanup, a rejected (keyed) + // address never evicts its prior owner -- register cannot reclaim or + // disrupt a victim's route on token auth alone. - logger.info`Sidecar ${sidecarId} registered with ${String(agentAddresses.length)} agents`; + logger.info`Sidecar ${sidecarId} registered; routed ${String(addrSet.size)} of ${String(agentAddresses.length)} address(es) (keyless first-deploy only)`; } async function handleReconnect( @@ -664,7 +715,8 @@ export function createSidecarRouter( // can receive frames while the ownership challenge is pending. Every // reconnect address -- session and workflow-derived alike -- enters // routing only through the verified path below, never unchallenged here. - handleRegister(ws, sidecarId, token, []); + // Empty address list, so the key-existence gate has nothing to await. + await handleRegister(ws, sidecarId, token, []); const conn = connections.get(ws); if (conn === undefined) return; From a041cc646d76c52540bc0b5b89a3c39dd7b0b1a6 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Mon, 6 Jul 2026 23:44:54 -0500 Subject: [PATCH 084/101] Fail closed on a key-lookup error during reconnect The reconnect challenge looks up each claimed address's public key with `await lookupKey(addr)` inside a `Promise.all`, unguarded, and the handler is dispatched as `void handleReconnect(...)`. A rejecting lookup -- a transient database failure on the production reconnect path, which every honest sidecar exercises -- floats out as an unhandled promise rejection that can terminate the hub. Catch the lookup error per address and fail closed: treat the address as unverifiable (null key), so it fails its challenge and stays unrouted, and log the failure. The hub keeps running and the sidecar retries on its next reconnect. This mirrors the register key-existence gate, which fails closed on the same error. --- .../src/ws/sidecar-handler.test.ts | 68 +++++++++++++++++++ .../hub-sessions/src/ws/sidecar-handler.ts | 18 +++-- 2 files changed, 81 insertions(+), 5 deletions(-) diff --git a/packages/hub-sessions/src/ws/sidecar-handler.test.ts b/packages/hub-sessions/src/ws/sidecar-handler.test.ts index fe7d38d0..140c9f0a 100644 --- a/packages/hub-sessions/src/ws/sidecar-handler.test.ts +++ b/packages/hub-sessions/src/ws/sidecar-handler.test.ts @@ -2242,6 +2242,74 @@ describe("SidecarRouter", () => { expect(failedFrame.reason).toBe("Unknown agent address"); }); + test("fails closed on a key-lookup error during reconnect instead of crashing", async () => { + // A rejecting lookup (e.g. a transient DB failure) on the production + // reconnect path must be caught and surfaced, not floated out of the + // void-dispatched handler as an unhandled rejection that could take down + // the hub. Fail closed: the address is treated as unverifiable, fails + // its challenge, and stays unrouted. (The test completing rather than + // hanging on an unhandled rejection is itself part of the assertion.) + const captured: { level: string; message: string }[] = []; + const savedConfig = getConfig(); + configureSync({ + reset: true, + sinks: { + capture: (record) => { + const message = Array.isArray(record.message) + ? record.message + .map((part) => + typeof part === "string" ? part : JSON.stringify(part), + ) + .join("") + : String(record.message); + captured.push({ level: record.level, message }); + }, + }, + loggers: [{ category: [], lowestLevel: "debug", sinks: ["capture"] }], + }); + try { + const router = createSidecarRouter({ + requestTimeoutMs: 5000, + lookups: { + lookupPublicKey: async () => { + throw new Error("db unavailable"); + }, + }, + }); + const ws = createMockWs(); + router.handleOpen(ws); + router.handleMessage( + ws, + JSON.stringify({ + type: "reconnect", + sidecarId: "sc-1", + token: "tok", + agentAddresses: ["agent@local"], + }), + ); + + await new Promise((r) => setTimeout(r, 50)); + + expect(router.getRoutableAddresses()).not.toContain("agent@local"); + const failedFrame = ws.sent + .map((s) => JSON.parse(s)) + .find((f: { type: string }) => f.type === "challenge.failed"); + expect(failedFrame?.address).toBe("agent@local"); + expect( + captured.some( + (l) => + l.level === "error" && l.message.includes("Key lookup failed"), + ), + ).toBe(true); + } finally { + if (savedConfig) { + configureSync({ reset: true, ...savedConfig }); + } else { + resetSync(); + } + } + }); + test("partial success routes verified addresses only", async () => { const kp1 = await generateKeyPair(); const kp2 = await generateKeyPair(); diff --git a/packages/hub-sessions/src/ws/sidecar-handler.ts b/packages/hub-sessions/src/ws/sidecar-handler.ts index 47ce9f27..e7a5234c 100644 --- a/packages/hub-sessions/src/ws/sidecar-handler.ts +++ b/packages/hub-sessions/src/ws/sidecar-handler.ts @@ -731,12 +731,20 @@ export function createSidecarRouter( addressIndex.set(addr, ws); } - // Look up stored public keys for all claimed addresses. + // Look up stored public keys for all claimed addresses. Fail closed on a + // lookup error (e.g. a transient DB failure): treat the address as + // unverifiable so it fails its challenge and stays unrouted, rather than + // letting the rejection float out of this void-dispatched handler as an + // unhandled rejection that could take down the hub. const keyLookups = await Promise.all( - agentAddresses.map(async (addr) => ({ - address: addr, - publicKeyHex: await lookupKey(addr), - })), + agentAddresses.map(async (addr) => { + try { + return { address: addr, publicKeyHex: await lookupKey(addr) }; + } catch (err) { + logger.error`Key lookup failed for ${addr} during reconnect: ${err instanceof Error ? err.message : String(err)}; failing closed`; + return { address: addr, publicKeyHex: null }; + } + }), ); // If the connection was closed or superseded while we were awaiting From 6e89b984c64d6c0b38062d76f095f48ed9c1102f Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Tue, 7 Jul 2026 00:20:07 -0500 Subject: [PATCH 085/101] Make register additive instead of replacing the address set `handleRegister` cleared the connection's entire owned set on every re-register and rebuilt it from the frame's addresses. Since the gate routes only keyless first-deploys, a re-register on a live connection dropped every keyed route the connection had proved via challenged reconnect -- the connection self-evicted its own routes until its next reconnect. A sidecar that first-deploys agent-a and later first-deploys agent-c would also lose agent-a's route to the second register. Make re-register additive: the connection inherits every address it already owns and ADDS the frame's keyless first-deploys. Register never drops an owned route; removal happens via undeploy or disconnect, not register-omission. The cross-connection ghost-cleanup still runs for the newly-claimed addresses (a first-deploy claimed from another ws still evicts that ws), but a connection's own inherited routes are left alone. This is the correct contract under the challenged model: a register frame no longer carries the sidecar's complete live set (keyed addresses arrive via reconnect), so an omitted address no longer means "removed." --- .../src/ws/sidecar-handler.test.ts | 41 +++++++++++- .../hub-sessions/src/ws/sidecar-handler.ts | 66 +++++++++---------- 2 files changed, 69 insertions(+), 38 deletions(-) diff --git a/packages/hub-sessions/src/ws/sidecar-handler.test.ts b/packages/hub-sessions/src/ws/sidecar-handler.test.ts index 140c9f0a..94aad06a 100644 --- a/packages/hub-sessions/src/ws/sidecar-handler.test.ts +++ b/packages/hub-sessions/src/ws/sidecar-handler.test.ts @@ -101,7 +101,10 @@ describe("SidecarRouter", () => { ]); }); - test("re-registration updates addresses", async () => { + test("re-registration adds addresses without dropping owned routes", async () => { + // Additive re-register: a second register ADDS its addresses without + // dropping those from the first. Removal happens via undeploy/disconnect, + // not register-omission (the frame no longer carries the full live set). const ws = createMockWs(); router.handleOpen(ws); router.handleMessage( @@ -126,7 +129,10 @@ describe("SidecarRouter", () => { ); await tick(); - expect(router.getRoutableAddresses()).toEqual(["agent-c@local"]); + expect(router.getRoutableAddresses().sort()).toEqual([ + "agent-a@local", + "agent-c@local", + ]); }); test("disconnect cleans up routing table", () => { @@ -316,6 +322,37 @@ describe("SidecarRouter", () => { expect(rogueWs.sent).toHaveLength(0); }); + test("a re-register does not drop an address already verified via reconnect", async () => { + // A connection that proved a keyed address via challenged reconnect keeps + // that route across a later register, even though the gate would refuse + // to re-add a keyed address: additive re-register inherits the owned set + // rather than replacing it. Removal is via undeploy/disconnect, not + // register-omission. + const kp = await generateKeyPair(); + const r = gatedRouter(hexEncode(kp.publicKey)); + const ws = createMockWs(); + await reconnectVerify(r, ws, kp.privateKey); + expect(r.getRoutableAddresses()).toContain(KEYED); + + // The same ws registers a keyless first-deploy. The keyed + // reconnect-verified route must survive alongside the new one. + r.handleMessage( + ws, + JSON.stringify({ + type: "register", + sidecarId: "owner", + token: "tok", + agentAddresses: ["fresh@local"], + }), + ); + await tick(); + + expect(r.getRoutableAddresses().sort()).toEqual( + [KEYED, "fresh@local"].sort(), + ); + expect(r.routeMail(KEYED, "dGVzdA==")).toBe(true); + }); + test("fails closed and surfaces an error when no lookup is configured", async () => { const captured: { level: string; message: string }[] = []; const savedConfig = getConfig(); diff --git a/packages/hub-sessions/src/ws/sidecar-handler.ts b/packages/hub-sessions/src/ws/sidecar-handler.ts index e7a5234c..babd2e5b 100644 --- a/packages/hub-sessions/src/ws/sidecar-handler.ts +++ b/packages/hub-sessions/src/ws/sidecar-handler.ts @@ -579,43 +579,26 @@ export function createSidecarRouter( routableAddresses.push(addr); } - // If this same ws is re-registering, drop its addressIndex entries so - // the new register/reconnect rebuilds the owned set (handleReconnect - // and the loop below re-add the addresses that remain owned). Do not - // drop connectorStates here: this branch is reached only on a same-ws - // re-register, where the harness is live and its connector state is - // current. A genuinely restarting harness opens a new ws (so existing - // is undefined) and its stale state is cleared by the cross-ws ghost - // loop below and by handleClose. + // Additive re-register: inherit every address this ws already owns and ADD + // the frame's keyless first-deploys. Register never drops an owned route -- + // an address proved via challenged reconnect stays routed, and removal + // happens via undeploy/disconnect, not register-omission. // - // An address this ws already proved via challenged reconnect (so it now - // carries a stored key) is dropped here and NOT re-added, because the gate - // above refuses a keyed address on register. A client that re-registers a - // keyed address on a live connection therefore self-evicts that route - // until its next reconnect. That is the intended contract -- keyed - // addresses route only via challenged reconnect, the honest client sends - // an empty register, and the harm is self-inflicted (a register frame can - // only name this connection's own routes, not another connection's). + // (Under the retired full-set model a register frame carried the sidecar's + // complete live set, so an omitted address meant "removed". Now the frame + // carries only keyless first-deploys the sidecar is adding -- keyed + // addresses arrive via reconnect -- so omission is meaningless, and a + // replace-on-register would wrongly drop an earlier first-deploy when a + // later one is registered, as well as any reconnect-verified keyed route.) const existing = connections.get(ws); - if (existing !== undefined) { - for (const addr of existing.agentAddresses) { - addressIndex.delete(addr); - } - for (const addr of existing.workflowAddresses) { - addressIndex.delete(addr); - } - } - - const addrSet = new Set(routableAddresses); - // The connection's workflow-address set starts empty and is populated - // only from VERIFIED reconnect addresses (post-challenge) and from - // `bindStepRoute`'s transient per-step routes -- never from an - // unauthenticated frame field. It backs `handleClose` reclamation. - const workflowSet = new Set(); - - // Clean up ghost entries from other connections that previously - // owned addresses this sidecar is now claiming. - for (const addr of addrSet) { + const inheritedAgent = new Set(existing?.agentAddresses); + const inheritedWorkflow = new Set(existing?.workflowAddresses); + + // Clean up ghost entries from OTHER connections for the newly-claimed + // addresses only. An inherited address is already owned by this ws + // (prevWs === ws), so it needs no eviction and its in-flight deploy must + // not be cancelled. + for (const addr of routableAddresses) { const prevWs = addressIndex.get(addr); if (prevWs !== undefined && prevWs !== ws) { const prevConn = connections.get(prevWs); @@ -640,6 +623,15 @@ export function createSidecarRouter( } } + // New keyless first-deploys join the inherited session set; the workflow + // set is inherited unchanged (register never adds to it -- a + // workflow-derived address routes only through the challenged reconnect). + for (const addr of routableAddresses) { + inheritedAgent.add(addr); + } + const addrSet = inheritedAgent; + const workflowSet = inheritedWorkflow; + const conn: SidecarConnection = { sidecarId, agentAddresses: addrSet, @@ -650,7 +642,9 @@ export function createSidecarRouter( }; connections.set(ws, conn); - for (const addr of addrSet) { + // Only the newly-claimed addresses need a routing write + queue discard; + // inherited addresses already point at this ws in addressIndex. + for (const addr of routableAddresses) { addressIndex.set(addr, ws); // Discard any disconnect queue — register has no identity // verification, so flushing to an unverified connection is unsafe. From b00e9f7fddd3a03c5d6cf80239b5921241bc640e Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Tue, 7 Jul 2026 01:17:35 -0500 Subject: [PATCH 086/101] Serialize per-connection frame dispatch to close the register race Gating register routing on an async key lookup means a frame that depends on the just-registered route -- connector.state.changed, mail, a pack push -- can arrive during the lookup window and be dropped, because addressIndex is not yet populated when its handler runs. Serialize each connection's frame dispatch: a frame that establishes or reads routing waits on a per-ws promise chain for earlier such frames to finish, so it observes their completed effects. Frames that are terminal responses to an already-issued outbound request (session.ack/error, agent.deploy.ack, agent.error, agent.undeploy.ack, repo.pack.ack/reject) plus liveness (ping) BYPASS the chain: they resolve the very promises an in-flight queued handler blocks on, so queuing them would deadlock the challenge round-trip (challenge.response's agent.reconnected reaction runs sendSourcesUpdate, which awaits a later session.ack). A frameBypassesQueue switch classifies every frame variant, with an assertNever default, so a new frame type is a compile error rather than a latent deadlock or a silent bypass hole. The bypass set is safe on two counts: every frame that can resolve a pending request is in it (no queued handler can wedge on a queued response), and no bypass handler reads addressIndex or connectorStates (so the poisoning-vector connector.state.changed stays queued and cannot be poisoned out of band). The deadlock guard is mutation-checked: routing session.ack through the queue instead of bypass wedges the reconnect, so the bypass classification is load-bearing, not a latency nicety. The routing-mechanics unit tests that asserted synchronously now settle the now-async dispatch first. --- .../src/ws/sidecar-handler.test.ts | 231 +++++++++++++++++- .../hub-sessions/src/ws/sidecar-handler.ts | 151 +++++++++--- 2 files changed, 339 insertions(+), 43 deletions(-) diff --git a/packages/hub-sessions/src/ws/sidecar-handler.test.ts b/packages/hub-sessions/src/ws/sidecar-handler.test.ts index 94aad06a..e8094fd3 100644 --- a/packages/hub-sessions/src/ws/sidecar-handler.test.ts +++ b/packages/hub-sessions/src/ws/sidecar-handler.test.ts @@ -153,7 +153,7 @@ describe("SidecarRouter", () => { expect(router.getRoutableAddresses()).toEqual([]); }); - test("invalid token closes connection", () => { + test("invalid token closes connection", async () => { const router = createSidecarRouter({ validateToken: () => false, }); @@ -168,6 +168,7 @@ describe("SidecarRouter", () => { agentAddresses: [], }), ); + await tick(); expect(ws.closed).toBe(true); expect(router.getConnectedSidecars()).toEqual([]); @@ -475,6 +476,191 @@ describe("SidecarRouter", () => { }); }); + describe("frame dispatch serialization (F1)", () => { + const STATE = { + threadRoot: "", + lastMessageId: "", + replyTo: "user@example.com", + cc: [], + }; + + async function driveReconnect( + r: ReturnType, + ws: ReturnType, + addr: string, + privateKey: Uint8Array, + ) { + r.handleMessage( + ws, + JSON.stringify({ + type: "reconnect", + sidecarId: "sc", + token: "tok", + agentAddresses: [addr], + }), + ); + await new Promise((res) => setTimeout(res, 50)); + const challenge = ws.sent + .map((s) => JSON.parse(s)) + .find((f: { type: string }) => f.type === "challenge"); + r.handleMessage( + ws, + JSON.stringify({ + type: "challenge.response", + responses: [ + { + address: addr, + signature: await signChallenge( + challenge.challenges[0].nonce, + addr, + privateKey, + ), + }, + ], + }), + ); + await new Promise((res) => setTimeout(res, 50)); + } + + test("a dependent frame after a non-empty register is processed, not dropped", async () => { + // The register key-existence gate awaits lookupPublicKey, so routing lands + // asynchronously. Per-ws serialization makes a following dependent frame + // (connector.state.changed for the just-registered keyless address) wait + // for that routing rather than be dropped because addressIndex was empty. + const r = createSidecarRouter({ + requestTimeoutMs: 500, + hubPublicKey: TEST_HUB_KEY, + lookups: { lookupPublicKey: async () => null }, + }); + const ws = createMockWs(); + r.handleOpen(ws); + // Non-empty register immediately followed by a dependent frame, with NO + // await between -- the follower must still observe the route. + r.handleMessage( + ws, + JSON.stringify({ + type: "register", + sidecarId: "sc", + token: "tok", + agentAddresses: ["fresh@local"], + }), + ); + r.handleMessage( + ws, + JSON.stringify({ + type: "connector.state.changed", + agentAddress: "fresh@local", + connectorState: STATE, + }), + ); + await tick(); + + expect(r.getConnectorState("fresh@local")).toEqual(STATE); + }); + + test("a rogue's dependent frame for a keyed victim is dropped, not poisoned", async () => { + // Serialization must NOT reintroduce the poisoning it exists to prevent: a + // rogue's non-empty register naming a KEYED victim is refused by the gate, + // so a connector.state.changed from the rogue in the same window still + // sees addressIndex.get(victim) !== rogueWs and is dropped; the victim's + // cached state is untouched. + const kp = await generateKeyPair(); + const victim = "victim@local"; + const r = createSidecarRouter({ + requestTimeoutMs: 5000, + hubPublicKey: TEST_HUB_KEY, + lookups: { + lookupPublicKey: async (addr) => + addr === victim ? hexEncode(kp.publicKey) : null, + }, + }); + const ownerWs = createMockWs(); + r.handleOpen(ownerWs); + await driveReconnect(r, ownerWs, victim, kp.privateKey); + r.handleMessage( + ownerWs, + JSON.stringify({ + type: "connector.state.changed", + agentAddress: victim, + connectorState: STATE, + }), + ); + await tick(); + expect(r.getConnectorState(victim)).toEqual(STATE); + + // Rogue: register naming the keyed victim + a poisoning frame, no await. + const rogueWs = createMockWs(); + r.handleOpen(rogueWs); + r.handleMessage( + rogueWs, + JSON.stringify({ + type: "register", + sidecarId: "rogue", + token: "tok", + agentAddresses: [victim], + }), + ); + r.handleMessage( + rogueWs, + JSON.stringify({ + type: "connector.state.changed", + agentAddress: victim, + connectorState: { ...STATE, replyTo: "attacker@evil" }, + }), + ); + await tick(); + + // Not poisoned: still the owner's state. + expect(r.getConnectorState(victim)).toEqual(STATE); + }); + + test("a reconnect whose reaction awaits a later session.ack does not deadlock", async () => { + // handleChallengeResponse awaits the agent.reconnected reaction, whose + // subscriber issues sendSourcesUpdate -> awaits a LATER session.ack frame. + // session.ack must BYPASS the per-ws chain, or the reconnect wedges behind + // its own in-flight handler. (The test completing rather than timing out + // is the assertion.) + const kp = await generateKeyPair(); + const addr = "agent@local"; + const r = createSidecarRouter({ + requestTimeoutMs: 5000, + hubPublicKey: TEST_HUB_KEY, + lookups: { + lookupPublicKey: async (a) => + a === addr ? hexEncode(kp.publicKey) : null, + }, + }); + let reactionDone = false; + r.events.on("agent.reconnected", async () => { + await r.sendSourcesUpdate(addr, TEST_SOURCES, TEST_DEFAULT_SOURCE); + reactionDone = true; + }); + const ws = createMockWs(); + r.handleOpen(ws); + await driveReconnect(r, ws, addr, kp.privateKey); + + // The reaction has sent a sources.update request and is awaiting its ack. + const sourcesFrame = ws.sent + .map((s) => JSON.parse(s)) + .find((f: { type: string }) => f.type === "sources.update"); + expect(sourcesFrame).toBeDefined(); + + // Answer with a session.ack (a later inbound frame). If it queued behind + // the still-in-flight handleChallengeResponse it would never resolve. + r.handleMessage( + ws, + JSON.stringify({ + type: "session.ack", + requestId: sourcesFrame.requestId, + }), + ); + await new Promise((res) => setTimeout(res, 50)); + + expect(reactionDone).toBe(true); + expect(r.getRoutableAddresses()).toContain(addr); + }); + }); + describe("workflow-address reconnect (challenged, at parity with launched agents)", () => { // A workflow-substrate deployment address (ins_dep_...) proves ownership // through the SAME Ed25519 challenge as a launched agent: the hub resolves @@ -802,6 +988,7 @@ describe("SidecarRouter", () => { recipients: ["receiver@local"], }), ); + await tick(); const delivered = lastSent(ws2); expect(delivered.type).toBe("mail.inbound"); @@ -832,7 +1019,7 @@ describe("SidecarRouter", () => { expect(delivered.type).toBe("mail.inbound"); }); - test("unroutable mail emits mail.outbound.undelivered", () => { + test("unroutable mail emits mail.outbound.undelivered", async () => { const outbound: { rawMessage: string; recipients: string[] }[] = []; const router = createSidecarRouter({ lookups: { lookupPublicKey: async () => null }, @@ -864,6 +1051,7 @@ describe("SidecarRouter", () => { recipients: ["external@remote"], }), ); + await tick(); expect(outbound).toHaveLength(1); expect(outbound[0]?.recipients).toEqual(["external@remote"]); @@ -1028,6 +1216,7 @@ describe("SidecarRouter", () => { agentAddresses: [], }), ); + await tick(); const config = { sessionId: "ses_test", @@ -1090,6 +1279,7 @@ describe("SidecarRouter", () => { agentAddresses: [], }), ); + await tick(); const config = { sessionId: "ses_test", @@ -1149,6 +1339,7 @@ describe("SidecarRouter", () => { agentAddresses: [], }), ); + await tick(); const config = { sessionId: "ses_test", @@ -1198,6 +1389,7 @@ describe("SidecarRouter", () => { agentAddresses: [], }), ); + await tick(); const config = { sessionId: "ses_test", @@ -1283,6 +1475,7 @@ describe("SidecarRouter", () => { agentAddresses: [], }), ); + await tick(); const config = { sessionId: "ses_test", @@ -1351,6 +1544,7 @@ describe("SidecarRouter", () => { agentAddresses: [], }), ); + await tick(); const config = { sessionId: "ses_test", @@ -1467,7 +1661,7 @@ describe("SidecarRouter", () => { }); describe("agent events", () => { - test("agent.event frames are forwarded to subscribers", () => { + test("agent.event frames are forwarded to subscribers", async () => { const events: { addr: string; sid: string; event: unknown }[] = []; const router = createSidecarRouter({ lookups: { lookupPublicKey: async () => null }, @@ -1497,6 +1691,7 @@ describe("SidecarRouter", () => { event: { type: "reactor.start", seq: 0, data: {} }, }), ); + await tick(); expect(events).toHaveLength(1); expect(events[0]?.addr).toBe("agent@local"); @@ -1507,7 +1702,7 @@ describe("SidecarRouter", () => { }); }); - test("agent.event frames are emitted on router.events", () => { + test("agent.event frames are emitted on router.events", async () => { const seen: { addr: string; sid: string }[] = []; const router = createSidecarRouter({ lookups: { lookupPublicKey: async () => null }, @@ -1536,6 +1731,7 @@ describe("SidecarRouter", () => { event: { type: "reactor.start", seq: 0, data: {} }, }), ); + await tick(); expect(seen).toEqual([{ addr: "agent@local", sid: "sess-1" }]); }); @@ -1600,6 +1796,7 @@ describe("SidecarRouter", () => { connectorState: state, }), ); + await tick(); expect(router.getConnectorState("agent@local")).toEqual(state); @@ -1612,6 +1809,7 @@ describe("SidecarRouter", () => { connectorState: null, }), ); + await tick(); expect(router.getConnectorState("agent@local")).toBeNull(); }); @@ -1656,6 +1854,7 @@ describe("SidecarRouter", () => { connectorState: state, }), ); + await tick(); expect(seen).toEqual([{ addr: "agent@local", state }]); }); @@ -1691,6 +1890,7 @@ describe("SidecarRouter", () => { }, }), ); + await tick(); expect(router.getConnectorState("agent@local")).not.toBeNull(); // A second sidecar registers claiming the same address without @@ -1743,6 +1943,7 @@ describe("SidecarRouter", () => { }, }), ); + await tick(); expect(router.getConnectorState("agent@local")).not.toBeNull(); @@ -1753,7 +1954,7 @@ describe("SidecarRouter", () => { }); describe("session subscriptions", () => { - test("subscriber receives events for its session", () => { + test("subscriber receives events for its session", async () => { const received: unknown[] = []; const ws = createMockWs(); router.handleOpen(ws); @@ -1778,12 +1979,13 @@ describe("SidecarRouter", () => { event: { type: "reactor.start", seq: 0, data: {} }, }), ); + await tick(); expect(received).toHaveLength(1); expect(received[0]).toEqual({ type: "reactor.start", seq: 0, data: {} }); }); - test("subscriber does not receive events for other agents", () => { + test("subscriber does not receive events for other agents", async () => { const received: unknown[] = []; const ws = createMockWs(); router.handleOpen(ws); @@ -1808,11 +2010,12 @@ describe("SidecarRouter", () => { event: { type: "reactor.start", seq: 0, data: {} }, }), ); + await tick(); expect(received).toHaveLength(0); }); - test("unsubscribe stops delivery", () => { + test("unsubscribe stops delivery", async () => { const received: unknown[] = []; const ws = createMockWs(); router.handleOpen(ws); @@ -1839,6 +2042,7 @@ describe("SidecarRouter", () => { event: { type: "reactor.start", seq: 0, data: {} }, }), ); + await tick(); unsub(); @@ -1851,12 +2055,13 @@ describe("SidecarRouter", () => { event: { type: "reactor.end", seq: 1, data: {} }, }), ); + await tick(); expect(received).toHaveLength(1); expect(received[0]).toEqual({ type: "reactor.start", seq: 0, data: {} }); }); - test("multiple subscribers receive the same event", () => { + test("multiple subscribers receive the same event", async () => { const received1: unknown[] = []; const received2: unknown[] = []; const ws = createMockWs(); @@ -1883,12 +2088,13 @@ describe("SidecarRouter", () => { event: { type: "reactor.start", seq: 0, data: {} }, }), ); + await tick(); expect(received1).toHaveLength(1); expect(received2).toHaveLength(1); }); - test("stale unsubscribe does not evict a later subscriber", () => { + test("stale unsubscribe does not evict a later subscriber", async () => { const ws = createMockWs(); router.handleOpen(ws); router.handleMessage( @@ -1920,11 +2126,12 @@ describe("SidecarRouter", () => { event: { type: "reactor.start", seq: 0, data: {} }, }), ); + await tick(); expect(received).toHaveLength(1); }); - test("subscriber that unsubscribes mid-dispatch does not drop later subscribers", () => { + test("subscriber that unsubscribes mid-dispatch does not drop later subscribers", async () => { const received: unknown[] = []; const ws = createMockWs(); router.handleOpen(ws); @@ -1953,6 +2160,7 @@ describe("SidecarRouter", () => { event: { type: "reactor.start", seq: 0, data: {} }, }), ); + await tick(); expect(received).toHaveLength(1); expect(received[0]).toEqual({ type: "reactor.start", seq: 0, data: {} }); @@ -2008,6 +2216,7 @@ describe("SidecarRouter", () => { agentAddresses: [], }), ); + await tick(); // A fresh provision routes to this sidecar during the restore window. const deployPromise = router.sendAgentDeploy( @@ -2089,6 +2298,7 @@ describe("SidecarRouter", () => { agentAddresses: [], }), ); + await tick(); const deployPromise = router.sendAgentDeploy( "window-agent@local", @@ -2119,6 +2329,7 @@ describe("SidecarRouter", () => { connectorState, }), ); + await tick(); expect(router.getConnectorState("window-agent@local")).toEqual( connectorState, ); diff --git a/packages/hub-sessions/src/ws/sidecar-handler.ts b/packages/hub-sessions/src/ws/sidecar-handler.ts index babd2e5b..c6193328 100644 --- a/packages/hub-sessions/src/ws/sidecar-handler.ts +++ b/packages/hub-sessions/src/ws/sidecar-handler.ts @@ -304,6 +304,16 @@ export function createSidecarRouter( // ws handle → liveness timer (reset on each ping from the sidecar) const livenessTimers = new Map>(); + // Per-ws serialization chain for QUEUE-class frames (see `frameBypassesQueue` + // for the split and the invariant behind it). A frame that establishes or + // reads routing waits for earlier queued frames on the same ws to complete, + // so it observes their finished effects -- most importantly an async + // register's routing write, which would otherwise land after a following + // connector.state.changed / mail / pack frame and silently drop it. It holds + // a single in-flight promise per ws (replaced each queued frame), cleared on + // close. + const messageChains = new Map>(); + // transferId → pending pack transfer (resolved by repo.pack.ack, rejected by repo.pack.reject) type PendingPack = { transferId: string; @@ -411,58 +421,131 @@ export function createSidecarRouter( } const frame = validated; + // Bypass frames (liveness + terminal responses to outbound requests) + // dispatch immediately: they resolve the very promises a queued handler + // may be blocked on, so queuing them would deadlock the round-trip. + if (frameBypassesQueue(frame)) { + // Guard so a bypass handler's failure -- a synchronous throw or an async + // ack handler's rejection -- is logged rather than floating out of the + // immediate dispatch. The async wrapper turns a synchronous throw into a + // rejection too, matching the queue path's .then/.catch coverage. + void (async () => dispatchFrame(ws, frame))().catch((err: unknown) => { + logger.warn`Frame handler failed for ${frame.type}: ${err instanceof Error ? err.message : String(err)}`; + }); + return; + } + // Everything else serializes per ws so a frame that establishes or reads + // routing observes earlier queued frames' completed effects. + const prev = messageChains.get(ws) ?? Promise.resolve(); + const next = prev + .then(() => dispatchFrame(ws, frame)) + .catch((err: unknown) => { + logger.warn`Frame handler failed for ${frame.type}: ${err instanceof Error ? err.message : String(err)}`; + }); + messageChains.set(ws, next); + } + + function assertNever(x: never): never { + throw new Error(`Unclassified sidecar frame type: ${JSON.stringify(x)}`); + } + + // Whether `frame` bypasses the per-ws serialization chain. Invariant: a frame + // bypasses IFF it is liveness (ping) OR a terminal response to an + // already-issued outbound request -- correlated purely by + // requestId/transferId/agentAddress in the pending maps, touching no routing + // state. Such a frame has no ordering obligation against new inbound frames + // (a response cannot resolve "too early" for a request that already went + // out), and it is exactly what in-flight queued handlers block on, so it MUST + // run out of band or the challenge round-trip deadlocks (challenge.response + // -> agent.reconnected reaction -> sendSourcesUpdate awaits a later + // session.ack). Every other frame establishes or reads routing, or carries an + // inbound payload whose order matters, so it queues. The exhaustive switch + + // assertNever makes adding a SidecarFrame variant without classifying it a + // compile error, not a latent deadlock or a silent bypass hole. + function frameBypassesQueue(frame: SidecarFrame): boolean { switch (frame.type) { + case "ping": + case "session.ack": + case "session.error": + case "agent.deploy.ack": + case "agent.error": + case "agent.undeploy.ack": + case "repo.pack.ack": + case "repo.pack.reject": + return true; case "register": - void handleRegister( + case "reconnect": + case "challenge.response": + case "mail.outbound": + case "agent.event": + case "connector.state.changed": + case "repo.pack.push": + case "repo.pack.done": + case "deploy.apply.error": + return false; + default: + return assertNever(frame); + } + } + + // Runs one frame's handler. Returns the handler's promise for async handlers + // so the per-ws chain can await bounded completion; sync handlers return + // void. Never awaits a promise that resolves on a LATER same-ws frame -- the + // only such await (the challenge round-trip's session.ack) is reached via a + // bypass frame, which does not queue. + function dispatchFrame( + ws: WsHandle, + frame: SidecarFrame, + ): void | Promise { + switch (frame.type) { + case "register": + return handleRegister( ws, frame.sidecarId, frame.token, frame.agentAddresses, ); - break; case "reconnect": - void handleReconnect( + return handleReconnect( ws, frame.sidecarId, frame.token, frame.agentAddresses, frame.deployRefs ?? {}, ); - break; case "challenge.response": - void handleChallengeResponse(ws, frame.responses); - break; + return handleChallengeResponse(ws, frame.responses); case "agent.deploy.ack": - void handleDeployAck(frame.agentAddress, frame.publicKey); - break; + return handleDeployAck(frame.agentAddress, frame.publicKey); case "agent.error": rejectDeployPending(frame.agentAddress, frame.error); rejectUndeployPending(frame.agentAddress, frame.error); - break; + return; case "agent.undeploy.ack": resolveUndeployPending(frame.agentAddress); - break; + return; case "ping": handlePing(ws); - break; + return; case "mail.outbound": if (frame.delivered !== true) { handleMailOutbound(frame.rawMessage, frame.recipients); - } else if (lookups.persistMail && frame.senderAddress) { - void handleMailPersist( + return; + } + if (lookups.persistMail && frame.senderAddress) { + return handleMailPersist( lookups.persistMail, frame.rawMessage, frame.senderAddress, frame.recipients, ); - } else if (frame.delivered === true) { - if (!frame.senderAddress) { - logger.warn`Dropping delivered mail.outbound frame with no senderAddress`; - } else { - logger.warn`Dropping delivered mail.outbound frame: no persistMail lookup configured`; - } } - break; + if (!frame.senderAddress) { + logger.warn`Dropping delivered mail.outbound frame with no senderAddress`; + } else { + logger.warn`Dropping delivered mail.outbound frame: no persistMail lookup configured`; + } + return; case "agent.event": events.emit("agent.event", { agentAddress: frame.agentAddress, @@ -470,39 +553,38 @@ export function createSidecarRouter( event: frame.event, }); dispatchToSubscribers(frame.agentAddress, frame.event); - break; + return; case "connector.state.changed": // Gate the cache write on the sending sidecar actually owning // the named agent. A misbehaving sidecar that knows another // agent's address could otherwise poison the cached state. if (addressIndex.get(frame.agentAddress) !== ws) { logger.warn`Dropping connector.state.changed for ${frame.agentAddress}: not registered to this sidecar`; - break; + return; } connectorStates.set(frame.agentAddress, frame.connectorState); events.emit("connector.state.changed", { agentAddress: frame.agentAddress, connectorState: frame.connectorState, }); - break; + return; case "session.ack": resolvePending(frame.requestId); - break; + return; case "session.error": rejectPending(frame.requestId, frame.error); - break; + return; case "repo.pack.ack": resolvePackPending(frame.transferId); - break; + return; case "repo.pack.reject": rejectPackPending(frame.transferId, frame.reason); - break; + return; case "repo.pack.push": handlePackPush(ws, frame); - break; + return; case "repo.pack.done": - void handlePackDone(ws, frame); - break; + return handlePackDone(ws, frame); case "deploy.apply.error": // Gate the failure emit on the sending sidecar actually // owning the named agent. A misbehaving sidecar that knows @@ -512,7 +594,7 @@ export function createSidecarRouter( // and any failure-driven rollback logic the hub runs. if (addressIndex.get(frame.agentAddress) !== ws) { logger.warn`Dropping deploy.apply.error for ${frame.agentAddress}: not registered to this sidecar`; - break; + return; } events.emit("deploy.apply.error", { agentAddress: frame.agentAddress, @@ -523,9 +605,9 @@ export function createSidecarRouter( ...(frame.package !== undefined ? { package: frame.package } : {}), occurredAt: frame.occurredAt, }); - break; + return; default: - logger.warn`Unknown frame type from sidecar: ${(frame as { type: string }).type}`; + return assertNever(frame); } } @@ -1133,6 +1215,9 @@ export function createSidecarRouter( livenessTimers.delete(ws); } + // Drop the per-ws serialization chain; no more frames will queue on it. + messageChains.delete(ws); + // Cancel any pending challenge for this connection. const challengeReq = pendingChallenges.get(ws); if (challengeReq !== undefined) { From f461c9fdb05e0fe260b17d0de0553c6af0f2989d Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Tue, 7 Jul 2026 09:49:59 -0500 Subject: [PATCH 087/101] Close the sidecar socket when a reconnect deploy-ref read fails The boot-time reconnect re-announce reads each deployment's deploy ref before sending the challenged reconnect frame. That read ran inside an unguarded async IIFE in the connection open handler, outside the message-queue tail catch, so a rejection (corrupt or unreadable ref state) became an unhandled promise: the reconnect frame was never sent, the sidecar's deployment routes silently vanished, and nothing was logged. Wrap the re-announce in a catch that logs the failure and closes the socket, forcing a clean reconnect retry on the existing 3s backoff. The send sits after the ref-read loop, so a throw skips it and no partial reconnect frame goes out. The test drives a rejecting deploy-ref read and asserts the socket is closed (a reconnect is scheduled) rather than the failure swallowed; it fails both when the catch is absent and when the catch logs without closing. --- packages/hub-agent/src/ws/hub-link.test.ts | 62 ++++++++++++++++++++++ packages/hub-agent/src/ws/hub-link.ts | 40 +++++++++----- 2 files changed, 89 insertions(+), 13 deletions(-) diff --git a/packages/hub-agent/src/ws/hub-link.test.ts b/packages/hub-agent/src/ws/hub-link.test.ts index 5e192598..d9ab76a4 100644 --- a/packages/hub-agent/src/ws/hub-link.test.ts +++ b/packages/hub-agent/src/ws/hub-link.test.ts @@ -458,6 +458,68 @@ describe("sidecar↔hub integration", () => { expect(env.router.getRoutableAddresses()).not.toContain(deploymentAddress); }); + test("a failing deploy-ref read closes the socket instead of silently dropping the reconnect", async () => { + const failEnv = startTestServer(); + + // Capture the reconnect callback so no real timer fires during the + // test. Its presence is also the proof we assert on: the re-announce + // catch closes the socket, and the close handler schedules a reconnect + // through this scheduler. Without the fix the deploy-ref rejection is + // an unhandled promise, no close occurs, and this stays null. + let pendingReconnect: (() => void) | null = null; + const fakeScheduleReconnect: ReconnectScheduler = (cb) => { + pendingReconnect = cb; + return () => { + pendingReconnect = null; + }; + }; + + const transport = createInMemoryTransport(); + const deploymentAddress = "ins_dep_reffail1@integration.interchange"; + // The deploy-ref read rejects the way a corrupt or unreadable ref + // would. The re-announce IIFE must catch it and close the socket + // rather than let the reconnect vanish with nothing logged. + const sessions: SessionManager = { + ...createMockSessionManager(), + getDeployRef: () => + Promise.reject(new Error("simulated corrupt deploy ref")), + }; + + const client = createHubLink({ + hubURL: `ws://localhost:${failEnv.server.port}/ws`, + sidecarId: "sc-ref-fail", + token: "test-token", + transport, + sessions, + ...withTestDeployBindings(), + getWorkflowAddresses: () => [deploymentAddress], + scheduleReconnect: fakeScheduleReconnect, + }); + + try { + client.connect(); + // On open the register frame is sent, then the re-announce IIFE + // reads the deploy ref and the read rejects. The catch closes the + // socket, and the close handler schedules a reconnect through the + // fake scheduler. Observing that pending callback is the proof the + // socket was closed rather than the failure swallowed. Without the + // fix the rejection is an unhandled promise, no close occurs, and + // this stays null until the wait times out. (The connect→reject→ + // close cycle is faster than a poll interval, so the transient + // connected state is deliberately not asserted -- the scheduled + // reconnect is the durable signal.) + await waitFor(() => pendingReconnect !== null); + + // No reconnect frame was ever sent, so the address never routed. + expect(failEnv.router.getRoutableAddresses()).not.toContain( + deploymentAddress, + ); + } finally { + client.close(); + await failEnv.server.stop(true); + } + }); + test("repo.pack.reject sent when applyDeployPack throws signature_invalid", async () => { const transport = createInMemoryTransport(); const sessions = createMockSessionManager(); diff --git a/packages/hub-agent/src/ws/hub-link.ts b/packages/hub-agent/src/ws/hub-link.ts index aff82b5f..295ed3bf 100644 --- a/packages/hub-agent/src/ws/hub-link.ts +++ b/packages/hub-agent/src/ws/hub-link.ts @@ -1144,21 +1144,35 @@ export function createHubLink(config: HubLinkConfig): HubLink { const restoredAddresses = getWorkflowAddresses(); if (restoredAddresses.length > 0) { void (async () => { - const deployRefs: Record = {}; - for (const address of restoredAddresses) { - const ref = await sessions.getDeployRef(address); - if (ref !== null) { - deployRefs[address] = ref; + try { + const deployRefs: Record = {}; + for (const address of restoredAddresses) { + const ref = await sessions.getDeployRef(address); + if (ref !== null) { + deployRefs[address] = ref; + } } + send({ + type: "reconnect", + sidecarId, + token, + agentAddresses: restoredAddresses, + ...(Object.keys(deployRefs).length > 0 ? { deployRefs } : {}), + }); + flush(); + } catch (err) { + // A failing deploy-ref read (corrupt or unreadable ref state) + // must not silently drop the reconnect. Without this frame the + // hub never re-challenges these addresses, so their routes + // vanish with nothing logged. Surface the failure and close the + // socket to force a clean reconnect retry. No partial reconnect + // frame is sent: the send sits after the loop, so a throw skips + // it. This IIFE runs in the `open` handler, outside the + // `messageQueue` tail-catch, so this catch is its only net. + const msg = err instanceof Error ? err.message : String(err); + logger.error`Deployment re-announce failed, closing connection: ${msg}`; + ws?.close(); } - send({ - type: "reconnect", - sidecarId, - token, - agentAddresses: restoredAddresses, - ...(Object.keys(deployRefs).length > 0 ? { deployRefs } : {}), - }); - flush(); })(); } }); From 25ef8f7893312562c5806bf78f3c20daeadfcd63 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Tue, 7 Jul 2026 10:13:07 -0500 Subject: [PATCH 088/101] Reap the respawned child when its credentials read fails The recycle respawn spawns the new workflow-process child and wires its IPC channels, then re-reads per-step credentials before the ready handshake. That read is a substrate call that can reject -- a grants file that became malformed is precisely the grant-refresh path recycle doubles as. It ran with no guard around the new handle, so on a rejection triggerRecycle unwound and the supervisor's recycle-failure teardown reaped only the PRIOR cohort (state.handle during recycling). The freshly-spawned child leaked its OS process and both IPC channels. Wrap the credentials read in a catch that reaps the new child (kill, then finalize the pumps) and rethrows, mirroring the spawn path, which already routes this same throw through its teardown owner. The reap that the handshake-failure path open-coded is extracted into reapUnreadyChild so both failure points share one kill-first sequence. --- .../src/supervisor/recycle.test.ts | 55 +++++++++ .../workflow-host/src/supervisor/recycle.ts | 116 +++++++++++++----- 2 files changed, 141 insertions(+), 30 deletions(-) diff --git a/packages/workflow-host/src/supervisor/recycle.test.ts b/packages/workflow-host/src/supervisor/recycle.test.ts index 2671006f..e4e051d8 100644 --- a/packages/workflow-host/src/supervisor/recycle.test.ts +++ b/packages/workflow-host/src/supervisor/recycle.test.ts @@ -244,6 +244,14 @@ async function seedStepGrants( ); } +async function corruptStepGrants( + baseDir: string, + repoId: RepoId, +): Promise { + const dir = path.join(baseDir, repoId.kind, repoId.id); + await fs.writeFile(path.join(dir, STEP_GRANTS_PATH), "{ this is not json"); +} + type FakeChild = { pid: number; channelId: string | undefined; @@ -1572,3 +1580,50 @@ describe("supervisor recycle: external drain phase guard", () => { await supervisor.shutdown(); }); }); + +describe("supervisor recycle: respawn credentials-read failure", () => { + test("a respawn whose credentials re-read throws reaps the freshly-spawned child", async () => { + const baseDir = await makeTempDir("recycle-credleak-"); + const ipcKeypair = await generateKeyPair(); + const mailBus = createMockMailBus(); + // sigtermExits so the reap's SIGTERM settles the fake child. + const tracker = createSpawnTracker({ sigtermExits: true }); + const { supervisor } = await spawnSupervisor({ + baseDir, + tracker, + mailBus, + ipcKeypair, + }); + expect(tracker.totalSpawns).toBe(1); + + // Corrupt the step grants so the recycle's pre-handshake + // assembleCredentialsSnapshot re-read throws -- the recycle doubles + // as the grant-refresh path, so a malformed grants file is exactly + // the read that fails here. + await corruptStepGrants( + baseDir, + defaultStepRepoId({ deploymentId: "deployment-x", stepId: "step-1" }), + ); + + let caught: unknown; + try { + await supervisor.recycle({ reason: "credentials-leak-regression" }); + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(Error); + expect(caught instanceof Error && caught.message).toMatch(/not valid JSON/); + + // The respawn spawned the new child before the credentials read + // threw. Its catch must reap that child rather than leak it: the + // supervisor's recycle-failure teardown reaps only the PRIOR cohort + // (state.handle during `recycling`), so a child that failed before + // installation is invisible to it. + expect(tracker.totalSpawns).toBe(2); + const newChild = tracker.children[1]; + if (newChild === undefined) { + throw new Error("tracker.children[1] missing"); + } + expect(newChild.killSignals.length).toBeGreaterThan(0); + }); +}); diff --git a/packages/workflow-host/src/supervisor/recycle.ts b/packages/workflow-host/src/supervisor/recycle.ts index 049125a8..a60a3d0a 100644 --- a/packages/workflow-host/src/supervisor/recycle.ts +++ b/packages/workflow-host/src/supervisor/recycle.ts @@ -378,21 +378,48 @@ export async function triggerRecycle( }); const eventPump = pumpEvents(eventIter, ctx.onInferenceEvent); + // The child handshake below is deadline-bounded and needs these timer + // bindings; the pre-handshake credentials-read reap needs them too, so + // derive them before that read. + const setTimer = ctx.setTimer ?? defaultSetTimer; + const clearTimer = ctx.clearTimer ?? defaultClearTimer; + // Re-read per-step credentials. A grants update that landed during // the previous child's lifetime is picked up here -- the recycle // doubles as the supervisor's grant-refresh path. The deploy tree // is not consulted; this read is against the `agent-state` repos // alone, whose contents are independent of `workflow.json`. - const credentialsSnapshot = await assembleCredentialsSnapshot({ - repoStore: ctx.bindings.repoStore, - principal: ctx.bindings.readPrincipal, - stepOrder: ctx.stepOrder, - deploymentId: ctx.bindings.deploymentId, - deriveStepAddress: ctx.bindings.deriveStepAddress, - ...(ctx.bindings.deriveStepRepoId !== undefined - ? { deriveStepRepoId: ctx.bindings.deriveStepRepoId } - : {}), - }); + // + // This is a substrate read that can reject -- a grants file that + // became malformed is precisely the recycle's grant-refresh path. The + // new child is already spawned and wired but not yet installed on + // `state`, so the supervisor's recycle-failure teardown (which reaps + // the PRIOR cohort) cannot see it; reap it here on failure or it + // leaks. The spawn path routes this same throw through its teardown + // owner. The try wraps only the awaited read: the handle and pumps it + // reaps are all constructed above, so the reap always has live + // handles. + let credentialsSnapshot: CredentialsSnapshot; + try { + credentialsSnapshot = await assembleCredentialsSnapshot({ + repoStore: ctx.bindings.repoStore, + principal: ctx.bindings.readPrincipal, + stepOrder: ctx.stepOrder, + deploymentId: ctx.bindings.deploymentId, + deriveStepAddress: ctx.bindings.deriveStepAddress, + ...(ctx.bindings.deriveStepRepoId !== undefined + ? { deriveStepRepoId: ctx.bindings.deriveStepRepoId } + : {}), + }); + } catch (cause) { + await reapUnreadyChild(handle, eventPump, controlIncoming, { + killTimeoutMs, + setTimer, + clearTimer, + phase: "credentials read failure", + }); + throw cause; + } // Steps 4 + 5: self-discover + resume. These run inside the child // before it emits `ready`; the supervisor waits, but bounded -- a child @@ -407,8 +434,6 @@ export async function triggerRecycle( // timer. The respawn is deadline-bounded on the child handshake, not on // that read. const readyTimeoutMs = ctx.readyTimeoutMs ?? DEFAULT_READY_TIMEOUT_MS; - const setTimer = ctx.setTimer ?? defaultSetTimer; - const clearTimer = ctx.clearTimer ?? defaultClearTimer; const readyOutcome = readyPromise.then( (info) => ({ kind: "ready" as const, info }), (err: unknown) => ({ kind: "failed" as const, err }), @@ -422,26 +447,15 @@ export async function triggerRecycle( if (readyRace.kind !== "ready") { // The new child was never installed on `state`, so the supervisor's // recycle-failure teardown (which reaps the PRIOR cohort) would leak - // it. Reap it here. Kill FIRST, then finalize the pumps: process - // death drives EOF on both channels, which unparks `waitForReady`'s - // in-flight `iter.next()` (letting `controlIncoming.return` complete) - // and ends `pumpEvents`. Awaiting either finalizer before the kill - // would hang behind the still-open channels. `killChildHandle` on an - // already-dead handle is a cheap no-op, so the `failed` path (a - // control-channel end does not guarantee the process died, since the - // event channel is separate) is reaped too, not just the timeout. - await killChildHandle(handle, killTimeoutMs, { - logger, + // it. Reap it here. `killChildHandle` on an already-dead handle is a + // cheap no-op, so the `failed` path (a control-channel end does not + // guarantee the process died, since the event channel is separate) + // is reaped too, not just the timeout. + await reapUnreadyChild(handle, eventPump, controlIncoming, { + killTimeoutMs, setTimer, clearTimer, - }); - void eventPump.catch((cause: unknown) => { - const message = cause instanceof Error ? cause.message : String(cause); - logger.warn`recycle: reaped child eventPump failed after handshake failure: ${message}`; - }); - void controlIncoming.return(undefined).catch((cause: unknown) => { - const message = cause instanceof Error ? cause.message : String(cause); - logger.warn`recycle: reaped child controlIncoming.return failed after handshake failure: ${message}`; + phase: "handshake failure", }); if (readyRace.kind === "timeout") { throw new Error( @@ -480,6 +494,48 @@ export async function triggerRecycle( }; } +/** + * Reap a respawned child that was spawned and wired but never installed + * on `state`. Such a child is invisible to the supervisor's + * recycle-failure teardown -- that path reaps the PRIOR cohort + * (`state.handle` during `recycling`) -- so a respawn that fails after + * the spawn must reap the new child here or it leaks its OS process and + * both IPC channels. + * + * Kill FIRST, then finalize the pumps: process death drives EOF on both + * channels, which unparks `waitForReady`'s in-flight `iter.next()` + * (letting `controlIncoming.return` complete) and ends `pumpEvents`. + * Awaiting either finalizer before the kill would hang behind the + * still-open channels. `killChildHandle` on an already-dead handle is a + * cheap no-op, so a child that died on its own -- not just one killed on + * timeout -- is reaped safely too. + */ +async function reapUnreadyChild( + handle: SubprocessHandle, + eventPump: Promise, + controlIncoming: AsyncGenerator, + deps: { + killTimeoutMs: number; + setTimer: (cb: () => void, ms: number) => unknown; + clearTimer: (handle: unknown) => void; + phase: string; + }, +): Promise { + await killChildHandle(handle, deps.killTimeoutMs, { + logger, + setTimer: deps.setTimer, + clearTimer: deps.clearTimer, + }); + void eventPump.catch((cause: unknown) => { + const message = cause instanceof Error ? cause.message : String(cause); + logger.warn`recycle: reaped child eventPump failed after ${deps.phase}: ${message}`; + }); + void controlIncoming.return(undefined).catch((cause: unknown) => { + const message = cause instanceof Error ? cause.message : String(cause); + logger.warn`recycle: reaped child controlIncoming.return failed after ${deps.phase}: ${message}`; + }); +} + /** * Iterate the new child's control-receive iterator until the `ready` * frame lands. Identical shape to the supervisor's spawn-time helper; From 5266d81ccdab0260c70726f18c7aa0b87d8e89fb Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Tue, 7 Jul 2026 10:26:51 -0500 Subject: [PATCH 089/101] Preserve the spawn failure cause when teardown also throws The spawn-failure catch tore the cohort down with an unguarded `await shutdownInternal(...)` before rethrowing the spawn cause. That teardown walks several individually-unguarded steps (mail unsubscribe, child kill, broadcaster dispose, accumulator stop); a throw in any of them propagated out of the catch and replaced the original spawn cause, hiding the real startup failure behind a secondary teardown error. Guard the teardown with a `.catch` that logs the secondary failure and lets the original `cause` rethrow, mirroring the recycle-failure catch that already preserves its cause this way. The test injects a throwing mail-unsubscribe so a ready-timeout spawn's teardown fails, and asserts the surfaced error is the ready timeout, not the teardown error. --- .../src/supervisor/recycle.test.ts | 51 +++++++++++++++++++ .../src/supervisor/supervisor.ts | 12 +++++ 2 files changed, 63 insertions(+) diff --git a/packages/workflow-host/src/supervisor/recycle.test.ts b/packages/workflow-host/src/supervisor/recycle.test.ts index e4e051d8..904dc93d 100644 --- a/packages/workflow-host/src/supervisor/recycle.test.ts +++ b/packages/workflow-host/src/supervisor/recycle.test.ts @@ -589,6 +589,57 @@ describe("supervisor spawn: failure cleanup", () => { "unregister:deployment-x@example.com", ); }); + + test("a teardown failure during spawn cleanup does not mask the spawn cause", async () => { + // Regression: the spawn-failure catch tears the cohort down through + // shutdownInternal, whose steps (mail unsubscribe, kill, dispose) are + // individually unguarded. A throw in teardown must not replace the + // original spawn cause -- the operator needs the real startup failure, + // not a secondary teardown error. + const baseDir = await makeTempDir("spawn-teardown-mask-"); + const ipcKeypair = await generateKeyPair(); + const baseMailBus = createMockMailBus(); + // The mail unsubscribe (called unguarded in shutdownInternal) throws, + // so the teardown itself fails while unwinding a failed spawn. + const mailBus: MailBusBindings = { + ...baseMailBus, + subscribeMailForAddress(address, handler) { + baseMailBus.subscribeMailForAddress(address, handler); + return () => { + throw new Error("mail unsubscribe boom during teardown"); + }; + }, + }; + // sigtermExits so the timeout path's kill settles the fake child. + const tracker = createSpawnTracker({ sigtermExits: true }); + await seedStepGrants( + baseDir, + defaultStepRepoId({ deploymentId: "deployment-x", stepId: "step-1" }), + [{ resource: "thing", action: "read" }], + ); + const bindings = await buildBindings({ + baseDir, + spawner: tracker.spawner, + mailBus, + ipcKeypair, + }); + const supervisor = createWorkflowSupervisor({ + ...bindings, + readyTimeoutMs: 20, + }); + + // Never drive `ready`, so the handshake times out. The spawn cause + // (ready timeout) must surface even though the teardown then throws + // its own error. + await expect( + supervisor.spawn({ + stepOrder: ["step-1"], + definitionHash: "def-hash-abc", + warmKeep: false, + onInferenceEvent: () => undefined, + }), + ).rejects.toThrow(/did not emit ready/); + }); }); describe("supervisor spawn: dynamic env", () => { diff --git a/packages/workflow-host/src/supervisor/supervisor.ts b/packages/workflow-host/src/supervisor/supervisor.ts index 3db0ef26..f4174809 100644 --- a/packages/workflow-host/src/supervisor/supervisor.ts +++ b/packages/workflow-host/src/supervisor/supervisor.ts @@ -1638,8 +1638,20 @@ export function createWorkflowSupervisor( }; } catch (cause) { const message = cause instanceof Error ? cause.message : String(cause); + // Guard the teardown so a throw inside `shutdownInternal` (an + // unguarded `mailUnsubscribe`, `handle.kill`, broadcaster dispose, + // or accumulator stop) cannot replace the original spawn `cause`. + // A masked cause would hide the real startup failure behind a + // secondary teardown error. Mirrors the recycle-failure catch, + // which preserves its cause the same way. await shutdownInternal({ reason: `spawn failed during startup: ${message}`, + }).catch((shutdownCause) => { + const inner = + shutdownCause instanceof Error + ? shutdownCause.message + : String(shutdownCause); + logger.error`shutdown after spawn failure also threw: ${inner}`; }); throw cause; } From d4913456c8f52dfff088f0bec18187eb8cf36ff0 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Tue, 7 Jul 2026 10:37:49 -0500 Subject: [PATCH 090/101] Attribute source-rotation rollback safety to frame serialization The source-rotation persist rolls back `currentSources` on a failed write. Its comment credited that safety to the hub awaiting each sources.update ack before sending the next -- but the hub does no such pacing; it dispatches rotations fire-and-forget. The real serializer is the sidecar's per-connection inbound-frame queue: each hub frame's handler runs to completion before the next begins, so no second rotation is ever in flight to have its committed table clobbered by the rollback. Rewrite the rollback comment to name the true mechanism, and mark the frame queue itself so its ordering reads as load-bearing. A future change that parallelized inbound-frame dispatch would break the rollback; the corrected comments make that dependency explicit at both ends. No behavior changes. --- apps/sidecar/src/workflow-host-wiring.ts | 14 ++++++++++---- packages/hub-agent/src/ws/hub-link.ts | 8 ++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/apps/sidecar/src/workflow-host-wiring.ts b/apps/sidecar/src/workflow-host-wiring.ts index 1a36e86d..704f82cb 100644 --- a/apps/sidecar/src/workflow-host-wiring.ts +++ b/apps/sidecar/src/workflow-host-wiring.ts @@ -1391,10 +1391,16 @@ export function createSidecarDeployRouter(deps: { buildDeploymentRecord(spec, rotated), ); } catch (cause) { - // Restoring unconditionally is safe because rotations are - // single-flight per deployment: the hub awaits each - // sources.update ack before sending the next, so prevSources - // cannot clobber a concurrently-committed later rotation. + // Restoring unconditionally is safe because rotations for one + // deployment are serialized by the sidecar's per-connection + // inbound-frame queue: each hub frame, sources.update + // included, runs its handler to completion on that queue + // before the next frame's handler starts, so no second + // rotation is in flight whose committed table this rollback + // could clobber. This does NOT rely on the hub pacing its + // sends -- the hub dispatches sources.update fire-and-forget; + // the sidecar frame queue is the sole serializer. Parallelizing + // inbound-frame dispatch would break this rollback. currentSources = prevSources; throw cause; } diff --git a/packages/hub-agent/src/ws/hub-link.ts b/packages/hub-agent/src/ws/hub-link.ts index 295ed3bf..b5114aa1 100644 --- a/packages/hub-agent/src/ws/hub-link.ts +++ b/packages/hub-agent/src/ws/hub-link.ts @@ -1188,6 +1188,14 @@ export function createHubLink(config: HubLinkConfig): HubLink { // Per-arm guards (mail/signal/drain) are the primary defence; // this catch is the belt-and-braces guarantee that no future // unguarded arm can wedge the link. + // + // This chain also serializes inbound frames: each frame's + // handler runs to completion before the next begins. A downstream + // invariant depends on that ordering -- the workflow + // source-rotation persist rolls back on failure assuming no second + // rotation is in flight, which holds only because sources.update + // frames are processed one at a time here. Parallelizing this + // dispatch would break that rollback. const data = event.data; messageQueue = messageQueue.then(() => handleMessage(data).catch((err: unknown) => { From ee90e339193ca2345c4df9d4d841020a1c0b50c1 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Tue, 7 Jul 2026 11:17:05 -0500 Subject: [PATCH 091/101] Reconcile the deployment key comment with the not-null address add The public_key column comment justified its nullability partly by a "pre-migration" deployment reading null. No such row can exist: the sibling address column was added NOT NULL with no default, which Postgres rejects on a non-empty table, and the repo's migration convention (matching the credential.provider_id add) is that the table is empty when the column lands. The two positions contradicted each other. Drop the impossible pre-migration case from the key comment -- the nullable window is only the real not-yet-acked state between deploy-start and deploy-ack -- and document on the address column why the NOT NULL add needs no default or backfill. Comment-only. --- packages/db/src/schema/workflow-deployments.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/db/src/schema/workflow-deployments.ts b/packages/db/src/schema/workflow-deployments.ts index ac5612cf..a8e5dbd8 100644 --- a/packages/db/src/schema/workflow-deployments.ts +++ b/packages/db/src/schema/workflow-deployments.ts @@ -35,13 +35,18 @@ export const workflowDeployment = pgTable( // than re-derived at read time so the reconnect ownership challenge can // look up the deployment's public key by address, symmetrically with the // `agent_instance` path. Unique so a double-insert fails loud. + // + // Added NOT NULL with no default and no backfill. Like the repo's other + // such adds (e.g. `credential.provider_id`), the migration relies on the + // table being empty when it runs: this table and the `address` column + // land one migration apart, both unreleased, so no populated row predates + // the column and none needs a backfilled address. address: text("address").notNull(), // The Ed25519 public key the sidecar minted for this deployment address, // persisted at deploy-ack. Nullable by design: the row is written at - // deploy-start and the key arrives at ack, so a not-yet-acked (or - // pre-migration) deployment reads `null` and its reconnect challenge - // fails closed -- the address stays unrouted rather than routing without - // ownership proof. + // deploy-start and the key arrives at ack, so a not-yet-acked deployment + // reads `null` and its reconnect challenge fails closed -- the address + // stays unrouted rather than routing without ownership proof. publicKey: text("public_key"), status: text("status") .$type() From 352f1efefe46cffa8cf9a04a2cdfc36b12e27293 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Tue, 7 Jul 2026 12:32:07 -0500 Subject: [PATCH 092/101] Rewrite the claim-check API comment for the delta write path The claim-check API header still described the four inbox/processing/ consumed operations as routing through `writeTreePreservingPrefix` with the per-address subtree as `preservePrefix` and the substrate replacing that subtree wholesale via `clearPrefix`. The operations were converted to `writeTreeDelta`: each scopes the change to the per-address prefix and returns a targeted set of `puts`/`deletes` from a `computeDelta` callback that reads the prior tree via `listDirOids`/`readBlobByOid`, and the substrate applies that delta over the prior tree rather than rewriting the subtree. Rewrite the header to describe the delta mechanism the code now uses. Comment-only. --- .../hub-sessions/src/workflow-run-kind.ts | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/packages/hub-sessions/src/workflow-run-kind.ts b/packages/hub-sessions/src/workflow-run-kind.ts index 047ba888..40eeb416 100644 --- a/packages/hub-sessions/src/workflow-run-kind.ts +++ b/packages/hub-sessions/src/workflow-run-kind.ts @@ -2400,8 +2400,8 @@ export const workflowRunAuthorize: AuthorizeFn = ( // --------------------------------------------------------------------- // Claim-check API. // -// Four operations layer on top of `RepoStore.writeTreePreservingPrefix` -// to give the workflow runtime a FIFO claim-check queue per address: +// Four operations layer on top of `RepoStore.writeTreeDelta` to give +// the workflow runtime a FIFO claim-check queue per address: // // enqueueInbox — append a new inbox entry for an inbound // message. @@ -2417,16 +2417,18 @@ export const workflowRunAuthorize: AuthorizeFn = ( // its `-` filename // key so FIFO ordering survives a crash. // -// All four route through `writeTreePreservingPrefix` with the -// per-address subtree as `preservePrefix`. The substrate serializes -// concurrent claim-check operations on the per-repo lock; the merge -// callback reads the prior address subtree directly via -// `isomorphic-git` (the substrate's `existing` parameter is a -// direct-children-only view, which does not see entries nested under -// inbox/processing/consumed), computes the new state, and returns the -// full set of files. The substrate's `clearPrefix` semantics replace -// the address subtree wholesale with the returned set, which is the -// atomic-commit guarantee these operations require. +// All four route through `writeTreeDelta`, scoped to the per-address +// subtree via `changedPathPrefixes`. The substrate serializes concurrent +// claim-check operations on the per-repo lock and invokes each +// operation's `computeDelta` callback with a `prior` view of the +// committed tree. The callback reads only what it needs directly -- +// `prior.listDirOids` for a directory's names and OIDs, and +// `prior.readBlobByOid` for a specific entry's bytes -- and returns a +// TARGETED delta (the `puts` and `deletes` for the paths that change), +// not the full subtree. The substrate applies that delta atomically over +// the prior tree, carrying every untouched entry forward by OID and +// landing the whole delta in a single commit, which is the atomic-commit +// guarantee these operations require. function claimCheckCommitRef(): string { // Every claim-check operation targets the same canonical ref used by From f2b34856d93ed1f3ffb503a7fc074fed7a5a15e6 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Tue, 7 Jul 2026 12:37:24 -0500 Subject: [PATCH 093/101] Correct two stale symbol fates in the execution-host design doc The design doc's symbol-fate section claimed two changes that never happened. It said `wrapHarnessAsSingleStepWorkflow` was renamed to `wrapHarnessAsSingleStepAgent` (a symbol that does not exist; the original name shipped) and that `deriveDeploymentId` was deleted (it is alive and called at four workflow-host wiring sites). The same non-existent `wrapHarnessAsSingleStepAgent` also appeared earlier, in the workflow-level defaulting section. Correct all three references to match the shipped symbols. Doc-only. --- docs/unified-execution-host-design.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/docs/unified-execution-host-design.md b/docs/unified-execution-host-design.md index 9bc27c9b..db31e089 100644 --- a/docs/unified-execution-host-design.md +++ b/docs/unified-execution-host-design.md @@ -737,8 +737,8 @@ oci-container`. A per-node declaration equal to or weaker than the current rung - **Workflow-level defaulting.** Unchanged from Decision 2: `defineWorkflow` normalizes an absent workflow-level `isolation` to `{ granularity: "per-tenant" }`. The single-agent launch path - (`wrapHarnessAsSingleStepAgent`) sets the workflow-level field from the launch - request / agent policy. + (`wrapHarnessAsSingleStepWorkflow`) sets the workflow-level field from the + launch request / agent policy. - **Validation.** `defineWorkflow`'s `normalize` validates every `isolation` occurrence — workflow-level and per-node — against the allowed `granularity` / `sandbox` sets, throwing a structured error on an unknown @@ -1013,17 +1013,16 @@ Every surviving "trivial"-named symbol is renamed to reflect that the single-step identity path is the canonical single-agent deploy, not a "trivial" special case — or deleted where the in-process branch it served is gone: -- `wrapHarnessAsSingleStepWorkflow` -> `wrapHarnessAsSingleStepAgent` - (`packages/workflow-deploy/src/orchestrator.ts`), still used to build the - one-step definition. +- `wrapHarnessAsSingleStepWorkflow` + (`packages/workflow-deploy/src/orchestrator.ts`) kept its name; still used to + build the one-step definition. - `buildTrivialApprovalSet` -> `buildSingleStepApprovalSet` (`packages/hub-sessions/src/session-service.ts`). - `isTrivialDeploy` / `trivialBindings` / the trivial orchestrator branch: **deleted** if single-step routes through the multi-step branch (likely); otherwise renamed `isSingleStepDeploy`. Confirm during build. -- `deriveDeploymentId` (`apps/sidecar/src/workflow-host-wiring.ts`): - deleted; call `deriveWorkflowRunRepoId` directly at the call sites (it is a - thin delegator). +- `deriveDeploymentId` (`apps/sidecar/src/workflow-host-wiring.ts`): kept; still + called at the workflow-host wiring sites. - `TrivialLaunch` / `trivialLaunch` / `trivialClaimedSlugSucceeded`: deleted with the in-process branch. From d1183f2b7219263d4aee32c671bd636a2dfd4f45 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Tue, 7 Jul 2026 12:39:24 -0500 Subject: [PATCH 094/101] Correct stale in-process-harness references after its removal Retiring the in-process (single-agent) harness deleted its build path and event forwarder, leaving `createDefaultHarnessBuilder` as a source-admission check only. Comments across three packages still asserted present-tense wiring to that removed machinery: that `default-harness.ts` reaches the tool-materialization module, that the per-step tool caps are the same ones "the in-process harness builder" uses, and that the step-invoker event filter mirrors a forwarder in `default-harness.ts` that no longer exists. Reword each to name the current single consumer (the workflow-process child's substrate factory) and the live source of the loader caps (the sidecar boot edge), and drop the broken forwarder pointers. Two honest "legacy in-process harness" behavior-lineage notes lose their now-stale tail too. Comment-only; the possibly-dead `assetRoot` default beside one of these comments is left for a separate change. --- apps/sidecar/src/step-agent-tools.ts | 4 +-- apps/sidecar/src/tool-materialization.ts | 32 +++++++++---------- .../sidecar/src/workflow-substrate-factory.ts | 9 +++--- packages/hub-sessions/src/session-service.ts | 7 ++-- .../src/adapters/step-invoker.ts | 6 ++-- 5 files changed, 26 insertions(+), 32 deletions(-) diff --git a/apps/sidecar/src/step-agent-tools.ts b/apps/sidecar/src/step-agent-tools.ts index 79141bf1..4e78295f 100644 --- a/apps/sidecar/src/step-agent-tools.ts +++ b/apps/sidecar/src/step-agent-tools.ts @@ -57,8 +57,8 @@ const INSTANCE_PREFIX = "ins_"; * Cache and registry caps the per-step tool loader needs. Resolved at * the sidecar boot edge from the existing `SIDECAR_CACHE_*` / * `SIDECAR_REGISTRY_*` config keys and threaded into the child through - * the substrate config, so the child's per-step materialization uses - * the same caps the in-process harness builder does. + * the substrate config, so the child's per-step materialization is + * bounded by those boot-edge-resolved caps. */ export interface StepToolCacheConfig { readonly cacheMaxBytes: number; diff --git a/apps/sidecar/src/tool-materialization.ts b/apps/sidecar/src/tool-materialization.ts index d7fb24d8..e0786ef0 100644 --- a/apps/sidecar/src/tool-materialization.ts +++ b/apps/sidecar/src/tool-materialization.ts @@ -2,18 +2,18 @@ // // The deploy-tree reader, the `@intx/tool-packaging` loader // construction, `applyAtomic`, and the active-deploy-id persistence -// ladder live here so both the in-process harness builder -// (`default-harness.ts`) and the workflow-process child's substrate -// factory (`workflow-substrate-factory.ts`) reach the same -// implementation. Keeping this in `apps/sidecar` (and out of -// `@intx/workflow-host`) preserves the portable orchestration -// package's host-agnostic layering: the child IS the sidecar binary, -// so the sidecar's tool runtime is already present in the child's -// address space without the portable package depending on it. +// ladder live here so the workflow-process child's substrate factory +// (`workflow-substrate-factory.ts`) reaches this implementation without +// the portable orchestration package depending on the sidecar's tool +// runtime. Keeping this in `apps/sidecar` (and out of +// `@intx/workflow-host`) preserves that package's host-agnostic +// layering: the child IS the sidecar binary, so the sidecar's tool +// runtime is already present in the child's address space. // // The module is deliberately free of any `@intx/harness` reactor -// dependency so the workflow-process child does not drag the -// in-process harness's transport/reactor ownership into its graph. +// dependency so the workflow-process child's boot graph does not pull +// in `@intx/harness`'s transport/reactor stack -- a forbidden-import +// guard enforces that boundary. import fs from "node:fs"; import path from "node:path"; @@ -172,10 +172,9 @@ interface MaterializedToolPackages { } // Exported for direct unit testing of the manifest-invalid gate -// (JSON.parse failure + arktype schema failure). The in-process -// harness builder reaches it through `createDefaultHarnessBuilder`; -// the workflow-process child reaches it through the substrate -// factory's per-step agent builder. +// (JSON.parse failure + arktype schema failure). The workflow-process +// child reaches it through the substrate factory's per-step agent +// builder. export async function materializeToolPackages(args: { rawManifestBytes: string | undefined; /** @@ -188,9 +187,8 @@ export async function materializeToolPackages(args: { /** * Workspace root the loader resolves `kind: "asset"` tarball mounts * against (`//...`). Defaults to - * `/workspace` -- the in-process harness builder's layout, - * where the deploy flow stages asset packs into the same dir the - * agent runs in. + * `/workspace` -- the default workspace layout, where the + * deploy flow stages asset packs into the same dir the agent runs in. * * The workflow-process child overrides this: it stages assets in the * step's LEGACY agent dir workspace (where the hub's asset-pack push diff --git a/apps/sidecar/src/workflow-substrate-factory.ts b/apps/sidecar/src/workflow-substrate-factory.ts index 5ab72ec7..e6060427 100644 --- a/apps/sidecar/src/workflow-substrate-factory.ts +++ b/apps/sidecar/src/workflow-substrate-factory.ts @@ -146,9 +146,9 @@ const SubstrateConfig = type({ STEP_INFERENCE_SOURCES: "string > 0", // Per-step tool-loader caps. The supervisor threads the boot edge's // resolved `SIDECAR_CACHE_MAX_BYTES` / `SIDECAR_REGISTRY_MAX_TARBALL_BYTES` - // through `substrateEnv`; the child's per-step tool materialization - // uses the same caps the in-process harness builder does. Validated - // as positive-finite-number strings at this boundary. + // through `substrateEnv` so the child's per-step tool materialization is + // bounded by the sidecar's boot-edge-resolved caps. Validated as + // positive-finite-number strings at this boundary. SIDECAR_CACHE_MAX_BYTES: "string > 0", SIDECAR_REGISTRY_MAX_TARBALL_BYTES: "string > 0", // JSON-encoded custom inference adapter manifest. Required: the boot @@ -641,8 +641,7 @@ export function createSidecarStepBuildEnv( // Feed the reactor the step's full ordered failover chain and pin // its initial source to element 0. The reactor resolves the initial // source by id and fails over forward through `sources`, so this - // restores cross-source failover inside the workflow-child, matching - // the legacy in-process harness. + // restores cross-source failover inside the workflow-child. sources, defaultSource: activeSource.id, storage, diff --git a/packages/hub-sessions/src/session-service.ts b/packages/hub-sessions/src/session-service.ts index 16a71c81..924f83cb 100644 --- a/packages/hub-sessions/src/session-service.ts +++ b/packages/hub-sessions/src/session-service.ts @@ -1030,10 +1030,9 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { // Pin the step's inference sources to the instance's FULL ordered source // chain so the workflow-process child's reactor fails over across it at - // runtime, matching the legacy in-process harness. The route already - // resolved and authorized `config.sources` against the tenant catalog, so - // it is pinned directly rather than re-run through the orchestrator's - // operator-approval gate. + // runtime. The route already resolved and authorized `config.sources` + // against the tenant catalog, so it is pinned directly rather than re-run + // through the orchestrator's operator-approval gate. // // Fail loud on the invariant the reactor depends on: the reactor resolves // its initial source by id (`defaultSource`) and fails over FORWARD-ONLY diff --git a/packages/workflow-host/src/adapters/step-invoker.ts b/packages/workflow-host/src/adapters/step-invoker.ts index e85ed524..eb751ee5 100644 --- a/packages/workflow-host/src/adapters/step-invoker.ts +++ b/packages/workflow-host/src/adapters/step-invoker.ts @@ -235,8 +235,7 @@ async function invokeColdStep( // `onEvent` sink; absent a sink the agent's `stream()` is never // consumed (stub agents whose `stream()` throws stay untouched). // - // `message.received` is the single intentional exclusion, matching - // the in-process harness's forwarder (`default-harness.ts`): it is an + // `message.received` is the single intentional exclusion: it is an // assembly-internal dequeue signal, and the hub-facing audit chain // expresses per-message work through the `message.run.started` / // `message.run.ended` bracket pair instead. The filter is an @@ -447,8 +446,7 @@ async function sendWithAbort( * Forwarding is best-effort observability: a sink that throws is * logged and swallowed so a downstream consumer's failure cannot abort * the step. A failure of the stream iterator itself (the agent's - * teardown surfacing through the iterator) is logged at warn, mirroring - * the in-process harness's forwarder. + * teardown surfacing through the iterator) is logged at warn. */ function subscribeAgentEvents( agent: Agent, From e381ebad7affed6f1d6bb49268d90c984f0a2440 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Tue, 7 Jul 2026 13:37:19 -0500 Subject: [PATCH 095/101] Reject an empty sources list at the sources-update boundary `SourcesUpdateFrame.sources` validated as an unconstrained `InferenceSource.array()`, but the doc asserted the list is non-empty with element 0 equal to `defaultSource` -- neither of which the boundary checked. The sibling deploy frame already tightened its per-step source arrays to `.atLeastLength(1)`. Tighten `sources` to `.atLeastLength(1)`, matching that sibling and the frame's own contract: an empty rotation has no source for the agent to swap to. This is safe against the live producer -- the hub's `pushInstanceSourceUpdate` returns early when there is no head source, so it never emits an empty list. The `defaultSource`-equality claim is producer-enforced only and cannot be expressed structurally, so the doc is reworded to say so rather than asserting a check that does not exist. --- packages/types/src/sidecar.test.ts | 31 ++++++++++++++++++++++++++++++ packages/types/src/sidecar.ts | 10 ++++++---- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/packages/types/src/sidecar.test.ts b/packages/types/src/sidecar.test.ts index fcd954ac..fdbcbc67 100644 --- a/packages/types/src/sidecar.test.ts +++ b/packages/types/src/sidecar.test.ts @@ -5,6 +5,7 @@ import { DeployApplyErrorCategory, DeployApplyErrorFrame, SidecarFrame, + SourcesUpdateFrame, } from "./sidecar"; describe("DeployApplyErrorCategory", () => { @@ -216,3 +217,33 @@ describe("AgentDeployFrame", () => { expect(result instanceof type.errors).toBe(true); }); }); + +describe("SourcesUpdateFrame", () => { + const source = { + id: "src_a", + provider: "openai", + baseURL: "https://api.openai.test", + apiKey: "sk-a", + model: "gpt-a", + }; + const base = { + type: "sources.update" as const, + requestId: "req_1", + agentAddress: "agt_1@example.test", + defaultSource: "src_a", + }; + + test("accepts a frame with a non-empty sources list", () => { + const result = SourcesUpdateFrame({ ...base, sources: [source] }); + expect(result instanceof type.errors).toBe(false); + }); + + test("rejects a frame whose sources list is empty", () => { + // The hub never emits an empty rotation -- `pushInstanceSourceUpdate` + // returns early when there is no head source -- so the boundary + // rejects an empty `sources` rather than accepting a rotation the + // agent could not swap to any live source. + const result = SourcesUpdateFrame({ ...base, sources: [] }); + expect(result instanceof type.errors).toBe(true); + }); +}); diff --git a/packages/types/src/sidecar.ts b/packages/types/src/sidecar.ts index deeef353..0e8044eb 100644 --- a/packages/types/src/sidecar.ts +++ b/packages/types/src/sidecar.ts @@ -361,15 +361,17 @@ export type PongFrame = typeof PongFrame.infer; /** * Push an updated inference-source list to a running single-step * deployment. The sidecar routes it to the deployment's supervisor, which - * delivers it to the warm agent and swaps its sources in place; element 0 - * of `sources` is the active source and must equal `defaultSource`. - * Responds with session.ack or session.error. + * delivers it to the warm agent and swaps its sources in place. `sources` + * is non-empty (validated at this boundary, mirroring the deploy frame's + * per-step source arrays). Element 0 is the active source; the producer + * sets `defaultSource` to its id -- that equality is producer-enforced, + * not checked here. Responds with session.ack or session.error. */ export const SourcesUpdateFrame = type({ type: "'sources.update'", requestId: "string", agentAddress: "string", - sources: InferenceSource.array(), + sources: InferenceSource.array().atLeastLength(1), defaultSource: "string", }); export type SourcesUpdateFrame = typeof SourcesUpdateFrame.infer; From 4541fa7a6270449fc9c94e523cd3da8025a6890f Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Tue, 7 Jul 2026 13:41:55 -0500 Subject: [PATCH 096/101] Assert the deploy public key shape in orchestrator tests Four orchestrator deploy tests asserted `result.publicKey` only with `toBeTruthy()`, which passes on any non-empty value. The sibling workflow-host-wiring tests already assert the minted key against `/^[0-9a-f]{64}$/`. Tighten these to the same hex-64 shape so a mangled or wrong-shaped key is caught, not just an absent one. --- .../workflow-deploy/src/orchestrator-fallback-source.test.ts | 4 ++-- .../workflow-deploy/src/orchestrator-nonagent-source.test.ts | 2 +- packages/workflow-deploy/src/orchestrator.test.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/workflow-deploy/src/orchestrator-fallback-source.test.ts b/packages/workflow-deploy/src/orchestrator-fallback-source.test.ts index 762b19c1..b2b540be 100644 --- a/packages/workflow-deploy/src/orchestrator-fallback-source.test.ts +++ b/packages/workflow-deploy/src/orchestrator-fallback-source.test.ts @@ -236,7 +236,7 @@ describe("pickStepInferenceSource (agent step)", () => { operatorApprovals: approvals, }); - expect(result.publicKey).toBeTruthy(); + expect(result.publicKey).toMatch(/^[0-9a-f]{64}$/); expect(deps.sources).toHaveLength(1); const sources = deps.sources[0]; if (sources === undefined) throw new Error("missing sources"); @@ -304,7 +304,7 @@ describe("pickStepInferenceSource (agent step)", () => { operatorApprovals: approvals, }); - expect(result.publicKey).toBeTruthy(); + expect(result.publicKey).toMatch(/^[0-9a-f]{64}$/); expect(deps.sources).toHaveLength(1); const sources = deps.sources[0]; if (sources === undefined) throw new Error("missing sources"); diff --git a/packages/workflow-deploy/src/orchestrator-nonagent-source.test.ts b/packages/workflow-deploy/src/orchestrator-nonagent-source.test.ts index 044b3094..5cd584c7 100644 --- a/packages/workflow-deploy/src/orchestrator-nonagent-source.test.ts +++ b/packages/workflow-deploy/src/orchestrator-nonagent-source.test.ts @@ -308,7 +308,7 @@ describe("pickStepInferenceSource (non-agent step)", () => { operatorApprovals: approvals, }); - expect(result.publicKey).toBeTruthy(); + expect(result.publicKey).toMatch(/^[0-9a-f]{64}$/); expect(deps.sources).toHaveLength(1); const sources = deps.sources[0]; if (sources === undefined) throw new Error("missing sources"); diff --git a/packages/workflow-deploy/src/orchestrator.test.ts b/packages/workflow-deploy/src/orchestrator.test.ts index 44d5475b..e98a47a9 100644 --- a/packages/workflow-deploy/src/orchestrator.test.ts +++ b/packages/workflow-deploy/src/orchestrator.test.ts @@ -629,7 +629,7 @@ describe("createWorkflowDeployOrchestrator", () => { throw new Error("missing step id"); } expect(call.sources[expectedStepId]).toBeDefined(); - expect(result.publicKey).toBeTruthy(); + expect(result.publicKey).toMatch(/^[0-9a-f]{64}$/); }); test("source-pin failure carries workflow.id and names the offending provider+model", async () => { From b6f790893490fce9a0d17dece0db7bfa9061900b Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Tue, 7 Jul 2026 14:22:58 -0500 Subject: [PATCH 097/101] Remove the dead deploy-apply error frame and emitter path The `deploy.apply.error` frame reported a failed tool-package apply from the sidecar to the hub. Its whole path is dead: the sidecar sender was already removed, production never wires the `emitDeployApplyError` callback (the step tool cache is built without it), and the hub-side event was declared with a permanently-empty subscriber set. So the frame type, the emitter type, the emit call sites, the hub receive/dispatch cases, and the empty event registration are all unreachable. Remove them: the DeployApplyErrorFrame type and its SidecarFrame union membership, the DeployApplyErrorEmitter type, the emitDeployApplyError parameter and its three call sites, the two dispatch switch cases, and the sidecar-events entry and subscriber set. The production behavior on a failed apply is unchanged -- the failure is still written to the durable rejected-apply audit and then thrown; only the never-consumed frame emission is gone. `DeployApplyErrorCategory` stays: it is the tool-packaging error taxonomy used by the loader and atomic-apply layers, independent of the frame. Stale prose describing the removed "frame channel" is updated to the throw/audit path, and the tests keep their throw-path coverage by asserting the durable audit instead of the removed emission. --- apps/sidecar/src/step-agent-tools.ts | 14 +-- apps/sidecar/src/tool-materialization.test.ts | 86 +++++++------------ apps/sidecar/src/tool-materialization.ts | 68 ++++----------- packages/hub-agent/src/deploy-tree.test.ts | 4 +- packages/hub-agent/src/deploy-tree.ts | 4 +- packages/hub-agent/src/harness-builder.ts | 12 --- packages/hub-agent/src/paths.ts | 3 - .../hub-sessions/src/ws/sidecar-events.ts | 27 +----- .../hub-sessions/src/ws/sidecar-handler.ts | 22 ----- packages/tool-packaging/README.md | 4 +- packages/tool-packaging/src/index.ts | 3 +- packages/types/src/sidecar.test.ts | 72 ---------------- packages/types/src/sidecar.ts | 41 +-------- 13 files changed, 61 insertions(+), 299 deletions(-) diff --git a/apps/sidecar/src/step-agent-tools.ts b/apps/sidecar/src/step-agent-tools.ts index 4e78295f..d1d5e4ed 100644 --- a/apps/sidecar/src/step-agent-tools.ts +++ b/apps/sidecar/src/step-agent-tools.ts @@ -37,11 +37,7 @@ import { type BaseEnv, type ToolBundle, } from "@intx/agent"; -import { - readDeployTree, - sanitizeAddress, - type DeployApplyErrorEmitter, -} from "@intx/hub-agent/paths"; +import { readDeployTree, sanitizeAddress } from "@intx/hub-agent/paths"; import { getLogger } from "@intx/log"; import type { LoadedToolFactory } from "@intx/tool-packaging"; import { resolveStepAddress } from "@intx/workflow-deploy"; @@ -182,8 +178,8 @@ export function stepDeployTreeDir(args: { * A deploy with no tool-package manifest yields empty factories -- the * legitimate `rawManifestBytes === undefined` case. A manifest that is * present but fails to load surfaces loudly through - * `materializeToolPackages` (the deploy.apply.error / throw path), - * never a silent empty-tools fallback that would mask a broken deploy. + * `materializeToolPackages` (the throw path), never a silent + * empty-tools fallback that would mask a broken deploy. */ export async function materializeStepTools(args: { dataDir: string; @@ -193,7 +189,6 @@ export async function materializeStepTools(args: { /** Per-step state root; cache + instance dir + workspace live under it. */ storeDir: string; cache: StepToolCacheConfig; - emitDeployApplyError?: DeployApplyErrorEmitter; }): Promise { const deployTreeDir = stepDeployTreeDir({ dataDir: args.dataDir, @@ -227,9 +222,6 @@ export async function materializeStepTools(args: { cacheRoot, cacheMaxBytes: args.cache.cacheMaxBytes, registryMaxTarballBytes: args.cache.registryMaxTarballBytes, - ...(args.emitDeployApplyError !== undefined - ? { emitDeployApplyError: args.emitDeployApplyError } - : { emitDeployApplyError: undefined }), }); return { factories: materialized.factories, diff --git a/apps/sidecar/src/tool-materialization.test.ts b/apps/sidecar/src/tool-materialization.test.ts index 46d13c3a..6139f61a 100644 --- a/apps/sidecar/src/tool-materialization.test.ts +++ b/apps/sidecar/src/tool-materialization.test.ts @@ -27,31 +27,9 @@ afterEach(async () => { ); }); -interface CapturedDeployApplyError { - attemptId: string; - previousDeployId: string; - category: string; - message: string; - occurredAt: string; -} - -function captureEmitter(): { - calls: CapturedDeployApplyError[]; - emit: (e: CapturedDeployApplyError) => void; -} { - const calls: CapturedDeployApplyError[] = []; - return { - calls, - emit: (e) => { - calls.push(e); - }, - }; -} - describe("materializeToolPackages — manifest.invalid gate", () => { test("returns empty factories when no manifest bytes supplied", async () => { const storeDir = await tempDir(); - const { calls, emit } = captureEmitter(); const result = await materializeToolPackages({ assetMounts: new Map(), rawManifestBytes: undefined, @@ -60,16 +38,13 @@ describe("materializeToolPackages — manifest.invalid gate", () => { cacheRoot: "/unused-by-manifest-invalid-gate", cacheMaxBytes: 1024 * 1024, registryMaxTarballBytes: 10 * 1024 * 1024, - emitDeployApplyError: emit, }); expect(result.factories).toEqual([]); expect(result.pluginFactories).toEqual([]); - expect(calls).toEqual([]); }); - test("JSON.parse failure persists the raw bytes verbatim and emits manifest.invalid", async () => { + test("JSON.parse failure throws and persists the raw bytes verbatim", async () => { const storeDir = await tempDir(); - const { calls, emit } = captureEmitter(); const corrupt = "{this is not valid json"; let caught: unknown; try { @@ -81,7 +56,6 @@ describe("materializeToolPackages — manifest.invalid gate", () => { cacheRoot: "/unused-by-manifest-invalid-gate", cacheMaxBytes: 1024 * 1024, registryMaxTarballBytes: 10 * 1024 * 1024, - emitDeployApplyError: emit, }); } catch (err) { caught = err; @@ -90,12 +64,6 @@ describe("materializeToolPackages — manifest.invalid gate", () => { expect(String(caught)).toMatch(/manifest\.invalid/); expect(String(caught)).toMatch(/JSON\.parse failed/); - expect(calls).toHaveLength(1); - const call = calls[0]; - if (call === undefined) throw new Error("emitter was not called"); - expect(call.category).toBe("manifest.invalid"); - expect(call.previousDeployId).toBe("none"); - const auditRoot = path.join(storeDir, "audit", "rejected-applies"); const attemptDirs = await fs.promises.readdir(auditRoot); expect(attemptDirs).toHaveLength(1); @@ -121,12 +89,13 @@ describe("materializeToolPackages — manifest.invalid gate", () => { ); expect(errorJson.category).toBe("manifest.invalid"); expect(errorJson.message).toMatch(/JSON\.parse failed/); - expect(errorJson.attemptId).toBe(call.attemptId); + expect(errorJson.previousDeployId).toBe("none"); + expect(typeof errorJson.attemptId).toBe("string"); + expect(errorJson.attemptId.length).toBeGreaterThan(0); }); - test("schema failure persists the raw bytes verbatim and emits manifest.invalid", async () => { + test("schema failure throws and persists the raw bytes verbatim", async () => { const storeDir = await tempDir(); - const { calls, emit } = captureEmitter(); // Valid JSON but wrong shape — arktype rejects it. const wrongShape = JSON.stringify({ schemaVersion: 99 }); let caught: unknown; @@ -139,7 +108,6 @@ describe("materializeToolPackages — manifest.invalid gate", () => { cacheRoot: "/unused-by-manifest-invalid-gate", cacheMaxBytes: 1024 * 1024, registryMaxTarballBytes: 10 * 1024 * 1024, - emitDeployApplyError: emit, }); } catch (err) { caught = err; @@ -148,11 +116,6 @@ describe("materializeToolPackages — manifest.invalid gate", () => { expect(String(caught)).toMatch(/manifest\.invalid/); expect(String(caught)).toMatch(/schema validation failed/); - expect(calls).toHaveLength(1); - const call = calls[0]; - if (call === undefined) throw new Error("emitter was not called"); - expect(call.category).toBe("manifest.invalid"); - const auditRoot = path.join(storeDir, "audit", "rejected-applies"); const attemptDirs = await fs.promises.readdir(auditRoot); expect(attemptDirs).toHaveLength(1); @@ -170,6 +133,11 @@ describe("materializeToolPackages — manifest.invalid gate", () => { "utf-8", ); expect(persisted).toBe(wrongShape); + + const errorJson = JSON.parse( + await fs.promises.readFile(path.join(attemptDir, "error.json"), "utf-8"), + ); + expect(errorJson.category).toBe("manifest.invalid"); }); describe("parseActiveDeployId", () => { @@ -290,12 +258,12 @@ describe("materializeToolPackages — manifest.invalid gate", () => { }); }); - test("boot reconciliation reads the dirty marker as previousDeployId", async () => { + test("boot reconciliation records the dirty marker as previousDeployId", async () => { // Seed an instance where the prior apply could not durably record // the committed deploy id and a dirty marker carries the truth. // The next apply (any apply — here, a manifest-invalid one) must - // surface the marker's id as `previousDeployId` in the failure - // frame, not the stale recorded id and not "none". + // surface the marker's id as `previousDeployId` in the audit + // record, not the stale recorded id and not "none". const storeDir = await tempDir(); const instanceDir = path.join(storeDir, "tool-packages"); await fs.promises.mkdir(instanceDir, { recursive: true }); @@ -308,7 +276,6 @@ describe("materializeToolPackages — manifest.invalid gate", () => { "v1:truth-from-marker", ); - const { calls, emit } = captureEmitter(); let caught: unknown; try { await materializeToolPackages({ @@ -319,21 +286,29 @@ describe("materializeToolPackages — manifest.invalid gate", () => { cacheRoot: "/unused-by-manifest-invalid-gate", cacheMaxBytes: 1024 * 1024, registryMaxTarballBytes: 10 * 1024 * 1024, - emitDeployApplyError: emit, }); } catch (err) { caught = err; } expect(caught).toBeInstanceOf(Error); - expect(calls).toHaveLength(1); - const call = calls[0]; - if (call === undefined) throw new Error("emitter was not called"); - expect(call.previousDeployId).toBe("truth-from-marker"); + + const auditRoot = path.join(storeDir, "audit", "rejected-applies"); + const attemptDirs = await fs.promises.readdir(auditRoot); + expect(attemptDirs).toHaveLength(1); + const attemptDirName = attemptDirs[0]; + if (attemptDirName === undefined) { + throw new Error("no attempt dir was created"); + } + const errorJson = JSON.parse( + await fs.promises.readFile( + path.join(auditRoot, attemptDirName, "error.json"), + "utf-8", + ), + ); + expect(errorJson.previousDeployId).toBe("truth-from-marker"); }); - test("manifest.invalid still throws when no emitDeployApplyError is wired", async () => { - // The hub-side emitter is optional (e.g., for tests or offline - // boots). The gate must still reject and surface to the caller. + test("manifest.invalid throws and writes an audit entry", async () => { const storeDir = await tempDir(); let caught: unknown; try { @@ -341,11 +316,10 @@ describe("materializeToolPackages — manifest.invalid gate", () => { assetMounts: new Map(), rawManifestBytes: "{ not json", storeDir, - agentAddress: "agent-no-emitter", + agentAddress: "agent-throws", cacheRoot: "/unused-by-manifest-invalid-gate", cacheMaxBytes: 1024 * 1024, registryMaxTarballBytes: 10 * 1024 * 1024, - emitDeployApplyError: undefined, }); } catch (err) { caught = err; diff --git a/apps/sidecar/src/tool-materialization.ts b/apps/sidecar/src/tool-materialization.ts index e0786ef0..08022ab1 100644 --- a/apps/sidecar/src/tool-materialization.ts +++ b/apps/sidecar/src/tool-materialization.ts @@ -19,7 +19,6 @@ import fs from "node:fs"; import path from "node:path"; import { type } from "arktype"; import { type AnnotatedPluginFactory } from "@intx/agent"; -import { type DeployApplyErrorEmitter } from "@intx/hub-agent/paths"; import { getLogger } from "@intx/log"; import { type LoadedToolFactory, @@ -201,7 +200,6 @@ export async function materializeToolPackages(args: { cacheRoot: string; cacheMaxBytes: number; registryMaxTarballBytes: number; - emitDeployApplyError: DeployApplyErrorEmitter | undefined; }): Promise { if (args.rawManifestBytes === undefined) { return { factories: [], pluginFactories: [] }; @@ -262,8 +260,8 @@ export async function materializeToolPackages(args: { // failures and arktype schema failures route through the same // `manifest.invalid` category so the hub sees a single failure // shape for malformed manifests regardless of the specific defect. - // The helper persists the rejected bytes + the failure payload and - // emits the WS frame, then returns the Error for the caller to throw. + // The helper persists the rejected bytes + the failure payload to + // the durable audit, then returns the Error for the caller to throw. // Returning instead of throwing lets the caller use `throw await ...`, // which TypeScript narrows control flow against without any // dead-code suffix at the call site. @@ -283,15 +281,6 @@ export async function materializeToolPackages(args: { occurredAt, }, }); - if (args.emitDeployApplyError !== undefined) { - args.emitDeployApplyError({ - attemptId, - previousDeployId, - category: "manifest.invalid", - message, - occurredAt, - }); - } return new Error( `tool-package apply rejected (manifest.invalid): ${reason}`, ); @@ -354,16 +343,6 @@ export async function materializeToolPackages(args: { manifestBytes: rawManifestBytes, failure: result, }); - if (args.emitDeployApplyError !== undefined) { - args.emitDeployApplyError({ - attemptId: result.attemptId, - previousDeployId: result.previousDeployId, - category: result.category, - message: result.message, - ...(result.package !== undefined ? { package: result.package } : {}), - occurredAt: result.occurredAt, - }); - } throw new Error( `tool-package apply rejected (${result.category}): ${result.message}`, ); @@ -377,11 +356,11 @@ export async function materializeToolPackages(args: { // `persistActiveDeployIdWithFallback` has already written the new id // through its no-fsync / dirty-marker degradation ladder, so the new // deploy is logically committed, but the id was not durably flushed. - // Route this through `apply.previous-rotation.failed` — the wire - // contract for that category says `previousDeployId` carries the - // **new** deploy id (the one now live) so the hub records the on-disk - // truth. Emit the failure frame and the audit entry, then throw so - // the harness tears down: the durability gap means the next apply + // Record this as `apply.previous-rotation.failed` — for that category + // `previousDeployId` carries the **new** deploy id (the one now live) + // so the durable audit records the on-disk truth. Write the audit + // entry, then throw so the harness tears + // down: the durability gap means the next apply // cannot trust `previousDeployId` until the next boot reconciles via // the dirty marker. const persistOutcome = await persistActiveDeployIdWithFallback( @@ -406,15 +385,6 @@ export async function materializeToolPackages(args: { occurredAt, }, }); - if (args.emitDeployApplyError !== undefined) { - args.emitDeployApplyError({ - attemptId, - previousDeployId: result.activeDeployId, - category: "apply.previous-rotation.failed", - message, - occurredAt, - }); - } throw new Error( `tool-package apply rejected (apply.previous-rotation.failed): ${message}`, { cause: err }, @@ -558,11 +528,11 @@ export async function clearDirtyMarker( * * Returns `{ degraded: false }` on full success. Returns * `{ degraded: true, error }` when the primary persist failed; the - * caller routes the existing `apply.previous-rotation.failed` frame on - * the back of it. + * caller records an `apply.previous-rotation.failed` audit entry and + * throws on the back of it. * * If the fallback persist also fails, on-disk state will diverge from - * the failure frame for one boot cycle. Boot-time reconciliation reads + * the recorded id for one boot cycle. Boot-time reconciliation reads * the dirty marker and prefers it. * * Exported for direct unit testing of the dirty-marker contract. @@ -590,7 +560,7 @@ export async function persistActiveDeployIdWithFallback( logger.warn`active-deploy-id dirty marker written to ${dirtyPath}; next boot will prefer it over the stale recorded id`; return { degraded: true, error: primary }; } catch (marker) { - logger.error`active-deploy-id dirty marker write also failed for ${activeIdFile}.dirty: ${marker instanceof Error ? marker.message : String(marker)}; on-disk state will diverge from the failure frame for one boot cycle`; + logger.error`active-deploy-id dirty marker write also failed for ${activeIdFile}.dirty: ${marker instanceof Error ? marker.message : String(marker)}; on-disk state will diverge from the recorded id for one boot cycle`; return { degraded: true, error: primary }; } } @@ -643,14 +613,14 @@ async function writeRejectedApplyAudit(args: { ); await fs.promises.mkdir(dir, { recursive: true }); // fsync the files and their parent directory before returning so the - // WS frame emitted by the caller is the second event in the durable - // sequence, not the first. Otherwise, a crash after the frame leaves - // the wire but before the audit bytes hit disk would have the hub - // know about a rejection the sidecar cannot prove the manifest for - // on replay. fsync failures are downgraded to warnings: the bytes - // are written either way, and a refusing fsync (rare networked-FS - // failure mode) should not turn into a second cascading failure on - // an already-failing apply. + // rejection's evidence is durable before the caller throws and tears + // the apply down. Otherwise, a crash after the throw begins + // propagating but before the audit bytes hit disk would leave a + // rejected apply the sidecar cannot prove the manifest for on replay. + // fsync failures are downgraded to warnings: the bytes are written + // either way, and a refusing fsync (rare networked-FS failure mode) + // should not turn into a second cascading failure on an already- + // failing apply. await fsyncWriteFile(path.join(dir, "manifest.json"), args.manifestBytes); await fsyncWriteFile( path.join(dir, "error.json"), diff --git a/packages/hub-agent/src/deploy-tree.test.ts b/packages/hub-agent/src/deploy-tree.test.ts index 838ce9fb..ce158f69 100644 --- a/packages/hub-agent/src/deploy-tree.test.ts +++ b/packages/hub-agent/src/deploy-tree.test.ts @@ -131,8 +131,8 @@ describe("readDeployTree", () => { test("returns corrupt-JSON bytes as-is for the caller to handle", async () => { // The reader does not parse; the caller's loader gate produces - // the manifest.invalid failure and routes it through the same - // deploy.apply.error frame channel as every other category. + // the manifest.invalid failure the same apply-rejection way as + // every other category. const dir = await tempDir(); await fs.promises.mkdir(path.join(dir, "deploy"), { recursive: true }); const garbage = "{this is not valid json"; diff --git a/packages/hub-agent/src/deploy-tree.ts b/packages/hub-agent/src/deploy-tree.ts index df27849a..7afd24c4 100644 --- a/packages/hub-agent/src/deploy-tree.ts +++ b/packages/hub-agent/src/deploy-tree.ts @@ -29,8 +29,8 @@ export type DeployTree = { * JSON parsing and arktype validation are the caller's * responsibility — the sidecar's harness builder does both inside * `materializeToolPackages` so a corrupt or schema-invalid manifest - * routes through the same `deploy.apply.error` frame channel as - * every other apply-time failure (category `manifest.invalid`). + * fails the apply loudly (category `manifest.invalid`) the same way + * as every other apply-time failure. * * Undefined means the manifest file is not present. */ diff --git a/packages/hub-agent/src/harness-builder.ts b/packages/hub-agent/src/harness-builder.ts index 3a20d535..b5e42001 100644 --- a/packages/hub-agent/src/harness-builder.ts +++ b/packages/hub-agent/src/harness-builder.ts @@ -6,7 +6,6 @@ // dependencies on the concrete inference packages the check consults. import type { InferenceSource } from "@intx/types/runtime"; -import type { DeployApplyErrorFrame } from "@intx/types/sidecar"; export type HarnessBuilder = { /** @@ -17,14 +16,3 @@ export type HarnessBuilder = { */ canBuildSource(source: InferenceSource): void; }; - -/** - * Callback the tool-package loader uses to emit a deploy-apply error - * frame back to the hub. The host (the sidecar's workflow-child tool - * materialization) translates this into the wire-level frame. Lives here - * as the dependency-light home the `@intx/hub-agent/paths` entry - * re-exports without pulling in the orchestrator module graph. - */ -export type DeployApplyErrorEmitter = ( - payload: Omit, -) => void; diff --git a/packages/hub-agent/src/paths.ts b/packages/hub-agent/src/paths.ts index d1ef1607..e95bbeaa 100644 --- a/packages/hub-agent/src/paths.ts +++ b/packages/hub-agent/src/paths.ts @@ -13,6 +13,3 @@ export { readDeployTree, type DeployTree } from "./deploy-tree"; export { sanitizeAddress } from "./agent-paths"; -// Type-only re-export: `export type` is erased under `verbatimModuleSyntax`, -// so it adds no runtime import edge to `harness-builder`. -export type { DeployApplyErrorEmitter } from "./harness-builder"; diff --git a/packages/hub-sessions/src/ws/sidecar-events.ts b/packages/hub-sessions/src/ws/sidecar-events.ts index 44553cc4..4b4bab6d 100644 --- a/packages/hub-sessions/src/ws/sidecar-events.ts +++ b/packages/hub-sessions/src/ws/sidecar-events.ts @@ -19,11 +19,7 @@ // wire layer already has both behaviors, and pretending otherwise // would silently change failure handling. -import type { - DeployApplyErrorCategory, - PackRejectReason, - RepoId, -} from "@intx/types/sidecar"; +import type { PackRejectReason, RepoId } from "@intx/types/sidecar"; import type { ConnectorThreadState } from "@intx/types/runtime"; import { getLogger } from "@intx/log"; @@ -112,26 +108,6 @@ export type SidecarEventMap = { "deploy.ref.stale": { agentAddress: string; }; - - /** Notification. Emitted when the sidecar reports that its - * tool-package apply pipeline rejected a deploy. `category` matches - * the closed `DeployApplyErrorCategory` enum; `package` is set - * when the failure implicates a specific manifest entry. - * `previousDeployId` is the atomicity contract — it is always the - * deploy id the instance was running before the rejected attempt - * (the instance keeps running that deploy untouched). Listeners - * are responsible for surfacing the failure to operators and/or - * the deploy lifecycle subsystem; the wire layer only delivers. - */ - "deploy.apply.error": { - agentAddress: string; - attemptId: string; - previousDeployId: string; - category: DeployApplyErrorCategory; - message: string; - package?: { name: string; version: string }; - occurredAt: string; - }; }; export type SidecarEventType = keyof SidecarEventMap; @@ -165,7 +141,6 @@ export function createSidecarEmitter(): SidecarEventEmitter { "agent.deploy.ack": new Set(), "agent.reconnected": new Set(), "deploy.ref.stale": new Set(), - "deploy.apply.error": new Set(), "connector.state.changed": new Set(), }; diff --git a/packages/hub-sessions/src/ws/sidecar-handler.ts b/packages/hub-sessions/src/ws/sidecar-handler.ts index c6193328..44a51316 100644 --- a/packages/hub-sessions/src/ws/sidecar-handler.ts +++ b/packages/hub-sessions/src/ws/sidecar-handler.ts @@ -481,7 +481,6 @@ export function createSidecarRouter( case "connector.state.changed": case "repo.pack.push": case "repo.pack.done": - case "deploy.apply.error": return false; default: return assertNever(frame); @@ -585,27 +584,6 @@ export function createSidecarRouter( return; case "repo.pack.done": return handlePackDone(ws, frame); - case "deploy.apply.error": - // Gate the failure emit on the sending sidecar actually - // owning the named agent. A misbehaving sidecar that knows - // another agent's address could otherwise drive the hub to - // record an apply failure against a deploy the other agent's - // sidecar still considers live, contaminating audit trails - // and any failure-driven rollback logic the hub runs. - if (addressIndex.get(frame.agentAddress) !== ws) { - logger.warn`Dropping deploy.apply.error for ${frame.agentAddress}: not registered to this sidecar`; - return; - } - events.emit("deploy.apply.error", { - agentAddress: frame.agentAddress, - attemptId: frame.attemptId, - previousDeployId: frame.previousDeployId, - category: frame.category, - message: frame.message, - ...(frame.package !== undefined ? { package: frame.package } : {}), - occurredAt: frame.occurredAt, - }); - return; default: return assertNever(frame); } diff --git a/packages/tool-packaging/README.md b/packages/tool-packaging/README.md index c002c246..dbaa6d77 100644 --- a/packages/tool-packaging/README.md +++ b/packages/tool-packaging/README.md @@ -85,8 +85,8 @@ and the sidecar's fetch can cause the apply to fail with The substrate's `writeTreePreservingPrefix` primitive makes PUTs and DELETEs against the asset atomic with respect to one another, but it does not couple them to the sidecar's deploy-apply read. The -intentional behavior is: the apply fails loudly through the existing -`deploy.apply.error` frame channel. There is no silent corruption — +intentional behavior is: the apply fails loudly as a rejected +deploy-apply. There is no silent corruption — the integrity check on the sidecar side guarantees that mismatched bytes are rejected — but the operator does see a failed apply if the asset is mutated concurrently with a launch. diff --git a/packages/tool-packaging/src/index.ts b/packages/tool-packaging/src/index.ts index 6b556292..aefd1e39 100644 --- a/packages/tool-packaging/src/index.ts +++ b/packages/tool-packaging/src/index.ts @@ -22,8 +22,7 @@ // applyAtomic({ manifest, loader, … }) — per-deploy-id apply // protocol that stages each deploy into its own never-renamed // directory and maps every loader failure category onto an -// `ApplyAtomicFailure` the caller routes to the -// `deploy.apply.error` frame channel. +// `ApplyAtomicFailure` the caller surfaces as a rejected apply. export { type ClosureResolver, diff --git a/packages/types/src/sidecar.test.ts b/packages/types/src/sidecar.test.ts index fdbcbc67..a0b6c16b 100644 --- a/packages/types/src/sidecar.test.ts +++ b/packages/types/src/sidecar.test.ts @@ -3,8 +3,6 @@ import { type } from "arktype"; import { AgentDeployFrame, DeployApplyErrorCategory, - DeployApplyErrorFrame, - SidecarFrame, SourcesUpdateFrame, } from "./sidecar"; @@ -38,76 +36,6 @@ describe("DeployApplyErrorCategory", () => { }); }); -describe("DeployApplyErrorFrame", () => { - const validFrame = { - type: "deploy.apply.error" as const, - agentAddress: "agt_1", - attemptId: "atp_1", - previousDeployId: "dpl_0", - category: "integrity.mismatch" as const, - message: "fetched bytes for left-pad@1.3.0 did not match pinned integrity", - occurredAt: "2026-06-05T20:00:00.000Z", - }; - - test("accepts a minimal valid frame", () => { - const result = DeployApplyErrorFrame(validFrame); - expect(result instanceof type.errors).toBe(false); - }); - - test("accepts a frame with package context", () => { - const result = DeployApplyErrorFrame({ - ...validFrame, - package: { name: "left-pad", version: "1.3.0" }, - }); - expect(result instanceof type.errors).toBe(false); - }); - - test("rejects a wrong type discriminator", () => { - const result = DeployApplyErrorFrame({ - ...validFrame, - type: "deploy.apply.ok", - }); - expect(result instanceof type.errors).toBe(true); - }); - - test("rejects an unknown category on the frame", () => { - const result = DeployApplyErrorFrame({ - ...validFrame, - category: "network.timeout", - }); - expect(result instanceof type.errors).toBe(true); - }); - - test("rejects a frame missing previousDeployId", () => { - const { previousDeployId: _omitted, ...partial } = validFrame; - const result = DeployApplyErrorFrame(partial); - expect(result instanceof type.errors).toBe(true); - }); - - test("rejects a package field with a missing version", () => { - const result = DeployApplyErrorFrame({ - ...validFrame, - package: { name: "left-pad" }, - }); - expect(result instanceof type.errors).toBe(true); - }); -}); - -describe("SidecarFrame union", () => { - test("accepts a DeployApplyErrorFrame as a member", () => { - const result = SidecarFrame({ - type: "deploy.apply.error", - agentAddress: "agt_1", - attemptId: "atp_1", - previousDeployId: "dpl_0", - category: "manifest.invalid", - message: "schemaVersion was '2', expected '1'", - occurredAt: "2026-06-05T20:00:00.000Z", - }); - expect(result instanceof type.errors).toBe(false); - }); -}); - describe("AgentDeployFrame", () => { const baseConfig = { sessionId: "ses_1", diff --git a/packages/types/src/sidecar.ts b/packages/types/src/sidecar.ts index 0e8044eb..a194527e 100644 --- a/packages/types/src/sidecar.ts +++ b/packages/types/src/sidecar.ts @@ -619,44 +619,6 @@ export const DeployApplyErrorCategory = type.enumerated( ); export type DeployApplyErrorCategory = typeof DeployApplyErrorCategory.infer; -/** - * Sidecar reports a failed deploy-apply attempt back to the hub. - * - * `previousDeployId` is the atomicity contract for every category - * except `apply.previous-rotation.failed`: it names the deploy the - * instance was running before the rejected attempt, and after this - * frame is emitted the instance continues to run that deploy untouched. - * - * For `apply.previous-rotation.failed` the field has inverted meaning. - * The new deploy was staged and its `active-deploy-id` commit was - * written through the degradation ladder (so the new deploy is live) - * before the persist was found to be non-durable; the field carries - * the **new** deploy id so the hub can record the on-disk truth rather - * than a stale pre-apply id. The atomicity contract is preserved in - * the sense that the field always reflects the deploy id the instance - * is now running — what shifts is whether "now" is pre-apply or - * post-apply, depending on whether the commit landed before the - * failure. Renaming the field to make this explicit would break the - * wire shape; the category's semantics document the override instead. - * - * `package` is set when the failure implicates a specific manifest - * entry; deploy-wide failures (manifest.invalid) omit it. - */ -export const DeployApplyErrorFrame = type({ - type: "'deploy.apply.error'", - agentAddress: "string", - attemptId: "string", - previousDeployId: "string", - category: DeployApplyErrorCategory, - message: "string", - "package?": type({ - name: "string", - version: "string", - }), - occurredAt: "string", -}); -export type DeployApplyErrorFrame = typeof DeployApplyErrorFrame.infer; - /** * Hub requests the sidecar to push its current agent state. The sidecar * responds by sending pack.push frames followed by pack.done using the @@ -688,8 +650,7 @@ export const SidecarFrame = RegisterFrame.or(ReconnectFrame) .or(PackPushFrame) .or(PackDoneFrame) .or(PackAckFrame) - .or(PackRejectFrame) - .or(DeployApplyErrorFrame); + .or(PackRejectFrame); export type SidecarFrame = typeof SidecarFrame.infer; /** All frame types the hub sends to the sidecar. */ From 610b8d6b54c8f117b558745accd8a3612849fffa Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Tue, 7 Jul 2026 14:38:21 -0500 Subject: [PATCH 098/101] Guard shutdown teardown so it always kills the child and stops `shutdownInternal` owns the invariant that teardown always kills the child and always reaches `stopped`, but three steps were unguarded while every neighbor already logs-and-continues: the recycle-policy stop, the mail unsubscribe, and the child kill. Because the unsubscribe runs before the kill, a throwing unsubscribe skipped the kill entirely -- orphaning the child process -- and, having thrown past the phase assignment, left the supervisor wedged in `stopping`. Both the shutdown and recycle-failure paths route through here, so both were exposed. Wrap each of the three steps in its own try/catch that logs at warn and continues (matching the existing unregisterAddress guard), and move the `state = { phase: "stopped" }` transition into a `finally` so it runs regardless. Independent guards so a throw in one step cannot skip a later one; the kill is always attempted and the supervisor always reaches `stopped`. This keeps the function's established best-effort-and-log contract -- it does not rethrow or aggregate, which would change the `shutdown()` and recycle callers and risk re-masking the original cause. Errors surface via `logger.warn`; nothing is silently swallowed. The tests inject a throwing unsubscribe and a throwing kill and assert the child is still killed and a second shutdown is an idempotent no-op. --- .../src/supervisor/recycle.test.ts | 86 +++++++++++++++++- .../src/supervisor/supervisor.ts | 88 +++++++++++++------ 2 files changed, 146 insertions(+), 28 deletions(-) diff --git a/packages/workflow-host/src/supervisor/recycle.test.ts b/packages/workflow-host/src/supervisor/recycle.test.ts index 904dc93d..730eb966 100644 --- a/packages/workflow-host/src/supervisor/recycle.test.ts +++ b/packages/workflow-host/src/supervisor/recycle.test.ts @@ -270,7 +270,10 @@ type SpawnTracker = { totalSpawns: number; }; -function createSpawnTracker(opts: { sigtermExits?: boolean }): SpawnTracker { +function createSpawnTracker(opts: { + sigtermExits?: boolean; + killThrows?: boolean; +}): SpawnTracker { const children: FakeChild[] = []; const spawnEnvs: Record[] = []; const spawner: SubprocessSpawner = ({ env }) => { @@ -307,6 +310,11 @@ function createSpawnTracker(opts: { sigtermExits?: boolean }): SpawnTracker { childToSupervisor.close(); child.resolveExit?.(0); } + // Settle the exit above BEFORE throwing so a shutdown awaiting + // `exited` does not hang; the throw exercises the kill-step guard. + if (opts.killThrows === true) { + throw new Error("child kill boom"); + } }, exited, }; @@ -1678,3 +1686,79 @@ describe("supervisor recycle: respawn credentials-read failure", () => { expect(newChild.killSignals.length).toBeGreaterThan(0); }); }); + +describe("supervisor shutdown: teardown robustness", () => { + test("shutdown still kills the child and reaches stopped when mail unsubscribe throws", async () => { + const baseDir = await makeTempDir("shutdown-unsub-throw-"); + const ipcKeypair = await generateKeyPair(); + const baseMailBus = createMockMailBus(); + // The mail unsubscribe runs BEFORE the child kill in shutdown. Left + // unguarded, a throw here skips the kill (child leak) and leaves the + // supervisor wedged in `stopping`. + const mailBus: MailBusBindings = { + ...baseMailBus, + subscribeMailForAddress(address, handler) { + baseMailBus.subscribeMailForAddress(address, handler); + return () => { + throw new Error("mail unsubscribe boom during shutdown"); + }; + }, + }; + const tracker = createSpawnTracker({ sigtermExits: true }); + const { supervisor } = await spawnSupervisor({ + baseDir, + tracker, + mailBus, + ipcKeypair, + }); + const child = tracker.children[0]; + if (child === undefined) { + throw new Error("tracker.children[0] missing"); + } + + // Shutdown must resolve despite the throwing unsubscribe. + await supervisor.shutdown(); + + // The kill step was reached even though the unsubscribe (which runs + // first) threw. + expect(child.killSignals.length).toBe(1); + + // The supervisor reached `stopped`, not wedged in `stopping`: a second + // shutdown is an idempotent no-op that returns without re-running the + // (throwing) teardown, so no further kill is issued. + await supervisor.shutdown(); + expect(child.killSignals.length).toBe(1); + }); + + test("shutdown reaches stopped when the child kill throws", async () => { + const baseDir = await makeTempDir("shutdown-kill-throw-"); + const ipcKeypair = await generateKeyPair(); + const mailBus = createMockMailBus(); + // The child kill throws (after settling exit). The `finally` must still + // drive the supervisor to `stopped` rather than leaving it in + // `stopping`. + const tracker = createSpawnTracker({ + sigtermExits: true, + killThrows: true, + }); + const { supervisor } = await spawnSupervisor({ + baseDir, + tracker, + mailBus, + ipcKeypair, + }); + const child = tracker.children[0]; + if (child === undefined) { + throw new Error("tracker.children[0] missing"); + } + + // Shutdown must resolve even though kill threw. + await supervisor.shutdown(); + expect(child.killSignals.length).toBe(1); + + // Reached `stopped`: the second shutdown no-ops rather than re-entering + // teardown (which would call the throwing kill again). + await supervisor.shutdown(); + expect(child.killSignals.length).toBe(1); + }); +}); diff --git a/packages/workflow-host/src/supervisor/supervisor.ts b/packages/workflow-host/src/supervisor/supervisor.ts index f4174809..d6e39c42 100644 --- a/packages/workflow-host/src/supervisor/supervisor.ts +++ b/packages/workflow-host/src/supervisor/supervisor.ts @@ -2047,35 +2047,69 @@ export function createWorkflowSupervisor( path only waits for the substrate write to settle. */ }); } - if (recyclePolicy !== null) { - recyclePolicy.stop(); - recyclePolicy = null; - } - spawnContext = null; - if ( - prior.phase === "starting" || - prior.phase === "running" || - prior.phase === "recycling" - ) { - if (prior.mailUnsubscribe !== null) prior.mailUnsubscribe(); - try { - bindings.mailBus.unregisterAddress(bindings.deploymentMailAddress); - } catch (cause) { - const message = cause instanceof Error ? cause.message : String(cause); - logger.warn`mail bus unregisterAddress threw: ${message}`; + // Teardown is best-effort-and-log-and-continue: every step is + // individually guarded so a throw in one (an unsubscribe callback, a + // policy stop, a kill) cannot skip the steps after it -- in + // particular, a throwing `mailUnsubscribe` must not prevent the child + // `kill`. The `finally` makes the `stopped` transition unconditional: + // whatever a step does, the supervisor must not be left wedged in + // `stopping`. This is the documented shutdown carve-out to the + // fail-loud rule -- leaking the child or wedging the supervisor is + // strictly worse than logging and continuing -- so errors surface via + // `logger.warn`, they are not silently swallowed. + try { + if (recyclePolicy !== null) { + try { + recyclePolicy.stop(); + } catch (cause) { + const message = + cause instanceof Error ? cause.message : String(cause); + logger.warn`recycle policy stop threw during shutdown: ${message}`; + } + recyclePolicy = null; } - prior.handle.kill(); - await prior.handle.exited.catch(() => { - /* swallowed: the host has already been told the deployment is - coming down; an error surfaced from the spawner is the - process exiting with a non-zero code, which is what the - shutdown path expects. */ - }); - await prior.eventPump.catch(() => { - /* swallowed for the same reason as above. */ - }); + spawnContext = null; + if ( + prior.phase === "starting" || + prior.phase === "running" || + prior.phase === "recycling" + ) { + if (prior.mailUnsubscribe !== null) { + try { + prior.mailUnsubscribe(); + } catch (cause) { + const message = + cause instanceof Error ? cause.message : String(cause); + logger.warn`mail unsubscribe threw during shutdown: ${message}`; + } + } + try { + bindings.mailBus.unregisterAddress(bindings.deploymentMailAddress); + } catch (cause) { + const message = + cause instanceof Error ? cause.message : String(cause); + logger.warn`mail bus unregisterAddress threw: ${message}`; + } + try { + prior.handle.kill(); + } catch (cause) { + const message = + cause instanceof Error ? cause.message : String(cause); + logger.warn`child kill threw during shutdown: ${message}`; + } + await prior.handle.exited.catch(() => { + /* swallowed: the host has already been told the deployment is + coming down; an error surfaced from the spawner is the + process exiting with a non-zero code, which is what the + shutdown path expects. */ + }); + await prior.eventPump.catch(() => { + /* swallowed for the same reason as above. */ + }); + } + } finally { + state = { phase: "stopped" }; } - state = { phase: "stopped" }; logger.info`supervisor shutdown complete (${opts.reason})`; } From 330a4d826028a9d53fd722a40f700184acad67f2 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Tue, 7 Jul 2026 14:40:59 -0500 Subject: [PATCH 099/101] Reflow three mid-sentence comment wraps Three comments broke mid-sentence with a dangling word on its own line (instance deploy-at-head, single-step deploy docstring, repo-store cache note). Reflow each so the sentence reads continuously. Comment-only. --- packages/hub-api/src/routes/instances.ts | 6 +++--- packages/hub-sessions/src/repo-store/store.ts | 5 ++--- packages/hub-sessions/src/session-service.ts | 6 +++--- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/packages/hub-api/src/routes/instances.ts b/packages/hub-api/src/routes/instances.ts index bdf0cbec..3aa29695 100644 --- a/packages/hub-api/src/routes/instances.ts +++ b/packages/hub-api/src/routes/instances.ts @@ -515,9 +515,9 @@ export function createInstanceRoutes({ // Deploy the instance as a single-step workflow at the head: it runs // as a supervised workflow-process child. The real `agentId` (row.id) // is passed so the child resolves the instance's skills and pinned - // tool packages. The - // returned head public key is surfaced separately via the sidecar's - // `agent.deploy.ack`, so the route discards it here. + // tool packages. The returned head public key is surfaced + // separately via the sidecar's `agent.deploy.ack`, so the route + // discards it here. await sessionService.deployInstanceAtHead({ agentAddress, agentId: row.id, diff --git a/packages/hub-sessions/src/repo-store/store.ts b/packages/hub-sessions/src/repo-store/store.ts index d669b346..2d50b84f 100644 --- a/packages/hub-sessions/src/repo-store/store.ts +++ b/packages/hub-sessions/src/repo-store/store.ts @@ -135,9 +135,8 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { // updateIndex / commit / listFiles, and reuses parsed packfile // indexes across object reads. The index-free tree assembly // (git.writeTree / git.writeBlob) uses no index and threads no cache. - // The store is - // the single writer under withRepoLock, so a long-lived per-repo - // cache never races a concurrent mutator. + // The store is the single writer under withRepoLock, so a long-lived + // per-repo cache never races a concurrent mutator. // // The cache stays a pure accelerator, never a second source of truth, // because the on-disk repo remains authoritative on every axis: diff --git a/packages/hub-sessions/src/session-service.ts b/packages/hub-sessions/src/session-service.ts index 924f83cb..074d8d1c 100644 --- a/packages/hub-sessions/src/session-service.ts +++ b/packages/hub-sessions/src/session-service.ts @@ -876,9 +876,9 @@ export function createSessionService(deps: SessionServiceDeps): SessionService { * workflow frame. The workflow frame makes the sidecar initialize the * head repo and spawn the workflow-process child; the follow-up pack * lands the head's deploy tree. Returns the supervisor's principal - * public key from the frame's - * ack. A workflow-frame launch always yields a deploy-ack key; its - * absence is a wiring bug, not a tolerable case. + * public key from the frame's ack. A workflow-frame launch always + * yields a deploy-ack key; its absence is a wiring bug, not a + * tolerable case. */ const deploySingleStepAtHead: DeploySingleStepFn = async (deployParams) => { const result = await executeLaunchPhases({ From 7699e4715cbb99cf1370e0708483a43b22b878b0 Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Tue, 7 Jul 2026 15:28:05 -0500 Subject: [PATCH 100/101] Make supervisor shutdown structurally total shutdownInternal guarded its teardown one step at a time, but that approach left an escape hatch: the pre-try steps (the accumulator-stop loop, the cohort abort/reject/dispose/wake block) ran outside the finally, so the documented "always reach stopped" invariant did not actually hold. The step-by-step guarding also left the spawn-cleanup regression test vacuous -- with the mail unsubscribe now guarded inside shutdownInternal, a thrown unsubscribe no longer reaches the spawn-catch the test meant to exercise. Stop enumerating throw sources; make the invariant structural. Run the whole teardown body in one try and move the two load-bearing actions -- the child kill and the phase transition to stopped -- into the finally, so a throw in any step still runs both. Make the one pre-try step that can actually throw total at its owner: guard the broadcaster dispose's per-listener onDispose loop (log at error, continue), covering both its shutdown and recycle call sites. Guard the drain-accumulator stop loop per-iteration. The steps that provably cannot throw (cohort abort, rejectCohortAwaiters, wakeDispatch) get no guards but sit inside the try so the invariant survives future edits. The spawn-failure catch's cause guard stays as defense-in-depth for a distinct invariant -- the spawn cause must survive the unwind -- and its comment is corrected to say so rather than list now-guarded steps. The spawn-cleanup test and its comments are repaired to match; the discriminating coverage of the teardown guards lives in the shutdown teardown-robustness tests that drive shutdownInternal directly. --- .../src/supervisor/recycle.test.ts | 30 ++- .../src/supervisor/supervisor.ts | 197 ++++++++++-------- .../src/supervisor/terminal-broadcaster.ts | 22 +- 3 files changed, 151 insertions(+), 98 deletions(-) diff --git a/packages/workflow-host/src/supervisor/recycle.test.ts b/packages/workflow-host/src/supervisor/recycle.test.ts index 730eb966..d4d2c1e9 100644 --- a/packages/workflow-host/src/supervisor/recycle.test.ts +++ b/packages/workflow-host/src/supervisor/recycle.test.ts @@ -598,17 +598,21 @@ describe("supervisor spawn: failure cleanup", () => { ); }); - test("a teardown failure during spawn cleanup does not mask the spawn cause", async () => { - // Regression: the spawn-failure catch tears the cohort down through - // shutdownInternal, whose steps (mail unsubscribe, kill, dispose) are - // individually unguarded. A throw in teardown must not replace the - // original spawn cause -- the operator needs the real startup failure, - // not a secondary teardown error. + test("a throwing teardown step during spawn cleanup is absorbed", async () => { + // On a failed spawn the spawn-catch unwinds the cohort through + // shutdownInternal. Its teardown steps are individually guarded and the + // child kill lives in its `finally`, so a throwing step (here the mail + // unsubscribe) is absorbed end to end: the spawn still rejects with the + // real ready-timeout cause rather than the secondary teardown error, + // and the child is still reaped. The spawn-catch guard that preserves + // the cause is now defense-in-depth -- shutdownInternal is total by + // construction -- so the discriminating coverage of the teardown guards + // themselves lives in the "supervisor shutdown: teardown robustness" + // block below, which drives shutdownInternal directly. const baseDir = await makeTempDir("spawn-teardown-mask-"); const ipcKeypair = await generateKeyPair(); const baseMailBus = createMockMailBus(); - // The mail unsubscribe (called unguarded in shutdownInternal) throws, - // so the teardown itself fails while unwinding a failed spawn. + // The mail unsubscribe throws during the spawn-failure unwind. const mailBus: MailBusBindings = { ...baseMailBus, subscribeMailForAddress(address, handler) { @@ -618,7 +622,7 @@ describe("supervisor spawn: failure cleanup", () => { }; }, }; - // sigtermExits so the timeout path's kill settles the fake child. + // sigtermExits so the ready-timeout reap settles the fake child. const tracker = createSpawnTracker({ sigtermExits: true }); await seedStepGrants( baseDir, @@ -647,6 +651,14 @@ describe("supervisor spawn: failure cleanup", () => { onInferenceEvent: () => undefined, }), ).rejects.toThrow(/did not emit ready/); + + // The child was reaped during the unwind despite the throwing + // unsubscribe -- a throwing teardown step leaves nothing orphaned. + const child = tracker.children[0]; + if (child === undefined) { + throw new Error("tracker.children[0] missing"); + } + expect(child.killSignals.length).toBeGreaterThan(0); }); }); diff --git a/packages/workflow-host/src/supervisor/supervisor.ts b/packages/workflow-host/src/supervisor/supervisor.ts index d6e39c42..9cc9c0f9 100644 --- a/packages/workflow-host/src/supervisor/supervisor.ts +++ b/packages/workflow-host/src/supervisor/supervisor.ts @@ -1638,12 +1638,12 @@ export function createWorkflowSupervisor( }; } catch (cause) { const message = cause instanceof Error ? cause.message : String(cause); - // Guard the teardown so a throw inside `shutdownInternal` (an - // unguarded `mailUnsubscribe`, `handle.kill`, broadcaster dispose, - // or accumulator stop) cannot replace the original spawn `cause`. - // A masked cause would hide the real startup failure behind a - // secondary teardown error. Mirrors the recycle-failure catch, - // which preserves its cause the same way. + // Defense-in-depth for a distinct invariant: the original spawn + // `cause` must survive the unwind. `shutdownInternal` is designed to + // be total and should not throw, but if it ever regresses this guard + // logs the secondary teardown error rather than letting it replace + // `cause` and hide the real startup failure. Mirrors the + // recycle-failure catch, which preserves its cause the same way. await shutdownInternal({ reason: `spawn failed during startup: ${message}`, }).catch((shutdownCause) => { @@ -1975,89 +1975,100 @@ export function createWorkflowSupervisor( if (state.phase === "idle" || state.phase === "stopped") return; const prior = state; state = { phase: "stopping" }; - // Stop every armed drainTimeout accumulator before tearing the - // child down. An accumulator left running would otherwise fire - // its `setTimeout` callback (or its terminal-event watcher's - // settle hook) against a shutdown-mid-flight supervisor; the - // explicit `stop()` makes the lifecycle deterministic. + // shutdownInternal is designed to be TOTAL: when a child is up it must + // always kill it and always reach `stopped`, no matter which teardown + // step throws. Rather than depend on every step being individually + // non-throwing (an approach that has already leaked an escape hatch), + // the whole teardown body runs inside one `try`, and the two + // load-bearing actions -- the child kill and the `phase = "stopped"` + // transition -- live in the `finally`, so a throw anywhere above them + // still runs both. This is the documented shutdown carve-out to the + // fail-loud rule: leaking the child or wedging the supervisor in + // `stopping` is strictly worse than logging and continuing, so the + // steps that can throw surface at `logger.warn` and execution proceeds. + // (`terminalCohortAbort.abort`, `rejectCohortAwaiters`, and + // `wakeDispatch` cannot throw, and the broadcaster's `dispose` is total + // by construction; they sit inside the `try` regardless so the + // invariant survives if that ever changes.) const accumulatorsToDispose = [...drainAccumulators.values()]; - for (const accumulator of accumulatorsToDispose) { - accumulator.stop(); - } - drainAccumulators.clear(); - if ( - prior.phase === "starting" || - prior.phase === "running" || - prior.phase === "recycling" - ) { - prior.terminalCohortAbort.abort(); - // Reject every pending merge round-trip and markConsumed waiter - // so handler closures awaiting them (including fire-and-forget - // `handleSubstrateWriteRequest` instances) cannot outlive the - // dying cohort. Without this, the `await new Promise` inside - // each handler would sit forever on a resolver the dying control - // channel will never invoke. - rejectCohortAwaiters("shutdown"); - // Dispose the cohort broadcaster so any minted iterator settles - // with `done: true` -- the dispatch loop's `waitForRunTerminal` - // and any drainTimeout watcher unblock through the same shutdown - // path the cohort abort drives. - prior.terminalBroadcaster.dispose(); - // Wake the dispatch loop so its `dispatchWake` await settles - // and the loop notices the cohort abort. Without the wake, the - // loop's `Promise.race` would sit on the wake promise until - // some other actor woke it. - wakeDispatch(); - } - // Await every accumulator's `disposed()` so a pending escalation - // commit or terminal-event watcher coroutine cannot outlive the - // supervisor and fire against torn-down bindings. - await Promise.all( - accumulatorsToDispose.map((a) => - a.disposed().catch(() => { - /* swallowed: each accumulator already logs its own failure. */ - }), - ), - ); - if ( - (prior.phase === "running" || prior.phase === "recycling") && - prior.dispatchLoop !== null - ) { - await prior.dispatchLoop.catch(() => { - /* swallowed: dispatch-loop failures are surfaced by the - loop's own logger; the shutdown path only waits for the - loop's last iteration to settle. */ - }); - } - if ( - (prior.phase === "starting" || - prior.phase === "running" || - prior.phase === "recycling") && - prior.replayDone !== null - ) { - // Await the spawn-time replayProcessingToInbox before tearing - // the bindings down. The replay's substrate write - // (`processing/` -> `inbox/` rename via a tree commit) must - // settle before the supervisor's exit; without the await the - // substrate I/O outlives the supervisor and a subsequent boot - // can observe a partially-applied replay. - await prior.replayDone.catch(() => { - /* swallowed: the replay's own catch already surfaces the - failure to the supervisor's warn channel; the shutdown - path only waits for the substrate write to settle. */ - }); - } - // Teardown is best-effort-and-log-and-continue: every step is - // individually guarded so a throw in one (an unsubscribe callback, a - // policy stop, a kill) cannot skip the steps after it -- in - // particular, a throwing `mailUnsubscribe` must not prevent the child - // `kill`. The `finally` makes the `stopped` transition unconditional: - // whatever a step does, the supervisor must not be left wedged in - // `stopping`. This is the documented shutdown carve-out to the - // fail-loud rule -- leaking the child or wedging the supervisor is - // strictly worse than logging and continuing -- so errors surface via - // `logger.warn`, they are not silently swallowed. try { + // Stop every armed drainTimeout accumulator before tearing the child + // down. An accumulator left running would otherwise fire its + // `setTimeout` callback (or its terminal-event watcher's settle hook) + // against a shutdown-mid-flight supervisor. Guard each `stop` so one + // throwing accumulator does not leave the rest armed. + for (const accumulator of accumulatorsToDispose) { + try { + accumulator.stop(); + } catch (cause) { + const message = + cause instanceof Error ? cause.message : String(cause); + logger.warn`drain accumulator stop threw during shutdown: ${message}`; + } + } + drainAccumulators.clear(); + if ( + prior.phase === "starting" || + prior.phase === "running" || + prior.phase === "recycling" + ) { + prior.terminalCohortAbort.abort(); + // Reject every pending merge round-trip and markConsumed waiter + // so handler closures awaiting them (including fire-and-forget + // `handleSubstrateWriteRequest` instances) cannot outlive the + // dying cohort. Without this, the `await new Promise` inside + // each handler would sit forever on a resolver the dying control + // channel will never invoke. + rejectCohortAwaiters("shutdown"); + // Dispose the cohort broadcaster so any minted iterator settles + // with `done: true` -- the dispatch loop's `waitForRunTerminal` + // and any drainTimeout watcher unblock through the same shutdown + // path the cohort abort drives. + prior.terminalBroadcaster.dispose(); + // Wake the dispatch loop so its `dispatchWake` await settles + // and the loop notices the cohort abort. Without the wake, the + // loop's `Promise.race` would sit on the wake promise until + // some other actor woke it. + wakeDispatch(); + } + // Await every accumulator's `disposed()` so a pending escalation + // commit or terminal-event watcher coroutine cannot outlive the + // supervisor and fire against torn-down bindings. + await Promise.all( + accumulatorsToDispose.map((a) => + a.disposed().catch(() => { + /* swallowed: each accumulator already logs its own failure. */ + }), + ), + ); + if ( + (prior.phase === "running" || prior.phase === "recycling") && + prior.dispatchLoop !== null + ) { + await prior.dispatchLoop.catch(() => { + /* swallowed: dispatch-loop failures are surfaced by the + loop's own logger; the shutdown path only waits for the + loop's last iteration to settle. */ + }); + } + if ( + (prior.phase === "starting" || + prior.phase === "running" || + prior.phase === "recycling") && + prior.replayDone !== null + ) { + // Await the spawn-time replayProcessingToInbox before tearing + // the bindings down. The replay's substrate write + // (`processing/` -> `inbox/` rename via a tree commit) must + // settle before the supervisor's exit; without the await the + // substrate I/O outlives the supervisor and a subsequent boot + // can observe a partially-applied replay. + await prior.replayDone.catch(() => { + /* swallowed: the replay's own catch already surfaces the + failure to the supervisor's warn channel; the shutdown + path only waits for the substrate write to settle. */ + }); + } if (recyclePolicy !== null) { try { recyclePolicy.stop(); @@ -2090,6 +2101,17 @@ export function createWorkflowSupervisor( cause instanceof Error ? cause.message : String(cause); logger.warn`mail bus unregisterAddress threw: ${message}`; } + } + } finally { + // Load-bearing: the child kill and the `stopped` transition run + // whatever happened above, so a throwing teardown step can neither + // leak the child nor wedge the supervisor in `stopping`. The kill is + // itself guarded so a throw here cannot re-escape the `finally`. + if ( + prior.phase === "starting" || + prior.phase === "running" || + prior.phase === "recycling" + ) { try { prior.handle.kill(); } catch (cause) { @@ -2107,7 +2129,6 @@ export function createWorkflowSupervisor( /* swallowed for the same reason as above. */ }); } - } finally { state = { phase: "stopped" }; } logger.info`supervisor shutdown complete (${opts.reason})`; diff --git a/packages/workflow-host/src/supervisor/terminal-broadcaster.ts b/packages/workflow-host/src/supervisor/terminal-broadcaster.ts index 050ca33b..22ea04cd 100644 --- a/packages/workflow-host/src/supervisor/terminal-broadcaster.ts +++ b/packages/workflow-host/src/supervisor/terminal-broadcaster.ts @@ -18,8 +18,16 @@ // directly so the consumer wakes up even if its own cohort-abort race // has not landed yet). +import { getLogger } from "@intx/log"; + import type { TerminalEventSource, TerminalRunEvent } from "./types"; +const logger = getLogger([ + "workflow-host", + "supervisor", + "terminal-broadcaster", +]); + type Listener = { onEvent: (event: TerminalRunEvent) => void; onDispose: () => void; @@ -203,7 +211,19 @@ export function createTerminalBroadcaster(): TerminalBroadcaster { } listenersByRunId.clear(); for (const listener of snapshot) { - listener.onDispose(); + // Guard each `onDispose` so one throwing listener does not skip + // the rest: dispose must finalise every minted iterator or the + // supervisor's teardown can leak a consumer awaiting a `next()` + // that never settles. A throw here is a listener bug, so surface + // it at error and continue -- this is what lets the shutdown path + // treat `dispose()` as total. + try { + listener.onDispose(); + } catch (cause) { + const message = + cause instanceof Error ? cause.message : String(cause); + logger.error`terminal broadcaster listener onDispose threw: ${message}`; + } } }, get disposed() { From 5179d03482668ae7b71ae8e9074e26dbed7e820d Mon Sep 17 00:00:00 2001 From: Alexander Guy Date: Tue, 7 Jul 2026 15:31:30 -0500 Subject: [PATCH 101/101] Reflow an awkward comment wrap in tool materialization The deploy-apply removal's reflow left "the harness tears / down" split across two lines mid-phrase. Reflow so the sentence reads continuously. Comment-only. --- apps/sidecar/src/tool-materialization.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/apps/sidecar/src/tool-materialization.ts b/apps/sidecar/src/tool-materialization.ts index 08022ab1..af623345 100644 --- a/apps/sidecar/src/tool-materialization.ts +++ b/apps/sidecar/src/tool-materialization.ts @@ -359,10 +359,9 @@ export async function materializeToolPackages(args: { // Record this as `apply.previous-rotation.failed` — for that category // `previousDeployId` carries the **new** deploy id (the one now live) // so the durable audit records the on-disk truth. Write the audit - // entry, then throw so the harness tears - // down: the durability gap means the next apply - // cannot trust `previousDeployId` until the next boot reconciles via - // the dirty marker. + // entry, then throw so the harness tears down: the durability gap + // means the next apply cannot trust `previousDeployId` until the next + // boot reconciles via the dirty marker. const persistOutcome = await persistActiveDeployIdWithFallback( instanceDir, activeIdFile,