From 0512a1156d8b851b2eae350efcc7f992bc9d1074 Mon Sep 17 00:00:00 2001 From: Trevor Walker Date: Tue, 1 Sep 2026 00:44:38 -0600 Subject: [PATCH 1/2] feat(prime): install managed distributions safely Closes #194 --- .../connection/ConnectionEnvironmentRow.tsx | 83 +- apps/server/src/auth/RpcAuthorization.ts | 2 + .../src/provider/Drivers/PrimeAgentDriver.ts | 17 +- .../provider/Layers/ProviderService.test.ts | 88 + .../src/provider/Layers/ProviderService.ts | 97 +- .../src/provider/Services/ProviderService.ts | 17 + .../prime/PrimeAgentDistributionVerifier.ts | 70 + .../prime/PrimeAgentManagedToolStore.test.ts | 986 ++++++++++ .../prime/PrimeAgentManagedToolStore.ts | 1667 +++++++++++++++++ .../provider/prime/PrimeManagedMaintenance.ts | 244 +++ apps/server/src/serverSettings.test.ts | 52 + apps/server/src/serverSettings.ts | 134 ++ apps/server/src/ws.ts | 15 + .../settings/ProviderInstanceCard.tsx | 230 +++ docs/README.md | 2 + docs/internals/glossary.md | 6 + .../prime-agent-distribution-verification.md | 26 +- docs/internals/prime-agent-managed-install.md | 85 + .../prime-agent-managed-rollback.md | 71 + docs/user/providers-prime-agent.md | 31 + packages/client-runtime/src/state/server.ts | 11 + packages/contracts/src/rpc.ts | 26 + packages/contracts/src/server.ts | 79 + 23 files changed, 4018 insertions(+), 21 deletions(-) create mode 100644 apps/server/src/provider/prime/PrimeAgentManagedToolStore.test.ts create mode 100644 apps/server/src/provider/prime/PrimeAgentManagedToolStore.ts create mode 100644 apps/server/src/provider/prime/PrimeManagedMaintenance.ts create mode 100644 docs/internals/prime-agent-managed-install.md create mode 100644 docs/operations/prime-agent-managed-rollback.md diff --git a/apps/mobile/src/features/connection/ConnectionEnvironmentRow.tsx b/apps/mobile/src/features/connection/ConnectionEnvironmentRow.tsx index 0c7053449..adac1d1ad 100644 --- a/apps/mobile/src/features/connection/ConnectionEnvironmentRow.tsx +++ b/apps/mobile/src/features/connection/ConnectionEnvironmentRow.tsx @@ -1,7 +1,8 @@ import { SymbolView } from "../../components/AppSymbol"; import { connectionStatusText } from "@t3tools/client-runtime/connection"; import type { AtomCommandResult } from "@t3tools/client-runtime/state/runtime"; -import type { EnvironmentId } from "@t3tools/contracts"; +import { useAtomValue } from "@effect/atom-react"; +import type { EnvironmentId, ProviderInstanceId } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; import { AsyncResult } from "effect/unstable/reactivity"; import { useCallback, useState } from "react"; @@ -12,6 +13,8 @@ import { AppText as Text, AppTextInput as TextInput } from "../../components/App import { cn } from "../../lib/cn"; import { copyTextWithHaptic } from "../../lib/copyTextWithHaptic"; import type { ConnectedEnvironmentSummary } from "../../state/remote-runtime-types"; +import { serverEnvironment } from "../../state/server"; +import { useEnvironmentQuery } from "../../state/query"; import { ConnectionStatusDot } from "./ConnectionStatusDot"; function connectionStatusLabel(environment: ConnectedEnvironmentSummary): string | null { @@ -22,6 +25,82 @@ function connectionStatusLabel(environment: ConnectedEnvironmentSummary): string }); } +function PrimeHostMaintenanceInstanceStatus(props: { + readonly environmentId: EnvironmentId; + readonly instanceId: ProviderInstanceId; + readonly label: string; + readonly distributionMessage: string | null; +}) { + const { data, error, isPending } = useEnvironmentQuery( + serverEnvironment.primeManagedMaintenance({ + environmentId: props.environmentId, + input: { instanceId: props.instanceId }, + }), + ); + const operation = data?.scheduled ?? data?.operation ?? null; + return ( + + {props.label} + + {data?.message ?? + (isPending ? "Reading host maintenance status." : (error ?? "Status unavailable."))} + + {props.distributionMessage ? ( + + {props.distributionMessage} + + ) : null} + {operation ? ( + + {operation.status.replaceAll("-", " ")} ยท {operation.message} + + ) : null} + {data?.guidance ? ( + {data.guidance} + ) : null} + + ); +} + +function PrimeHostMaintenanceStatus(props: { readonly environmentId: EnvironmentId }) { + const config = useAtomValue(serverEnvironment.configValueAtom(props.environmentId)); + const primeProviders = + config?.providers.filter((provider) => provider.driver === "primeAgent") ?? []; + return ( + + + Prime host maintenance + + {primeProviders.length > 0 ? ( + primeProviders.map((provider) => ( + + )) + ) : ( + + {config === null + ? "Connect to read Prime maintenance status." + : "This environment reports no configured Prime Agent instance."} + + )} + + Install, update, rollback, switch back, and cleanup are host operations. Open Provider + Settings in Pylon web or desktop for this environment. Active work is never interrupted. + + + ); +} + export function ConnectionEnvironmentRow(props: { readonly environment: ConnectedEnvironmentSummary; readonly expanded: boolean; @@ -163,6 +242,8 @@ export function ConnectionEnvironmentRow(props: { )} + + {props.environment.isRelayManaged ? null : ( - inspectPrimeAgentDistribution( + return yield* Effect.promise(async () => { + const managedReceipt = await resolvePrimeManagedBuildReceiptTarget({ + stateDir: serverConfig.stateDir, + packageRoot: publicPackage.packageRoot, + }); + return await inspectPrimeAgentDistribution( { - stateDir: serverConfig.stateDir, - instanceId, + stateDir: managedReceipt?.stateDir ?? serverConfig.stateDir, + instanceId: managedReceipt?.instanceId ?? instanceId, packageRoot: publicPackage.packageRoot, platform: hostPlatform, checkedAt: snapshot.checkedAt, ...(enableUpdateChecks === undefined ? {} : { enableUpdateChecks }), }, { loadLatestVerifiedPublication }, - ), - ); + ); + }); }).pipe( Effect.catchCause(() => Effect.succeed({ diff --git a/apps/server/src/provider/Layers/ProviderService.test.ts b/apps/server/src/provider/Layers/ProviderService.test.ts index 4308d104f..4e7ce6024 100644 --- a/apps/server/src/provider/Layers/ProviderService.test.ts +++ b/apps/server/src/provider/Layers/ProviderService.test.ts @@ -1160,6 +1160,94 @@ it.effect( }).pipe(Effect.provide(NodeServices.layer)), ); +it.effect("ProviderServiceLive fences starts and inventories exact instance quiescence", () => { + const codex = makeFakeCodexAdapter(); + const cursor = makeFakeCodexAdapter(CURSOR_DRIVER); + const registry = makeAdapterRegistryMock({ + [CODEX_DRIVER]: codex.adapter, + [CURSOR_DRIVER]: cursor.adapter, + }); + const runtimeRepositoryLayer = ProviderSessionRuntime.layer.pipe( + Layer.provide(SqlitePersistenceMemory), + ); + const directoryLayer = ProviderSessionDirectoryLive.pipe(Layer.provide(runtimeRepositoryLayer)); + const providerLayer = Layer.merge( + makeProviderServiceLive().pipe( + Layer.provide(Layer.succeed(ProviderAdapterRegistry.ProviderAdapterRegistry, registry)), + Layer.provide(directoryLayer), + Layer.provide(defaultServerSettingsLayer), + Layer.provide(serverConfigTestLayer), + Layer.provide(AnalyticsService.layerTest), + Layer.provide( + Layer.succeed( + ProviderEventLoggers.ProviderEventLoggers, + ProviderEventLoggers.NoOpProviderEventLoggers, + ), + ), + ), + directoryLayer, + ); + + return Effect.gen(function* () { + const provider = yield* ProviderService.ProviderService; + const activeThread = asThreadId("thread-maintenance-active"); + yield* provider.startSession(activeThread, { + provider: CODEX_DRIVER, + providerInstanceId: codexInstanceId, + threadId: activeThread, + cwd: "/tmp/project-maintenance-active", + runtimeMode: "full-access", + }); + assert.deepInclude(yield* provider.reserveProviderMaintenance!(codexInstanceId), { + status: "busy", + }); + + const quiescentInstanceId = ProviderInstanceId.make("cursor"); + const reserved = yield* provider.reserveProviderMaintenance!(quiescentInstanceId); + assert.equal(reserved.status, "reserved"); + if (reserved.status !== "reserved") return; + const fencedThread = asThreadId("thread-maintenance-fenced"); + const fencedError = yield* provider + .startSession(fencedThread, { + provider: CURSOR_DRIVER, + providerInstanceId: quiescentInstanceId, + threadId: fencedThread, + cwd: "/tmp/project-maintenance-fenced", + runtimeMode: "full-access", + }) + .pipe(Effect.flip); + assert.instanceOf(fencedError, ProviderValidationError); + assert.include(fencedError.message, "fenced for scheduled host maintenance"); + + const directory = yield* ProviderSessionDirectory.ProviderSessionDirectory; + const recoveryThread = asThreadId("thread-maintenance-recovery-fenced"); + yield* directory.upsert({ + threadId: recoveryThread, + provider: CURSOR_DRIVER, + providerInstanceId: quiescentInstanceId, + runtimeMode: "full-access", + }); + cursor.startSession.mockClear(); + const recoveryError = yield* provider + .sendTurn({ threadId: recoveryThread, input: "must not recover", attachments: [] }) + .pipe(Effect.flip); + assert.instanceOf(recoveryError, ProviderValidationError); + assert.include(recoveryError.message, "fenced for scheduled host maintenance"); + assert.equal(cursor.startSession.mock.calls.length, 0); + + yield* provider.releaseProviderMaintenance!(reserved.reservation); + yield* provider.startSession(fencedThread, { + provider: CURSOR_DRIVER, + providerInstanceId: quiescentInstanceId, + threadId: fencedThread, + cwd: "/tmp/project-maintenance-fenced", + runtimeMode: "full-access", + }); + yield* provider.stopSession({ threadId: fencedThread }); + yield* provider.stopSession({ threadId: activeThread }); + }).pipe(Effect.provide(Layer.merge(providerLayer, NodeServices.layer))); +}); + routing.layer("ProviderServiceLive routing", (it) => { it.effect("reclaims start reservations after long historical-thread churn", () => Effect.gen(function* () { diff --git a/apps/server/src/provider/Layers/ProviderService.ts b/apps/server/src/provider/Layers/ProviderService.ts index 29e2b44c8..fc4b08888 100644 --- a/apps/server/src/provider/Layers/ProviderService.ts +++ b/apps/server/src/provider/Layers/ProviderService.ts @@ -348,6 +348,27 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( Effect.asVoid, ) : Effect.void; + const instanceMaintenanceState = yield* SynchronizedRef.make( + new Map(), + ); + const beginInstanceStart = (instanceId: ProviderInstanceId) => + SynchronizedRef.modify(instanceMaintenanceState, (current) => { + const state = current.get(instanceId) ?? { pendingStarts: 0 }; + if (state.fenceToken !== undefined) return [false, current] as const; + const next = new Map(current); + next.set(instanceId, { ...state, pendingStarts: state.pendingStarts + 1 }); + return [true, next] as const; + }); + const finishInstanceStart = (instanceId: ProviderInstanceId) => + SynchronizedRef.update(instanceMaintenanceState, (current) => { + const state = current.get(instanceId); + if (!state) return current; + const next = new Map(current); + const pendingStarts = Math.max(0, state.pendingStarts - 1); + if (pendingStarts === 0 && state.fenceToken === undefined) next.delete(instanceId); + else next.set(instanceId, { ...state, pendingStarts }); + return next; + }); const reserveStartSession = (threadId: ThreadId) => SynchronizedRef.modify(startReservations, (current) => { const previous = current.get(threadId); @@ -882,10 +903,16 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( } as const; } + if (!(yield* beginInstanceStart(instanceId))) { + return yield* toValidationError( + input.operation, + `Provider instance '${instanceId}' is fenced for scheduled host maintenance.`, + ); + } const recovered = yield* recoverSessionForThread({ binding, operation: input.operation, - }); + }).pipe(Effect.ensuring(finishInstanceStart(instanceId))); return { adapter: recovered.adapter, instanceId, @@ -942,6 +969,12 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( "ProviderService.startSession", parsed, ); + if (!(yield* beginInstanceStart(resolvedInstanceId))) { + return yield* toValidationError( + "ProviderService.startSession", + `Provider instance '${resolvedInstanceId}' is fenced for scheduled host maintenance.`, + ); + } let metricProvider = parsed.provider ?? String(resolvedInstanceId); yield* Effect.annotateCurrentSpan({ "provider.operation": "start-session", @@ -1130,7 +1163,10 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( }), ), ) - .pipe(Effect.ensuring(releaseStartReservation(threadId, reservation.token))); + .pipe( + Effect.ensuring(releaseStartReservation(threadId, reservation.token)), + Effect.ensuring(finishInstanceStart(resolvedInstanceId)), + ); }, ); @@ -2382,6 +2418,61 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( }, ); + const releaseProviderMaintenance: ProviderServiceMethod<"releaseProviderMaintenance"> = ( + reservation, + ) => + SynchronizedRef.update(instanceMaintenanceState, (current) => { + const entry = [...current.entries()].find( + ([, state]) => state.fenceToken === reservation.token, + ); + if (!entry) return current; + const [instanceId, state] = entry; + const next = new Map(current); + if (state.pendingStarts === 0) next.delete(instanceId); + else next.set(instanceId, { pendingStarts: state.pendingStarts }); + return next; + }); + + const reserveProviderMaintenance: ProviderServiceMethod<"reserveProviderMaintenance"> = Effect.fn( + "reserveProviderMaintenance", + )(function* (instanceId) { + const token = NodeCrypto.randomUUID(); + const fenced = yield* SynchronizedRef.modify(instanceMaintenanceState, (current) => { + const state = current.get(instanceId) ?? { pendingStarts: 0 }; + if (state.fenceToken !== undefined || state.pendingStarts > 0) { + return [false, current] as const; + } + const next = new Map(current); + next.set(instanceId, { pendingStarts: 0, fenceToken: token }); + return [true, next] as const; + }); + if (!fenced) { + return { + status: "busy", + reasons: ["a provider session start or another maintenance reservation is pending"], + } as const; + } + const reservation = { token }; + const sessions = yield* listSessionsForInstance(instanceId).pipe( + Effect.onError(() => releaseProviderMaintenance(reservation)), + ); + const activeIncarnation = [...currentSessionIncarnations.values()].some( + (incarnation) => incarnation.instanceId === instanceId, + ); + if (sessions.length > 0 || activeIncarnation) { + yield* releaseProviderMaintenance(reservation); + return { + status: "busy", + reasons: [ + sessions.some((session) => session.activeTurnId !== undefined) + ? "an active or admitted provider turn exists" + : "an active provider session or owned runtime exists", + ], + } as const; + } + return { status: "reserved", reservation } as const; + }); + const getCapabilities: ProviderServiceMethod<"getCapabilities"> = (instanceId) => registry.getByInstance(instanceId).pipe(Effect.map((adapter) => adapter.capabilities)); @@ -2560,6 +2651,8 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( listSessions, getSessionContinuation, listSessionsForInstance, + reserveProviderMaintenance, + releaseProviderMaintenance, getCapabilities, getInstanceInfo, rollbackConversation, diff --git a/apps/server/src/provider/Services/ProviderService.ts b/apps/server/src/provider/Services/ProviderService.ts index 93b6318b3..c324e9d82 100644 --- a/apps/server/src/provider/Services/ProviderService.ts +++ b/apps/server/src/provider/Services/ProviderService.ts @@ -66,6 +66,14 @@ import type { ProviderInstanceRoutingInfo } from "./ProviderAdapterRegistry.ts"; /** * ProviderServiceShape - Service API for provider session and turn orchestration. */ +export interface ProviderMaintenanceReservation { + readonly token: string; +} + +export type ProviderMaintenanceReservationResult = + | { readonly status: "reserved"; readonly reservation: ProviderMaintenanceReservation } + | { readonly status: "busy"; readonly reasons: ReadonlyArray }; + export interface ProviderServiceShape { /** * Start a provider session. @@ -212,6 +220,15 @@ export interface ProviderServiceShape { instanceId: ProviderInstanceId, ) => Effect.Effect, ProviderServiceError>; + /** Atomically fences new starts/admissions, then inventories exact instance quiescence. */ + readonly reserveProviderMaintenance?: ( + instanceId: ProviderInstanceId, + ) => Effect.Effect; + + readonly releaseProviderMaintenance?: ( + reservation: ProviderMaintenanceReservation, + ) => Effect.Effect; + /** * Read capabilities for the adapter bound to a configured provider instance. */ diff --git a/apps/server/src/provider/prime/PrimeAgentDistributionVerifier.ts b/apps/server/src/provider/prime/PrimeAgentDistributionVerifier.ts index 9cd05b7be..e49830b04 100644 --- a/apps/server/src/provider/prime/PrimeAgentDistributionVerifier.ts +++ b/apps/server/src/provider/prime/PrimeAgentDistributionVerifier.ts @@ -1547,6 +1547,7 @@ async function loadPublicationFixtureForRelease( channel: ServerProviderDistributionChannel, release: GitHubRelease, dependencies: PrimeDistributionNetworkDependencies, + options: { readonly includeRootArtifact?: boolean } = {}, ): Promise { let previewRelease = release; let stableManifestBytes: Buffer | undefined; @@ -1586,6 +1587,14 @@ async function loadPublicationFixtureForRelease( ) { throw new Error("Stable release must contain only its signed singleton manifest."); } + const rootAsset = parsedRelease.assets.find((asset) => asset.package === "prime-agent"); + if (!rootAsset) throw new Error("Release manifest has no Prime root artifact."); + const rootArtifactBytes = options.includeRootArtifact + ? await dependencies.fetchBytes( + releaseAsset(previewRelease, rootAsset.file).browser_download_url, + MAX_ROOT_ARTIFACT_BYTES, + ) + : undefined; const subjectDigests = new Set([ ...parsedRelease.assets.map((asset) => asset.sha256), sha256(releaseManifestBytes), @@ -1606,6 +1615,7 @@ async function loadPublicationFixtureForRelease( releaseManifestBytes, previewManifestBytes, ...(stableManifestBytes ? { stableManifestBytes } : {}), + ...(rootArtifactBytes ? { rootArtifactBytes } : {}), attestationBundlesBySubjectSha256, }; } @@ -1680,6 +1690,66 @@ export function makeLatestPrimePublicationLoader( }; } +export interface VerifiedPrimePublicationBundle { + readonly publication: VerifiedPrimePublication; + readonly rootArtifactBytes: Buffer; +} + +/** + * Loads the exact signed channel/build manifests, attestations, and root artifact. Verification + * succeeds before the root bytes leave this boundary. Installers must still treat the archive as + * hostile input and validate it before extraction. + */ +export function makeLatestPrimePublicationBundleLoader( + dependencies: PrimeDistributionNetworkDependencies = makePrimeDistributionNetworkDependencies(), +): (channel: ServerProviderDistributionChannel) => Promise { + return async (channel) => { + const raw = await dependencies.fetchJson( + `https://api.github.com/repos/${PRIME_DISTRIBUTION_REPOSITORY}/releases?per_page=100`, + MAX_RELEASE_RESPONSE_BYTES, + ); + const releases = decodeGitHubReleases(raw) + .filter( + (release) => + !release.draft && + release.immutable && + (channel === "preview" + ? /^pylon-build-g[0-9a-f]{12}-r[1-9][0-9]*$/u.test(release.tag_name) + : /^pylon-stable-[0-9]{6}-g[0-9a-f]{12}-r[1-9][0-9]*$/u.test(release.tag_name)), + ) + .slice(0, MAX_FEED_CANDIDATES); + if (releases.length === 0) throw new Error(`No immutable ${channel} publication exists.`); + const trustedRoot = await dependencies.getTrustedRoot(); + const verified: VerifiedPrimePublication[] = []; + for (const release of releases) { + try { + // Authenticate manifests and every attested artifact digest first. Untrusted root bytes are + // not downloaded until the latest exact signed publication has been selected. + const fixture = await loadPublicationFixtureForRelease(channel, release, dependencies); + verified.push( + await verifyPrimePublicationFixture(fixture, { + verifyBundle: async (bundle, expected) => + verifyPrimeSigstoreBundle(bundle, trustedRoot, expected), + verifySourcePolicy: (expected) => verifyRemoteSourcePolicy(expected, dependencies), + }), + ); + } catch { + // One malformed or unrelated release must not hide a later exact candidate. + } + } + const publication = verified.toSorted((left, right) => right.sequence - left.sequence)[0]; + if (!publication) throw new Error(`No exact signed ${channel} publication verified.`); + const rootArtifactBytes = await dependencies.fetchBytes( + `${PRIME_DISTRIBUTION_REPOSITORY_URL}/releases/download/${publication.buildId}/${publication.rootAsset}`, + MAX_ROOT_ARTIFACT_BYTES, + ); + if (sha256(rootArtifactBytes) !== publication.rootSha256) { + throw new Error("Prime root artifact does not match its exact signed digest."); + } + return { publication, rootArtifactBytes }; + }; +} + /** * A real immutable fixture gate for bridge CI. It is deliberately fail-closed: callers must supply * every byte and bundle through {@link verifyPrimePublicationFixture}; no marker or package metadata diff --git a/apps/server/src/provider/prime/PrimeAgentManagedToolStore.test.ts b/apps/server/src/provider/prime/PrimeAgentManagedToolStore.test.ts new file mode 100644 index 000000000..8de8c4b3d --- /dev/null +++ b/apps/server/src/provider/prime/PrimeAgentManagedToolStore.test.ts @@ -0,0 +1,986 @@ +// @effect-diagnostics nodeBuiltinImport:off +import * as NodeCrypto from "node:crypto"; +import * as NodeFSP from "node:fs/promises"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import * as NodeZlib from "node:zlib"; +import { afterEach, describe, expect, it, vi } from "vite-plus/test"; + +import { + assertPrimeAttestationBinding, + canonicalPrimeDistributionJson, + PRIME_DISTRIBUTION_REF, + PRIME_DISTRIBUTION_REPOSITORY, + PRIME_DISTRIBUTION_REPOSITORY_URL, + PRIME_PREVIEW_MANIFEST, + PRIME_RELEASE_MANIFEST, + type ExpectedPrimeAttestation, + type PrimePublicationFixture, + type PrimeSlsaStatement, + verifyPrimePublicationFixture, +} from "./PrimeAgentDistributionVerifier.ts"; +import { + PRIME_MANAGED_TOOL_DIRECTORY, + PrimeAgentManagedToolStore, + resolvePrimeManagedBuildReceiptTarget, + type PrimeManagedBinding, + type PrimeManagedPublicationBundle, + type PrimeManagedToolStoreDependencies, +} from "./PrimeAgentManagedToolStore.ts"; + +const temporaryDirectories: string[] = []; +afterEach(async () => { + vi.restoreAllMocks(); + await Promise.all( + temporaryDirectories + .splice(0) + .map((directory) => NodeFSP.rm(directory, { recursive: true, force: true })), + ); +}); + +const digest = (algorithm: "sha256" | "sha512", value: Buffer): string => + NodeCrypto.createHash(algorithm).update(value).digest("hex"); + +function tarOctal(value: number, length: number): Buffer { + return Buffer.from(`${value.toString(8).padStart(length - 1, "0")}\0`, "ascii"); +} + +function tarHeader(input: { + readonly path: string; + readonly type?: "file" | "directory" | "symlink" | "hardlink"; + readonly bytes?: Buffer; + readonly mode?: number; + readonly link?: string; +}): Buffer { + const header = Buffer.alloc(512); + header.write(input.path, 0, 100, "utf8"); + tarOctal(input.mode ?? (input.type === "directory" ? 0o755 : 0o644), 8).copy(header, 100); + tarOctal(0, 8).copy(header, 108); + tarOctal(0, 8).copy(header, 116); + tarOctal(input.bytes?.byteLength ?? 0, 12).copy(header, 124); + tarOctal(0, 12).copy(header, 136); + header.fill(0x20, 148, 156); + const type = input.type ?? "file"; + header[156] = + type === "file" ? 0x30 : type === "directory" ? 0x35 : type === "symlink" ? 0x32 : 0x31; + if (input.link) header.write(input.link, 157, 100, "utf8"); + header.write("ustar\0", 257, 6, "ascii"); + header.write("00", 263, 2, "ascii"); + let checksum = 0; + for (const byte of header) checksum += byte; + Buffer.from(`${checksum.toString(8).padStart(6, "0")}\0 `, "ascii").copy(header, 148); + return header; +} + +function makeTarGz( + entries: ReadonlyArray<{ + readonly path: string; + readonly type?: "file" | "directory" | "symlink" | "hardlink"; + readonly bytes?: Buffer; + readonly mode?: number; + readonly link?: string; + }>, +): Buffer { + const parts: Buffer[] = []; + for (const entry of entries) { + parts.push(tarHeader(entry)); + if (entry.bytes) { + parts.push(entry.bytes); + const padding = (512 - (entry.bytes.byteLength % 512)) % 512; + if (padding) parts.push(Buffer.alloc(padding)); + } + } + parts.push(Buffer.alloc(1024)); + return NodeZlib.gzipSync(Buffer.concat(parts), { level: 9 }); +} + +function distributionMetadata(sourceCommit: string, sourceTree: string) { + return { + schemaVersion: 1, + repository: PRIME_DISTRIBUTION_REPOSITORY_URL, + sourceCommit, + sourceTree, + buildId: `pylon-build-g${sourceCommit.slice(0, 12)}-r1`, + recipeRevision: 1, + node: "22.23.2", + npm: "11.10.1", + packageLockSha256: "e".repeat(64), + }; +} + +function safeRootTarball(sourceCommit: string, sourceTree: string, version = "1.0.0"): Buffer { + const packageJson = Buffer.from( + `${JSON.stringify({ + name: "prime-agent", + version, + type: "module", + bin: { "prime-agent": "dist/bundle/cli.js" }, + scripts: { postinstall: "node postinstall.cjs" }, + pylonDistribution: distributionMetadata(sourceCommit, sourceTree), + })}\n`, + ); + return makeTarGz([ + { path: "package/", type: "directory", mode: 0o755 }, + { path: "package/package.json", bytes: packageJson, mode: 0o644 }, + { path: "package/postinstall.cjs", bytes: Buffer.from("throw new Error('must-not-run');\n") }, + { path: "package/dist/", type: "directory", mode: 0o755 }, + { path: "package/dist/bundle/", type: "directory", mode: 0o755 }, + { + path: "package/dist/bundle/cli.js", + bytes: Buffer.from("#!/usr/bin/env node\nconsole.log('safe fixture');\n"), + mode: 0o755, + }, + ]); +} + +function statementFor(expected: ExpectedPrimeAttestation): PrimeSlsaStatement { + return { + _type: "https://in-toto.io/Statement/v1", + subject: expected.subjects.map((subject) => ({ + name: subject.name, + digest: { sha256: subject.sha256 }, + })), + predicateType: "https://slsa.dev/provenance/v1", + predicate: { + buildDefinition: { + buildType: "https://actions.github.io/buildtypes/workflow/v1", + externalParameters: { + workflow: { + repository: PRIME_DISTRIBUTION_REPOSITORY_URL, + path: expected.workflow, + ref: PRIME_DISTRIBUTION_REF, + }, + }, + internalParameters: { + github: { + event_name: expected.event, + repository_id: "1349002285", + repository_owner_id: "11325514", + runner_environment: "github-hosted", + }, + }, + resolvedDependencies: [ + { + uri: `git+${PRIME_DISTRIBUTION_REPOSITORY_URL}@${PRIME_DISTRIBUTION_REF}`, + digest: { gitCommit: expected.sourceCommit }, + }, + ], + }, + runDetails: { + builder: { + id: `${PRIME_DISTRIBUTION_REPOSITORY_URL}/${expected.workflow}@${PRIME_DISTRIBUTION_REF}`, + }, + metadata: { + invocationId: `https://github.com/${PRIME_DISTRIBUTION_REPOSITORY}/actions/runs/${expected.workflowRunId ?? "9100"}/attempts/1`, + }, + }, + }, + }; +} + +const acceptedLocalVerifier = { + verifyBundle: async (_bundle: unknown, expected: ExpectedPrimeAttestation) => { + const statement = statementFor(expected); + assertPrimeAttestationBinding(statement, expected); + return statement; + }, + verifySourcePolicy: async () => {}, +}; + +async function publicationBundle( + input: { + readonly channel?: "stable" | "preview"; + readonly sequence?: number; + readonly commitDigit?: string; + readonly rootBytes?: Buffer; + } = {}, +): Promise { + const channel = input.channel ?? "stable"; + const sequence = input.sequence ?? 1; + const commitDigit = input.commitDigit ?? "a"; + const sourceCommit = commitDigit.repeat(40); + const sourceTree = (commitDigit === "f" ? "e" : "f").repeat(40); + const policyCommit = "c".repeat(40); + const buildId = `pylon-build-g${sourceCommit.slice(0, 12)}-r1`; + const version = "1.0.0"; + const rootBytes = input.rootBytes ?? safeRootTarball(sourceCommit, sourceTree, version); + const artifactBytes = new Map([ + [`pylon-prime-agent-${version}.tgz`, rootBytes], + [`pylon-prime-agent-ai-${version}.tgz`, Buffer.from(`ai-${commitDigit}`)], + [`pylon-prime-agent-core-${version}.tgz`, Buffer.from(`core-${commitDigit}`)], + [`pylon-prime-agent-tui-${version}.tgz`, Buffer.from(`tui-${commitDigit}`)], + ]); + const packages = new Map([ + [`pylon-prime-agent-${version}.tgz`, "prime-agent"], + [`pylon-prime-agent-ai-${version}.tgz`, "@earendil-works/pi-ai"], + [`pylon-prime-agent-core-${version}.tgz`, "@earendil-works/pi-agent-core"], + [`pylon-prime-agent-tui-${version}.tgz`, "@earendil-works/pi-tui"], + ]); + const assets = [...artifactBytes] + .map(([file, bytes]) => ({ + package: packages.get(file)!, + file, + size: bytes.byteLength, + sha256: digest("sha256", bytes), + sha512: digest("sha512", bytes), + })) + .toSorted((left, right) => left.file.localeCompare(right.file)); + const source = { + repository: PRIME_DISTRIBUTION_REPOSITORY_URL, + commit: sourceCommit, + tree: sourceTree, + }; + const releaseManifest = { + schemaVersion: 1, + source, + build: { + id: buildId, + recipeRevision: 1, + node: "22.23.2", + npm: "11.10.1", + lockfile: { file: "package-lock.json", sha256: "e".repeat(64) }, + assetBaseUrl: `${PRIME_DISTRIBUTION_REPOSITORY_URL}/releases/download/${buildId}`, + }, + package: { name: "prime-agent", command: "prime-agent", version, minimumNode: "22.8.0" }, + assets, + attestationSubjects: assets.map((asset) => ({ + name: asset.file, + digest: { sha256: asset.sha256, sha512: asset.sha512 }, + })), + }; + const releaseManifestBytes = Buffer.from(canonicalPrimeDistributionJson(releaseManifest)); + const previewManifest = { + schemaVersion: 1, + channel: "preview", + repository: PRIME_DISTRIBUTION_REPOSITORY_URL, + publicationPolicyRevision: 1, + sequenceEpoch: 1, + sequence, + workflowRunId: String(9000 + sequence), + build: { + tag: buildId, + id: buildId, + recipeRevision: 1, + source, + releaseManifest: { + file: PRIME_RELEASE_MANIFEST, + sha256: digest("sha256", releaseManifestBytes), + }, + }, + assets: assets.map(({ file, size, sha256, sha512 }) => ({ file, size, sha256, sha512 })), + }; + const previewManifestBytes = Buffer.from(canonicalPrimeDistributionJson(previewManifest)); + const stableManifest = { + schemaVersion: 1, + channel: "stable", + repository: PRIME_DISTRIBUTION_REPOSITORY_URL, + sequence, + tag: `pylon-stable-${String(sequence).padStart(6, "0")}-g${sourceCommit.slice(0, 12)}-r1`, + history: { + highWater: sequence - 1, + previous: + sequence === 1 + ? null + : { + tag: `pylon-stable-${String(sequence - 1).padStart(6, "0")}-g${"b".repeat(12)}-r1`, + sha256: "1".repeat(64), + }, + }, + build: { + previewSequence: { + sequenceEpoch: 1, + sequence, + workflowRunId: String(9000 + sequence), + }, + previewTag: buildId, + id: buildId, + recipeRevision: 1, + publicationPolicyRevision: 1, + source, + releaseManifest: { + file: PRIME_RELEASE_MANIFEST, + sha256: digest("sha256", releaseManifestBytes), + }, + previewManifest: { + file: PRIME_PREVIEW_MANIFEST, + sha256: digest("sha256", previewManifestBytes), + }, + assets: previewManifest.assets, + }, + promotion: { + kind: "promote", + policyCommit, + policyTree: "d".repeat(40), + publicationPolicyRevision: 1, + }, + revocations: [], + }; + const stableManifestBytes = Buffer.from(canonicalPrimeDistributionJson(stableManifest)); + const attestationBundlesBySubjectSha256 = new Map>(); + for (const subject of [ + ...assets.map((asset) => asset.sha256), + digest("sha256", releaseManifestBytes), + digest("sha256", previewManifestBytes), + digest("sha256", stableManifestBytes), + ]) { + attestationBundlesBySubjectSha256.set(subject, [{ acceptedLocalProof: subject }]); + } + const fixture: PrimePublicationFixture = { + channel, + releaseManifestBytes, + previewManifestBytes, + ...(channel === "stable" ? { stableManifestBytes } : {}), + rootArtifactBytes: rootBytes, + attestationBundlesBySubjectSha256, + }; + const publication = await verifyPrimePublicationFixture(fixture, acceptedLocalVerifier); + return { publication, rootArtifactBytes: rootBytes }; +} + +async function makeHarness( + input: { + readonly bundle?: PrimeManagedPublicationBundle; + readonly busy?: boolean; + readonly installMode?: "seam" | "production"; + readonly crashAfterCommitOnce?: boolean; + readonly installationBarrier?: () => Promise; + } = {}, +) { + const stateDir = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "pylon-prime-managed-")); + temporaryDirectories.push(stateDir); + const stock = NodePath.join(stateDir, "stock-prime-agent"); + await NodeFSP.writeFile(stock, "stock-byte-for-byte\n", { mode: 0o755 }); + let currentBundle = input.bundle ?? (await publicationBundle()); + let loaderError: Error | undefined; + let binding: PrimeManagedBinding = { binaryPath: stock, generation: "binding-0" }; + let busy = input.busy ?? false; + let generation = 0; + let crashAfterCommitOnce = input.crashAfterCommitOnce ?? false; + const reservations = new Set(); + const dependencies: PrimeManagedToolStoreDependencies = { + loadLatestVerifiedPublication: async () => { + if (loaderError) throw loaderError; + return currentBundle; + }, + readBinding: async () => binding, + reserveQuiescentBinding: async (_instanceId, expected) => { + if ( + expected.generation !== binding.generation || + expected.binaryPath !== binding.binaryPath + ) { + throw new Error("Provider binding changed before the maintenance fence was acquired."); + } + if (busy) return { status: "busy", reasons: ["active provider session"] }; + const token = `reservation-${generation}`; + reservations.add(token); + return { status: "reserved", reservation: { token } }; + }, + commitBinding: async ({ expected, binaryPath, reservation }) => { + if (!reservations.has(reservation.token)) throw new Error("Missing exact maintenance fence."); + if ( + expected.generation !== binding.generation || + expected.binaryPath !== binding.binaryPath + ) { + throw new Error("Provider binding changed while maintenance was fenced."); + } + generation += 1; + binding = { binaryPath, generation: `binding-${generation}` }; + if (crashAfterCommitOnce) { + crashAfterCommitOnce = false; + throw new Error("simulated crash after binding CAS"); + } + return binding; + }, + releaseReservation: async ({ token }) => { + reservations.delete(token); + }, + ...(input.installMode === "production" + ? {} + : { + installVerifiedArchive: async ({ extractedPackagePath, prefixPath }) => { + await input.installationBarrier?.(); + const packageRoot = NodePath.join(prefixPath, "node_modules", "prime-agent"); + const binDirectory = NodePath.join(prefixPath, "node_modules", ".bin"); + await NodeFSP.mkdir(NodePath.dirname(packageRoot), { recursive: true }); + await NodeFSP.cp(extractedPackagePath, packageRoot, { recursive: true }); + await NodeFSP.chmod(NodePath.join(packageRoot, "dist", "bundle", "cli.js"), 0o755); + await NodeFSP.mkdir(binDirectory, { recursive: true }); + await NodeFSP.symlink( + "../prime-agent/dist/bundle/cli.js", + NodePath.join(binDirectory, "prime-agent"), + ); + }, + }), + now: () => "2026-09-01T00:00:00.000Z", + }; + const store = new PrimeAgentManagedToolStore({ stateDir, platform: "linux", dependencies }); + await store.initialize(); + return { + stateDir, + stock, + store, + get binding() { + return binding; + }, + setBundle(bundle: PrimeManagedPublicationBundle) { + loaderError = undefined; + currentBundle = bundle; + }, + setOffline(message = "offline") { + loaderError = new Error(message); + }, + setBusy(value: boolean) { + busy = value; + }, + setExternalBinding(binaryPath: string) { + generation += 1; + binding = { binaryPath, generation: `binding-${generation}` }; + }, + }; +} + +function commandId(prefix: string): string { + return `${prefix}-${NodeCrypto.randomBytes(5).toString("hex")}`; +} + +describe("Pylon-managed Prime tool store", () => { + it("installs stable by default, requires explicit preview opt-in, and never changes stock bytes", async () => { + const harness = await makeHarness(); + const stockBefore = await NodeFSP.readFile(harness.stock); + const installed = await harness.store.command({ + commandId: commandId("stable-install"), + instanceId: "primeAgent", + action: "install", + }); + expect(installed).toMatchObject({ status: "succeeded", channel: "stable" }); + expect(harness.binding.binaryPath).toMatch(/node_modules\/\.bin\/prime-agent$/u); + expect(await NodeFSP.readFile(harness.stock)).toEqual(stockBefore); + expect(await harness.store.status("primeAgent")).toMatchObject({ + mode: "managed", + selectedBuildId: installed.buildId, + }); + + const preview = await publicationBundle({ channel: "preview", sequence: 2, commitDigit: "b" }); + harness.setBundle(preview); + await expect( + harness.store.command({ + commandId: commandId("preview-missing-opt-in"), + instanceId: "primeAgent", + action: "update", + channel: "preview", + }), + ).rejects.toThrow(/explicit preview opt-in/u); + const accepted = await harness.store.command({ + commandId: commandId("preview-opt-in"), + instanceId: "primeAgent", + action: "update", + channel: "preview", + allowPreview: true, + }); + expect(accepted).toMatchObject({ status: "succeeded", channel: "preview" }); + expect(await NodeFSP.readFile(harness.stock)).toEqual(stockBefore); + }); + + it("installs the verified bundled CLI offline without executing its lifecycle script", async () => { + const harness = await makeHarness({ installMode: "production" }); + const receipt = await harness.store.command({ + commandId: "offline-layout", + instanceId: "primeAgent", + action: "install", + }); + expect(receipt.status).toBe("succeeded"); + const launcher = await NodeFSP.readlink(harness.binding.binaryPath); + expect(launcher).toBe("../prime-agent/dist/bundle/cli.js"); + const receiptTarget = await resolvePrimeManagedBuildReceiptTarget({ + stateDir: harness.stateDir, + packageRoot: NodePath.join(NodePath.dirname(harness.binding.binaryPath), "..", "prime-agent"), + }); + expect(receiptTarget).toMatchObject({ + instanceId: `managed-build:${receipt.buildId}`, + }); + expect( + await NodeFSP.readFile( + NodePath.join(NodePath.dirname(harness.binding.binaryPath), launcher), + "utf8", + ), + ).toContain("safe fixture"); + }); + + it("publishes durable progress to other clients while installation is still running", async () => { + let releaseInstallation!: () => void; + const installationReleased = new Promise((resolve) => { + releaseInstallation = resolve; + }); + let reportInstallStart!: () => void; + const installStarted = new Promise((resolve) => { + reportInstallStart = resolve; + }); + const harness = await makeHarness({ + installationBarrier: async () => { + reportInstallStart(); + await installationReleased; + }, + }); + const running = harness.store.command({ + commandId: "observable-progress", + instanceId: "primeAgent", + action: "install", + }); + await installStarted; + await expect(harness.store.status("primeAgent")).resolves.toMatchObject({ + operation: { + commandId: "observable-progress", + status: "installing", + }, + }); + releaseInstallation(); + await expect(running).resolves.toMatchObject({ status: "succeeded" }); + }); + + it("reconciles a crash after settings CAS from the durable selection journal", async () => { + const harness = await makeHarness({ crashAfterCommitOnce: true }); + const interrupted = await harness.store.command({ + commandId: "crash-after-cas", + instanceId: "primeAgent", + action: "install", + }); + expect(interrupted.status).toBe("failed"); + expect(harness.binding.binaryPath).toContain(interrupted.buildId!); + + await harness.store.initialize(); + const recovered = await harness.store.status("primeAgent"); + expect(recovered).toMatchObject({ + mode: "managed", + selectedBuildId: interrupted.buildId, + operation: { commandId: "crash-after-cas", status: "succeeded" }, + }); + }); + + it("updates side by side, rolls back only to verified receipt-owned bytes, switches back, and prunes exact builds", async () => { + const first = await publicationBundle({ sequence: 1, commitDigit: "a" }); + const harness = await makeHarness({ bundle: first }); + const stockBefore = await NodeFSP.readFile(harness.stock); + const install = await harness.store.command({ + commandId: commandId("install"), + instanceId: "primeAgent", + action: "install", + }); + const second = await publicationBundle({ sequence: 2, commitDigit: "b" }); + harness.setBundle(second); + const update = await harness.store.command({ + commandId: commandId("update"), + instanceId: "primeAgent", + action: "update", + }); + expect(update.buildId).not.toBe(install.buildId); + expect((await harness.store.status("primeAgent")).availableBuilds).toHaveLength(2); + + const rollback = await harness.store.command({ + commandId: commandId("rollback"), + instanceId: "primeAgent", + action: "rollback", + buildId: install.buildId!, + }); + expect(rollback).toMatchObject({ status: "succeeded", buildId: install.buildId }); + expect(harness.binding.binaryPath).toContain(install.buildId!); + + const stock = await harness.store.command({ + commandId: commandId("stock"), + instanceId: "primeAgent", + action: "use-stock", + }); + expect(stock.status).toBe("succeeded"); + expect(harness.binding.binaryPath).toBe(harness.stock); + expect(await NodeFSP.readFile(harness.stock)).toEqual(stockBefore); + + const cleanup = await harness.store.command({ + commandId: commandId("cleanup"), + instanceId: "primeAgent", + action: "cleanup", + }); + expect(cleanup.status).toBe("succeeded"); + expect((await harness.store.status("primeAgent")).availableBuilds).toEqual([]); + await expect( + harness.store.command({ + commandId: commandId("rollback-pruned"), + instanceId: "primeAgent", + action: "rollback", + buildId: install.buildId!, + }), + ).resolves.toMatchObject({ status: "failed" }); + expect(await NodeFSP.readFile(harness.stock)).toEqual(stockBefore); + }); + + it("treats an external provider-path edit as the new configured stock binding", async () => { + const harness = await makeHarness(); + const installed = await harness.store.command({ + commandId: "managed-before-external-edit", + instanceId: "primeAgent", + action: "install", + }); + const custom = NodePath.join(harness.stateDir, "custom-prime-agent"); + await NodeFSP.writeFile(custom, "custom-stock-bytes\n", { mode: 0o755 }); + harness.setExternalBinding(custom); + + await expect( + harness.store.command({ + commandId: "stock-after-external-edit", + instanceId: "primeAgent", + action: "use-stock", + }), + ).resolves.toMatchObject({ status: "succeeded" }); + expect(harness.binding.binaryPath).toBe(custom); + + await expect( + harness.store.command({ + commandId: "rollback-after-external-edit", + instanceId: "primeAgent", + action: "rollback", + buildId: installed.buildId!, + }), + ).resolves.toMatchObject({ status: "succeeded" }); + await harness.store.command({ + commandId: "restore-new-custom-stock", + instanceId: "primeAgent", + action: "use-stock", + }); + expect(harness.binding.binaryPath).toBe(custom); + }); + + it("schedules an exact binding while busy and commits after the instance drains", async () => { + const harness = await makeHarness({ busy: true }); + const scheduled = await harness.store.command({ + commandId: "busy-update", + instanceId: "primeAgent", + action: "install", + }); + expect(scheduled.status).toBe("waiting-for-quiescence"); + expect(harness.binding.binaryPath).toBe(harness.stock); + expect((await harness.store.status("primeAgent")).scheduled).toMatchObject({ + commandId: "busy-update", + }); + + harness.setBusy(false); + const drained = await harness.store.drain("primeAgent"); + expect(drained).toMatchObject({ commandId: "busy-update", status: "succeeded" }); + expect(harness.binding.binaryPath).toContain(drained!.buildId!); + expect((await harness.store.status("primeAgent")).scheduled).toBeNull(); + }); + + it("supersedes an older scheduled switch with the latest explicit command", async () => { + const harness = await makeHarness({ busy: true }); + const first = await harness.store.command({ + commandId: "scheduled-first", + instanceId: "primeAgent", + action: "install", + }); + expect(first.status).toBe("waiting-for-quiescence"); + const second = await harness.store.command({ + commandId: "scheduled-latest", + instanceId: "primeAgent", + action: "update", + }); + expect(second.status).toBe("waiting-for-quiescence"); + const status = await harness.store.status("primeAgent"); + expect(status.scheduled?.commandId).toBe("scheduled-latest"); + expect(status.operation).toMatchObject({ commandId: "scheduled-latest" }); + await expect( + harness.store.command({ + commandId: "scheduled-first", + instanceId: "primeAgent", + action: "install", + }), + ).resolves.toMatchObject({ status: "failed", message: expect.stringContaining("superseded") }); + }); + + it("recovers download/extraction leftovers without selecting partial bytes", async () => { + const stateDir = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "pylon-prime-crash-")); + temporaryDirectories.push(stateDir); + const root = NodePath.join(stateDir, ...PRIME_MANAGED_TOOL_DIRECTORY.split("/")); + await NodeFSP.mkdir(NodePath.join(root, ".staging-crashed", "partial"), { + recursive: true, + mode: 0o700, + }); + await NodeFSP.mkdir(NodePath.join(root, `pylon-build-g${"a".repeat(12)}-r1`, "partial"), { + recursive: true, + mode: 0o700, + }); + let io = 0; + const binding = { binaryPath: "/stock", generation: "0" }; + const store = new PrimeAgentManagedToolStore({ + stateDir, + platform: "linux", + dependencies: { + loadLatestVerifiedPublication: async () => { + io += 1; + throw new Error("not used"); + }, + readBinding: async () => binding, + reserveQuiescentBinding: async () => ({ status: "busy", reasons: [] }), + commitBinding: async () => binding, + releaseReservation: async () => {}, + }, + }); + await store.initialize(); + expect(await NodeFSP.readdir(root)).not.toContain(".staging-crashed"); + expect(await NodeFSP.readdir(root)).not.toContain(`pylon-build-g${"a".repeat(12)}-r1`); + expect((await store.status("primeAgent")).selectedBuildId).toBeNull(); + expect(io).toBe(0); + }); + + it.each([ + { + name: "path traversal", + archive: () => makeTarGz([{ path: "package/../escape", bytes: Buffer.from("x") }]), + message: /escapes|violates/u, + }, + { + name: "symlink", + archive: () => makeTarGz([{ path: "package/link", type: "symlink", link: "/tmp/outside" }]), + message: /symlink|unsupported/u, + }, + { + name: "hardlink", + archive: () => makeTarGz([{ path: "package/link", type: "hardlink", link: "package.json" }]), + message: /hardlink|unsupported/u, + }, + { + name: "case collision", + archive: () => + makeTarGz([ + { path: "package/File", bytes: Buffer.from("a") }, + { path: "package/file", bytes: Buffer.from("b") }, + ]), + message: /case-colliding/u, + }, + { + name: "wrong package", + archive: () => { + const commit = "a".repeat(40); + const tree = "f".repeat(40); + return makeTarGz([ + { path: "package/", type: "directory" }, + { + path: "package/package.json", + bytes: Buffer.from( + JSON.stringify({ + name: "not-prime-agent", + version: "1.0.0", + bin: { "prime-agent": "dist/bundle/cli.js" }, + pylonDistribution: distributionMetadata(commit, tree), + }), + ), + }, + { path: "package/dist/bundle/cli.js", bytes: Buffer.from("safe"), mode: 0o755 }, + ]); + }, + message: /wrong package/u, + }, + { + name: "wrong bin", + archive: () => { + const commit = "a".repeat(40); + const tree = "f".repeat(40); + return makeTarGz([ + { path: "package/", type: "directory" }, + { + path: "package/package.json", + bytes: Buffer.from( + JSON.stringify({ + name: "prime-agent", + version: "1.0.0", + bin: { prime: "attack.js" }, + pylonDistribution: distributionMetadata(commit, tree), + }), + ), + }, + { path: "package/attack.js", bytes: Buffer.from("attack"), mode: 0o755 }, + ]); + }, + message: /binary identity/u, + }, + { + name: "unexpected install script", + archive: () => { + const commit = "a".repeat(40); + const tree = "f".repeat(40); + return makeTarGz([ + { path: "package/", type: "directory" }, + { + path: "package/package.json", + bytes: Buffer.from( + JSON.stringify({ + name: "prime-agent", + version: "1.0.0", + bin: { "prime-agent": "dist/bundle/cli.js" }, + scripts: { preinstall: "node attack.js" }, + pylonDistribution: distributionMetadata(commit, tree), + }), + ), + }, + { path: "package/dist/bundle/cli.js", bytes: Buffer.from("safe") }, + ]); + }, + message: /unexpected install script/u, + }, + ])( + "rejects a cryptographically accepted malicious archive: $name", + async ({ archive, message }) => { + const bundle = await publicationBundle({ rootBytes: archive() }); + const harness = await makeHarness({ bundle }); + const result = await harness.store.command({ + commandId: commandId("malicious"), + instanceId: "primeAgent", + action: "install", + }); + expect(result.status).toBe("failed"); + expect(result.message).toMatch(message); + expect(harness.binding.binaryPath).toBe(harness.stock); + expect((await harness.store.status("primeAgent")).availableBuilds).toEqual([]); + }, + ); + + it("rejects digest mismatch and signed replay, while offline failure keeps the selected verified build", async () => { + const current = await publicationBundle({ sequence: 2, commitDigit: "b" }); + const harness = await makeHarness({ bundle: current }); + const installed = await harness.store.command({ + commandId: commandId("install-current"), + instanceId: "primeAgent", + action: "install", + }); + const selectedPath = harness.binding.binaryPath; + + const tampered = { + ...current, + rootArtifactBytes: Buffer.from("tampered after verification"), + }; + harness.setBundle(tampered); + const digestFailure = await harness.store.command({ + commandId: commandId("tampered"), + instanceId: "primeAgent", + action: "update", + }); + expect(digestFailure).toMatchObject({ status: "failed" }); + expect(digestFailure.message).toMatch(/digest/u); + + harness.setBundle(await publicationBundle({ sequence: 1, commitDigit: "a" })); + const replay = await harness.store.command({ + commandId: commandId("replay"), + instanceId: "primeAgent", + action: "update", + }); + expect(replay).toMatchObject({ status: "failed" }); + expect(replay.message).toMatch(/replay|downgrade/u); + expect(harness.binding.binaryPath).toBe(selectedPath); + expect((await harness.store.status("primeAgent")).selectedBuildId).toBe(installed.buildId); + + harness.setOffline("offline"); + const offline = await harness.store.command({ + commandId: commandId("offline"), + instanceId: "primeAgent", + action: "update", + }); + expect(offline).toMatchObject({ status: "failed" }); + expect(offline.message).toMatch(/offline/u); + expect(harness.binding.binaryPath).toBe(selectedPath); + expect((await harness.store.status("primeAgent")).selectedBuildId).toBe(installed.buildId); + }); + + it("keeps a verified newer channel high-water when its archive fails safe installation", async () => { + const sourceCommit = "d".repeat(40); + const sourceTree = "f".repeat(40); + const invalidRoot = makeTarGz([ + { path: "package/", type: "directory", mode: 0o755 }, + { + path: "package/package.json", + bytes: Buffer.from( + JSON.stringify({ + name: "wrong-signed-package", + version: "1.0.0", + bin: { "prime-agent": "dist/bundle/cli.js" }, + pylonDistribution: distributionMetadata(sourceCommit, sourceTree), + }), + ), + }, + ]); + const newer = await publicationBundle({ + sequence: 3, + commitDigit: "d", + rootBytes: invalidRoot, + }); + const harness = await makeHarness({ bundle: newer }); + await expect( + harness.store.command({ + commandId: "newer-invalid-archive", + instanceId: "primeAgent", + action: "install", + }), + ).resolves.toMatchObject({ status: "failed" }); + + harness.setBundle(await publicationBundle({ sequence: 2, commitDigit: "b" })); + await expect( + harness.store.command({ + commandId: "older-after-invalid", + instanceId: "primeAgent", + action: "install", + }), + ).resolves.toMatchObject({ + status: "failed", + message: expect.stringMatching(/replay|downgrade/u), + }); + expect(harness.binding.binaryPath).toBe(harness.stock); + }); + + it("deduplicates the same multi-client command and rejects command-id collisions", async () => { + const harness = await makeHarness(); + const input = { + commandId: "shared-command", + instanceId: "primeAgent", + action: "install" as const, + }; + const [left, right] = await Promise.all([ + harness.store.command(input), + harness.store.command(input), + ]); + expect(left).toEqual(right); + expect((await harness.store.status("primeAgent")).availableBuilds).toHaveLength(1); + await expect(harness.store.command({ ...input, action: "use-stock" })).rejects.toThrow( + /reused with different input/u, + ); + }); + + it("does zero download/install/runtime IO on native Windows and gives exact WSL2 guidance", async () => { + const stateDir = NodePath.join(NodeOS.tmpdir(), `pylon-native-win-${NodeCrypto.randomUUID()}`); + let io = 0; + expect( + () => + new PrimeAgentManagedToolStore({ + stateDir, + platform: "win32", + dependencies: { + loadLatestVerifiedPublication: async () => { + io += 1; + throw new Error("must not run"); + }, + readBinding: async () => { + io += 1; + return { binaryPath: "prime-agent", generation: "0" }; + }, + reserveQuiescentBinding: async () => { + io += 1; + return { status: "busy", reasons: [] }; + }, + commitBinding: async () => { + io += 1; + return { binaryPath: "prime-agent", generation: "0" }; + }, + releaseReservation: async () => { + io += 1; + }, + }, + }), + ).toThrow(/WSL2.*no download or install/u); + expect(io).toBe(0); + await expect(NodeFSP.lstat(stateDir)).rejects.toMatchObject({ code: "ENOENT" }); + }); +}); diff --git a/apps/server/src/provider/prime/PrimeAgentManagedToolStore.ts b/apps/server/src/provider/prime/PrimeAgentManagedToolStore.ts new file mode 100644 index 000000000..8cdbfecee --- /dev/null +++ b/apps/server/src/provider/prime/PrimeAgentManagedToolStore.ts @@ -0,0 +1,1667 @@ +// @effect-diagnostics nodeBuiltinImport:off +// @effect-diagnostics globalTimers:off +// @effect-diagnostics globalDate:off +import type { ServerProviderDistributionChannel } from "@t3tools/contracts"; +import * as NodeCrypto from "node:crypto"; +import * as NodeFS from "node:fs"; +import * as NodeFSP from "node:fs/promises"; +import * as NodePath from "node:path"; +import * as NodeZlib from "node:zlib"; + +import { + inspectPrimeAgentDistribution, + persistPrimeManagedReceipt, + type VerifiedPrimePublication, +} from "./PrimeAgentDistributionVerifier.ts"; + +export const PRIME_MANAGED_TOOL_DIRECTORY = "provider-tools/prime-agent"; +export const PRIME_MANAGED_STATE_FILE = "managed-tool-state-v1.json"; +export const PRIME_MANAGED_BUILD_FILE = "pylon-managed-build-v1.json"; + +const BUILD_ID = /^pylon-build-g[0-9a-f]{12}-r[1-9][0-9]*$/u; +const PACKAGE_VERSION = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/u; +const ROOT_ASSET = /^pylon-prime-agent-\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?\.tgz$/u; +const COMMAND_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u; +const MAX_ROOT_ARCHIVE_BYTES = 256 * 1024 * 1024; +const MAX_UNCOMPRESSED_BYTES = 768 * 1024 * 1024; +const MAX_ARCHIVE_ENTRIES = 30_000; +const MAX_ARCHIVE_FILE_BYTES = 128 * 1024 * 1024; +const MAX_ARCHIVE_PATH_BYTES = 512; +const MAX_PACKAGE_JSON_BYTES = 256 * 1024; +const MAX_STORED_OPERATIONS = 512; +const RECEIPT_STATE_DIRECTORY = ".pylon-managed-receipt"; +const RECEIPT_INSTANCE_PREFIX = "managed-build:"; +const STAGING_PREFIX = ".staging-"; + +export type PrimeManagedAction = "install" | "update" | "rollback" | "use-stock" | "cleanup"; +export type PrimeManagedOperationStatus = + | "queued" + | "downloading" + | "verifying" + | "installing" + | "waiting-for-quiescence" + | "switching" + | "succeeded" + | "failed"; + +export interface PrimeManagedBinding { + readonly binaryPath: string; + /** An opaque generation over the complete provider binding, not only binaryPath. */ + readonly generation: string; +} + +export interface PrimeManagedReservation { + readonly token: string; +} + +export type PrimeManagedReservationResult = + | { readonly status: "reserved"; readonly reservation: PrimeManagedReservation } + | { readonly status: "busy"; readonly reasons: ReadonlyArray }; + +export interface PrimeManagedPublicationBundle { + readonly publication: VerifiedPrimePublication; + readonly rootArtifactBytes: Buffer; +} + +export interface PrimeManagedToolStoreDependencies { + readonly loadLatestVerifiedPublication: ( + channel: ServerProviderDistributionChannel, + ) => Promise; + readonly readBinding: (instanceId: string) => Promise; + /** + * The reservation must atomically fence new admissions/session starts for the instance, then + * prove that it has no active or pending admission, turn, session, owned process, or SDK context. + */ + readonly reserveQuiescentBinding: ( + instanceId: string, + expected: PrimeManagedBinding, + ) => Promise; + /** Compare-and-set the complete expected binding while the quiescence fence is held. */ + readonly commitBinding: (input: { + readonly instanceId: string; + readonly expected: PrimeManagedBinding; + readonly binaryPath: string; + readonly reservation: PrimeManagedReservation; + }) => Promise; + readonly releaseReservation: (reservation: PrimeManagedReservation) => Promise; + /** Test seam. Production builds an offline layout directly from the verified bundled CLI. */ + readonly installVerifiedArchive?: (input: { + readonly archivePath: string; + readonly extractedPackagePath: string; + readonly prefixPath: string; + readonly publication: VerifiedPrimePublication; + }) => Promise; + readonly now?: () => string; +} + +export interface PrimeManagedCommandInput { + readonly commandId: string; + readonly instanceId: string; + readonly action: PrimeManagedAction; + readonly channel?: ServerProviderDistributionChannel; + readonly allowPreview?: boolean; + readonly buildId?: string; + readonly scheduleIfBusy?: boolean; +} + +export interface PrimeManagedCommandReceipt { + readonly commandId: string; + readonly instanceId: string; + readonly action: PrimeManagedAction; + readonly status: PrimeManagedOperationStatus; + readonly channel: ServerProviderDistributionChannel | null; + readonly buildId: string | null; + readonly message: string; + readonly startedAt: string; + readonly finishedAt: string | null; +} + +export interface PrimeManagedInstalledBuild { + readonly buildId: string; + readonly channel: ServerProviderDistributionChannel; + readonly sequence: number; + readonly binaryPath: string; + readonly packageRoot: string; +} + +export interface PrimeManagedInstanceStatus { + readonly instanceId: string; + readonly mode: "stock" | "managed"; + readonly selectedBuildId: string | null; + readonly channel: ServerProviderDistributionChannel | null; + readonly availableBuilds: ReadonlyArray; + readonly scheduled: PrimeManagedCommandReceipt | null; + readonly operation: PrimeManagedCommandReceipt | null; + readonly message: string; +} + +interface StoredSelection { + readonly mode: "stock" | "managed"; + readonly selectedBuildId: string | null; + readonly channel: ServerProviderDistributionChannel | null; + readonly stockBinaryPath: string; + readonly binding: PrimeManagedBinding; +} + +interface StoredScheduled { + readonly commandId: string; + readonly instanceId: string; + readonly action: PrimeManagedAction; + readonly expected: PrimeManagedBinding; + readonly targetBinaryPath: string; + readonly buildId: string | null; + readonly channel: ServerProviderDistributionChannel | null; +} + +interface StoredHighWater { + readonly sequenceEpoch: 1; + readonly sequence: number; + readonly buildId: string; +} + +interface StoredSelectionIntent { + readonly commandId: string; + readonly instanceId: string; + readonly action: PrimeManagedAction; + readonly expected: PrimeManagedBinding; + readonly targetBinaryPath: string; + readonly stockBinaryPath: string; + readonly buildId: string | null; + readonly channel: ServerProviderDistributionChannel | null; +} + +interface StoredState { + readonly schemaVersion: 1; + readonly revision: number; + readonly selections: Record; + readonly selectionIntents: Record; + readonly scheduled: Record; + readonly operations: Record< + string, + PrimeManagedCommandReceipt & { readonly fingerprint: string } + >; + readonly latestOperationIds: Record; + readonly highWater: Partial>; +} + +interface BuildMarker { + readonly schemaVersion: 1; + readonly buildId: string; + readonly channel: ServerProviderDistributionChannel; + readonly sequenceEpoch: 1; + readonly sequence: number; + readonly rootSha256: string; + readonly packageRoot: string; + readonly binaryPath: string; +} + +interface TarEntry { + readonly path: string; + readonly kind: "file" | "directory"; + readonly mode: number; + readonly bytes?: Buffer; +} + +function emptyState(): StoredState { + return { + schemaVersion: 1, + revision: 0, + selections: {}, + selectionIntents: {}, + scheduled: {}, + operations: {}, + latestOperationIds: {}, + highWater: {}, + }; +} + +function sha256(value: NodeJS.ArrayBufferView | string): string { + return NodeCrypto.createHash("sha256").update(value).digest("hex"); +} + +function isErrno(cause: unknown, code: string): boolean { + return ( + typeof cause === "object" && + cause !== null && + "code" in cause && + (cause as { readonly code?: unknown }).code === code + ); +} + +function stableJson(value: unknown): string { + const canonical = (input: unknown): unknown => { + if (input === null || typeof input === "string" || typeof input === "boolean") return input; + if (typeof input === "number" && Number.isFinite(input)) return input; + if (Array.isArray(input)) return input.map(canonical); + if (typeof input !== "object") throw new Error("Managed Prime state is not JSON data."); + return Object.fromEntries( + Object.entries(input as Readonly>) + .toSorted(([left], [right]) => left.localeCompare(right)) + .map(([key, entry]) => [key, canonical(entry)]), + ); + }; + return `${JSON.stringify(canonical(value), null, 2)}\n`; +} + +function validateCommand(input: PrimeManagedCommandInput): void { + if (!COMMAND_ID.test(input.commandId)) + throw new Error("Prime maintenance command id is invalid."); + if (!input.instanceId.trim()) throw new Error("Prime provider instance id is required."); + if (input.channel === "preview" && input.allowPreview !== true) { + throw new Error("Prime preview builds require explicit preview opt-in."); + } + if (input.allowPreview === true && input.channel !== "preview") { + throw new Error("Preview opt-in is valid only with the preview channel."); + } + if (input.action === "rollback" && (!input.buildId || !BUILD_ID.test(input.buildId))) { + throw new Error("Prime rollback requires an exact verified managed build id."); + } + if (input.action !== "rollback" && input.buildId !== undefined) { + throw new Error("A build id is accepted only for explicit rollback."); + } + if ( + (input.action === "use-stock" || input.action === "cleanup" || input.action === "rollback") && + input.channel !== undefined + ) { + throw new Error(`Prime ${input.action} does not accept a channel.`); + } +} + +function commandFingerprint(input: PrimeManagedCommandInput): string { + return sha256(stableJson(input)); +} + +async function ensurePrivateDirectory(path: string): Promise { + await NodeFSP.mkdir(path, { recursive: true, mode: 0o700 }); + const info = await NodeFSP.lstat(path); + const uid = process.getuid?.(); + if ( + !info.isDirectory() || + info.isSymbolicLink() || + (uid !== undefined && info.uid !== uid) || + (info.mode & 0o077) !== 0 + ) { + throw new Error(`Managed Prime directory is not a private real directory: ${path}`); + } +} + +async function readBoundedRegularFile(path: string, maxBytes: number): Promise { + let handle: NodeFSP.FileHandle; + try { + handle = await NodeFSP.open( + path, + NodeFS.constants.O_RDONLY | (NodeFS.constants.O_NOFOLLOW ?? 0), + ); + } catch (cause) { + if (isErrno(cause, "ENOENT")) return undefined; + throw cause; + } + try { + const before = await handle.stat({ bigint: true }); + if (!before.isFile() || before.size < 1n || before.size > BigInt(maxBytes)) { + throw new Error(`Managed Prime file is not one bounded regular file: ${path}`); + } + const bytes = Buffer.alloc(Number(before.size)); + let offset = 0; + while (offset < bytes.byteLength) { + const read = await handle.read(bytes, offset, bytes.byteLength - offset, offset); + if (read.bytesRead === 0) throw new Error(`Managed Prime file was truncated: ${path}`); + offset += read.bytesRead; + } + const after = await handle.stat({ bigint: true }); + const pathAfter = await NodeFSP.lstat(path, { bigint: true }); + if ( + before.dev !== after.dev || + before.ino !== after.ino || + before.size !== after.size || + before.mtimeNs !== after.mtimeNs || + after.dev !== pathAfter.dev || + after.ino !== pathAfter.ino || + !pathAfter.isFile() + ) { + throw new Error(`Managed Prime file changed while it was read: ${path}`); + } + return bytes; + } finally { + await handle.close(); + } +} + +async function syncDirectory(path: string): Promise { + const handle = await NodeFSP.open(path, NodeFS.constants.O_RDONLY); + try { + await handle.sync(); + } finally { + await handle.close(); + } +} + +async function writeJsonAtomically(directory: string, file: string, value: unknown): Promise { + const temporary = NodePath.join( + directory, + `.${file}.${process.pid}.${NodeCrypto.randomBytes(10).toString("hex")}.tmp`, + ); + const target = NodePath.join(directory, file); + const handle = await NodeFSP.open( + temporary, + NodeFS.constants.O_WRONLY | NodeFS.constants.O_CREAT | NodeFS.constants.O_EXCL, + 0o600, + ); + try { + await handle.writeFile(stableJson(value), "utf8"); + await handle.sync(); + } finally { + await handle.close(); + } + try { + await NodeFSP.rename(temporary, target); + await syncDirectory(directory); + } catch (cause) { + await NodeFSP.rm(temporary, { force: true }); + throw cause; + } +} + +function decodeStoredState(value: unknown): StoredState { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error("Managed Prime state is not an object."); + } + const state = value as Partial; + if ( + state.schemaVersion !== 1 || + !Number.isSafeInteger(state.revision) || + typeof state.selections !== "object" || + state.selections === null || + (state.selectionIntents !== undefined && + (typeof state.selectionIntents !== "object" || state.selectionIntents === null)) || + typeof state.scheduled !== "object" || + state.scheduled === null || + typeof state.operations !== "object" || + state.operations === null || + (state.latestOperationIds !== undefined && + (typeof state.latestOperationIds !== "object" || state.latestOperationIds === null)) || + typeof state.highWater !== "object" || + state.highWater === null + ) { + throw new Error("Managed Prime state has an unsupported schema."); + } + return { + ...(state as StoredState), + selectionIntents: state.selectionIntents ?? {}, + latestOperationIds: state.latestOperationIds ?? {}, + }; +} + +async function gunzipBounded(bytes: Buffer): Promise { + if (bytes.byteLength < 1 || bytes.byteLength > MAX_ROOT_ARCHIVE_BYTES) { + throw new Error("Prime root tarball exceeds its bounded compressed size."); + } + return await new Promise((resolve, reject) => { + const gunzip = NodeZlib.createGunzip(); + const chunks: Buffer[] = []; + let total = 0; + gunzip.on("data", (chunk: Buffer) => { + total += chunk.byteLength; + if (total > MAX_UNCOMPRESSED_BYTES) { + gunzip.destroy(new Error("Prime root tarball exceeds its bounded expanded size.")); + return; + } + chunks.push(Buffer.from(chunk)); + }); + gunzip.once("error", reject); + gunzip.once("end", () => resolve(Buffer.concat(chunks, total))); + gunzip.end(bytes); + }); +} + +function tarString(block: Buffer, offset: number, length: number): string { + const slice = block.subarray(offset, offset + length); + const zero = slice.indexOf(0); + return slice.subarray(0, zero < 0 ? slice.length : zero).toString("utf8"); +} + +function tarNumber(block: Buffer, offset: number, length: number, field: string): number { + const text = tarString(block, offset, length).trim(); + if (!/^[0-7]+$/u.test(text)) throw new Error(`Prime tarball has an invalid ${field}.`); + const value = Number.parseInt(text, 8); + if (!Number.isSafeInteger(value) || value < 0) { + throw new Error(`Prime tarball has an out-of-range ${field}.`); + } + return value; +} + +function verifyTarChecksum(block: Buffer): void { + const recorded = tarNumber(block, 148, 8, "header checksum"); + let sum = 0; + for (let index = 0; index < block.length; index += 1) { + sum += index >= 148 && index < 156 ? 0x20 : block[index]!; + } + if (sum !== recorded) throw new Error("Prime tarball header checksum is invalid."); +} + +function safeArchivePath(raw: string): string { + if (!raw || Buffer.byteLength(raw) > MAX_ARCHIVE_PATH_BYTES || raw.includes("\\")) { + throw new Error("Prime tarball contains an invalid path."); + } + if (raw.startsWith("/") || /^[A-Za-z]:/u.test(raw) || raw.includes("\0")) { + throw new Error("Prime tarball contains an absolute path."); + } + const parts = raw.split("/").filter((part) => part.length > 0); + if ( + parts.length < 1 || + parts[0] !== "package" || + parts.some((part) => part === "." || part === ".." || part.normalize("NFC") !== part) + ) { + throw new Error("Prime tarball path escapes or violates its one package root."); + } + return parts.join("/"); +} + +async function parsePrimeTarball(bytes: Buffer): Promise> { + const tar = await gunzipBounded(bytes); + const entries: TarEntry[] = []; + const collisionKeys = new Map(); + let offset = 0; + let zeroBlocks = 0; + let expandedFiles = 0; + while (offset + 512 <= tar.byteLength) { + const block = tar.subarray(offset, offset + 512); + offset += 512; + if (block.every((byte) => byte === 0)) { + zeroBlocks += 1; + if (zeroBlocks >= 2) break; + continue; + } + if (zeroBlocks !== 0) throw new Error("Prime tarball contains data after an end marker."); + verifyTarChecksum(block); + const name = tarString(block, 0, 100); + const prefix = tarString(block, 345, 155); + const path = safeArchivePath(prefix ? `${prefix}/${name}` : name); + const type = block[156] ?? 0; + const kind = type === 0 || type === 0x30 ? "file" : type === 0x35 ? "directory" : undefined; + if (!kind) { + throw new Error( + "Prime tarball contains a symlink, hardlink, device, extension, or other unsupported entry.", + ); + } + const mode = tarNumber(block, 100, 8, "mode"); + if ((mode & ~0o777) !== 0) throw new Error("Prime tarball contains privileged mode bits."); + const size = tarNumber(block, 124, 12, "entry size"); + if (kind === "directory" && size !== 0) { + throw new Error("Prime tarball directory contains bytes."); + } + if (size > MAX_ARCHIVE_FILE_BYTES || offset + size > tar.byteLength) { + throw new Error("Prime tarball entry exceeds its bounded size."); + } + expandedFiles += size; + if (expandedFiles > MAX_UNCOMPRESSED_BYTES || entries.length >= MAX_ARCHIVE_ENTRIES) { + throw new Error("Prime tarball exceeds its bounded entry set."); + } + const collisionKey = path.normalize("NFC").toLocaleLowerCase("en-US"); + const prior = collisionKeys.get(collisionKey); + if (prior !== undefined) { + throw new Error(`Prime tarball contains a duplicate or case-colliding path: ${prior}`); + } + collisionKeys.set(collisionKey, path); + entries.push({ + path, + kind, + mode, + ...(kind === "file" ? { bytes: Buffer.from(tar.subarray(offset, offset + size)) } : {}), + }); + offset += Math.ceil(size / 512) * 512; + } + if (zeroBlocks < 2 || tar.subarray(offset).some((byte) => byte !== 0)) { + throw new Error("Prime tarball has no exact zero-padded end marker."); + } + return entries; +} + +function packageManifestFromEntries(entries: ReadonlyArray): { + readonly manifest: Readonly>; + readonly cliPath: string; +} { + const packageJson = entries.find( + (entry) => entry.path === "package/package.json" && entry.kind === "file", + ); + if (!packageJson?.bytes || packageJson.bytes.byteLength > MAX_PACKAGE_JSON_BYTES) { + throw new Error("Prime tarball lacks one bounded package.json."); + } + let manifest: unknown; + try { + manifest = JSON.parse(packageJson.bytes.toString("utf8")) as unknown; + } catch (cause) { + throw new Error("Prime tarball package.json is invalid JSON.", { cause }); + } + if (typeof manifest !== "object" || manifest === null || Array.isArray(manifest)) { + throw new Error("Prime tarball package.json is not an object."); + } + const record = manifest as Readonly>; + const bin = record.bin; + if ( + record.name !== "prime-agent" || + typeof record.version !== "string" || + typeof record.pylonDistribution !== "object" || + record.pylonDistribution === null || + typeof bin !== "object" || + bin === null || + Array.isArray(bin) || + Object.keys(bin).length !== 1 || + (bin as Readonly>)["prime-agent"] !== "dist/bundle/cli.js" + ) { + throw new Error("Prime tarball has the wrong package or prime-agent binary identity."); + } + const scripts = record.scripts; + if (scripts !== undefined) { + if ( + typeof scripts !== "object" || + scripts === null || + Array.isArray(scripts) || + Object.keys(scripts).length !== 1 || + (scripts as Readonly>).postinstall !== "node postinstall.cjs" + ) { + throw new Error("Prime tarball contains an unexpected install script."); + } + if ( + !entries.some((entry) => entry.path === "package/postinstall.cjs" && entry.kind === "file") + ) { + throw new Error("Prime tarball declares its inert postinstall file but omits it."); + } + } + const cliPath = "package/dist/bundle/cli.js"; + const cli = entries.find((entry) => entry.path === cliPath && entry.kind === "file"); + if (!cli?.bytes || cli.bytes.byteLength < 1 || (cli.mode & 0o111) === 0) { + throw new Error("Prime tarball omits an executable exact public CLI entry."); + } + return { manifest: record, cliPath }; +} + +function assertPackageMatchesPublication( + manifest: Readonly>, + publication: VerifiedPrimePublication, +): void { + const metadata = manifest.pylonDistribution; + const record = + typeof metadata === "object" && metadata !== null && !Array.isArray(metadata) + ? (metadata as Readonly>) + : undefined; + if ( + manifest.version !== publication.packageVersion || + record?.buildId !== publication.buildId || + record.sourceCommit !== publication.sourceCommit || + record.sourceTree !== publication.sourceTree || + record.recipeRevision !== publication.recipeRevision || + publication.rootAsset !== `pylon-prime-agent-${publication.packageVersion}.tgz` + ) { + throw new Error("Prime tarball package metadata conflicts with the verified publication."); + } +} + +async function makeContainedDirectory(root: string, relative: string): Promise { + let current = root; + for (const part of relative.split("/").filter(Boolean)) { + current = NodePath.join(current, part); + try { + await NodeFSP.mkdir(current, { mode: 0o700 }); + } catch (cause) { + if (!isErrno(cause, "EEXIST")) throw cause; + } + const info = await NodeFSP.lstat(current); + if (!info.isDirectory() || info.isSymbolicLink()) { + throw new Error("Prime extraction encountered a non-directory path component."); + } + } +} + +async function extractPrimeEntries( + root: string, + entries: ReadonlyArray, +): Promise { + await ensurePrivateDirectory(root); + for (const entry of entries.toSorted((left, right) => left.path.localeCompare(right.path))) { + const relative = entry.path; + const target = NodePath.join(root, ...relative.split("/")); + const expectedPrefix = `${NodePath.resolve(root)}${NodePath.sep}`; + if (!NodePath.resolve(target).startsWith(expectedPrefix)) { + throw new Error("Prime extraction target escapes its staging root."); + } + if (entry.kind === "directory") { + await makeContainedDirectory(root, relative); + await NodeFSP.chmod(target, entry.mode & 0o777); + continue; + } + await makeContainedDirectory(root, NodePath.posix.dirname(relative)); + const handle = await NodeFSP.open( + target, + NodeFS.constants.O_WRONLY | + NodeFS.constants.O_CREAT | + NodeFS.constants.O_EXCL | + (NodeFS.constants.O_NOFOLLOW ?? 0), + entry.mode & 0o777, + ); + try { + await handle.writeFile(entry.bytes!); + await handle.sync(); + } finally { + await handle.close(); + } + } + const directories = new Set([root]); + for (const entry of entries) { + const parts = entry.path.split("/"); + const directoryParts = entry.kind === "directory" ? parts : parts.slice(0, -1); + for (let length = 1; length <= directoryParts.length; length += 1) { + directories.add(NodePath.join(root, ...directoryParts.slice(0, length))); + } + } + for (const directory of [...directories].toSorted((left, right) => right.length - left.length)) { + await syncDirectory(directory); + } + const packageRoot = NodePath.join(root, "package"); + const realPackageRoot = await NodeFSP.realpath(packageRoot); + if (realPackageRoot !== packageRoot) { + throw new Error("Prime extracted package root is not canonical."); + } + return packageRoot; +} + +async function installVerifiedPackageTree(input: { + readonly extractedPackagePath: string; + readonly prefixPath: string; +}): Promise { + // The signed root artifact already contains the bundled CLI. Building the managed layout + // ourselves keeps installation offline and byte-bounded: no package manager, lifecycle script, + // registry resolution, or dependency download can run after verification. + await ensurePrivateDirectory(input.prefixPath); + const nodeModules = NodePath.join(input.prefixPath, "node_modules"); + const binDirectory = NodePath.join(nodeModules, ".bin"); + await NodeFSP.mkdir(nodeModules, { mode: 0o700 }); + await NodeFSP.mkdir(binDirectory, { mode: 0o700 }); + const packageRoot = NodePath.join(nodeModules, "prime-agent"); + await NodeFSP.rename(input.extractedPackagePath, packageRoot); + const binaryPath = NodePath.join(binDirectory, "prime-agent"); + await NodeFSP.symlink("../prime-agent/dist/bundle/cli.js", binaryPath); + await syncDirectory(binDirectory); + await syncDirectory(nodeModules); + await syncDirectory(input.prefixPath); +} + +async function validateInstalledLauncher(input: { + readonly prefixPath: string; + readonly expectedManifest: Readonly>; +}): Promise<{ readonly binaryPath: string; readonly packageRoot: string }> { + const packageRoot = NodePath.join(input.prefixPath, "node_modules", "prime-agent"); + const canonicalPackageRoot = await NodeFSP.realpath(packageRoot); + if (canonicalPackageRoot !== packageRoot) + throw new Error("Installed Prime package root is not canonical."); + const installedManifestBytes = await readBoundedRegularFile( + NodePath.join(packageRoot, "package.json"), + MAX_PACKAGE_JSON_BYTES, + ); + if (!installedManifestBytes) throw new Error("Installed Prime package.json is missing."); + const installedManifest = JSON.parse(installedManifestBytes.toString("utf8")) as unknown; + if (stableJson(installedManifest) !== stableJson(input.expectedManifest)) { + throw new Error("Installed Prime package identity differs from the verified root tarball."); + } + const binaryPath = NodePath.join(input.prefixPath, "node_modules", ".bin", "prime-agent"); + const launcher = await NodeFSP.lstat(binaryPath); + const expectedCli = NodePath.join(packageRoot, "dist", "bundle", "cli.js"); + if (launcher.isSymbolicLink()) { + const link = await NodeFSP.readlink(binaryPath); + if (NodePath.isAbsolute(link) || (await NodeFSP.realpath(binaryPath)) !== expectedCli) { + throw new Error("Managed Prime launcher points outside the verified package root."); + } + } else if (!launcher.isFile()) { + throw new Error("Managed Prime launcher is not a POSIX file or contained symlink."); + } + await NodeFSP.access(binaryPath, NodeFS.constants.X_OK); + return { binaryPath, packageRoot }; +} + +function markerFor( + publication: VerifiedPrimePublication, + installed: PrimeManagedInstalledBuild, +): BuildMarker { + return { + schemaVersion: 1, + buildId: publication.buildId, + channel: publication.channel, + sequenceEpoch: publication.sequenceEpoch, + sequence: publication.sequence, + rootSha256: publication.rootSha256, + packageRoot: installed.packageRoot, + binaryPath: installed.binaryPath, + }; +} + +function decodeBuildMarker(value: unknown): BuildMarker { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error("Managed Prime build marker is invalid."); + } + const marker = value as Partial; + if ( + marker.schemaVersion !== 1 || + typeof marker.buildId !== "string" || + !BUILD_ID.test(marker.buildId) || + (marker.channel !== "stable" && marker.channel !== "preview") || + marker.sequenceEpoch !== 1 || + !Number.isSafeInteger(marker.sequence) || + typeof marker.rootSha256 !== "string" || + !/^[0-9a-f]{64}$/u.test(marker.rootSha256) || + typeof marker.packageRoot !== "string" || + typeof marker.binaryPath !== "string" + ) { + throw new Error("Managed Prime build marker has an unsupported schema."); + } + return marker as BuildMarker; +} + +export async function resolvePrimeManagedBuildReceiptTarget(input: { + readonly stateDir: string; + readonly packageRoot: string; +}): Promise<{ readonly stateDir: string; readonly instanceId: string } | undefined> { + const canonicalStateDir = await NodeFSP.realpath(NodePath.resolve(input.stateDir)); + const canonicalPackageRoot = await NodeFSP.realpath(NodePath.resolve(input.packageRoot)); + const managedRoot = NodePath.join(canonicalStateDir, ...PRIME_MANAGED_TOOL_DIRECTORY.split("/")); + const relative = NodePath.relative(managedRoot, canonicalPackageRoot); + const parts = relative.split(NodePath.sep); + if ( + parts.length !== 4 || + !BUILD_ID.test(parts[0]!) || + parts[1] !== "prefix" || + parts[2] !== "node_modules" || + parts[3] !== "prime-agent" + ) { + return undefined; + } + const buildId = parts[0]!; + return { + stateDir: NodePath.join(managedRoot, buildId, RECEIPT_STATE_DIRECTORY), + instanceId: `${RECEIPT_INSTANCE_PREFIX}${buildId}`, + }; +} + +export class PrimeAgentManagedToolStore { + readonly #stateDir: string; + readonly #root: string; + readonly #statePath: string; + readonly #platform: NodeJS.Platform; + readonly #dependencies: PrimeManagedToolStoreDependencies; + #queue: Promise = Promise.resolve(); + + constructor(input: { + readonly stateDir: string; + readonly platform: NodeJS.Platform; + readonly dependencies: PrimeManagedToolStoreDependencies; + }) { + if (input.platform === "win32") { + throw new Error( + "Native Windows Prime managed install/update is unavailable. Run Pylon and Prime Agent inside WSL2; no download or install was started.", + ); + } + this.#stateDir = NodeFS.realpathSync.native(NodePath.resolve(input.stateDir)); + this.#root = NodePath.join(this.#stateDir, ...PRIME_MANAGED_TOOL_DIRECTORY.split("/")); + this.#statePath = NodePath.join(this.#root, PRIME_MANAGED_STATE_FILE); + this.#platform = input.platform; + this.#dependencies = input.dependencies; + } + + async initialize(): Promise { + await this.#exclusive(async () => { + await ensurePrivateDirectory(this.#root); + await this.#recoverTemporaryEntries(); + await this.#reconcileSelectionIntents(); + }); + } + + async command(input: PrimeManagedCommandInput): Promise { + validateCommand(input); + return await this.#exclusive(async () => { + await this.#reconcileSelectionIntents(); + return await this.#runCommand(input); + }); + } + + async drain(instanceId: string): Promise { + return await this.#exclusive(async () => { + await this.#reconcileSelectionIntents(); + let state = await this.#readState(); + const scheduled = state.scheduled[instanceId]; + if (!scheduled) return null; + const operation = state.operations[scheduled.commandId]; + if (!operation) throw new Error("Scheduled Prime maintenance lost its command receipt."); + try { + await this.#validateSelectionTarget(state, scheduled); + return await this.#trySelection({ + state, + receipt: operation, + expected: scheduled.expected, + targetBinaryPath: scheduled.targetBinaryPath, + buildId: scheduled.buildId, + channel: scheduled.channel, + scheduleIfBusy: true, + }); + } catch (cause) { + state = await this.#readState(); + const { [instanceId]: _scheduled, ...remainingScheduled } = state.scheduled; + const cleared = { + ...state, + revision: state.revision + 1, + scheduled: remainingScheduled, + } satisfies StoredState; + await this.#writeState(cleared); + return await this.#finishOperation(cleared, operation, { + status: "failed", + message: cause instanceof Error ? cause.message : String(cause), + }); + } + }); + } + + async status(instanceId: string): Promise { + // State files and promoted build directories are atomic. Keep this read outside the command + // queue so every client can observe durable download/verify/install/wait/switch progress while + // the environment-owning command continues. + const state = await this.#readState(); + const selection = state.selections[instanceId]; + const availableBuilds = await this.#listVerifiedBuilds(); + const scheduledState = state.scheduled[instanceId]; + const scheduled = scheduledState ? (state.operations[scheduledState.commandId] ?? null) : null; + const latestOperationId = state.latestOperationIds[instanceId]; + const operation = + scheduled ?? + (latestOperationId + ? (state.operations[latestOperationId] ?? null) + : (Object.values(state.operations) + .filter((candidate) => candidate.instanceId === instanceId) + .toSorted((left, right) => right.startedAt.localeCompare(left.startedAt))[0] ?? null)); + const mode = selection?.mode ?? "stock"; + return { + instanceId, + mode, + selectedBuildId: selection?.selectedBuildId ?? null, + channel: selection?.channel ?? null, + availableBuilds, + scheduled, + operation, + message: + scheduled !== null + ? "Prime host maintenance is scheduled. It will switch only after the provider instance drains completely." + : mode === "managed" + ? `This environment uses verified Pylon-managed Prime build ${selection!.selectedBuildId}.` + : "This environment uses its stock or configured Prime Agent binary.", + }; + } + + async #runCommand(input: PrimeManagedCommandInput): Promise { + await ensurePrivateDirectory(this.#root); + let state = await this.#readState(); + const fingerprint = commandFingerprint(input); + const prior = state.operations[input.commandId]; + if (prior) { + if (prior.fingerprint !== fingerprint) { + throw new Error("Prime maintenance command id was reused with different input."); + } + return prior; + } + if (input.action !== "cleanup") { + state = await this.#supersedeScheduled(state, input.instanceId, input.commandId); + } + const now = this.#now(); + let receipt: PrimeManagedCommandReceipt & { readonly fingerprint: string } = { + commandId: input.commandId, + instanceId: input.instanceId, + action: input.action, + status: "queued", + channel: + input.channel ?? + (input.action === "install" || input.action === "update" ? "stable" : null), + buildId: input.buildId ?? null, + message: "Prime host maintenance is queued.", + startedAt: now, + finishedAt: null, + fingerprint, + }; + state = await this.#storeOperation(state, receipt); + try { + if (input.action === "cleanup") { + receipt = await this.#updateOperation(state, receipt, { + status: "installing", + message: "Pruning exact unreferenced receipt-owned Prime builds.", + }); + state = await this.#readState(); + const removed = await this.#cleanup(state); + return await this.#finishOperation(state, receipt, { + status: "succeeded", + message: removed.length + ? `Removed unreferenced managed builds: ${removed.join(", ")}.` + : "No unreferenced receipt-owned managed Prime build needed cleanup.", + }); + } + + const expected = await this.#dependencies.readBinding(input.instanceId); + state = await this.#reconcileExternalBinding(state, input.instanceId, expected); + let targetBinaryPath: string; + let buildId: string | null = null; + let channel: ServerProviderDistributionChannel | null = null; + if (input.action === "install" || input.action === "update") { + channel = input.channel ?? "stable"; + receipt = await this.#updateOperation(state, receipt, { + status: "downloading", + channel, + message: `Downloading the exact signed ${channel} Prime publication.`, + }); + const bundle = await this.#dependencies.loadLatestVerifiedPublication(channel); + receipt = await this.#updateOperation(await this.#readState(), receipt, { + status: "verifying", + channel, + buildId: bundle.publication.buildId, + message: + "Verifying publication provenance, digest, and the root archive before extraction.", + }); + this.#assertPublicationCandidate(await this.#readState(), bundle.publication); + // Observing a newer exact signed publication advances replay protection even when its + // archive later fails safe extraction or installation. A bad release must not reopen an + // implicit downgrade path to an older channel head. + state = await this.#recordHighWater(await this.#readState(), bundle.publication); + receipt = await this.#updateOperation(state, receipt, { + status: "installing", + channel, + buildId: bundle.publication.buildId, + message: "Extracting the verified bundled CLI into a new side-by-side build.", + }); + const installed = await this.#install(bundle); + targetBinaryPath = installed.binaryPath; + buildId = installed.buildId; + receipt = await this.#updateOperation(await this.#readState(), receipt, { + status: "installing", + channel, + buildId, + message: `Verified managed Prime build ${buildId} is staged side by side.`, + }); + } else if (input.action === "rollback") { + const installed = await this.#readVerifiedBuild(input.buildId!); + targetBinaryPath = installed.binaryPath; + buildId = installed.buildId; + channel = installed.channel; + } else { + const selection = state.selections[input.instanceId]; + if (!selection || selection.mode === "stock") { + return await this.#finishOperation(state, receipt, { + status: "succeeded", + message: "Prime already uses its stock or configured binary.", + }); + } + targetBinaryPath = selection.stockBinaryPath; + } + return await this.#trySelection({ + state: await this.#readState(), + receipt, + expected, + targetBinaryPath, + buildId, + channel, + scheduleIfBusy: input.scheduleIfBusy !== false, + }); + } catch (cause) { + const current = await this.#readState(); + return await this.#finishOperation(current, receipt, { + status: "failed", + message: cause instanceof Error ? cause.message : String(cause), + }); + } + } + + async #trySelection(input: { + readonly state: StoredState; + readonly receipt: PrimeManagedCommandReceipt & { readonly fingerprint: string }; + readonly expected: PrimeManagedBinding; + readonly targetBinaryPath: string; + readonly buildId: string | null; + readonly channel: ServerProviderDistributionChannel | null; + readonly scheduleIfBusy: boolean; + }): Promise { + let state = input.state; + let receipt = await this.#updateOperation(state, input.receipt, { + status: "waiting-for-quiescence", + buildId: input.buildId, + channel: input.channel, + message: "Waiting for exact provider-instance quiescence before changing its binary.", + }); + state = await this.#readState(); + const reservation = await this.#dependencies.reserveQuiescentBinding( + receipt.instanceId, + input.expected, + ); + if (reservation.status === "busy") { + if (!input.scheduleIfBusy) { + throw new Error(`Prime maintenance is blocked: ${reservation.reasons.join("; ")}`); + } + const scheduled: StoredScheduled = { + commandId: receipt.commandId, + instanceId: receipt.instanceId, + action: receipt.action, + expected: input.expected, + targetBinaryPath: input.targetBinaryPath, + buildId: input.buildId, + channel: input.channel, + }; + const next: StoredState = { + ...state, + revision: state.revision + 1, + scheduled: { ...state.scheduled, [receipt.instanceId]: scheduled }, + operations: { + ...state.operations, + [receipt.commandId]: { + ...receipt, + status: "waiting-for-quiescence", + message: `Scheduled until the instance drains: ${reservation.reasons.join("; ")}`, + }, + }, + }; + await this.#writeState(next); + return next.operations[receipt.commandId]!; + } + try { + receipt = await this.#updateOperation(state, receipt, { + status: "switching", + message: "The instance is fenced and quiescent. Switching its exact binary binding.", + }); + state = await this.#readState(); + const previous = state.selections[receipt.instanceId]; + const intent: StoredSelectionIntent = { + commandId: receipt.commandId, + instanceId: receipt.instanceId, + action: receipt.action, + expected: input.expected, + targetBinaryPath: input.targetBinaryPath, + stockBinaryPath: previous?.stockBinaryPath ?? input.expected.binaryPath, + buildId: input.buildId, + channel: input.channel, + }; + const intentState: StoredState = { + ...state, + revision: state.revision + 1, + selectionIntents: { ...state.selectionIntents, [receipt.instanceId]: intent }, + }; + // This journal is synced before CAS. A crash on either side of the settings write can + // therefore reconcile the exact observed binding without guessing or selecting partial bytes. + await this.#writeState(intentState); + const committed = await this.#dependencies.commitBinding({ + instanceId: receipt.instanceId, + expected: input.expected, + binaryPath: input.targetBinaryPath, + reservation: reservation.reservation, + }); + const next = await this.#commitSelectionState(await this.#readState(), intent, committed); + return await this.#finishOperation(next, receipt, { + status: "succeeded", + message: + input.buildId === null + ? "Prime now uses the stock or configured binary. No global bytes were changed." + : `${receipt.action === "rollback" ? "Rolled back" : "Selected"} verified managed Prime build ${input.buildId}.`, + }); + } finally { + await this.#dependencies.releaseReservation(reservation.reservation); + } + } + + async #reconcileExternalBinding( + state: StoredState, + instanceId: string, + current: PrimeManagedBinding, + ): Promise { + const selected = state.selections[instanceId]; + if ( + !selected || + (selected.binding.generation === current.generation && + selected.binding.binaryPath === current.binaryPath) + ) { + return state; + } + const next: StoredState = { + ...state, + revision: state.revision + 1, + selections: { + ...state.selections, + [instanceId]: { + mode: "stock", + selectedBuildId: null, + channel: null, + stockBinaryPath: current.binaryPath, + binding: current, + }, + }, + }; + await this.#writeState(next); + return next; + } + + async #validateSelectionTarget( + state: StoredState, + target: Pick< + StoredSelectionIntent, + "instanceId" | "action" | "targetBinaryPath" | "buildId" | "channel" + >, + ): Promise { + if (target.buildId !== null) { + const build = await this.#readVerifiedBuild(target.buildId); + if ( + target.action === "use-stock" || + target.targetBinaryPath !== build.binaryPath || + target.channel !== build.channel + ) { + throw new Error("Persisted Prime maintenance target conflicts with its verified build."); + } + return; + } + const selection = state.selections[target.instanceId]; + if ( + target.action !== "use-stock" || + target.channel !== null || + !selection || + target.targetBinaryPath !== selection.stockBinaryPath + ) { + throw new Error("Persisted Prime stock switch conflicts with its recorded original binding."); + } + } + + async #commitSelectionState( + state: StoredState, + intent: StoredSelectionIntent, + binding: PrimeManagedBinding, + ): Promise { + const selection: StoredSelection = { + mode: intent.buildId === null ? "stock" : "managed", + selectedBuildId: intent.buildId, + channel: intent.channel, + stockBinaryPath: intent.stockBinaryPath, + binding, + }; + const { [intent.instanceId]: _intent, ...remainingIntents } = state.selectionIntents; + const { [intent.instanceId]: _scheduled, ...remainingScheduled } = state.scheduled; + const next: StoredState = { + ...state, + revision: state.revision + 1, + selections: { ...state.selections, [intent.instanceId]: selection }, + selectionIntents: remainingIntents, + scheduled: remainingScheduled, + }; + await this.#writeState(next); + return next; + } + + async #reconcileSelectionIntents(): Promise { + let state = await this.#readState(); + for (const intent of Object.values(state.selectionIntents)) { + await this.#validateSelectionTarget(state, intent); + const current = await this.#dependencies.readBinding(intent.instanceId); + const operation = state.operations[intent.commandId]; + if (current.binaryPath === intent.targetBinaryPath) { + state = await this.#commitSelectionState(state, intent, current); + if (operation) { + const recovered = { + ...operation, + status: "succeeded" as const, + message: + intent.buildId === null + ? "Recovered the completed switch to the stock or configured Prime binary." + : `Recovered the completed switch to verified managed Prime build ${intent.buildId}.`, + finishedAt: this.#now(), + }; + state = await this.#storeOperation(state, recovered); + } + continue; + } + const { [intent.instanceId]: _intent, ...remainingIntents } = state.selectionIntents; + const { [intent.instanceId]: _scheduled, ...remainingScheduled } = state.scheduled; + const expectedStillSelected = + current.binaryPath === intent.expected.binaryPath && + current.generation === intent.expected.generation; + const nextOperation = operation + ? { + ...operation, + status: "failed" as const, + message: expectedStillSelected + ? "Prime maintenance was interrupted before the atomic binding switch." + : "Prime maintenance was superseded by a different provider binding before recovery.", + finishedAt: this.#now(), + } + : undefined; + const next: StoredState = { + ...state, + revision: state.revision + 1, + selectionIntents: remainingIntents, + scheduled: remainingScheduled, + operations: nextOperation + ? { ...state.operations, [intent.commandId]: nextOperation } + : state.operations, + }; + await this.#writeState(next); + state = next; + } + } + + async #supersedeScheduled( + state: StoredState, + instanceId: string, + replacementCommandId: string, + ): Promise { + const scheduled = state.scheduled[instanceId]; + if (!scheduled || scheduled.commandId === replacementCommandId) return state; + const prior = state.operations[scheduled.commandId]; + const { [instanceId]: _scheduled, ...remainingScheduled } = state.scheduled; + const next: StoredState = { + ...state, + revision: state.revision + 1, + scheduled: remainingScheduled, + operations: prior + ? { + ...state.operations, + [scheduled.commandId]: { + ...prior, + status: "failed", + message: `Prime maintenance was superseded by command ${replacementCommandId}.`, + finishedAt: this.#now(), + }, + } + : state.operations, + }; + await this.#writeState(next); + return next; + } + + async #install(bundle: PrimeManagedPublicationBundle): Promise { + const publication = bundle.publication; + if ( + !BUILD_ID.test(publication.buildId) || + !PACKAGE_VERSION.test(publication.packageVersion) || + !ROOT_ASSET.test(publication.rootAsset) || + (publication.channel !== "stable" && publication.channel !== "preview") || + publication.sequenceEpoch !== 1 || + !Number.isSafeInteger(publication.sequence) || + publication.sequence < 1 + ) { + throw new Error("Verified publication identity is invalid."); + } + if ( + bundle.rootArtifactBytes.byteLength < 1 || + bundle.rootArtifactBytes.byteLength > MAX_ROOT_ARCHIVE_BYTES || + sha256(bundle.rootArtifactBytes) !== publication.rootSha256 + ) { + throw new Error("Prime root tarball digest does not match the verified publication."); + } + const finalDirectory = NodePath.join(this.#root, publication.buildId); + try { + return await this.#readVerifiedBuild(publication.buildId); + } catch (cause) { + if (!isErrno(cause, "ENOENT")) { + const exists = await NodeFSP.lstat(finalDirectory).then( + () => true, + (error) => (isErrno(error, "ENOENT") ? false : Promise.reject(error)), + ); + if (exists) throw cause; + } + } + + const entries = await parsePrimeTarball(bundle.rootArtifactBytes); + const packageIdentity = packageManifestFromEntries(entries); + assertPackageMatchesPublication(packageIdentity.manifest, publication); + const staging = NodePath.join( + this.#root, + `${STAGING_PREFIX}${publication.buildId}-${NodeCrypto.randomBytes(10).toString("hex")}`, + ); + await NodeFSP.mkdir(staging, { mode: 0o700 }); + let promoted = false; + try { + const archivePath = NodePath.join(staging, publication.rootAsset); + const archiveHandle = await NodeFSP.open( + archivePath, + NodeFS.constants.O_WRONLY | NodeFS.constants.O_CREAT | NodeFS.constants.O_EXCL, + 0o600, + ); + try { + await archiveHandle.writeFile(bundle.rootArtifactBytes); + await archiveHandle.sync(); + } finally { + await archiveHandle.close(); + } + const extractedPackagePath = await extractPrimeEntries( + NodePath.join(staging, "verified-source"), + entries, + ); + const prefixPath = NodePath.join(staging, "prefix"); + if (this.#dependencies.installVerifiedArchive) { + await this.#dependencies.installVerifiedArchive({ + archivePath, + extractedPackagePath, + prefixPath, + publication, + }); + } else { + await installVerifiedPackageTree({ extractedPackagePath, prefixPath }); + } + const installedInStage = await validateInstalledLauncher({ + prefixPath, + expectedManifest: packageIdentity.manifest, + }); + await NodeFSP.rm(NodePath.join(staging, "verified-source"), { recursive: true, force: true }); + await NodeFSP.rm(archivePath, { force: true }); + await syncDirectory(staging); + await NodeFSP.rename(staging, finalDirectory); + promoted = true; + await syncDirectory(this.#root); + const installed = { + buildId: publication.buildId, + channel: publication.channel, + sequence: publication.sequence, + binaryPath: installedInStage.binaryPath.replace(staging, finalDirectory), + packageRoot: installedInStage.packageRoot.replace(staging, finalDirectory), + } satisfies PrimeManagedInstalledBuild; + await persistPrimeManagedReceipt({ + stateDir: NodePath.join(finalDirectory, RECEIPT_STATE_DIRECTORY), + instanceId: `${RECEIPT_INSTANCE_PREFIX}${publication.buildId}`, + packageRoot: installed.packageRoot, + platform: this.#platform, + publication, + }); + await writeJsonAtomically( + finalDirectory, + PRIME_MANAGED_BUILD_FILE, + markerFor(publication, installed), + ); + await syncDirectory(finalDirectory); + return await this.#readVerifiedBuild(publication.buildId); + } catch (cause) { + if (promoted) { + // A promoted directory without a verified receipt+marker is never selectable. Recovery + // removes it on the next command; leave it in place if removal itself is unsafe. + await this.#removeIncompleteBuild(finalDirectory).catch(() => undefined); + } + throw cause; + } finally { + if (!promoted) await this.#removeExactTemporary(staging).catch(() => undefined); + } + } + + async #readVerifiedBuild(buildId: string): Promise { + if (!BUILD_ID.test(buildId)) throw new Error("Managed Prime build id is invalid."); + const directory = NodePath.join(this.#root, buildId); + const directoryInfo = await NodeFSP.lstat(directory); + if (!directoryInfo.isDirectory() || directoryInfo.isSymbolicLink()) { + throw new Error("Managed Prime build path is not a real directory."); + } + if ((await NodeFSP.realpath(directory)) !== directory) { + throw new Error("Managed Prime build directory is not canonical."); + } + const bytes = await readBoundedRegularFile( + NodePath.join(directory, PRIME_MANAGED_BUILD_FILE), + 64 * 1024, + ); + if (!bytes) throw new Error("Managed Prime build has no complete marker."); + const marker = decodeBuildMarker(JSON.parse(bytes.toString("utf8")) as unknown); + if ( + marker.buildId !== buildId || + marker.packageRoot !== NodePath.join(directory, "prefix", "node_modules", "prime-agent") || + marker.binaryPath !== + NodePath.join(directory, "prefix", "node_modules", ".bin", "prime-agent") + ) { + throw new Error("Managed Prime build marker escapes or conflicts with its build directory."); + } + await NodeFSP.access(marker.binaryPath, NodeFS.constants.X_OK); + const inspection = await inspectPrimeAgentDistribution( + { + stateDir: NodePath.join(directory, RECEIPT_STATE_DIRECTORY), + instanceId: `${RECEIPT_INSTANCE_PREFIX}${buildId}`, + packageRoot: marker.packageRoot, + platform: this.#platform, + checkedAt: this.#now(), + enableUpdateChecks: false, + }, + { + loadLatestVerifiedPublication: async () => { + throw new Error("Managed build validation does not use the network."); + }, + }, + ); + if ( + inspection.classification !== "pylon-managed" || + inspection.buildId !== buildId || + inspection.channel !== marker.channel || + inspection.sequence !== marker.sequence + ) { + throw new Error(`Managed Prime build ${buildId} is not owned by its exact #193 receipt.`); + } + return { + buildId, + channel: marker.channel, + sequence: marker.sequence, + binaryPath: marker.binaryPath, + packageRoot: marker.packageRoot, + }; + } + + async #listVerifiedBuilds(): Promise> { + await ensurePrivateDirectory(this.#root); + const entries = await NodeFSP.readdir(this.#root, { withFileTypes: true }); + const builds: PrimeManagedInstalledBuild[] = []; + for (const entry of entries) { + if (!BUILD_ID.test(entry.name)) continue; + try { + builds.push(await this.#readVerifiedBuild(entry.name)); + } catch { + // Partial or invalid directories are not selectable and never become cleanup authority. + } + } + return builds.toSorted((left, right) => right.sequence - left.sequence); + } + + #assertPublicationCandidate(state: StoredState, publication: VerifiedPrimePublication): void { + const prior = state.highWater[publication.channel]; + if (!prior) return; + if ( + publication.sequenceEpoch !== prior.sequenceEpoch || + publication.sequence < prior.sequence || + (publication.sequence === prior.sequence && publication.buildId !== prior.buildId) + ) { + throw new Error("Signed Prime channel replay or implicit downgrade was rejected."); + } + } + + async #recordHighWater( + state: StoredState, + publication: VerifiedPrimePublication, + ): Promise { + this.#assertPublicationCandidate(state, publication); + const prior = state.highWater[publication.channel]; + if (prior?.sequence === publication.sequence && prior.buildId === publication.buildId) + return state; + const next: StoredState = { + ...state, + revision: state.revision + 1, + highWater: { + ...state.highWater, + [publication.channel]: { + sequenceEpoch: publication.sequenceEpoch, + sequence: publication.sequence, + buildId: publication.buildId, + }, + }, + }; + await this.#writeState(next); + return next; + } + + async #cleanup(state: StoredState): Promise> { + const referenced = new Set(); + for (const selection of Object.values(state.selections)) { + if (selection.mode === "managed" && selection.selectedBuildId) + referenced.add(selection.selectedBuildId); + } + for (const scheduled of Object.values(state.scheduled)) { + if (scheduled.buildId) referenced.add(scheduled.buildId); + } + const removed: string[] = []; + const entries = await NodeFSP.readdir(this.#root, { withFileTypes: true }); + for (const entry of entries) { + if (!BUILD_ID.test(entry.name) || referenced.has(entry.name)) continue; + const path = NodePath.join(this.#root, entry.name); + const info = await NodeFSP.lstat(path); + if (!info.isDirectory() || info.isSymbolicLink() || (await NodeFSP.realpath(path)) !== path) { + continue; + } + try { + await this.#readVerifiedBuild(entry.name); + } catch { + continue; + } + await NodeFSP.rm(path, { recursive: true, force: false }); + removed.push(entry.name); + } + if (removed.length) await syncDirectory(this.#root); + return removed.toSorted(); + } + + async #recoverTemporaryEntries(): Promise { + const entries = await NodeFSP.readdir(this.#root, { withFileTypes: true }); + for (const entry of entries) { + const path = NodePath.join(this.#root, entry.name); + if (entry.name.startsWith(STAGING_PREFIX)) { + await this.#removeExactTemporary(path); + continue; + } + if (!BUILD_ID.test(entry.name) || !entry.isDirectory() || entry.isSymbolicLink()) continue; + try { + await this.#readVerifiedBuild(entry.name); + } catch { + await this.#removeIncompleteBuild(path); + } + } + } + + async #removeExactTemporary(path: string): Promise { + if ( + NodePath.dirname(path) !== this.#root || + !NodePath.basename(path).startsWith(STAGING_PREFIX) + ) { + throw new Error("Refusing to remove a non-staging Prime path."); + } + const info = await NodeFSP.lstat(path).catch((cause) => { + if (isErrno(cause, "ENOENT")) return undefined; + throw cause; + }); + if (!info) return; + if (!info.isDirectory() || info.isSymbolicLink() || (await NodeFSP.realpath(path)) !== path) { + throw new Error("Refusing to follow a Prime staging link during recovery."); + } + await NodeFSP.rm(path, { recursive: true, force: false }); + } + + async #removeIncompleteBuild(path: string): Promise { + if (NodePath.dirname(path) !== this.#root || !BUILD_ID.test(NodePath.basename(path))) { + throw new Error("Refusing to remove an unexpected Prime build path."); + } + const info = await NodeFSP.lstat(path); + if (!info.isDirectory() || info.isSymbolicLink() || (await NodeFSP.realpath(path)) !== path) { + throw new Error("Refusing to follow an incomplete Prime build link."); + } + const marker = await readBoundedRegularFile( + NodePath.join(path, PRIME_MANAGED_BUILD_FILE), + 64 * 1024, + ); + if (marker) { + // A complete marker turns deletion into explicit cleanup, which first verifies ownership. + throw new Error("Refusing recovery cleanup of a marked Prime build."); + } + await NodeFSP.rm(path, { recursive: true, force: false }); + } + + async #readState(): Promise { + await ensurePrivateDirectory(this.#root); + const bytes = await readBoundedRegularFile(this.#statePath, 2 * 1024 * 1024); + if (!bytes) return emptyState(); + return decodeStoredState(JSON.parse(bytes.toString("utf8")) as unknown); + } + + async #writeState(state: StoredState): Promise { + await writeJsonAtomically(this.#root, PRIME_MANAGED_STATE_FILE, state); + } + + async #storeOperation( + state: StoredState, + receipt: PrimeManagedCommandReceipt & { readonly fingerprint: string }, + ): Promise { + const latestOperationIds = { + ...state.latestOperationIds, + [receipt.instanceId]: receipt.commandId, + }; + const operations = { ...state.operations, [receipt.commandId]: receipt }; + const protectedIds = new Set([ + receipt.commandId, + ...Object.values(latestOperationIds), + ...Object.values(state.scheduled).map((scheduled) => scheduled.commandId), + ...Object.values(state.selectionIntents).map((intent) => intent.commandId), + ]); + const removable = Object.values(operations) + .filter((operation) => !protectedIds.has(operation.commandId)) + .toSorted( + (left, right) => + left.startedAt.localeCompare(right.startedAt) || + left.commandId.localeCompare(right.commandId), + ); + for (const operation of removable) { + if (Object.keys(operations).length <= MAX_STORED_OPERATIONS) break; + delete operations[operation.commandId]; + } + const next: StoredState = { + ...state, + revision: state.revision + 1, + operations, + latestOperationIds, + }; + await this.#writeState(next); + return next; + } + + async #updateOperation( + state: StoredState, + receipt: PrimeManagedCommandReceipt & { readonly fingerprint: string }, + patch: Partial, + ): Promise { + const nextReceipt = { ...receipt, ...patch, fingerprint: receipt.fingerprint }; + await this.#storeOperation(state, nextReceipt); + return nextReceipt; + } + + async #finishOperation( + state: StoredState, + receipt: PrimeManagedCommandReceipt & { readonly fingerprint: string }, + patch: Pick, + ): Promise { + const finished = { + ...receipt, + ...patch, + finishedAt: patch.status === "waiting-for-quiescence" ? null : this.#now(), + }; + await this.#storeOperation(state, finished); + return finished; + } + + #now(): string { + return this.#dependencies.now?.() ?? new Date().toISOString(); + } + + async #exclusive(work: () => Promise): Promise { + const previous = this.#queue; + let release!: () => void; + this.#queue = new Promise((resolve) => { + release = resolve; + }); + await previous.catch(() => undefined); + try { + return await work(); + } finally { + release(); + } + } +} diff --git a/apps/server/src/provider/prime/PrimeManagedMaintenance.ts b/apps/server/src/provider/prime/PrimeManagedMaintenance.ts new file mode 100644 index 000000000..45b3034f7 --- /dev/null +++ b/apps/server/src/provider/prime/PrimeManagedMaintenance.ts @@ -0,0 +1,244 @@ +import { + ProviderInstanceId, + ServerPrimeManagedMaintenanceError, + type ServerPrimeManagedCommandInput, + type ServerPrimeManagedCommandReceipt, + type ServerPrimeManagedMaintenance, +} from "@t3tools/contracts"; +import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Stream from "effect/Stream"; + +import { ServerConfig } from "../../config.ts"; +import { ServerSettingsService } from "../../serverSettings.ts"; +import { ProviderRegistry } from "../Services/ProviderRegistry.ts"; +import { ProviderService } from "../Services/ProviderService.ts"; +import { + PrimeAgentManagedToolStore, + type PrimeManagedCommandReceipt, +} from "./PrimeAgentManagedToolStore.ts"; +import { + makeLatestPrimePublicationBundleLoader, + makePrimeDistributionNetworkDependencies, +} from "./PrimeAgentDistributionVerifier.ts"; + +export interface PrimeManagedMaintenanceShape { + readonly status: ( + instanceId: ProviderInstanceId, + ) => Effect.Effect; + readonly run: ( + input: ServerPrimeManagedCommandInput, + ) => Effect.Effect; +} + +export class PrimeManagedMaintenance extends Context.Service< + PrimeManagedMaintenance, + PrimeManagedMaintenanceShape +>()("t3/provider/prime/PrimeManagedMaintenance") {} + +function maintenanceError(instanceId: ProviderInstanceId, cause: unknown) { + return new ServerPrimeManagedMaintenanceError({ + instanceId, + reason: cause instanceof Error ? cause.message : String(cause), + }); +} + +function contractReceipt(receipt: PrimeManagedCommandReceipt): ServerPrimeManagedCommandReceipt { + return { + commandId: receipt.commandId, + instanceId: ProviderInstanceId.make(receipt.instanceId), + action: receipt.action, + status: receipt.status, + channel: receipt.channel, + buildId: receipt.buildId, + message: receipt.message, + startedAt: receipt.startedAt, + finishedAt: receipt.finishedAt, + }; +} + +export const make = Effect.fn("PrimeManagedMaintenance.make")(function* () { + const config = yield* ServerConfig; + const platform = yield* HostProcessPlatform; + const settings = yield* ServerSettingsService; + const providerService = yield* ProviderService; + const providerRegistry = yield* ProviderRegistry; + const runtimeContext = yield* Effect.context(); + const runPromise = Effect.runPromiseWith(runtimeContext); + + if (platform !== "win32") { + if ( + !settings.readPrimeAgentBinaryBinding || + !settings.compareAndSetPrimeAgentBinaryPath || + !providerService.reserveProviderMaintenance || + !providerService.releaseProviderMaintenance + ) { + return yield* Effect.die( + new Error("Prime managed maintenance requires exact settings CAS and provider fences."), + ); + } + } + + if (platform === "win32") { + const guidance = + "Native Windows Prime managed install/update is unavailable. Install and run Pylon with Prime Agent inside WSL2; the Linux environment owns all downloads and runtime files."; + return PrimeManagedMaintenance.of({ + status: (_instanceId) => + Effect.succeed({ + supported: false, + controlsAvailable: false, + mode: "stock", + selectedBuildId: null, + channel: null, + availableBuilds: [], + scheduled: null, + operation: null, + message: guidance, + guidance, + }), + run: (input) => Effect.fail(maintenanceError(input.instanceId, new Error(guidance))), + }); + } + + const readPrimeAgentBinaryBinding = settings.readPrimeAgentBinaryBinding!; + const compareAndSetPrimeAgentBinaryPath = settings.compareAndSetPrimeAgentBinaryPath!; + const reserveProviderMaintenance = providerService.reserveProviderMaintenance!; + const releaseProviderMaintenance = providerService.releaseProviderMaintenance!; + + const loadLatestVerifiedPublication = makeLatestPrimePublicationBundleLoader( + makePrimeDistributionNetworkDependencies({ + tufCachePath: `${config.stateDir}/sigstore-tuf`, + }), + ); + const store = new PrimeAgentManagedToolStore({ + stateDir: config.stateDir, + platform, + dependencies: { + loadLatestVerifiedPublication, + readBinding: async (instanceId) => { + const binding = await runPromise( + readPrimeAgentBinaryBinding(instanceId).pipe(Effect.orDie), + ); + if (!binding) throw new Error("The target is not a configured Prime Agent instance."); + return binding; + }, + reserveQuiescentBinding: async (instanceId, expected) => { + const current = await runPromise( + readPrimeAgentBinaryBinding(instanceId).pipe(Effect.orDie), + ); + if ( + !current || + current.generation !== expected.generation || + current.binaryPath !== expected.binaryPath + ) { + throw new Error( + "The exact Prime provider binding changed before maintenance could fence it.", + ); + } + return await runPromise( + reserveProviderMaintenance(ProviderInstanceId.make(instanceId)).pipe(Effect.orDie), + ); + }, + commitBinding: async ({ instanceId, expected, binaryPath, reservation: _reservation }) => { + const committed = await runPromise( + compareAndSetPrimeAgentBinaryPath({ instanceId, expected, binaryPath }).pipe( + Effect.orDie, + ), + ); + if (!committed) { + throw new Error( + "The exact Prime provider binding changed; the staged build was not selected.", + ); + } + return committed; + }, + releaseReservation: (reservation) => runPromise(releaseProviderMaintenance(reservation)), + }, + }); + yield* Effect.tryPromise({ + try: () => store.initialize(), + catch: (cause) => maintenanceError(ProviderInstanceId.make("primeAgent"), cause), + }); + + const refreshAfterDrain = (instanceId: ProviderInstanceId) => + Effect.tryPromise({ + try: () => store.drain(instanceId), + catch: (cause) => maintenanceError(instanceId, cause), + }).pipe( + Effect.flatMap((receipt) => + receipt?.status === "succeeded" + ? providerRegistry.refreshInstance(instanceId).pipe(Effect.ignore) + : Effect.void, + ), + Effect.catch((cause) => + Effect.logWarning("Scheduled Prime host maintenance did not drain.", { + instanceId, + cause, + }), + ), + ); + yield* providerService.streamEvents.pipe( + Stream.filter( + (event) => + event.type === "session.exited" && + event.provider === "primeAgent" && + event.providerInstanceId !== undefined, + ), + Stream.runForEach((event) => refreshAfterDrain(event.providerInstanceId!)), + Effect.forkScoped, + ); + + const status: PrimeManagedMaintenanceShape["status"] = (instanceId) => + Effect.tryPromise({ + try: async () => { + const result = await store.status(instanceId); + return { + supported: true, + controlsAvailable: true, + mode: result.mode, + selectedBuildId: result.selectedBuildId, + channel: result.channel, + availableBuilds: result.availableBuilds.map((build) => ({ + buildId: build.buildId, + channel: build.channel, + sequence: build.sequence, + binaryPath: build.binaryPath, + })), + scheduled: result.scheduled ? contractReceipt(result.scheduled) : null, + operation: result.operation ? contractReceipt(result.operation) : null, + message: result.message, + guidance: null, + } satisfies ServerPrimeManagedMaintenance; + }, + catch: (cause) => maintenanceError(instanceId, cause), + }); + + const run: PrimeManagedMaintenanceShape["run"] = (input) => + Effect.tryPromise({ + try: async () => + contractReceipt( + await store.command({ + commandId: input.commandId, + instanceId: input.instanceId, + action: input.action, + ...(input.channel === undefined ? {} : { channel: input.channel }), + ...(input.allowPreview === undefined ? {} : { allowPreview: input.allowPreview }), + ...(input.buildId === undefined ? {} : { buildId: input.buildId }), + ...(input.scheduleIfBusy === undefined ? {} : { scheduleIfBusy: input.scheduleIfBusy }), + }), + ), + catch: (cause) => maintenanceError(input.instanceId, cause), + }).pipe( + Effect.tap((receipt) => + receipt.status === "succeeded" + ? providerRegistry.refreshInstance(input.instanceId).pipe(Effect.ignore) + : Effect.void, + ), + ); + + return PrimeManagedMaintenance.of({ status, run }); +}); + +export const layer = Layer.effect(PrimeManagedMaintenance, make()); diff --git a/apps/server/src/serverSettings.test.ts b/apps/server/src/serverSettings.test.ts index 45b424797..fcddb2b7c 100644 --- a/apps/server/src/serverSettings.test.ts +++ b/apps/server/src/serverSettings.test.ts @@ -1064,4 +1064,56 @@ it.layer(NodeServices.layer)("server settings", (it) => { ); }).pipe(Effect.provide(makeServerSettingsLayer())), ); + it.effect( + "compare-and-sets the complete Prime instance binding without changing sibling fields", + () => + Effect.gen(function* () { + const serverSettings = yield* ServerSettingsModule.ServerSettingsService; + const instanceId = ProviderInstanceId.make("prime_work"); + yield* serverSettings.updateSettings({ + providerInstances: { + [instanceId]: { + driver: ProviderDriverKind.make("primeAgent"), + displayName: "Prime Work", + environment: [{ name: "PRIME_PROFILE", value: "work", sensitive: false }], + config: { binaryPath: "/opt/prime/stock", homePath: "/tmp/prime-work" }, + }, + }, + }); + const readBinding = serverSettings.readPrimeAgentBinaryBinding!; + const compareAndSet = serverSettings.compareAndSetPrimeAgentBinaryPath!; + const expected = yield* readBinding(instanceId); + assert.isDefined(expected); + const committed = yield* compareAndSet({ + instanceId, + expected, + binaryPath: "/tmp/pylon-managed/.bin/prime-agent", + }); + assert.isDefined(committed); + const selected = yield* serverSettings.getSettings; + assert.deepInclude(selected.providerInstances[instanceId], { + driver: ProviderDriverKind.make("primeAgent"), + displayName: "Prime Work", + environment: [{ name: "PRIME_PROFILE", value: "work", sensitive: false }], + config: { + binaryPath: "/tmp/pylon-managed/.bin/prime-agent", + homePath: "/tmp/prime-work", + }, + }); + + yield* serverSettings.updateSettings({ + providerInstances: { + ...selected.providerInstances, + [instanceId]: { ...selected.providerInstances[instanceId]!, displayName: "Changed" }, + }, + }); + assert.isUndefined( + yield* compareAndSet({ + instanceId, + expected: committed, + binaryPath: "/tmp/pylon-managed/other/.bin/prime-agent", + }), + ); + }).pipe(Effect.provide(makeServerSettingsLayer())), + ); }); diff --git a/apps/server/src/serverSettings.ts b/apps/server/src/serverSettings.ts index 5a8650b7e..3ef7afc7f 100644 --- a/apps/server/src/serverSettings.ts +++ b/apps/server/src/serverSettings.ts @@ -24,6 +24,7 @@ import { ServerSettingsError, type ServerSettingsPatch, } from "@t3tools/contracts"; +import * as NodeCrypto from "node:crypto"; import * as Cache from "effect/Cache"; import * as Cause from "effect/Cause"; import * as Context from "effect/Context"; @@ -162,6 +163,76 @@ export function redactServerSettingsForClient(settings: ServerSettings): ServerS return { ...settings, providerInstances }; } +function hashPrimeAgentBinding(value: unknown): string { + const canonicalize = (input: unknown): unknown => { + if (Array.isArray(input)) return input.map(canonicalize); + if (typeof input !== "object" || input === null) return input; + return Object.fromEntries( + Object.entries(input as Readonly>) + .toSorted(([left], [right]) => left.localeCompare(right)) + .map(([key, entry]) => [key, canonicalize(entry)]), + ); + }; + return NodeCrypto.createHash("sha256") + .update(JSON.stringify(canonicalize(value))) + .digest("hex"); +} + +export interface PrimeAgentBinaryBinding { + readonly binaryPath: string; + readonly generation: string; +} + +function primeAgentBinaryBinding( + settings: ServerSettings, + instanceId: string, +): PrimeAgentBinaryBinding | undefined { + const explicit = settings.providerInstances[ProviderInstanceId.make(instanceId)]; + if (explicit) { + if (explicit.driver !== "primeAgent") return undefined; + const config = + typeof explicit.config === "object" && + explicit.config !== null && + !Array.isArray(explicit.config) + ? (explicit.config as Readonly>) + : {}; + return { + binaryPath: typeof config.binaryPath === "string" ? config.binaryPath : "", + generation: hashPrimeAgentBinding(explicit), + }; + } + if (instanceId !== "primeAgent") return undefined; + const legacy = settings.providers.primeAgent; + return { + binaryPath: legacy.binaryPath, + generation: hashPrimeAgentBinding(legacy), + }; +} + +function patchPrimeAgentBinaryPath( + settings: ServerSettings, + instanceId: string, + binaryPath: string, +): ServerSettingsPatch | undefined { + const explicit = settings.providerInstances[ProviderInstanceId.make(instanceId)]; + if (explicit) { + if (explicit.driver !== "primeAgent") return undefined; + const config = + typeof explicit.config === "object" && + explicit.config !== null && + !Array.isArray(explicit.config) + ? explicit.config + : {}; + return { + providerInstances: { + ...settings.providerInstances, + [instanceId]: { ...explicit, config: { ...config, binaryPath } }, + }, + }; + } + return instanceId === "primeAgent" ? { providers: { primeAgent: { binaryPath } } } : undefined; +} + export class ServerSettingsService extends Context.Service< ServerSettingsService, { @@ -179,6 +250,18 @@ export class ServerSettingsService extends Context.Service< patch: ServerSettingsPatch, ) => Effect.Effect; + /** Read the persisted complete Prime instance binding generation. */ + readonly readPrimeAgentBinaryBinding?: ( + instanceId: string, + ) => Effect.Effect; + + /** Atomically compare-and-set only this Prime instance's binary path. */ + readonly compareAndSetPrimeAgentBinaryPath?: (input: { + readonly instanceId: string; + readonly expected: PrimeAgentBinaryBinding; + readonly binaryPath: string; + }) => Effect.Effect; + /** Stream of settings change events. */ readonly streamChanges: Stream.Stream; @@ -221,6 +304,29 @@ const makeTest = (overrides: DeepPartial = {}) => Effect.tap((nextSettings) => Ref.set(currentSettingsRef, nextSettings)), Effect.map(resolveTextGenerationProvider), ), + readPrimeAgentBinaryBinding: (instanceId) => + Ref.get(currentSettingsRef).pipe( + Effect.map((settings) => primeAgentBinaryBinding(settings, instanceId)), + ), + compareAndSetPrimeAgentBinaryPath: (input) => + Ref.get(currentSettingsRef).pipe( + Effect.flatMap((current) => { + const binding = primeAgentBinaryBinding(current, input.instanceId); + if ( + binding?.generation !== input.expected.generation || + binding.binaryPath !== input.expected.binaryPath + ) { + return Effect.void.pipe(Effect.as(undefined as PrimeAgentBinaryBinding | undefined)); + } + const patch = patchPrimeAgentBinaryPath(current, input.instanceId, input.binaryPath); + if (!patch) + return Effect.void.pipe(Effect.as(undefined as PrimeAgentBinaryBinding | undefined)); + return normalizeServerSettings(applyServerSettingsPatch(current, patch)).pipe( + Effect.tap((next) => Ref.set(currentSettingsRef, next)), + Effect.map((next) => primeAgentBinaryBinding(next, input.instanceId)), + ); + }), + ), streamChanges: Stream.empty, subscribeChanges: Effect.succeed(Stream.empty), } satisfies ServerSettingsService["Service"]; @@ -751,6 +857,34 @@ const make = Effect.gen(function* () { return resolveTextGenerationProvider(materialized); }), ), + readPrimeAgentBinaryBinding: (instanceId) => + getSettingsFromCache.pipe( + Effect.map((settings) => primeAgentBinaryBinding(settings, instanceId)), + ), + compareAndSetPrimeAgentBinaryPath: (input) => + writeSemaphore.withPermits(1)( + Effect.gen(function* () { + const current = yield* getSettingsFromCache; + const binding = primeAgentBinaryBinding(current, input.instanceId); + if ( + binding?.generation !== input.expected.generation || + binding.binaryPath !== input.expected.binaryPath + ) { + return undefined; + } + const patch = patchPrimeAgentBinaryPath(current, input.instanceId, input.binaryPath); + if (!patch) return undefined; + const nextPersisted = yield* persistProviderEnvironmentSecrets( + current, + applyServerSettingsPatch(current, patch), + ); + const next = yield* normalizeServerSettings(nextPersisted); + yield* writeSettingsAtomically(next); + yield* Cache.set(settingsCache, cacheKey, next); + yield* emitChange(next); + return primeAgentBinaryBinding(next, input.instanceId); + }), + ), get streamChanges() { return materializeChanges(Stream.fromPubSub(changesPubSub)); }, diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index 4295c1fa9..cf299dcda 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -95,6 +95,7 @@ import { import * as ProviderRegistry from "./provider/Services/ProviderRegistry.ts"; import * as ProviderService from "./provider/Services/ProviderService.ts"; import * as ProviderMaintenanceRunner from "./provider/providerMaintenanceRunner.ts"; +import * as PrimeManagedMaintenance from "./provider/prime/PrimeManagedMaintenance.ts"; import { toProviderMessageSessionAgentError } from "./provider/providerMessageSessionAgentRpcError.ts"; import { toProviderSessionCompactionError } from "./provider/providerSessionCompactionRpcError.ts"; import { toProviderRefineSessionHarnessError } from "./provider/providerRefineSessionHarnessRpcError.ts"; @@ -452,6 +453,7 @@ const makeWsRpcLayer = ( const providerService = yield* ProviderService.ProviderService; const sideQuestionOwnership = makeSessionSideQuestionOwnership(); const providerMaintenanceRunner = yield* ProviderMaintenanceRunner.ProviderMaintenanceRunner; + const primeManagedMaintenance = yield* PrimeManagedMaintenance.PrimeManagedMaintenance; const providerLogin = yield* ProviderLoginCoordinator; const serverSelfUpdate = yield* ServerSelfUpdate.ServerSelfUpdate; const config = yield* ServerConfig.ServerConfig; @@ -1843,6 +1845,18 @@ const makeWsRpcLayer = ( "rpc.aggregate": "server", }, ), + [WS_METHODS.serverGetPrimeManagedMaintenance]: (input) => + observeRpcEffect( + WS_METHODS.serverGetPrimeManagedMaintenance, + primeManagedMaintenance.status(input.instanceId), + { "rpc.aggregate": "server" }, + ), + [WS_METHODS.serverRunPrimeManagedMaintenance]: (input) => + observeRpcEffect( + WS_METHODS.serverRunPrimeManagedMaintenance, + primeManagedMaintenance.run(input), + { "rpc.aggregate": "server" }, + ), [WS_METHODS.serverStartProviderLogin]: (input) => observeRpcEffect(WS_METHODS.serverStartProviderLogin, providerLogin.start(input), { "rpc.aggregate": "server", @@ -2755,6 +2769,7 @@ export const websocketRpcRouteLayer = Layer.unwrap( makeWsRpcLayer(session, clientOrigin, previewAutomationBroker).pipe( Layer.provideMerge(RpcSerialization.layerJson), Layer.provide(ProviderMaintenanceRunner.layer), + Layer.provide(PrimeManagedMaintenance.layer), Layer.provide( ProviderLoginCoordinatorLive.pipe(Layer.provide(ProviderLoginSessionsLive)), ), diff --git a/apps/web/src/components/settings/ProviderInstanceCard.tsx b/apps/web/src/components/settings/ProviderInstanceCard.tsx index b27235912..4efdb1d29 100644 --- a/apps/web/src/components/settings/ProviderInstanceCard.tsx +++ b/apps/web/src/components/settings/ProviderInstanceCard.tsx @@ -21,6 +21,8 @@ import { type ProviderInstanceEnvironmentVariable, type ProviderInstanceId, type ProviderDriverKind, + type ServerPrimeManagedAction, + type ServerPrimeManagedMaintenance, type ServerProvider, type ServerProviderModel, } from "@t3tools/contracts"; @@ -32,6 +34,13 @@ import { getProviderUnavailablePresentation, normalizeProviderAccentColor, } from "../../providerInstances"; +import { serverEnvironment } from "../../state/server"; +import { useEnvironmentQuery } from "../../state/query"; +import { useAtomCommand } from "../../state/use-atom-command"; +import { + isAtomCommandInterrupted, + squashAtomCommandFailure, +} from "@t3tools/client-runtime/state/runtime"; import { ProviderSignInDialog } from "./ProviderSignInDialog"; import type { EnvironmentId } from "@t3tools/contracts"; import { Badge } from "../ui/badge"; @@ -377,6 +386,218 @@ function ProviderEnvironmentSection(props: { ); } +let primeManagedCommandSequence = 0; +function primeManagedCommandId(action: ServerPrimeManagedAction): string { + primeManagedCommandSequence += 1; + return `prime-managed:${action}:${Date.now()}:${primeManagedCommandSequence}`; +} + +function PrimeManagedMaintenanceSection(props: { + readonly environmentId: EnvironmentId | undefined; + readonly instanceId: ProviderInstanceId; + readonly readOnly: boolean; + readonly distributionMessage: string | null; +}) { + const target = + props.environmentId === undefined + ? null + : serverEnvironment.primeManagedMaintenance({ + environmentId: props.environmentId, + input: { instanceId: props.instanceId }, + }); + const { data, error, isPending, refresh } = useEnvironmentQuery(target); + const runMaintenance = useAtomCommand(serverEnvironment.runPrimeManagedMaintenance, { + reportFailure: false, + }); + const refreshProviders = useAtomCommand(serverEnvironment.refreshProviders, { + reportFailure: false, + }); + const [runningAction, setRunningAction] = useState(null); + const [commandError, setCommandError] = useState(null); + const [previewConfirmed, setPreviewConfirmed] = useState(false); + + const run = async ( + action: ServerPrimeManagedAction, + options: { readonly channel?: "stable" | "preview"; readonly buildId?: string } = {}, + ) => { + if (props.environmentId === undefined || runningAction !== null) return; + setRunningAction(action); + setCommandError(null); + const result = await runMaintenance({ + environmentId: props.environmentId, + input: { + commandId: primeManagedCommandId(action), + instanceId: props.instanceId, + action, + ...(options.channel ? { channel: options.channel } : {}), + ...(options.channel === "preview" ? { allowPreview: true } : {}), + ...(options.buildId ? { buildId: options.buildId } : {}), + scheduleIfBusy: true, + }, + }); + setRunningAction(null); + if (result._tag === "Failure") { + if (!isAtomCommandInterrupted(result)) { + const cause = squashAtomCommandFailure(result); + setCommandError(cause instanceof Error ? cause.message : "Prime maintenance failed."); + } + return; + } + await refresh(); + await refreshProviders({ environmentId: props.environmentId, input: {} }); + }; + + const maintenance = data as ServerPrimeManagedMaintenance | null; + const queryError = error; + const selectedBuildId = maintenance?.selectedBuildId ?? null; + const operation = maintenance?.scheduled ?? maintenance?.operation ?? null; + const canWrite = maintenance?.controlsAvailable === true && !props.readOnly; + const stableAction: ServerPrimeManagedAction = + maintenance?.mode === "managed" ? "update" : "install"; + + return ( +
+
+
+

Pylon-managed Prime

+ +
+

+ {maintenance?.message ?? + (isPending + ? "Reading host maintenance status." + : "Host maintenance status is unavailable.")} +

+ {props.distributionMessage ? ( +

+ {props.distributionMessage} +

+ ) : null} + {maintenance?.guidance ? ( +

{maintenance.guidance}

+ ) : null} + {operation ? ( +

+ {operation.status.replaceAll("-", " ")} + {` ยท ${operation.message}`} +

+ ) : null} + {queryError || commandError ? ( +

+ {commandError ?? queryError} Check the environment connection and retry. The selected + Prime binary was not changed unless the status above confirms the switch. +

+ ) : null} +
+ + {maintenance?.supported === false ? null : ( +
+
+ + {maintenance?.mode === "managed" ? ( + + ) : null} + +
+ + + + + {maintenance && maintenance.availableBuilds.length > 0 ? ( +
+

+ Verified rollback builds +

+ {maintenance.availableBuilds.map((build) => ( +
+ + {build.buildId} ยท {build.channel} #{build.sequence} + + {build.buildId === selectedBuildId ? ( + + Selected + + ) : ( + + )} +
+ ))} +
+ ) : null} +
+ )} +
+ ); +} + interface ProviderInstanceCardProps { readonly instanceId: ProviderInstanceId; /** @@ -1048,6 +1269,15 @@ export function ProviderInstanceCard({ /> + {instance.driver === "primeAgent" ? ( + + ) : null} +
/ + pylon-managed-build-v1.json + .pylon-managed-receipt/ + prefix/node_modules/prime-agent/ + prefix/node_modules/.bin/prime-agent +``` + +A build directory is immutable after promotion. Its build id is the signed content identity, not a +SemVer choice. The build marker binds the package root, POSIX launcher, channel sequence, and root +artifact digest. The adjacent #193 receipt independently authenticates ownership and the exact package +root. Rollback and cleanup revalidate both offline before trusting a directory. + +The installer downloads the exact manifests, attestations, and root tarball through the #193 loader. +It verifies provenance and the root digest before parsing any archive header. Extraction uses a +bounded no-follow tar reader. It rejects absolute and traversing paths, Unicode and case collisions, +duplicates, links, devices, extensions, privileged modes, excess entries or bytes, the wrong package +or bin identity, and unexpected lifecycle scripts. + +The published root contains the bundled CLI. Pylon constructs the private `node_modules` package and +relative `.bin/prime-agent` link directly from the verified extracted tree. It does not invoke a +package manager, resolve a registry dependency, execute `postinstall`, or run artifact bytes during +installation. Staging directories are fsynced and renamed to the final build id only after launcher +validation. Startup removes exact unmarked staging or incomplete directories; marked builds require +receipt-owned cleanup. + +## Selection transaction + +Selection changes only the target Prime provider instance's complete settings binding. The server: + +1. reads the binary path and opaque binding generation; +2. fences new session starts for that instance; +3. inventories pending starts, admissions, active turns, adapter sessions, owned daemons/runtime + sessions, and loaded session incarnations; +4. returns a durable scheduled receipt when any exact owner is active; +5. writes and fsyncs a selection intent journal; +6. compare-and-sets the complete expected binding to the new launcher while the fence is held; +7. records the returned binding and selected build, then clears the intent and schedule; +8. releases the fence and refreshes the provider instance. + +The intent closes the crash window around settings CAS. Recovery reads the observed binding. If it is +the exact target, recovery records the completed selection. If it is still the complete expected +binding, recovery records an interrupted pre-switch failure. A different binding is treated as a +superseding user/settings change. A later explicit command supersedes an older scheduled command and +marks its receipt terminal rather than leaving two apparent pending switches. + +Distinct package roots and quiescent switching prevent a daemon from one build from sharing an +imported SDK module cache from another. Runtime capability still comes only from frozen SDK metadata +and exact post-attach negotiation. Managed distribution identity does not enable a native mode. + +## Commands and clients + +`serverGetPrimeManagedMaintenance` is read-scoped. `serverRunPrimeManagedMaintenance` is +operate-scoped. Both target one environment and provider instance, so local, remote, relay, tunnel, +multi-environment, and multi-client paths share the same server serialization and receipts. Command +ids are idempotent: reusing one with different input fails. Actions are install, update, rollback, +use-stock, and cleanup. Preview additionally requires `channel: preview` plus `allowPreview: true`. + +Web and desktop Provider Settings expose status, signed stable and explicit-preview actions, progress +and terminal errors, exact rollback builds, switch-back, and cleanup. Mobile reads status for every +Prime instance on each connected environment and directs host changes to web or desktop Provider +Settings. Native Windows returns WSL2 guidance before filesystem, network, provider, or runtime I/O. + +## Replay, offline, and cleanup rules + +Each channel has a persisted signed high-water `(sequenceEpoch, sequence, buildId)`. Lower sequences +and a different build at the same sequence fail. Rollback is allowed only as an explicit selection of +an already installed receipt-owned build; it does not lower the channel high-water. An offline feed +cannot change the selected build and is reported as a failure. + +Cleanup computes references from managed selections and scheduled switches. It ignores unrecognized, +linked, incomplete, or invalid-receipt directories and removes only an unreferenced build that passes +full offline marker and receipt validation. diff --git a/docs/operations/prime-agent-managed-rollback.md b/docs/operations/prime-agent-managed-rollback.md new file mode 100644 index 000000000..a0a8e6b62 --- /dev/null +++ b/docs/operations/prime-agent-managed-rollback.md @@ -0,0 +1,71 @@ +# Prime Agent managed rollback + +Use this runbook when a Pylon-managed Prime build is unhealthy and an environment must select a +previously verified build or return to its stock/configured Prime binary. + +## Safety rules + +- Do not edit `managed-tool-state-v1.json`, provider settings, build markers, or receipts by hand. +- Do not replace a build directory or `.bin/prime-agent` link in place. +- Do not delete a build until no provider selection or scheduled switch references it. +- Do not stop a user turn for maintenance. Let the exact provider instance drain. +- Do not run npm, pnpm, yarn, bun, or Homebrew against a managed build path. + +The server command owns the provider fence, complete-binding compare-and-set, selection journal, and +provider refresh. Manual file or settings changes bypass those guarantees. + +## Roll back to a verified build + +1. Open **Settings โ†’ Providers** and select the affected environment and Prime Agent instance. +2. Read the **Pylon-managed Prime** status. Record the selected build id, channel, sequence, operation + message, and any scheduled command. +3. Under **Verified rollback builds**, choose **Roll back** on the intended build. +4. If Pylon reports **waiting for quiescence**, leave the command scheduled. Confirm that no new work + is started on that instance and wait for its current admission, turn, session, daemon, and loaded + runtime context to exit. +5. Confirm the status changes to **succeeded** and the chosen build shows **Selected**. +6. Start a new Prime thread and verify provider readiness. Existing work must not be used as a + maintenance probe. + +A rollback selects an already installed build whose marker and private #193 receipt validate offline. +It does not lower the signed channel high-water. A missing build is not downloaded implicitly. + +## Return to stock/configured Prime + +1. Choose **Use stock/configured Prime** for the affected instance. +2. Wait for quiescence if scheduled. +3. Confirm the status says the instance uses its stock or configured binary. +4. Verify that the configured stock installation still works outside Pylon if the incident requires + that check. + +This action restores the binary path captured before the first managed selection. It does not install, +upgrade, rewrite, or remove stock bytes. + +## Cleanup after recovery + +Use **Prune unreferenced builds** only after the desired selection is healthy. Cleanup validates +ownership again and removes only unreferenced receipt-owned build directories. A selected build or a +build referenced by a scheduled switch is retained. Unknown, linked, partial, or invalid-receipt paths +are not cleanup authority and need separate investigation. + +## Failure and crash recovery + +- **Feed offline, rate limited, or invalid:** Keep the current verified build. Do not infer update + availability from SemVer or a package registry. +- **Replay or implicit downgrade rejected:** Do not clear high-water state. Use explicit rollback to an + installed verified build, or investigate the signed channel publication. +- **Interrupted staging:** Restart the Pylon server. Startup removes exact unmarked staging and + incomplete build directories without changing the selected binding. +- **Interrupted around selection:** Restart the Pylon server or reopen maintenance status. The durable + selection intent reconciles whether the complete settings CAS occurred and records a terminal + receipt. +- **Binding changed or command superseded:** Re-read Provider Settings. A user or another client chose + a newer binding. Submit a new command only after confirming the intended target. +- **Invalid receipt or marker:** Do not select or delete the directory manually. Preserve it for + diagnosis and choose a different verified build or stock. +- **Native Windows:** No managed I/O should have started. Run Pylon and Prime Agent inside WSL2 and + operate on that Linux environment. + +For server-side diagnosis, correlate the maintenance error and trace id with the selected environment. +The managed store is below that environment's runtime `userdata/provider-tools/prime-agent/` path. +Read it only for incident evidence; use the RPC/UI commands for changes. diff --git a/docs/user/providers-prime-agent.md b/docs/user/providers-prime-agent.md index c8428037f..aa2f22eda 100644 --- a/docs/user/providers-prime-agent.md +++ b/docs/user/providers-prime-agent.md @@ -66,6 +66,37 @@ ACP compatibility mode instead, because the daemon API cannot safely preserve ar arguments. Pylon shows that fallback in the provider status rather than silently discarding the arguments. +### Optional Pylon-managed installation + +Stock or configured Prime remains the default. To opt in, open **Settings โ†’ Providers โ†’ Prime Agent** +and use **Install stable** under **Pylon-managed Prime**. Pylon downloads and verifies the exact signed +publication on the selected environment host, installs it beside other builds, and changes only this +Prime provider instance's binary path. It does not overwrite or remove a global npm, pnpm, yarn, bun, +Homebrew, or standalone Prime installation. + +Stable is the default managed channel. Preview requires checking the preview warning before +**Install/update preview** becomes available. Signed channel sequence and build identity determine +updates; the package version does not. If the signed feed is offline or invalid, Pylon keeps the +current verified build selected and shows the failure instead of guessing that an update is available. + +Updates stage a new build before switching. If this Prime instance has an active admission, turn, +session, daemon, or loaded SDK runtime, Pylon schedules the switch until that exact instance drains. +It never interrupts a turn for maintenance. The same controls work when Settings connects to the host +locally, remotely, through a relay, or through a tunnel. + +Use the build list to roll back to an already verified build. **Use stock/configured Prime** restores +the binary path that was configured before managed installation. **Prune unreferenced builds** removes +only verified Pylon-owned builds that no provider selection or scheduled switch references. It never +touches the stock installation. + +Pylon Mobile shows each connected environment's Prime host-maintenance status under **Settings โ†’ +Environments**. Use web or desktop Provider Settings for install, update, rollback, switch-back, and +cleanup controls. + +Native Windows does not perform a managed Prime download or install. Install and run Pylon and Prime +Agent inside WSL2, connect to that Linux environment, and use its Linux path. macOS and Linux use their +native managed build only after exact runtime negotiation succeeds. + ## Turn Completion A Prime turn can contain several assistant segments around tool work. Pylon keeps those segments in diff --git a/packages/client-runtime/src/state/server.ts b/packages/client-runtime/src/state/server.ts index 00c3291c7..6ad071a5a 100644 --- a/packages/client-runtime/src/state/server.ts +++ b/packages/client-runtime/src/state/server.ts @@ -797,6 +797,17 @@ export function createServerEnvironmentAtoms( scheduler: configScheduler, concurrency: configConcurrency, }), + primeManagedMaintenance: createEnvironmentRpcQueryAtomFamily(runtime, { + label: "environment-data:server:prime-managed-maintenance", + tag: WS_METHODS.serverGetPrimeManagedMaintenance, + staleTimeMs: 0, + }), + runPrimeManagedMaintenance: createEnvironmentRpcCommand(runtime, { + label: "environment-data:server:run-prime-managed-maintenance", + tag: WS_METHODS.serverRunPrimeManagedMaintenance, + scheduler: configScheduler, + concurrency: configConcurrency, + }), startProviderLogin: createEnvironmentRpcCommand(runtime, { label: "environment-data:server:start-provider-login", tag: WS_METHODS.serverStartProviderLogin, diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts index 0b10ac2b2..d854fa23c 100644 --- a/packages/contracts/src/rpc.ts +++ b/packages/contracts/src/rpc.ts @@ -216,6 +216,10 @@ import { ServerProviderLoginStartInput, ServerProviderLoginSubmitInput, ServerProviderUpdateInput, + ServerPrimeManagedCommandInput, + ServerPrimeManagedCommandReceipt, + ServerPrimeManagedMaintenance, + ServerPrimeManagedMaintenanceError, ServerLifecycleStreamEvent, ServerRemoveKeybindingInput, ServerRemoveKeybindingResult, @@ -341,6 +345,8 @@ export const WS_METHODS = { serverGetConfig: "server.getConfig", serverRefreshProviders: "server.refreshProviders", serverUpdateProvider: "server.updateProvider", + serverGetPrimeManagedMaintenance: "server.getPrimeManagedMaintenance", + serverRunPrimeManagedMaintenance: "server.runPrimeManagedMaintenance", serverStartProviderLogin: "server.startProviderLogin", serverSubmitProviderLoginCode: "server.submitProviderLoginCode", serverCancelProviderLogin: "server.cancelProviderLogin", @@ -582,6 +588,24 @@ export const WsServerUpdateProviderRpc = Rpc.make(WS_METHODS.serverUpdateProvide error: Schema.Union([ServerProviderUpdateError, EnvironmentAuthorizationError]), }); +export const WsServerGetPrimeManagedMaintenanceRpc = Rpc.make( + WS_METHODS.serverGetPrimeManagedMaintenance, + { + payload: Schema.Struct({ instanceId: ProviderInstanceId }), + success: ServerPrimeManagedMaintenance, + error: Schema.Union([ServerPrimeManagedMaintenanceError, EnvironmentAuthorizationError]), + }, +); + +export const WsServerRunPrimeManagedMaintenanceRpc = Rpc.make( + WS_METHODS.serverRunPrimeManagedMaintenance, + { + payload: ServerPrimeManagedCommandInput, + success: ServerPrimeManagedCommandReceipt, + error: Schema.Union([ServerPrimeManagedMaintenanceError, EnvironmentAuthorizationError]), + }, +); + /** * Sign in to a provider account from the client. * @@ -1283,6 +1307,8 @@ export const WsRpcGroup = RpcGroup.make( WsProviderSetSessionAutoCompactionRpc, WsProviderRefineSessionHarnessRpc, WsServerUpdateProviderRpc, + WsServerGetPrimeManagedMaintenanceRpc, + WsServerRunPrimeManagedMaintenanceRpc, WsServerStartProviderLoginRpc, WsServerSubmitProviderLoginCodeRpc, WsServerCancelProviderLoginRpc, diff --git a/packages/contracts/src/server.ts b/packages/contracts/src/server.ts index 1f9170e58..1262c5e5a 100644 --- a/packages/contracts/src/server.ts +++ b/packages/contracts/src/server.ts @@ -212,6 +212,85 @@ export type ServerProviderDistributionClassification = export const ServerProviderDistributionChannel = Schema.Literals(["preview", "stable"]); export type ServerProviderDistributionChannel = typeof ServerProviderDistributionChannel.Type; +export const ServerPrimeManagedAction = Schema.Literals([ + "install", + "update", + "rollback", + "use-stock", + "cleanup", +]); +export type ServerPrimeManagedAction = typeof ServerPrimeManagedAction.Type; + +export const ServerPrimeManagedOperationStatus = Schema.Literals([ + "queued", + "downloading", + "verifying", + "installing", + "waiting-for-quiescence", + "switching", + "succeeded", + "failed", +]); +export type ServerPrimeManagedOperationStatus = typeof ServerPrimeManagedOperationStatus.Type; + +export const ServerPrimeManagedCommandReceipt = Schema.Struct({ + commandId: TrimmedNonEmptyString, + instanceId: ProviderInstanceId, + action: ServerPrimeManagedAction, + status: ServerPrimeManagedOperationStatus, + channel: Schema.NullOr(ServerProviderDistributionChannel), + buildId: Schema.NullOr(TrimmedNonEmptyString), + message: TrimmedNonEmptyString, + startedAt: IsoDateTime, + finishedAt: Schema.NullOr(IsoDateTime), +}); +export type ServerPrimeManagedCommandReceipt = typeof ServerPrimeManagedCommandReceipt.Type; + +export const ServerPrimeManagedInstalledBuild = Schema.Struct({ + buildId: TrimmedNonEmptyString, + channel: ServerProviderDistributionChannel, + sequence: Schema.Int.check(Schema.isGreaterThan(0)), + binaryPath: TrimmedNonEmptyString, +}); +export type ServerPrimeManagedInstalledBuild = typeof ServerPrimeManagedInstalledBuild.Type; + +export const ServerPrimeManagedMaintenance = Schema.Struct({ + supported: Schema.Boolean, + controlsAvailable: Schema.Boolean, + mode: Schema.Literals(["stock", "managed"]), + selectedBuildId: Schema.NullOr(TrimmedNonEmptyString), + channel: Schema.NullOr(ServerProviderDistributionChannel), + availableBuilds: Schema.Array(ServerPrimeManagedInstalledBuild), + scheduled: Schema.NullOr(ServerPrimeManagedCommandReceipt), + operation: Schema.NullOr(ServerPrimeManagedCommandReceipt), + message: TrimmedNonEmptyString, + guidance: Schema.NullOr(TrimmedNonEmptyString), +}); +export type ServerPrimeManagedMaintenance = typeof ServerPrimeManagedMaintenance.Type; + +export const ServerPrimeManagedCommandInput = Schema.Struct({ + commandId: TrimmedNonEmptyString, + instanceId: ProviderInstanceId, + action: ServerPrimeManagedAction, + channel: Schema.optional(ServerProviderDistributionChannel), + allowPreview: Schema.optional(Schema.Boolean), + buildId: Schema.optional(TrimmedNonEmptyString), + scheduleIfBusy: Schema.optional(Schema.Boolean), +}); +export type ServerPrimeManagedCommandInput = typeof ServerPrimeManagedCommandInput.Type; + +export class ServerPrimeManagedMaintenanceError extends Schema.TaggedErrorClass()( + "ServerPrimeManagedMaintenanceError", + { + instanceId: ProviderInstanceId, + reason: TrimmedNonEmptyString, + }, +) { + override get message(): string { + return this.reason; + } +} + /** * Signed build identity for provider distributions that publish one. * From 3634ce63ad4be66100c77a50fbf084f3b7ca0a55 Mon Sep 17 00:00:00 2001 From: Trevor Walker Date: Tue, 1 Sep 2026 11:24:57 -0600 Subject: [PATCH 2/2] fix(prime): preserve managed install bindings Refs #194 --- .../PrimeAgentDistributionVerifier.test.ts | 233 +++++++++ .../prime/PrimeAgentDistributionVerifier.ts | 469 ++++++++++++++---- .../prime/PrimeAgentManagedToolStore.test.ts | 158 ++++++ .../prime/PrimeAgentManagedToolStore.ts | 182 ++++++- .../provider/prime/PrimeManagedMaintenance.ts | 13 + apps/server/src/serverSettings.test.ts | 5 + apps/server/src/serverSettings.ts | 28 ++ .../prime-agent-distribution-verification.md | 14 +- docs/internals/prime-agent-managed-install.md | 19 +- 9 files changed, 977 insertions(+), 144 deletions(-) diff --git a/apps/server/src/provider/prime/PrimeAgentDistributionVerifier.test.ts b/apps/server/src/provider/prime/PrimeAgentDistributionVerifier.test.ts index 467adee91..b9bfdf25b 100644 --- a/apps/server/src/provider/prime/PrimeAgentDistributionVerifier.test.ts +++ b/apps/server/src/provider/prime/PrimeAgentDistributionVerifier.test.ts @@ -11,6 +11,9 @@ import { authenticatePrimeManagedState, canonicalPrimeDistributionJson, inspectPrimeAgentDistribution, + invalidatePrimeDistributionCache, + makeLatestPrimePublicationBundleLoader, + makeLatestPrimePublicationLoader, PRIME_DISTRIBUTION_REF, PRIME_DISTRIBUTION_REPOSITORY, PRIME_DISTRIBUTION_REPOSITORY_URL, @@ -27,6 +30,7 @@ import { primeDistributionStateDirectory, requireRealPrimePublicationFixture, type ExpectedPrimeAttestation, + type PrimeDistributionNetworkDependencies, type PrimeManagedStatePayload, type PrimePublicationFixture, type PrimeSlsaStatement, @@ -37,6 +41,7 @@ import { const temporaryDirectories: string[] = []; afterEach(async () => { vi.restoreAllMocks(); + invalidatePrimeDistributionCache(); await Promise.all( temporaryDirectories .splice(0) @@ -256,6 +261,113 @@ const validVerification = { verifySourcePolicy: async () => {}, }; +function githubAsset(id: number, name: string, url: string) { + return { id, name, size: 1, browser_download_url: url }; +} + +function makeNetworkHarness(input: { + readonly channel: "preview" | "stable"; + readonly candidates: number; + readonly key: string; + readonly now?: () => number; + readonly failureTtlMs?: number; + readonly rateLimitTtlMs?: number; + readonly refreshReuseMs?: number; + readonly failListOnceWith403?: boolean; +}) { + const fixture = syntheticPublication(input.channel); + const releaseManifest = JSON.parse(fixture.releaseManifestBytes.toString("utf8")) as { + assets: ReadonlyArray<{ readonly file: string }>; + }; + const stableManifest = fixture.stableManifestBytes + ? (JSON.parse(fixture.stableManifestBytes.toString("utf8")) as { readonly tag: string }) + : undefined; + const validTag = input.channel === "preview" ? BUILD_ID : stableManifest!.tag; + const bytes = new Map(); + const previewAssets = [ + githubAsset(1, PRIME_RELEASE_MANIFEST, "fixture://valid/release"), + githubAsset(2, PRIME_PREVIEW_MANIFEST, "fixture://valid/preview"), + ...releaseManifest.assets.map((asset, index) => + githubAsset(index + 3, asset.file, `fixture://valid/${asset.file}`), + ), + ]; + bytes.set("fixture://valid/release", fixture.releaseManifestBytes); + bytes.set("fixture://valid/preview", fixture.previewManifestBytes); + bytes.set( + `${PRIME_DISTRIBUTION_REPOSITORY_URL}/releases/download/${BUILD_ID}/${fixture.verified.rootAsset}`, + fixture.rootArtifactBytes!, + ); + const validPreviewRelease = { + id: 10_000, + tag_name: BUILD_ID, + draft: false, + prerelease: true, + immutable: true, + assets: previewAssets, + }; + bytes.set("fixture://valid/stable", fixture.stableManifestBytes ?? Buffer.alloc(0)); + const validRelease = + input.channel === "preview" + ? validPreviewRelease + : { + id: 20_000, + tag_name: validTag, + draft: false, + prerelease: false, + immutable: true, + assets: [githubAsset(20_001, "pylon-stable-channel-v1.json", "fixture://valid/stable")], + }; + // Duplicate valid entries let the test exercise the exact maximum candidate work without needing + // twelve unrelated cryptographic fixtures. GitHub ids are distinct; immutable tags/content match. + const releases = Array.from({ length: input.candidates }, (_, index) => ({ + ...validRelease, + id: validRelease.id + index, + })); + + let jsonRequests = 0; + let byteRequests = 0; + let trustedRootRequests = 0; + let listRequests = 0; + const dependencyShape = (): PrimeDistributionNetworkDependencies => ({ + cache: { + key: input.key, + ...(input.now ? { now: input.now } : {}), + ...(input.failureTtlMs === undefined ? {} : { failureTtlMs: input.failureTtlMs }), + ...(input.rateLimitTtlMs === undefined ? {} : { rateLimitTtlMs: input.rateLimitTtlMs }), + ...(input.refreshReuseMs === undefined ? {} : { refreshReuseMs: input.refreshReuseMs }), + }, + fetchJson: async (url) => { + jsonRequests += 1; + if (url.endsWith("/releases?per_page=100")) { + listRequests += 1; + if (input.failListOnceWith403 && listRequests === 1) { + throw new Error("Pylon distribution fetch failed with HTTP 403."); + } + return releases; + } + if (url.includes("/releases/tags/")) return validPreviewRelease; + if (url.includes("/attestations/")) return { attestations: [{ bundle: { proof: url } }] }; + throw new Error(`Unexpected JSON request: ${url}`); + }, + fetchBytes: async (url) => { + byteRequests += 1; + const value = bytes.get(url); + if (!value) throw new Error(`Unexpected byte request: ${url}`); + return value; + }, + getTrustedRoot: async () => { + trustedRootRequests += 1; + return {} as Awaited>; + }, + verifyBundle: validVerification.verifyBundle, + verifySourcePolicy: validVerification.verifySourcePolicy, + }); + return { + dependencyShape, + counts: () => ({ jsonRequests, byteRequests, trustedRootRequests, listRequests }), + }; +} + async function makePackage(input?: { readonly version?: string; readonly metadata?: unknown; @@ -536,6 +648,127 @@ describe("Pylon Prime publication verification", () => { } }); + it.each([ + { channel: "stable" as const, candidates: 1, jsonRequests: 4, byteRequests: 3 }, + { channel: "stable" as const, candidates: 7, jsonRequests: 22, byteRequests: 21 }, + { channel: "preview" as const, candidates: 12, jsonRequests: 13, byteRequests: 24 }, + ])( + "bounds $channel verification requests across $candidates feed candidates", + async ({ channel, candidates, jsonRequests, byteRequests }) => { + const harness = makeNetworkHarness({ + channel, + candidates, + key: `request-count-${channel}-${candidates}`, + }); + const loader = makeLatestPrimePublicationLoader(harness.dependencyShape()); + await expect(loader(channel)).resolves.toMatchObject({ channel }); + expect(harness.counts()).toEqual({ + jsonRequests, + byteRequests, + trustedRootRequests: 1, + listRequests: 1, + }); + }, + ); + + it("single-flights concurrent multi-instance status and maintenance loaders process-wide", async () => { + const harness = makeNetworkHarness({ channel: "stable", candidates: 1, key: "concurrent" }); + const leftStatus = makeLatestPrimePublicationLoader(harness.dependencyShape()); + const rightStatus = makeLatestPrimePublicationLoader(harness.dependencyShape()); + const maintenance = makeLatestPrimePublicationBundleLoader(harness.dependencyShape()); + const [left, right, bundle] = await Promise.all([ + leftStatus("stable"), + rightStatus("stable"), + maintenance("stable", { refresh: true }), + ]); + expect(left).toEqual(right); + expect(bundle.publication).toEqual(left); + expect(digest("sha256", bundle.rootArtifactBytes)).toBe(left.rootSha256); + expect(harness.counts()).toEqual({ + jsonRequests: 4, + byteRequests: 4, + trustedRootRequests: 1, + listRequests: 1, + }); + + await makeLatestPrimePublicationLoader(harness.dependencyShape())("stable"); + await makeLatestPrimePublicationBundleLoader(harness.dependencyShape())("stable", { + refresh: true, + }); + expect(harness.counts()).toEqual({ + jsonRequests: 4, + byteRequests: 4, + trustedRootRequests: 1, + listRequests: 1, + }); + }); + + it("caches 403 failures for the bounded retry TTL, then retries", async () => { + let now = 0; + const harness = makeNetworkHarness({ + channel: "preview", + candidates: 1, + key: "rate-limit", + now: () => now, + rateLimitTtlMs: 100, + failListOnceWith403: true, + }); + const loader = makeLatestPrimePublicationLoader(harness.dependencyShape()); + await expect(loader("preview")).rejects.toThrow(/HTTP 403/u); + await expect(loader("preview", { refresh: true })).rejects.toThrow(/HTTP 403/u); + now = 99; + await expect(loader("preview")).rejects.toThrow(/HTTP 403/u); + expect(harness.counts().listRequests).toBe(1); + + now = 100; + await expect(loader("preview")).resolves.toMatchObject({ channel: "preview" }); + expect(harness.counts()).toEqual({ + jsonRequests: 3, + byteRequests: 2, + trustedRootRequests: 1, + listRequests: 2, + }); + }); + + it("reuses just-fresh status for maintenance and supports exact cache invalidation", async () => { + let now = 0; + const key = "explicit-refresh-and-invalidation"; + const harness = makeNetworkHarness({ + channel: "preview", + candidates: 1, + key, + now: () => now, + refreshReuseMs: 50, + }); + const status = makeLatestPrimePublicationLoader(harness.dependencyShape()); + const maintenance = makeLatestPrimePublicationBundleLoader(harness.dependencyShape()); + await status("preview"); + await maintenance("preview", { refresh: true }); + expect(harness.counts()).toEqual({ + jsonRequests: 2, + byteRequests: 3, + trustedRootRequests: 1, + listRequests: 1, + }); + + now = 51; + await status("preview", { refresh: true }); + expect(harness.counts()).toEqual({ + jsonRequests: 4, + byteRequests: 5, + trustedRootRequests: 2, + listRequests: 2, + }); + invalidatePrimeDistributionCache({ key, channel: "preview" }); + await status("preview"); + expect(harness.counts()).toEqual({ + jsonRequests: 6, + byteRequests: 7, + trustedRootRequests: 3, + listRequests: 3, + }); + }); + it("keeps the real immutable fixture gate fail-closed until all exact inputs exist", () => { expect(() => requireRealPrimePublicationFixture({})).toThrow(/immutable preview\/stable/u); expect(() => diff --git a/apps/server/src/provider/prime/PrimeAgentDistributionVerifier.ts b/apps/server/src/provider/prime/PrimeAgentDistributionVerifier.ts index e49830b04..0168d5064 100644 --- a/apps/server/src/provider/prime/PrimeAgentDistributionVerifier.ts +++ b/apps/server/src/provider/prime/PrimeAgentDistributionVerifier.ts @@ -1,5 +1,6 @@ // @effect-diagnostics globalFetch:off // @effect-diagnostics nodeBuiltinImport:off +// @effect-diagnostics globalDate:off import { bundleFromJSON, assertBundleLatest, isBundleWithDsseEnvelope } from "@sigstore/bundle"; import { X509Certificate } from "@sigstore/core"; import { getTrustedRoot } from "@sigstore/tuf"; @@ -39,6 +40,12 @@ const MAX_RELEASE_RESPONSE_BYTES = 4 * 1024 * 1024; const MAX_ROOT_ARTIFACT_BYTES = 256 * 1024 * 1024; const MAX_FEED_CANDIDATES = 12; const FETCH_TIMEOUT_MS = 12_000; +const DISTRIBUTION_CACHE_TTL_MS = 15 * 60_000; +const DISTRIBUTION_BUNDLE_CACHE_TTL_MS = 30_000; +const DISTRIBUTION_FAILURE_TTL_MS = 15_000; +const DISTRIBUTION_RATE_LIMIT_TTL_MS = 60_000; +const DISTRIBUTION_MAX_RATE_LIMIT_TTL_MS = 5 * 60_000; +const DISTRIBUTION_REFRESH_REUSE_MS = 30_000; const SHA256 = Schema.String.check(Schema.isPattern(/^[0-9a-f]{64}$/)); const SHA512 = Schema.String.check(Schema.isPattern(/^[0-9a-f]{128}$/)); @@ -769,24 +776,38 @@ async function verifyExpectedSubjectBundles( bundlesByDigest: PrimePublicationFixture["attestationBundlesBySubjectSha256"], dependencies: PrimePublicationVerificationDependencies, ): Promise { + // One GitHub attestation can (and Pylon publications do) bind the whole subject set. Query and + // verify that multi-subject provenance once instead of spending one REST request and one + // cryptographic verification per subject. + const candidates: unknown[] = []; + const seenObjects = new Set(); for (const subject of expected.subjects) { - const bundles = bundlesByDigest.get(subject.sha256); - if (!bundles || bundles.length < 1 || bundles.length > 20) { - throw new Error(`No bounded Sigstore bundle set exists for ${subject.name}.`); - } - let verified = false; - for (const bundle of bundles) { - try { - await dependencies.verifyBundle(bundle, expected); - verified = true; - } catch { - // Public repositories can accumulate unrelated attestations. Only one exact, fully verified - // Pylon signer/source/subject binding is required for this digest. + for (const bundle of bundlesByDigest.get(subject.sha256) ?? []) { + if (seenObjects.has(bundle)) continue; + seenObjects.add(bundle); + candidates.push(bundle); + if (candidates.length > 20) { + throw new Error("Pylon publication has an unbounded Sigstore bundle set."); } } - if (!verified) - throw new Error(`No valid Pylon Sigstore attestation exists for ${subject.name}.`); } + if (candidates.length === 0) { + throw new Error( + `No bounded Sigstore bundle set exists for ${expected.subjects[0]?.name ?? "publication"}.`, + ); + } + for (const bundle of candidates) { + try { + await dependencies.verifyBundle(bundle, expected); + return; + } catch { + // Public repositories can accumulate unrelated attestations. Only one exact, fully verified + // Pylon signer/source/complete-subject binding is required. + } + } + throw new Error( + `No valid Pylon Sigstore attestation exists for ${expected.subjects[0]?.name ?? "publication"}.`, + ); } export async function verifyPrimePublicationFixture( @@ -1443,10 +1464,49 @@ export async function inspectPrimeAgentDistribution( }); } +export interface PrimeDistributionCacheOptions { + /** Separate fixture/test caches without weakening the process-wide production repository cache. */ + readonly key: string; + readonly now?: () => number; + readonly successTtlMs?: number; + readonly bundleTtlMs?: number; + readonly failureTtlMs?: number; + readonly rateLimitTtlMs?: number; + readonly refreshReuseMs?: number; +} + export interface PrimeDistributionNetworkDependencies { readonly fetchJson: (url: string, maxBytes: number) => Promise; readonly fetchBytes: (url: string, maxBytes: number) => Promise; readonly getTrustedRoot: () => Promise; + /** Production loaders share this cache identity across driver and maintenance instances. */ + readonly cache?: PrimeDistributionCacheOptions; + /** Test seams. Production always performs the server-owned verification functions above. */ + readonly verifyBundle?: PrimePublicationVerificationDependencies["verifyBundle"]; + readonly verifySourcePolicy?: PrimePublicationVerificationDependencies["verifySourcePolicy"]; +} + +class PrimeDistributionHttpError extends Error { + readonly status: number; + readonly retryAfterMs: number | undefined; + + constructor(status: number, retryAfterMs: number | undefined) { + super(`Pylon distribution fetch failed with HTTP ${status}.`); + this.status = status; + this.retryAfterMs = retryAfterMs; + } +} + +function retryAfterMilliseconds(response: Response): number | undefined { + const retryAfter = response.headers.get("retry-after"); + if (retryAfter) { + const seconds = Number(retryAfter); + if (Number.isFinite(seconds) && seconds >= 0) return seconds * 1_000; + const instant = Date.parse(retryAfter); + if (Number.isFinite(instant)) return Math.max(0, instant - Date.now()); + } + const reset = Number(response.headers.get("x-ratelimit-reset")); + return Number.isFinite(reset) ? Math.max(0, reset * 1_000 - Date.now()) : undefined; } async function boundedFetch(url: string, maxBytes: number, accept: string): Promise { @@ -1471,7 +1531,7 @@ async function boundedFetch(url: string, maxBytes: number, accept: string): Prom throw new Error("Pylon distribution fetch followed an untrusted redirect."); } if (!response.ok || !response.body) { - throw new Error(`Pylon distribution fetch failed with HTTP ${response.status}.`); + throw new PrimeDistributionHttpError(response.status, retryAfterMilliseconds(response)); } const length = Number(response.headers.get("content-length")); if (Number.isFinite(length) && length > maxBytes) { @@ -1494,9 +1554,13 @@ async function boundedFetch(url: string, maxBytes: number, accept: string): Prom } export function makePrimeDistributionNetworkDependencies( - options: { readonly tufCachePath?: string } = {}, + options: { + readonly tufCachePath?: string; + readonly cache?: Omit; + } = {}, ): PrimeDistributionNetworkDependencies { return { + cache: { key: PRIME_DISTRIBUTION_REPOSITORY, ...options.cache }, fetchJson: async (url, maxBytes) => { const bytes = await boundedFetch(url, maxBytes, "application/vnd.github+json"); return JSON.parse(bytes.toString("utf8")) as unknown; @@ -1595,21 +1659,22 @@ async function loadPublicationFixtureForRelease( MAX_ROOT_ARTIFACT_BYTES, ) : undefined; - const subjectDigests = new Set([ + const previewSubjectDigests = [ ...parsedRelease.assets.map((asset) => asset.sha256), sha256(releaseManifestBytes), sha256(previewManifestBytes), - ...(stableManifestBytes ? [sha256(stableManifestBytes)] : []), - ]); - const attestationBundlesBySubjectSha256 = new Map>(); - await Promise.all( - [...subjectDigests].map(async (digest) => { - attestationBundlesBySubjectSha256.set( - digest, - await fetchAttestationBundles(digest, dependencies), - ); - }), + ]; + const previewBundles = await fetchAttestationBundles(sha256(previewManifestBytes), dependencies); + const attestationBundlesBySubjectSha256 = new Map>( + previewSubjectDigests.map((digest) => [digest, previewBundles]), ); + if (stableManifestBytes) { + const stableDigest = sha256(stableManifestBytes); + attestationBundlesBySubjectSha256.set( + stableDigest, + await fetchAttestationBundles(stableDigest, dependencies), + ); + } return { channel, releaseManifestBytes, @@ -1649,50 +1714,220 @@ async function verifyRemoteSourcePolicy( } } -export function makeLatestPrimePublicationLoader( - dependencies: PrimeDistributionNetworkDependencies = makePrimeDistributionNetworkDependencies(), -): PrimeDistributionInspectionDependencies["loadLatestVerifiedPublication"] { - return async (channel) => { - const raw = await dependencies.fetchJson( - `https://api.github.com/repos/${PRIME_DISTRIBUTION_REPOSITORY}/releases?per_page=100`, - MAX_RELEASE_RESPONSE_BYTES, - ); - const releases = decodeGitHubReleases(raw) - .filter( - (release) => - !release.draft && - release.immutable && - (channel === "preview" - ? /^pylon-build-g[0-9a-f]{12}-r[1-9][0-9]*$/u.test(release.tag_name) - : /^pylon-stable-[0-9]{6}-g[0-9a-f]{12}-r[1-9][0-9]*$/u.test(release.tag_name)), - ) - .slice(0, MAX_FEED_CANDIDATES); - if (releases.length === 0) throw new Error(`No immutable ${channel} publication exists.`); - const trustedRoot = await dependencies.getTrustedRoot(); - const verified: VerifiedPrimePublication[] = []; - for (const release of releases) { - try { - const fixture = await loadPublicationFixtureForRelease(channel, release, dependencies); - verified.push( - await verifyPrimePublicationFixture(fixture, { - verifyBundle: async (bundle, expected) => - verifyPrimeSigstoreBundle(bundle, trustedRoot, expected), - verifySourcePolicy: (expected) => verifyRemoteSourcePolicy(expected, dependencies), - }), - ); - } catch { - // One malformed, draft-like, or unrelated release must not hide a later exact candidate. - } +export interface VerifiedPrimePublicationBundle { + readonly publication: VerifiedPrimePublication; + readonly rootArtifactBytes: Buffer; +} + +export interface PrimeDistributionLoadOptions { + /** Revalidate an old success, but reuse a just-fresh success and every live failure backoff. */ + readonly refresh?: boolean; +} + +type CachedPrimeDistributionResult = + | { + readonly status: "success"; + readonly value: T; + readonly storedAt: number; + readonly expiresAt: number; } - const latest = verified.toSorted((left, right) => right.sequence - left.sequence)[0]; - if (!latest) throw new Error(`No exact signed ${channel} publication verified.`); - return latest; + | { + readonly status: "failure"; + readonly error: unknown; + readonly storedAt: number; + readonly expiresAt: number; + }; + +interface PrimeDistributionChannelCache { + publication?: CachedPrimeDistributionResult; + publicationFlight?: Promise; + bundle?: CachedPrimeDistributionResult; + bundleFlight?: Promise; +} + +const primeDistributionChannelCaches = new Map(); + +function distributionCacheKey( + dependencies: PrimeDistributionNetworkDependencies, + channel: ServerProviderDistributionChannel, +): string | undefined { + return dependencies.cache ? `${dependencies.cache.key}:${channel}` : undefined; +} + +function cacheNow(dependencies: PrimeDistributionNetworkDependencies): number { + return dependencies.cache?.now?.() ?? Date.now(); +} + +function isRateLimitFailure(cause: unknown): boolean { + return ( + (cause instanceof PrimeDistributionHttpError && + (cause.status === 403 || cause.status === 429)) || + (cause instanceof Error && /HTTP (?:403|429)\b/u.test(cause.message)) + ); +} + +function failureCacheTtl( + cause: unknown, + dependencies: PrimeDistributionNetworkDependencies, +): number { + const configured = dependencies.cache; + if (!isRateLimitFailure(cause)) { + return Math.max(1, configured?.failureTtlMs ?? DISTRIBUTION_FAILURE_TTL_MS); + } + const requested = cause instanceof PrimeDistributionHttpError ? cause.retryAfterMs : undefined; + return Math.min( + DISTRIBUTION_MAX_RATE_LIMIT_TTL_MS, + Math.max(configured?.rateLimitTtlMs ?? DISTRIBUTION_RATE_LIMIT_TTL_MS, requested ?? 0), + ); +} + +function reusableCachedResult( + result: CachedPrimeDistributionResult | undefined, + dependencies: PrimeDistributionNetworkDependencies, + options: PrimeDistributionLoadOptions, +): CachedPrimeDistributionResult | undefined { + if (!result) return undefined; + const now = cacheNow(dependencies); + if (now >= result.expiresAt) return undefined; + if (result.status === "failure") return result; + const refreshReuseMs = dependencies.cache?.refreshReuseMs ?? DISTRIBUTION_REFRESH_REUSE_MS; + return options.refresh && now - result.storedAt >= refreshReuseMs ? undefined : result; +} + +function cachedValue(result: CachedPrimeDistributionResult): T { + if (result.status === "failure") throw result.error; + return result.value; +} + +/** Test/operations seam. Normal refreshes rely on TTL and never need explicit invalidation. */ +export function invalidatePrimeDistributionCache( + input: { + readonly channel?: ServerProviderDistributionChannel; + readonly key?: string; + } = {}, +): void { + for (const key of primeDistributionChannelCaches.keys()) { + const separator = key.lastIndexOf(":"); + const cacheKey = key.slice(0, separator); + const channel = key.slice(separator + 1); + if ( + (input.key === undefined || input.key === cacheKey) && + (input.channel === undefined || input.channel === channel) + ) { + primeDistributionChannelCaches.delete(key); + } + } +} + +async function loadLatestPrimePublicationUncached( + channel: ServerProviderDistributionChannel, + dependencies: PrimeDistributionNetworkDependencies, +): Promise { + const raw = await dependencies.fetchJson( + `https://api.github.com/repos/${PRIME_DISTRIBUTION_REPOSITORY}/releases?per_page=100`, + MAX_RELEASE_RESPONSE_BYTES, + ); + const releases = decodeGitHubReleases(raw) + .filter( + (release) => + !release.draft && + release.immutable && + (channel === "preview" + ? /^pylon-build-g[0-9a-f]{12}-r[1-9][0-9]*$/u.test(release.tag_name) + : /^pylon-stable-[0-9]{6}-g[0-9a-f]{12}-r[1-9][0-9]*$/u.test(release.tag_name)), + ) + .slice(0, MAX_FEED_CANDIDATES); + if (releases.length === 0) throw new Error(`No immutable ${channel} publication exists.`); + const trustedRoot = await dependencies.getTrustedRoot(); + const verified: VerifiedPrimePublication[] = []; + const sourcePolicies = new Map>(); + const verifySourcePolicyOnce = (expected: PrimeSourcePolicyExpectation) => { + const key = `${expected.commit}:${expected.tree}:${expected.workflow}:${expected.publicationPolicyRevision}`; + const current = sourcePolicies.get(key); + if (current) return current; + const started = dependencies.verifySourcePolicy + ? dependencies.verifySourcePolicy(expected) + : verifyRemoteSourcePolicy(expected, dependencies); + sourcePolicies.set(key, started); + return started; }; + for (const release of releases) { + try { + const fixture = await loadPublicationFixtureForRelease(channel, release, dependencies); + verified.push( + await verifyPrimePublicationFixture(fixture, { + verifyBundle: + dependencies.verifyBundle ?? + (async (bundle, expected) => verifyPrimeSigstoreBundle(bundle, trustedRoot, expected)), + verifySourcePolicy: verifySourcePolicyOnce, + }), + ); + } catch (cause) { + if (isRateLimitFailure(cause)) throw cause; + // One malformed, draft-like, or unrelated release must not hide a later exact candidate. + } + } + const latest = verified.toSorted((left, right) => right.sequence - left.sequence)[0]; + if (!latest) throw new Error(`No exact signed ${channel} publication verified.`); + return latest; } -export interface VerifiedPrimePublicationBundle { - readonly publication: VerifiedPrimePublication; - readonly rootArtifactBytes: Buffer; +async function loadCachedPrimePublication( + channel: ServerProviderDistributionChannel, + dependencies: PrimeDistributionNetworkDependencies, + options: PrimeDistributionLoadOptions, +): Promise { + const key = distributionCacheKey(dependencies, channel); + if (!key) return await loadLatestPrimePublicationUncached(channel, dependencies); + const cache = primeDistributionChannelCaches.get(key) ?? {}; + primeDistributionChannelCaches.set(key, cache); + const cached = reusableCachedResult(cache.publication, dependencies, options); + if (cached) return cachedValue(cached); + if (cache.publicationFlight) return await cache.publicationFlight; + const startedAt = cacheNow(dependencies); + const flight = loadLatestPrimePublicationUncached(channel, dependencies) + .then((value) => { + const current = primeDistributionChannelCaches.get(key); + if (current !== cache) return value; + const previousBuildId = + current.publication?.status === "success" ? current.publication.value.buildId : undefined; + current.publication = { + status: "success", + value, + storedAt: startedAt, + expiresAt: startedAt + (dependencies.cache?.successTtlMs ?? DISTRIBUTION_CACHE_TTL_MS), + }; + if (previousBuildId !== undefined && previousBuildId !== value.buildId) delete current.bundle; + primeDistributionChannelCaches.set(key, current); + return value; + }) + .catch((error: unknown) => { + const current = primeDistributionChannelCaches.get(key); + if (current !== cache) throw error; + current.publication = { + status: "failure", + error, + storedAt: startedAt, + expiresAt: startedAt + failureCacheTtl(error, dependencies), + }; + primeDistributionChannelCaches.set(key, current); + throw error; + }) + .finally(() => { + const current = primeDistributionChannelCaches.get(key); + if (current?.publicationFlight === flight) delete current.publicationFlight; + }); + cache.publicationFlight = flight; + return await flight; +} + +export function makeLatestPrimePublicationLoader( + dependencies: PrimeDistributionNetworkDependencies = makePrimeDistributionNetworkDependencies(), +): ( + channel: ServerProviderDistributionChannel, + options?: PrimeDistributionLoadOptions, +) => Promise { + return (channel, options = {}) => loadCachedPrimePublication(channel, dependencies, options); } /** @@ -1702,51 +1937,65 @@ export interface VerifiedPrimePublicationBundle { */ export function makeLatestPrimePublicationBundleLoader( dependencies: PrimeDistributionNetworkDependencies = makePrimeDistributionNetworkDependencies(), -): (channel: ServerProviderDistributionChannel) => Promise { - return async (channel) => { - const raw = await dependencies.fetchJson( - `https://api.github.com/repos/${PRIME_DISTRIBUTION_REPOSITORY}/releases?per_page=100`, - MAX_RELEASE_RESPONSE_BYTES, - ); - const releases = decodeGitHubReleases(raw) - .filter( - (release) => - !release.draft && - release.immutable && - (channel === "preview" - ? /^pylon-build-g[0-9a-f]{12}-r[1-9][0-9]*$/u.test(release.tag_name) - : /^pylon-stable-[0-9]{6}-g[0-9a-f]{12}-r[1-9][0-9]*$/u.test(release.tag_name)), - ) - .slice(0, MAX_FEED_CANDIDATES); - if (releases.length === 0) throw new Error(`No immutable ${channel} publication exists.`); - const trustedRoot = await dependencies.getTrustedRoot(); - const verified: VerifiedPrimePublication[] = []; - for (const release of releases) { - try { - // Authenticate manifests and every attested artifact digest first. Untrusted root bytes are - // not downloaded until the latest exact signed publication has been selected. - const fixture = await loadPublicationFixtureForRelease(channel, release, dependencies); - verified.push( - await verifyPrimePublicationFixture(fixture, { - verifyBundle: async (bundle, expected) => - verifyPrimeSigstoreBundle(bundle, trustedRoot, expected), - verifySourcePolicy: (expected) => verifyRemoteSourcePolicy(expected, dependencies), - }), - ); - } catch { - // One malformed or unrelated release must not hide a later exact candidate. +): ( + channel: ServerProviderDistributionChannel, + options?: PrimeDistributionLoadOptions, +) => Promise { + return async (channel, options = {}) => { + const publication = await loadCachedPrimePublication(channel, dependencies, options); + const key = distributionCacheKey(dependencies, channel); + const loadBundle = async () => { + const rootArtifactBytes = await dependencies.fetchBytes( + `${PRIME_DISTRIBUTION_REPOSITORY_URL}/releases/download/${publication.buildId}/${publication.rootAsset}`, + MAX_ROOT_ARTIFACT_BYTES, + ); + if (sha256(rootArtifactBytes) !== publication.rootSha256) { + throw new Error("Prime root artifact does not match its exact signed digest."); } + return { publication, rootArtifactBytes }; + }; + if (!key) return await loadBundle(); + const cache = primeDistributionChannelCaches.get(key) ?? {}; + primeDistributionChannelCaches.set(key, cache); + const cached = reusableCachedResult(cache.bundle, dependencies, options); + if (cached?.status === "success" && cached.value.publication.buildId === publication.buildId) { + return cached.value; } - const publication = verified.toSorted((left, right) => right.sequence - left.sequence)[0]; - if (!publication) throw new Error(`No exact signed ${channel} publication verified.`); - const rootArtifactBytes = await dependencies.fetchBytes( - `${PRIME_DISTRIBUTION_REPOSITORY_URL}/releases/download/${publication.buildId}/${publication.rootAsset}`, - MAX_ROOT_ARTIFACT_BYTES, - ); - if (sha256(rootArtifactBytes) !== publication.rootSha256) { - throw new Error("Prime root artifact does not match its exact signed digest."); - } - return { publication, rootArtifactBytes }; + if (cached?.status === "failure") throw cached.error; + if (cache.bundleFlight) return await cache.bundleFlight; + const startedAt = cacheNow(dependencies); + const flight = loadBundle() + .then((value) => { + const current = primeDistributionChannelCaches.get(key); + if (current !== cache) return value; + current.bundle = { + status: "success", + value, + storedAt: startedAt, + expiresAt: + startedAt + (dependencies.cache?.bundleTtlMs ?? DISTRIBUTION_BUNDLE_CACHE_TTL_MS), + }; + primeDistributionChannelCaches.set(key, current); + return value; + }) + .catch((error: unknown) => { + const current = primeDistributionChannelCaches.get(key); + if (current !== cache) throw error; + current.bundle = { + status: "failure", + error, + storedAt: startedAt, + expiresAt: startedAt + failureCacheTtl(error, dependencies), + }; + primeDistributionChannelCaches.set(key, current); + throw error; + }) + .finally(() => { + const current = primeDistributionChannelCaches.get(key); + if (current?.bundleFlight === flight) delete current.bundleFlight; + }); + cache.bundleFlight = flight; + return await flight; }; } diff --git a/apps/server/src/provider/prime/PrimeAgentManagedToolStore.test.ts b/apps/server/src/provider/prime/PrimeAgentManagedToolStore.test.ts index 8de8c4b3d..89bb06cf4 100644 --- a/apps/server/src/provider/prime/PrimeAgentManagedToolStore.test.ts +++ b/apps/server/src/provider/prime/PrimeAgentManagedToolStore.test.ts @@ -355,6 +355,10 @@ async function makeHarness( let binding: PrimeManagedBinding = { binaryPath: stock, generation: "binding-0" }; let busy = input.busy ?? false; let generation = 0; + let ownedRuntimeBuildReferences: ReadonlyArray<{ + readonly instanceId: string; + readonly buildId: string; + }> = []; let crashAfterCommitOnce = input.crashAfterCommitOnce ?? false; const reservations = new Set(); const dependencies: PrimeManagedToolStoreDependencies = { @@ -363,6 +367,8 @@ async function makeHarness( return currentBundle; }, readBinding: async () => binding, + listBindings: async () => [{ instanceId: "primeAgent", binding }], + listOwnedRuntimeBuildReferences: async () => ownedRuntimeBuildReferences, reserveQuiescentBinding: async (_instanceId, expected) => { if ( expected.generation !== binding.generation || @@ -436,6 +442,15 @@ async function makeHarness( generation += 1; binding = { binaryPath, generation: `binding-${generation}` }; }, + bumpBindingGeneration() { + generation += 1; + binding = { ...binding, generation: `binding-${generation}` }; + }, + setOwnedRuntimeBuildReferences( + references: ReadonlyArray<{ readonly instanceId: string; readonly buildId: string }>, + ) { + ownedRuntimeBuildReferences = references; + }, }; } @@ -611,6 +626,132 @@ describe("Pylon-managed Prime tool store", () => { expect(await NodeFSP.readFile(harness.stock)).toEqual(stockBefore); }); + it("preserves a managed selection across an unrelated display/home generation edit, then restores stock and cleans up", async () => { + const harness = await makeHarness(); + const stockBefore = await NodeFSP.readFile(harness.stock); + const installed = await harness.store.command({ + commandId: "install-before-unrelated-settings-edit", + instanceId: "primeAgent", + action: "install", + }); + expect(installed.status).toBe("succeeded"); + harness.bumpBindingGeneration(); + + const useStock = await harness.store.command({ + commandId: "stock-after-unrelated-settings-edit", + instanceId: "primeAgent", + action: "use-stock", + }); + expect(useStock.status).toBe("succeeded"); + expect(harness.binding.binaryPath).toBe(harness.stock); + + const cleanup = await harness.store.command({ + commandId: "cleanup-after-unrelated-settings-edit", + instanceId: "primeAgent", + action: "cleanup", + }); + expect(cleanup.status).toBe("succeeded"); + expect((await harness.store.status("primeAgent")).availableBuilds).toEqual([]); + expect(await NodeFSP.readFile(harness.stock)).toEqual(stockBefore); + }); + + it("takes the provider maintenance fence and refuses cleanup while a runtime is active", async () => { + const harness = await makeHarness(); + await harness.store.command({ + commandId: "install-before-active-cleanup", + instanceId: "primeAgent", + action: "install", + }); + await harness.store.command({ + commandId: "stock-before-active-cleanup", + instanceId: "primeAgent", + action: "use-stock", + }); + harness.setBusy(true); + const cleanup = await harness.store.command({ + commandId: "cleanup-while-runtime-active", + instanceId: "primeAgent", + action: "cleanup", + }); + expect(cleanup).toMatchObject({ + status: "failed", + message: expect.stringMatching(/blocked.*active provider session/u), + }); + expect((await harness.store.status("primeAgent")).availableBuilds).toHaveLength(1); + }); + + it("keeps unselected bytes referenced by a loaded owned runtime context until it unloads", async () => { + const harness = await makeHarness(); + const installed = await harness.store.command({ + commandId: "install-before-owned-context", + instanceId: "primeAgent", + action: "install", + }); + await harness.store.command({ + commandId: "stock-before-owned-context", + instanceId: "primeAgent", + action: "use-stock", + }); + harness.setOwnedRuntimeBuildReferences([ + { instanceId: "primeAgent", buildId: installed.buildId! }, + ]); + await harness.store.command({ + commandId: "cleanup-with-owned-context", + instanceId: "primeAgent", + action: "cleanup", + }); + expect((await harness.store.status("primeAgent")).availableBuilds).toHaveLength(1); + + harness.setOwnedRuntimeBuildReferences([]); + await harness.store.command({ + commandId: "cleanup-after-owned-context", + instanceId: "primeAgent", + action: "cleanup", + }); + expect((await harness.store.status("primeAgent")).availableBuilds).toEqual([]); + }); + + it("never deletes a build selected by an authoritative binding even when stored selection mode is corrupt", async () => { + const harness = await makeHarness(); + const installed = await harness.store.command({ + commandId: "install-before-corrupt-mode", + instanceId: "primeAgent", + action: "install", + }); + const statePath = NodePath.join( + harness.stateDir, + ...PRIME_MANAGED_TOOL_DIRECTORY.split("/"), + "managed-tool-state-v1.json", + ); + const state = JSON.parse(await NodeFSP.readFile(statePath, "utf8")) as { + selections: Record>; + }; + state.selections.primeAgent = { + ...state.selections.primeAgent, + mode: "stock", + selectedBuildId: null, + channel: null, + }; + await NodeFSP.writeFile(statePath, `${JSON.stringify(state, null, 2)}\n`); + + const cleanup = await harness.store.command({ + commandId: "cleanup-with-corrupt-mode", + instanceId: "primeAgent", + action: "cleanup", + }); + expect(cleanup.status).toBe("succeeded"); + expect((await harness.store.status("primeAgent")).availableBuilds).toEqual([ + expect.objectContaining({ buildId: installed.buildId }), + ]); + const useStock = await harness.store.command({ + commandId: "false-stock-with-corrupt-mode", + instanceId: "primeAgent", + action: "use-stock", + }); + expect(useStock).toMatchObject({ status: "failed" }); + expect(harness.binding.binaryPath).toContain(installed.buildId!); + }); + it("treats an external provider-path edit as the new configured stock binding", async () => { const harness = await makeHarness(); const installed = await harness.store.command({ @@ -620,6 +761,7 @@ describe("Pylon-managed Prime tool store", () => { }); const custom = NodePath.join(harness.stateDir, "custom-prime-agent"); await NodeFSP.writeFile(custom, "custom-stock-bytes\n", { mode: 0o755 }); + const customBefore = await NodeFSP.readFile(custom); harness.setExternalBinding(custom); await expect( @@ -645,6 +787,12 @@ describe("Pylon-managed Prime tool store", () => { action: "use-stock", }); expect(harness.binding.binaryPath).toBe(custom); + await harness.store.command({ + commandId: "cleanup-after-external-stock", + instanceId: "primeAgent", + action: "cleanup", + }); + expect(await NodeFSP.readFile(custom)).toEqual(customBefore); }); it("schedules an exact binding while busy and commits after the instance drains", async () => { @@ -716,6 +864,8 @@ describe("Pylon-managed Prime tool store", () => { throw new Error("not used"); }, readBinding: async () => binding, + listBindings: async () => [{ instanceId: "primeAgent", binding }], + listOwnedRuntimeBuildReferences: async () => [], reserveQuiescentBinding: async () => ({ status: "busy", reasons: [] }), commitBinding: async () => binding, releaseReservation: async () => {}, @@ -966,6 +1116,14 @@ describe("Pylon-managed Prime tool store", () => { io += 1; return { binaryPath: "prime-agent", generation: "0" }; }, + listBindings: async () => { + io += 1; + return []; + }, + listOwnedRuntimeBuildReferences: async () => { + io += 1; + return []; + }, reserveQuiescentBinding: async () => { io += 1; return { status: "busy", reasons: [] }; diff --git a/apps/server/src/provider/prime/PrimeAgentManagedToolStore.ts b/apps/server/src/provider/prime/PrimeAgentManagedToolStore.ts index 8cdbfecee..bdb75a28a 100644 --- a/apps/server/src/provider/prime/PrimeAgentManagedToolStore.ts +++ b/apps/server/src/provider/prime/PrimeAgentManagedToolStore.ts @@ -50,6 +50,16 @@ export interface PrimeManagedBinding { readonly generation: string; } +export interface PrimeManagedBindingEntry { + readonly instanceId: string; + readonly binding: PrimeManagedBinding; +} + +export interface PrimeManagedRuntimeBuildReference { + readonly instanceId: string; + readonly buildId: string; +} + export interface PrimeManagedReservation { readonly token: string; } @@ -66,8 +76,15 @@ export interface PrimeManagedPublicationBundle { export interface PrimeManagedToolStoreDependencies { readonly loadLatestVerifiedPublication: ( channel: ServerProviderDistributionChannel, + options?: { readonly refresh?: boolean }, ) => Promise; readonly readBinding: (instanceId: string) => Promise; + /** Every authoritative configured Prime binding, including disabled and legacy instances. */ + readonly listBindings: () => Promise>; + /** Build ids held by loaded provider adapters/SDK/daemon ownership contexts. */ + readonly listOwnedRuntimeBuildReferences: () => Promise< + ReadonlyArray + >; /** * The reservation must atomically fence new admissions/session starts for the instance, then * prove that it has no active or pending admission, turn, session, owned process, or SDK context. @@ -951,7 +968,9 @@ export class PrimeAgentManagedToolStore { channel, message: `Downloading the exact signed ${channel} Prime publication.`, }); - const bundle = await this.#dependencies.loadLatestVerifiedPublication(channel); + const bundle = await this.#dependencies.loadLatestVerifiedPublication(channel, { + refresh: true, + }); receipt = await this.#updateOperation(await this.#readState(), receipt, { status: "verifying", channel, @@ -987,12 +1006,22 @@ export class PrimeAgentManagedToolStore { } else { const selection = state.selections[input.instanceId]; if (!selection || selection.mode === "stock") { + if (await this.#managedBuildIdForBinaryPath(expected.binaryPath)) { + throw new Error( + "Prime still selects a receipt-owned managed launcher, but its original stock binding is unavailable. Refusing to report a stock switch.", + ); + } return await this.#finishOperation(state, receipt, { status: "succeeded", message: "Prime already uses its stock or configured binary.", }); } targetBinaryPath = selection.stockBinaryPath; + if (await this.#managedBuildIdForBinaryPath(targetBinaryPath)) { + throw new Error( + "The recorded original Prime binding points into the managed tool store; refusing a false stock switch.", + ); + } } return await this.#trySelection({ state: await this.#readState(), @@ -1093,6 +1122,15 @@ export class PrimeAgentManagedToolStore { binaryPath: input.targetBinaryPath, reservation: reservation.reservation, }); + if (committed.binaryPath !== input.targetBinaryPath) { + throw new Error("Prime settings CAS did not select the exact requested binary binding."); + } + if ( + input.buildId === null && + (await this.#managedBuildIdForBinaryPath(committed.binaryPath)) + ) { + throw new Error("Prime still selects managed bytes; the stock switch did not complete."); + } const next = await this.#commitSelectionState(await this.#readState(), intent, committed); return await this.#finishOperation(next, receipt, { status: "succeeded", @@ -1119,6 +1157,23 @@ export class PrimeAgentManagedToolStore { ) { return state; } + if ( + selected.mode === "managed" && + selected.selectedBuildId !== null && + current.binaryPath === selected.binding.binaryPath && + (await this.#managedBuildIdForBinaryPath(current.binaryPath)) === selected.selectedBuildId + ) { + const next: StoredState = { + ...state, + revision: state.revision + 1, + selections: { + ...state.selections, + [instanceId]: { ...selected, binding: current }, + }, + }; + await this.#writeState(next); + return next; + } const next: StoredState = { ...state, revision: state.revision + 1, @@ -1492,34 +1547,113 @@ export class PrimeAgentManagedToolStore { return next; } - async #cleanup(state: StoredState): Promise> { - const referenced = new Set(); - for (const selection of Object.values(state.selections)) { - if (selection.mode === "managed" && selection.selectedBuildId) - referenced.add(selection.selectedBuildId); + async #managedBuildIdForBinaryPath( + binaryPath: string, + builds?: ReadonlyArray, + ): Promise { + const available = builds ?? (await this.#listVerifiedBuilds()); + const absolute = NodePath.resolve(binaryPath); + let canonical: string | undefined; + try { + canonical = await NodeFSP.realpath(absolute); + } catch { + // A missing external path is not cleanup authority. Exact managed launcher strings still are. } - for (const scheduled of Object.values(state.scheduled)) { - if (scheduled.buildId) referenced.add(scheduled.buildId); + for (const build of available) { + if (absolute === NodePath.resolve(build.binaryPath)) return build.buildId; + if (canonical && canonical === (await NodeFSP.realpath(build.binaryPath))) + return build.buildId; } - const removed: string[] = []; - const entries = await NodeFSP.readdir(this.#root, { withFileTypes: true }); - for (const entry of entries) { - if (!BUILD_ID.test(entry.name) || referenced.has(entry.name)) continue; - const path = NodePath.join(this.#root, entry.name); - const info = await NodeFSP.lstat(path); - if (!info.isDirectory() || info.isSymbolicLink() || (await NodeFSP.realpath(path)) !== path) { - continue; + return undefined; + } + + async #cleanup(state: StoredState): Promise> { + const bindings = await this.#dependencies.listBindings(); + const duplicateInstance = bindings.find( + (entry, index) => + bindings.findIndex((candidate) => candidate.instanceId === entry.instanceId) !== index, + ); + if (duplicateInstance) { + throw new Error(`Prime cleanup received duplicate binding ${duplicateInstance.instanceId}.`); + } + const reservations: PrimeManagedReservation[] = []; + try { + for (const entry of bindings) { + const reserved = await this.#dependencies.reserveQuiescentBinding( + entry.instanceId, + entry.binding, + ); + if (reserved.status === "busy") { + throw new Error( + `Prime cleanup is blocked by ${entry.instanceId}: ${reserved.reasons.join("; ")}`, + ); + } + reservations.push(reserved.reservation); } - try { - await this.#readVerifiedBuild(entry.name); - } catch { - continue; + + const fencedBindings = await this.#dependencies.listBindings(); + const expected = bindings + .map( + (entry) => + [entry.instanceId, entry.binding.binaryPath, entry.binding.generation] as const, + ) + .toSorted(([left], [right]) => left.localeCompare(right)); + const observed = fencedBindings + .map( + (entry) => + [entry.instanceId, entry.binding.binaryPath, entry.binding.generation] as const, + ) + .toSorted(([left], [right]) => left.localeCompare(right)); + if (JSON.stringify(expected) !== JSON.stringify(observed)) { + throw new Error("Prime settings bindings changed while cleanup acquired its fences."); + } + + const builds = await this.#listVerifiedBuilds(); + const referenced = new Set(); + for (const selection of Object.values(state.selections)) { + if (selection.selectedBuildId) referenced.add(selection.selectedBuildId); + } + for (const scheduled of Object.values(state.scheduled)) { + if (scheduled.buildId) referenced.add(scheduled.buildId); + } + for (const entry of fencedBindings) { + const buildId = await this.#managedBuildIdForBinaryPath(entry.binding.binaryPath, builds); + if (buildId) referenced.add(buildId); + } + const reservedInstances = new Set(fencedBindings.map((entry) => entry.instanceId)); + for (const runtime of await this.#dependencies.listOwnedRuntimeBuildReferences()) { + if (!reservedInstances.has(runtime.instanceId)) { + throw new Error( + `Prime cleanup is blocked by an owned runtime context for unconfigured instance ${runtime.instanceId}.`, + ); + } + if (BUILD_ID.test(runtime.buildId)) referenced.add(runtime.buildId); + } + + const removed: string[] = []; + for (const build of builds) { + if (referenced.has(build.buildId)) continue; + const path = NodePath.join(this.#root, build.buildId); + const info = await NodeFSP.lstat(path); + if ( + !info.isDirectory() || + info.isSymbolicLink() || + (await NodeFSP.realpath(path)) !== path + ) { + continue; + } + // Re-verify immediately before deletion. Only exact receipt-owned bytes are cleanup targets. + await this.#readVerifiedBuild(build.buildId); + await NodeFSP.rm(path, { recursive: true, force: false }); + removed.push(build.buildId); + } + if (removed.length) await syncDirectory(this.#root); + return removed.toSorted(); + } finally { + for (const reservation of reservations.toReversed()) { + await this.#dependencies.releaseReservation(reservation); } - await NodeFSP.rm(path, { recursive: true, force: false }); - removed.push(entry.name); } - if (removed.length) await syncDirectory(this.#root); - return removed.toSorted(); } async #recoverTemporaryEntries(): Promise { diff --git a/apps/server/src/provider/prime/PrimeManagedMaintenance.ts b/apps/server/src/provider/prime/PrimeManagedMaintenance.ts index 45b3034f7..05926c2ba 100644 --- a/apps/server/src/provider/prime/PrimeManagedMaintenance.ts +++ b/apps/server/src/provider/prime/PrimeManagedMaintenance.ts @@ -71,6 +71,7 @@ export const make = Effect.fn("PrimeManagedMaintenance.make")(function* () { if (platform !== "win32") { if ( !settings.readPrimeAgentBinaryBinding || + !settings.listPrimeAgentBinaryBindings || !settings.compareAndSetPrimeAgentBinaryPath || !providerService.reserveProviderMaintenance || !providerService.releaseProviderMaintenance @@ -103,6 +104,7 @@ export const make = Effect.fn("PrimeManagedMaintenance.make")(function* () { } const readPrimeAgentBinaryBinding = settings.readPrimeAgentBinaryBinding!; + const listPrimeAgentBinaryBindings = settings.listPrimeAgentBinaryBindings!; const compareAndSetPrimeAgentBinaryPath = settings.compareAndSetPrimeAgentBinaryPath!; const reserveProviderMaintenance = providerService.reserveProviderMaintenance!; const releaseProviderMaintenance = providerService.releaseProviderMaintenance!; @@ -124,6 +126,17 @@ export const make = Effect.fn("PrimeManagedMaintenance.make")(function* () { if (!binding) throw new Error("The target is not a configured Prime Agent instance."); return binding; }, + listBindings: () => runPromise(listPrimeAgentBinaryBindings.pipe(Effect.orDie)), + listOwnedRuntimeBuildReferences: async () => { + const providers = await runPromise(providerRegistry.getProviders); + return providers.flatMap((provider) => + provider.driver === "primeAgent" && + provider.distribution?.classification === "pylon-managed" && + provider.distribution.buildId !== null + ? [{ instanceId: provider.instanceId, buildId: provider.distribution.buildId }] + : [], + ); + }, reserveQuiescentBinding: async (instanceId, expected) => { const current = await runPromise( readPrimeAgentBinaryBinding(instanceId).pipe(Effect.orDie), diff --git a/apps/server/src/serverSettings.test.ts b/apps/server/src/serverSettings.test.ts index fcddb2b7c..2d89cb988 100644 --- a/apps/server/src/serverSettings.test.ts +++ b/apps/server/src/serverSettings.test.ts @@ -1081,9 +1081,14 @@ it.layer(NodeServices.layer)("server settings", (it) => { }, }); const readBinding = serverSettings.readPrimeAgentBinaryBinding!; + const listBindings = serverSettings.listPrimeAgentBinaryBindings!; const compareAndSet = serverSettings.compareAndSetPrimeAgentBinaryPath!; const expected = yield* readBinding(instanceId); assert.isDefined(expected); + assert.deepEqual( + (yield* listBindings).find((entry) => entry.instanceId === instanceId), + { instanceId, binding: expected }, + ); const committed = yield* compareAndSet({ instanceId, expected, diff --git a/apps/server/src/serverSettings.ts b/apps/server/src/serverSettings.ts index 3ef7afc7f..f8ea230ec 100644 --- a/apps/server/src/serverSettings.ts +++ b/apps/server/src/serverSettings.ts @@ -183,6 +183,11 @@ export interface PrimeAgentBinaryBinding { readonly generation: string; } +export interface PrimeAgentBinaryBindingEntry { + readonly instanceId: string; + readonly binding: PrimeAgentBinaryBinding; +} + function primeAgentBinaryBinding( settings: ServerSettings, instanceId: string, @@ -209,6 +214,22 @@ function primeAgentBinaryBinding( }; } +function primeAgentBinaryBindings( + settings: ServerSettings, +): ReadonlyArray { + const entries: PrimeAgentBinaryBindingEntry[] = []; + for (const [instanceId, instance] of Object.entries(settings.providerInstances)) { + if (instance.driver !== "primeAgent") continue; + const binding = primeAgentBinaryBinding(settings, instanceId); + if (binding) entries.push({ instanceId, binding }); + } + if (!("primeAgent" in settings.providerInstances)) { + const binding = primeAgentBinaryBinding(settings, "primeAgent"); + if (binding) entries.push({ instanceId: "primeAgent", binding }); + } + return entries.toSorted((left, right) => left.instanceId.localeCompare(right.instanceId)); +} + function patchPrimeAgentBinaryPath( settings: ServerSettings, instanceId: string, @@ -255,6 +276,12 @@ export class ServerSettingsService extends Context.Service< instanceId: string, ) => Effect.Effect; + /** Read every authoritative Prime binding from one settings snapshot. */ + readonly listPrimeAgentBinaryBindings?: Effect.Effect< + ReadonlyArray, + ServerSettingsError + >; + /** Atomically compare-and-set only this Prime instance's binary path. */ readonly compareAndSetPrimeAgentBinaryPath?: (input: { readonly instanceId: string; @@ -861,6 +888,7 @@ const make = Effect.gen(function* () { getSettingsFromCache.pipe( Effect.map((settings) => primeAgentBinaryBinding(settings, instanceId)), ), + listPrimeAgentBinaryBindings: getSettingsFromCache.pipe(Effect.map(primeAgentBinaryBindings)), compareAndSetPrimeAgentBinaryPath: (input) => writeSemaphore.withPermits(1)( Effect.gen(function* () { diff --git a/docs/internals/prime-agent-distribution-verification.md b/docs/internals/prime-agent-distribution-verification.md index d9a075d92..3a41aa4a3 100644 --- a/docs/internals/prime-agent-distribution-verification.md +++ b/docs/internals/prime-agent-distribution-verification.md @@ -34,10 +34,16 @@ SLSA bindings: - one stable-manifest subject for stable promotion. The fetch and trusted-root functions are injected. Production keeps Sigstore TUF cache data below the -Pylon runtime state directory rather than writing to an unrelated user cache. Focused tests use -deterministic manifests and a cryptographic-verifier seam, then exercise certificate and SLSA binding separately. Bridge CI can -supply the first immutable artifact set through the fail-closed real-fixture gate. The gate has no -skip or metadata-only success mode. +Pylon runtime state directory rather than writing to an unrelated user cache. Verified channel, +publication, attestation, and installer-bundle results use one process-wide repository/channel TTL +and single flight shared by provider status and host maintenance. Short failure and rate-limit TTLs +bound retry traffic; explicit maintenance refreshes still reuse a just-fresh status result. One +multi-subject preview attestation is fetched and verified once for its exact subject set rather than +once per subject. Candidate and request counts remain bounded. + +Focused tests use deterministic manifests and a cryptographic-verifier seam, then exercise certificate +and SLSA binding separately. Bridge CI can supply the first immutable artifact set through the +fail-closed real-fixture gate. The gate has no skip or metadata-only success mode. ## Private managed state diff --git a/docs/internals/prime-agent-managed-install.md b/docs/internals/prime-agent-managed-install.md index a81f24cf3..a3470c665 100644 --- a/docs/internals/prime-agent-managed-install.md +++ b/docs/internals/prime-agent-managed-install.md @@ -52,9 +52,12 @@ Selection changes only the target Prime provider instance's complete settings bi The intent closes the crash window around settings CAS. Recovery reads the observed binding. If it is the exact target, recovery records the completed selection. If it is still the complete expected -binding, recovery records an interrupted pre-switch failure. A different binding is treated as a -superseding user/settings change. A later explicit command supersedes an older scheduled command and -marks its receipt terminal rather than leaving two apparent pending switches. +binding, recovery records an interrupted pre-switch failure. A different binary path is treated as a +superseding user/settings change. A generation-only change caused by another field preserves the +managed mode, build id, channel, and original stock path when the binary still names the exact +receipt-owned launcher. `Use stock` then compare-and-sets that original path and never reports success +while a managed launcher remains selected. A later explicit command supersedes an older scheduled +command and marks its receipt terminal rather than leaving two apparent pending switches. Distinct package roots and quiescent switching prevent a daemon from one build from sharing an imported SDK module cache from another. Runtime capability still comes only from frozen SDK metadata @@ -80,6 +83,10 @@ and a different build at the same sequence fail. Rollback is allowed only as an an already installed receipt-owned build; it does not lower the channel high-water. An offline feed cannot change the selected build and is reported as a failure. -Cleanup computes references from managed selections and scheduled switches. It ignores unrecognized, -linked, incomplete, or invalid-receipt directories and removes only an unreferenced build that passes -full offline marker and receipt validation. +Cleanup fences every authoritative configured Prime binding with the same maintenance reservation +used for switching and refuses the operation while a configured instance is active. Under those +fences it derives references from every settings binding, managed selection, scheduled switch, and +loaded or owned runtime build context. Binding paths protect exact or canonical managed launchers even +when stored selection mode is corrupt. Cleanup ignores unrecognized, linked, incomplete, or +invalid-receipt directories and removes only an unreferenced build that passes full offline marker and +receipt validation. External Prime installations are never cleanup targets.