From 1868c053a11c5d4fc8e2ee0e84013d551a19254c Mon Sep 17 00:00:00 2001 From: Naveen Kurra Date: Mon, 5 Oct 2026 12:33:33 -0400 Subject: [PATCH 01/19] Add DigitalOcean Managed Agents as a sandbox backend Adds a tenth sandbox backend that runs each qm computer as a Managed Agents microVM. Sessions are created with agent: none, which asks for a bare sandbox with no managed agent loop, no event-translation runtime and no model credential of its own, so qm keeps its own agent loop and its own approval gates. The backend talks to two planes. Session lifecycle is bearer-authenticated REST under /v2/agents/sessions, with a DigitalOcean IAM token that carries the team identity owning the sessions. Exec and file transfer take a different route: a WebSocket port-forward to guest port 8443, bridged to a local TCP listener, carrying gRPC to the sandbox-agent that every microVM runs. That path has no request deadline, so a command is bounded by qm's SANDBOX_TIMEOUT_SEC rather than by a control-plane buffer. Transfer inside the guest is confined to /workspace, so a scope's home lives at /workspace/home. Sessions are created allowing bash outright, because the provider would otherwise hold every command at its own approval gate behind qm's. Select it with SANDBOX_BACKEND=do-managed-agents and DO_AGENTS_API_TOKEN. Set DO_AGENTS_SNAPSHOT_S3_BUCKET so a scope's home survives losing its session: the provider exposes session checkpoints, but qm does not capture them yet, so a portable tar is the only recovery path. The proto is vendored rather than consumed from a published SDK. That is a stopgap: the REST exec endpoint buffers in the control plane, so it clamps to four minutes and 1 MiB and carries no per-command environment, which rules it out for a general sandbox. When streaming exec reaches the public edge, the vendored proto, the tunnel and the two gRPC dependencies all go away. --- cli/src/secrets.ts | 8 + package-lock.json | 4 + package.json | 6 + scripts/dev/cli.ts | 2 +- scripts/dev/lib/sandbox.ts | 38 + scripts/dev/lib/types.ts | 3 +- scripts/managed-agents-sandbox-smoke.ts | 122 ++++ scripts/managed-agents-turn-smoke.ts | 52 ++ src/config.ts | 58 +- src/deployment/secret-schema.ts | 3 + src/sandbox/managed-agents-client.ts | 671 ++++++++++++++++++ .../managed-agents-sandbox-agent.proto | 102 +++ src/sandbox/managed-agents-sandbox.ts | 533 ++++++++++++++ src/sandbox/managed-agents-tunnel.ts | 151 ++++ src/sandbox/sandbox-routing.ts | 11 +- src/wiring.ts | 46 +- test/live-slack-sandbox-providers.test.ts | 3 +- .../live-slack/scenarios-sandbox-providers.ts | 1 + 18 files changed, 1804 insertions(+), 10 deletions(-) create mode 100644 scripts/managed-agents-sandbox-smoke.ts create mode 100644 scripts/managed-agents-turn-smoke.ts create mode 100644 src/sandbox/managed-agents-client.ts create mode 100644 src/sandbox/managed-agents-sandbox-agent.proto create mode 100644 src/sandbox/managed-agents-sandbox.ts create mode 100644 src/sandbox/managed-agents-tunnel.ts diff --git a/cli/src/secrets.ts b/cli/src/secrets.ts index c02dc5c7de..6560efe630 100644 --- a/cli/src/secrets.ts +++ b/cli/src/secrets.ts @@ -170,6 +170,14 @@ export const FIRST_PARTY_SECRET_SPECS: readonly SecretSpec[] = [ required: { when: { kind: "sandbox-backend", backend: "e2b" } }, description: "E2B API key used by the e2b sandbox backend (from e2b.dev dashboard).", }, + { + name: "DO_AGENTS_API_TOKEN", + service: "core", + required: { when: { kind: "sandbox-backend", backend: "do-managed-agents" } }, + description: + "DigitalOcean IAM token for the DigitalOcean Managed Agents sandbox backend. Carries the team identity that owns the sandbox sessions, and authenticates both the session REST calls and the port-forward tunnel that carries exec and file transfer.", + generate: "doctl auth init # then create a personal access token at cloud.digitalocean.com/account/api/tokens", + }, { name: "MODAL_TOKEN_ID", service: "core", diff --git a/package-lock.json b/package-lock.json index 6683c716fd..5a60ebb9db 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,6 +21,8 @@ "@earendil-works/pi-ai": "0.82.0", "@earendil-works/pi-coding-agent": "https://github.com/yc-software/qm/releases/download/vendored-pi-coding-agent-0.82.0-security.5/earendil-works-pi-coding-agent-0.82.0-qm-security.5.tgz", "@fly/sprites": "0.2.3", + "@grpc/grpc-js": "^1.14.5", + "@grpc/proto-loader": "^0.8.1", "@openai/codex": "0.156.1", "@opencode-ai/plugin": "1.18.31", "@opencode-ai/sdk": "1.18.31", @@ -45,6 +47,7 @@ "socks-proxy-agent": "10.1.0", "tar-stream": "^3.2.0", "typebox": "^1.1.38", + "ws": "^8.21.1", "zod": "4.4.3" }, "devDependencies": { @@ -53,6 +56,7 @@ "@types/node": "^24.0.0", "@types/pg": "^8.11.10", "@types/tar-stream": "^3.1.4", + "@types/ws": "^8.18.1", "@yc-software/qm": "file:./cli", "esbuild": "0.28.1", "eslint": "^10.4.1", diff --git a/package.json b/package.json index 61e22feb32..04f900dca9 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,8 @@ "smoke:service-cred": "node scripts/service-credential-livetest.ts", "smoke:aws-sandbox": "node scripts/aws-sandbox-smoke.ts", "smoke:local-sandbox": "node scripts/local-sandbox-smoke.ts", + "smoke:managed-agents-sandbox": "node --env-file-if-exists=.env scripts/managed-agents-sandbox-smoke.ts", + "smoke:managed-agents-turn": "node --env-file-if-exists=.env scripts/managed-agents-turn-smoke.ts", "build:modal-image": "node scripts/modal-build-image.ts", "build:e2b-template": "node scripts/e2b-build-template.ts", "build:aws-image": "node scripts/aws-build-sandbox-image.ts", @@ -77,6 +79,8 @@ "@earendil-works/pi-ai": "0.82.0", "@earendil-works/pi-coding-agent": "https://github.com/yc-software/qm/releases/download/vendored-pi-coding-agent-0.82.0-security.5/earendil-works-pi-coding-agent-0.82.0-qm-security.5.tgz", "@fly/sprites": "0.2.3", + "@grpc/grpc-js": "^1.14.5", + "@grpc/proto-loader": "^0.8.1", "@openai/codex": "0.156.1", "@opencode-ai/plugin": "1.18.31", "@opencode-ai/sdk": "1.18.31", @@ -101,6 +105,7 @@ "socks-proxy-agent": "10.1.0", "tar-stream": "^3.2.0", "typebox": "^1.1.38", + "ws": "^8.21.1", "zod": "4.4.3" }, "overrides": { @@ -126,6 +131,7 @@ "@types/node": "^24.0.0", "@types/pg": "^8.11.10", "@types/tar-stream": "^3.1.4", + "@types/ws": "^8.18.1", "@yc-software/qm": "file:./cli", "devbar.sh": "1.1.0", "esbuild": "0.28.1", diff --git a/scripts/dev/cli.ts b/scripts/dev/cli.ts index a0436ab0fd..f5614ade4d 100644 --- a/scripts/dev/cli.ts +++ b/scripts/dev/cli.ts @@ -671,7 +671,7 @@ async function main(): Promise { }); default: console.error( - "usage: dev [up|down|status|restart|canary|logs|doctor] [--json] [--force] [--rotate] [--strict] [--sandbox local|sprites|smolmachines|e2b|porter|agent37|superserve|auto] [--surface web|slack|both] [--no-slack] [--no-watch] [--org id] [--fix]", + "usage: dev [up|down|status|restart|canary|logs|doctor] [--json] [--force] [--rotate] [--strict] [--sandbox local|sprites|smolmachines|e2b|porter|agent37|superserve|do-managed-agents|auto] [--surface web|slack|both] [--no-slack] [--no-watch] [--org id] [--fix]", ); return EXIT.usage; } diff --git a/scripts/dev/lib/sandbox.ts b/scripts/dev/lib/sandbox.ts index 9d48432748..3dc0b35122 100644 --- a/scripts/dev/lib/sandbox.ts +++ b/scripts/dev/lib/sandbox.ts @@ -190,6 +190,44 @@ export async function resolveSandbox(opts: { return { backend: "superserve", env, detail: `Superserve (template ${template})`, publicApiUrl: apiUrl, warnings }; } + if (backend === "do-managed-agents") { + const managedAgentsToken = opts.baseEnv.DO_AGENTS_API_TOKEN?.trim(); + if (!managedAgentsToken) + throw new Error( + "--sandbox do-managed-agents requires DO_AGENTS_API_TOKEN in the environment (a DigitalOcean API token for a team with Managed Agents access)", + ); + let managedAgentsApiUrl = opts.baseEnv.PUBLIC_API_URL || null; + if (!managedAgentsApiUrl) { + managedAgentsApiUrl = await startQuickTunnel(opts.corePort, opts.lock, opts.log); + if (!managedAgentsApiUrl) + warnings.push( + "cloudflared tunnel didn't come up -- agent self-API (crons/sends) won't be reachable from the sandbox", + ); + } + const env: Record = { + SANDBOX_BACKEND: "do-managed-agents", + DO_AGENTS_API_TOKEN: managedAgentsToken, + DO_AGENTS_NAME_PREFIX: opts.baseEnv.DO_AGENTS_NAME_PREFIX || "qmdev", + }; + if (opts.baseEnv.DO_AGENTS_API_BASE_URL) env.DO_AGENTS_API_BASE_URL = opts.baseEnv.DO_AGENTS_API_BASE_URL; + if (opts.baseEnv.DO_AGENTS_TEMPLATE) env.DO_AGENTS_TEMPLATE = opts.baseEnv.DO_AGENTS_TEMPLATE; + if (opts.baseEnv.DO_AGENTS_SIZE_SLUG) env.DO_AGENTS_SIZE_SLUG = opts.baseEnv.DO_AGENTS_SIZE_SLUG; + if (opts.baseEnv.DO_AGENTS_EGRESS_PROXY_URL) + env.DO_AGENTS_EGRESS_PROXY_URL = opts.baseEnv.DO_AGENTS_EGRESS_PROXY_URL; + else + warnings.push( + "DO_AGENTS_EGRESS_PROXY_URL unset -- do-managed-agents sandbox runs with NO egress enforcement; set it to QA the forced-proxy path", + ); + if (managedAgentsApiUrl) env.PUBLIC_API_URL = managedAgentsApiUrl; + return { + backend: "do-managed-agents", + env, + detail: `Managed Agents (${opts.baseEnv.DO_AGENTS_API_BASE_URL || "api.digitalocean.com"})`, + publicApiUrl: managedAgentsApiUrl, + warnings, + }; + } + if (backend === "porter") { const porterToken = opts.baseEnv.PORTER_DEPLOY_API_TOKEN; const projectId = opts.baseEnv.PORTER_DEPLOY_PROJECT_ID; diff --git a/scripts/dev/lib/types.ts b/scripts/dev/lib/types.ts index f1491f7832..f96294a41d 100644 --- a/scripts/dev/lib/types.ts +++ b/scripts/dev/lib/types.ts @@ -104,7 +104,8 @@ export interface StatusReport { children: Record; } -export type DevSandboxBackend = "local" | "sprites" | "smolmachines" | "e2b" | "porter" | "agent37" | "superserve"; +export type DevSandboxBackend = + "local" | "sprites" | "smolmachines" | "e2b" | "porter" | "agent37" | "superserve" | "do-managed-agents"; export type DevSandboxChoice = DevSandboxBackend | "auto"; export interface BootSpec { diff --git a/scripts/managed-agents-sandbox-smoke.ts b/scripts/managed-agents-sandbox-smoke.ts new file mode 100644 index 0000000000..96ba31e6f1 --- /dev/null +++ b/scripts/managed-agents-sandbox-smoke.ts @@ -0,0 +1,122 @@ +#!/usr/bin/env node +import assert from "node:assert/strict"; +import { randomBytes } from "node:crypto"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createManagedAgentsSandbox } from "../src/sandbox/managed-agents-sandbox.ts"; +import { createSdkManagedAgentsClient } from "../src/sandbox/managed-agents-client.ts"; +import { createLocalWorkspaceStore } from "../src/workspace/workspace-store.ts"; +import { scopeId } from "../src/types.ts"; + +const apiToken = process.env.DO_AGENTS_API_TOKEN; +const apiBaseUrl = process.env.DO_AGENTS_API_BASE_URL; +const template = process.env.DO_AGENTS_TEMPLATE ?? ""; +const agent = process.env.DO_AGENTS_AGENT; +const keep = process.env.DO_AGENTS_SMOKE_KEEP === "1"; + +const log = (...a: unknown[]) => console.log("[managed-agents-smoke]", ...a); + +async function main(): Promise { + if (!apiToken) throw new Error("DO_AGENTS_API_TOKEN is required"); + + const client = createSdkManagedAgentsClient({ + apiToken, + ...(apiBaseUrl ? { apiBaseUrl } : {}), + ...(agent ? { agent } : {}), + ...(template ? { template } : {}), + }); + + const prefix = `qms${randomBytes(3).toString("hex")}`; + const sandbox = createManagedAgentsSandbox( + createLocalWorkspaceStore(mkdtempSync(join(tmpdir(), "managed-agents-smoke-ws-"))), + { + client, + namePrefix: prefix, + }, + ); + const scope = scopeId("personal", "smoke"); + const layers = [{ scopeId: scope, mountPath: "/", mode: "rw" as const }]; + + log(`provisioning a qm agent computer (prefix ${prefix})...`); + const handle = await sandbox.provision(layers, { env: { QM_SMOKE: "1" } }); + log("provisioned", handle.id, "coldStart", handle.coldStart); + + try { + log("running a command through the qm Sandbox interface..."); + const r = await sandbox.run(handle, "pwd; echo QM_SMOKE=$QM_SMOKE; whoami"); + assert.equal(r.code, 0, `run exited ${r.code}: ${r.stderr}`); + log("stdout:", JSON.stringify(r.stdout)); + assert.match(r.stdout, /QM_SMOKE=1/); + + log("writing and reading a workspace file through qm..."); + await sandbox.writeFile(handle, "notes/hello.txt", "written by qm"); + assert.equal(await sandbox.readFile(handle, "notes/hello.txt"), "written by qm"); + log("file round-trip OK"); + + log("checking the toolchain qm expects..."); + const binaryName = (entry: string) => entry.trim().split(/\s+/)[0] ?? ""; + const declaredPresent = (sandbox.profile.spec?.tools ?? []).map(binaryName).filter(Boolean); + const declaredAbsent = (sandbox.profile.spec?.notInstalled ?? []).map(binaryName).filter(Boolean); + const plumbingNeeds = ["sh", "timeout", "tar", "base64", "sha256sum", "setsid", "mkfifo", "cut"]; + const probe = [...new Set([...plumbingNeeds, ...declaredPresent, ...declaredAbsent])]; + const tools = await sandbox.run( + handle, + `for t in ${probe.join(" ")}; do printf '%s=%s ' $t $(command -v $t >/dev/null 2>&1 && echo yes || echo NO); done; echo`, + ); + log(tools.stdout.trim()); + const onPath = new Set( + tools.stdout + .trim() + .split(/\s+/) + .flatMap((pair) => { + const [name, state] = pair.split("="); + return name && state === "yes" ? [name] : []; + }), + ); + for (const t of plumbingNeeds) + assert.ok(onPath.has(t), `qm's own file and process plumbing needs ${t}, which the guest does not have`); + for (const t of declaredPresent) + assert.ok(onPath.has(t), `the computer profile advertises ${t}, which the guest does not have`); + for (const t of declaredAbsent) + assert.ok(!onPath.has(t), `the computer profile says ${t} is absent, but the guest has it`); + + log("checking the guest carries no managed coding agent..."); + const bare = await sandbox.run( + handle, + "for t in codex claude claude-code opencode cursor hermes; do command -v $t >/dev/null && echo AGENT_ON_PATH=$t; done; echo ---; ps -eo comm= | sort -u | tr '\\n' ' '", + ); + log("guest:", bare.stdout.trim()); + assert.doesNotMatch(bare.stdout, /AGENT_ON_PATH=/, "a bare sandbox must carry no agent CLI"); + + log("checking the workspace survives a pause and resume..."); + await sandbox.run(handle, "echo persisted > survives-pause.txt"); + await sandbox.teardown(handle, {}); + const resumed = await sandbox.provision(layers, { env: { QM_SMOKE: "1" } }); + const after = await sandbox.run(resumed, "cat survives-pause.txt"); + assert.match(after.stdout, /persisted/, "the workspace did not survive a pause and resume"); + log("workspace intact after pause and resume"); + + if (process.env.DO_AGENTS_SMOKE_LONG === "1") { + log("checking a command survives past the 4-minute REST exec clamp..."); + const long = await sandbox.run(resumed, "sleep 250; echo past-the-clamp", { timeoutMs: 400_000 }); + assert.equal(long.code, 0, `long run exited ${long.code}: ${long.stderr}`); + assert.match(long.stdout, /past-the-clamp/); + log("250s command completed over the tunnel"); + } + + log("ALL CHECKS PASSED"); + } finally { + if (keep) { + log("DO_AGENTS_SMOKE_KEEP=1 — leaving the sandbox alive"); + } else { + log("tearing down and destroying the scope..."); + await sandbox.teardown(handle, { destroy: true }).catch((e: unknown) => log("teardown failed:", e)); + } + } +} + +main().catch((e: unknown) => { + console.error("[managed-agents-smoke] FAILED", e); + process.exit(1); +}); diff --git a/scripts/managed-agents-turn-smoke.ts b/scripts/managed-agents-turn-smoke.ts new file mode 100644 index 0000000000..8be7ba7ade --- /dev/null +++ b/scripts/managed-agents-turn-smoke.ts @@ -0,0 +1,52 @@ +#!/usr/bin/env node +import { randomUUID } from "node:crypto"; +import { stagingApiHeaders } from "../src/deployment/postdeploy-smoke.ts"; + +const core = process.env.CORE_URL ?? "http://localhost:8081"; +const principal = process.env.PRINCIPAL ?? "naveenkurra"; +const sourceSecret = process.env.CORE_SIGNING_SECRET ?? ""; +const portalSecret = process.env.PORTAL_IDENTITY_SECRET ?? ""; + +const log = (...a: unknown[]) => console.log("[managed-agents-turn]", ...a); + +async function main(): Promise { + if (!sourceSecret || !portalSecret) throw new Error("CORE_SIGNING_SECRET and PORTAL_IDENTITY_SECRET are required"); + + const nonce = randomUUID(); + const body = JSON.stringify({ + surface: "web", + actor: { externalId: principal }, + conversation: { kind: "dm", threadRef: `web:${principal}:managed-agents-turn-${nonce}` }, + text: "Use your execute tool to run exactly this shell command in your agent computer: `echo DO_AGENTS_TURN_OK; uname -sr; pwd; whoami`. Then reply with the exact stdout you got back and nothing else.", + origin: { kind: "human" }, + addressed: true, + skipMemory: true, + idempotencyKey: nonce, + }); + + const path = "/v1/turns"; + const headers = await stagingApiHeaders(principal, sourceSecret, portalSecret, "POST", path, body); + + log(`POST ${core}${path} (this provisions a Managed Agents microVM on the first execute)...`); + const started = Date.now(); + const res = await fetch(`${core}${path}`, { + method: "POST", + headers: { ...headers, "content-type": "application/json" }, + body, + signal: AbortSignal.timeout(600_000), + }); + const text = await res.text(); + log(`HTTP ${res.status} in ${Math.round((Date.now() - started) / 1000)}s`); + if (!res.ok) throw new Error(`turn failed: ${text.slice(0, 2000)}`); + + const turn = JSON.parse(text) as { status?: string; sessionId?: string; reply?: string }; + log("status:", turn.status, "session:", turn.sessionId); + log("reply:", JSON.stringify(turn.reply)); + if (!turn.reply?.includes("DO_AGENTS_TURN_OK")) throw new Error("the model reply does not contain DO_AGENTS_TURN_OK"); + log("REAL QM TURN EXECUTED IN A Managed Agents SANDBOX"); +} + +main().catch((e: unknown) => { + console.error("[managed-agents-turn] FAILED", e); + process.exit(1); +}); diff --git a/src/config.ts b/src/config.ts index df35cd61df..b55f0203b9 100644 --- a/src/config.ts +++ b/src/config.ts @@ -3,6 +3,7 @@ import { isStrongSigningSecret } from "./auth/source-auth.ts"; import { parseSandboxCapabilityTtlMs } from "./auth/capability-token.ts"; import { parseScopeId } from "./types.ts"; import type { LegacySandboxScopeDefaults } from "./sandbox/sandbox-resource-upgrade.ts"; +import type { SandboxBackendName } from "./sandbox/sandbox-routing.ts"; import { existsSync, readdirSync } from "node:fs"; import { parseProviderBaseUrl, @@ -73,9 +74,8 @@ export interface Config { securityPosture: SecurityPosture; sharingPosture: SharingPosture; legacySandboxScopeDefaults?: LegacySandboxScopeDefaults; - sandboxBackend: "aws" | "local" | "sprites" | "smolmachines" | "e2b" | "modal" | "porter" | "agent37" | "superserve"; - sandboxSecondaryBackend?: - "aws" | "local" | "sprites" | "smolmachines" | "e2b" | "modal" | "porter" | "agent37" | "superserve"; + sandboxBackend: SandboxBackendName; + sandboxSecondaryBackend?: SandboxBackendName; deployProvider: "docker" | "aws" | "fly" | "porter"; egressServiceHosts?: string[]; brandingDefault?: OrgBranding; @@ -205,6 +205,7 @@ export interface Config { agent37Sandbox: Agent37SandboxEnv; superserveSandbox: SuperserveSandboxEnv; e2bSandbox: E2bSandboxEnv; + managedAgentsSandbox: ManagedAgentsSandboxEnv; modalSandbox: ModalSandboxEnv; porterSandbox: PorterSandboxEnv; porterDeploy: PorterDeployEnv; @@ -431,6 +432,40 @@ function e2bSandboxEnv(env: NodeJS.ProcessEnv): E2bSandboxEnv { }; } +interface ManagedAgentsSandboxEnv { + apiToken?: string; + apiBaseUrl?: string; + template?: string; + namePrefix?: string; + sizeSlug?: string; + idleTimeoutSec?: number; + egressProxyUrl?: string; + snapshotS3Bucket?: string; + snapshotIntervalSec?: number; + defaultTimeoutSec?: number; +} + +function managedAgentsSandboxEnv(env: NodeJS.ProcessEnv): ManagedAgentsSandboxEnv { + return { + ...(env.DO_AGENTS_API_TOKEN ? { apiToken: env.DO_AGENTS_API_TOKEN } : {}), + ...(env.DO_AGENTS_API_BASE_URL ? { apiBaseUrl: env.DO_AGENTS_API_BASE_URL } : {}), + ...(env.DO_AGENTS_TEMPLATE ? { template: env.DO_AGENTS_TEMPLATE } : {}), + ...(env.DO_AGENTS_NAME_PREFIX ? { namePrefix: env.DO_AGENTS_NAME_PREFIX } : {}), + ...(env.DO_AGENTS_SIZE_SLUG ? { sizeSlug: env.DO_AGENTS_SIZE_SLUG } : {}), + ...(numEnvStrict("DO_AGENTS_IDLE_TIMEOUT_SEC", env.DO_AGENTS_IDLE_TIMEOUT_SEC) !== undefined + ? { idleTimeoutSec: numEnvStrict("DO_AGENTS_IDLE_TIMEOUT_SEC", env.DO_AGENTS_IDLE_TIMEOUT_SEC) } + : {}), + ...(env.DO_AGENTS_EGRESS_PROXY_URL ? { egressProxyUrl: env.DO_AGENTS_EGRESS_PROXY_URL } : {}), + ...(env.DO_AGENTS_SNAPSHOT_S3_BUCKET ? { snapshotS3Bucket: env.DO_AGENTS_SNAPSHOT_S3_BUCKET } : {}), + ...(numEnvStrict("DO_AGENTS_SNAPSHOT_INTERVAL_SEC", env.DO_AGENTS_SNAPSHOT_INTERVAL_SEC) !== undefined + ? { snapshotIntervalSec: numEnvStrict("DO_AGENTS_SNAPSHOT_INTERVAL_SEC", env.DO_AGENTS_SNAPSHOT_INTERVAL_SEC) } + : {}), + ...(numEnvStrict("SANDBOX_TIMEOUT_SEC", env.SANDBOX_TIMEOUT_SEC) !== undefined + ? { defaultTimeoutSec: numEnvStrict("SANDBOX_TIMEOUT_SEC", env.SANDBOX_TIMEOUT_SEC) } + : {}), + }; +} + interface ModalSandboxEnv { nativeSnapshotsEnabled?: boolean; nativeSnapshotIntervalSec?: number; @@ -999,6 +1034,7 @@ export function enabledSandboxBackends(config: Config): Array b porter: (env) => sandboxBackendSelected(env, "porter"), agent37: (env) => sandboxBackendSelected(env, "agent37"), superserve: (env) => sandboxBackendSelected(env, "superserve"), + "do-managed-agents": (env) => sandboxBackendSelected(env, "do-managed-agents"), "porter-deploy": (env) => env.DEPLOY_PROVIDER === "porter", "fly-shared-deploy": (env) => env.DEPLOY_PROVIDER === "fly" && Boolean(env.FLY_DEPLOY_SHARED_APP_NAME?.trim()), "fly-deploy": (env) => env.DEPLOY_PROVIDER === "fly", diff --git a/src/sandbox/managed-agents-client.ts b/src/sandbox/managed-agents-client.ts new file mode 100644 index 0000000000..ef35565e67 --- /dev/null +++ b/src/sandbox/managed-agents-client.ts @@ -0,0 +1,671 @@ +import { fileURLToPath } from "node:url"; +import { fetchWithRetry } from "../util/async.ts"; +import { errMessage, withRequestId } from "../util/errors.ts"; +import { openManagedAgentsTunnel, SANDBOX_AGENT_PORT, type ManagedAgentsTunnel } from "./managed-agents-tunnel.ts"; + +export interface ManagedAgentsCommandResult { + stdout: string; + stderr: string; + exitCode: number; + hitlRejected?: boolean; +} + +interface ManagedAgentsRunOpts { + timeoutMs?: number; + workdir?: string; + env?: Record; +} + +export type ManagedAgentsSessionState = + "unspecified" | "provisioning" | "ready" | "detached" | "destroying" | "destroyed" | "failed" | "paused"; + +export interface ManagedAgentsSessionSummary { + sessionId: string; + sandboxId: string; + name: string; + state: ManagedAgentsSessionState; +} + +export interface ManagedAgentsSessionInfo extends ManagedAgentsSessionSummary { + createdAtMs?: number; + errorMessage?: string; + sizeSlug?: string; + template?: string; +} + +export interface ManagedAgentsSession { + readonly sessionId: string; + readonly sandboxId: string; + runCommand(command: string, opts?: ManagedAgentsRunOpts): Promise; + readFileBytes(absPath: string): Promise; + writeFileBytes(absPath: string, data: Uint8Array): Promise; + pause(): Promise; + resume(): Promise; + kill(): Promise; +} + +export class ManagedAgentsSandboxGoneError extends Error { + constructor(sessionId: string, detail: string) { + super(`do-managed-agents session ${sessionId} is gone: ${detail}`); + this.name = "ManagedAgentsSandboxGoneError"; + } +} + +export class ManagedAgentsCommandLostError extends Error { + constructor(sessionId: string, detail: string) { + super( + `do-managed-agents session ${sessionId} was lost while a command was running (${detail}); the command may have partially executed and was not retried`, + ); + this.name = "ManagedAgentsCommandLostError"; + } +} + +export class ManagedAgentsHitlRejectedError extends Error { + constructor(sessionId: string) { + super( + `do-managed-agents rejected a command in session ${sessionId} at its own approval gate; set the session permissions to allow bash so qm's approval gates stay authoritative`, + ); + this.name = "ManagedAgentsHitlRejectedError"; + } +} + +interface ManagedAgentsCreateOpts { + name: string; + egressAllow?: string[]; +} + +export interface ManagedAgentsClient { + readonly nativePause?: boolean; + info(sessionId: string): Promise; + create(opts: ManagedAgentsCreateOpts): Promise; + connect(sessionId: string): Promise; + list(name: string): Promise; + kill(sessionId: string): Promise; +} + +export interface SdkManagedAgentsClientOptions { + apiToken: string; + apiBaseUrl?: string; + template?: string; + agent?: string; + sizeSlug?: string; + idleTimeoutSec?: number; + maxCommandMs?: number; + stallProbeMs?: number; + egressProxyUrl?: string; +} + +const DEFAULT_API_BASE_URL = "https://api.digitalocean.com"; +const DEFAULT_TEMPLATE = ""; +const DEFAULT_AGENT = "none"; +const DEFAULT_MAX_COMMAND_MS = 3600_000; +const CREATE_TIMEOUT_MS = 120_000; +const REQUEST_TIMEOUT_MS = 30_000; +const READY_TIMEOUT_MS = 300_000; +const READY_POLL_MS = 2_000; +const STALL_PROBE_MS = 45_000; +const READY_TRANSIENT_LIMIT = 5; +const UPLOAD_CHUNK_BYTES = 256 * 1024; +const DOWNLOAD_CHUNK_BYTES = 256 * 1024; +const MAX_EXEC_OUTPUT_BYTES = 16 * 1024 * 1024; +const CONNECT_TIMEOUT_MS = 30_000; +const GRPC_NOT_FOUND = 5; +const GRPC_INTERNAL = 13; +const GRPC_UNAVAILABLE = 14; + +const PROTO_PATH = fileURLToPath(new URL("./managed-agents-sandbox-agent.proto", import.meta.url)); + +const STATE_BY_WIRE: Record = { + SESSION_STATUS_UNSPECIFIED: "unspecified", + SESSION_STATUS_PROVISIONING: "provisioning", + SESSION_STATUS_READY: "ready", + SESSION_STATUS_DETACHED: "detached", + SESSION_STATUS_DESTROYING: "destroying", + SESSION_STATUS_DESTROYED: "destroyed", + SESSION_STATUS_FAILED: "failed", + SESSION_STATUS_PAUSED: "paused", +}; + +const GONE_STATES: ReadonlySet = new Set([ + "destroying", + "destroyed", + "failed", +]); + +const USABLE_STATES: ReadonlySet = new Set(["ready", "detached"]); + +export const managedAgentsSessionState = (wire: string | undefined): ManagedAgentsSessionState => + STATE_BY_WIRE[wire ?? ""] ?? "unspecified"; + +export function managedAgentsEgressAllow(egressProxyUrl: string): string[] { + const host = new URL(egressProxyUrl).hostname.replace(/^\[|\]$/g, ""); + if (!host) throw new Error(`DO_AGENTS_EGRESS_PROXY_URL ${egressProxyUrl} has no host to put on the egress allowlist`); + return [host]; +} + +export function managedAgentsManifest(opts: { + name: string; + template: string; + agent?: string; + sizeSlug?: string; + idleTimeoutSec?: number; + egressAllow?: readonly string[]; +}): string { + const lines = [`name: ${JSON.stringify(opts.name)}`, `agent: ${JSON.stringify(opts.agent ?? DEFAULT_AGENT)}`]; + if (opts.template) lines.push(`template: ${JSON.stringify(opts.template)}`); + if (opts.sizeSlug) lines.push(`size: ${JSON.stringify(opts.sizeSlug)}`); + if (opts.idleTimeoutSec) lines.push(`idle_timeout: ${JSON.stringify(`${opts.idleTimeoutSec}s`)}`); + if (opts.egressAllow?.length) { + lines.push("egress:"); + for (const host of opts.egressAllow) lines.push(` - ${JSON.stringify(host)}`); + } + lines.push("permissions:", " default: allow", " rules:", " - tool: bash", " action: allow"); + return `${lines.join("\n")}\n`; +} + +interface WireSession { + id?: string; + session_id?: string; + sandbox_id?: string; + name?: string; + status?: string; + template?: string; + size_slug?: string; + error_message?: string; + created_at?: string; +} + +const toSummary = (s: WireSession): ManagedAgentsSessionSummary => ({ + sessionId: s.session_id ?? s.id ?? "", + sandboxId: s.sandbox_id ?? "", + name: s.name ?? "", + state: managedAgentsSessionState(s.status), +}); + +const toInfo = (s: WireSession): ManagedAgentsSessionInfo => { + const createdAtMs = s.created_at ? Date.parse(s.created_at) : Number.NaN; + return { + ...toSummary(s), + ...(Number.isFinite(createdAtMs) ? { createdAtMs } : {}), + ...(s.error_message ? { errorMessage: s.error_message } : {}), + ...(s.size_slug ? { sizeSlug: s.size_slug } : {}), + ...(s.template ? { template: s.template } : {}), + }; +}; + +interface GrpcCallError { + code?: number; + details?: string; + message?: string; +} + +const grpcCode = (err: unknown): number | undefined => + typeof err === "object" && err !== null ? (err as GrpcCallError).code : undefined; + +const isGuestGone = (err: unknown): boolean => { + const code = grpcCode(err); + return code === GRPC_NOT_FOUND || code === GRPC_UNAVAILABLE; +}; + +const isMissingPath = (err: unknown): boolean => { + const code = grpcCode(err); + if (code === GRPC_NOT_FOUND) return true; + if (code !== GRPC_INTERNAL) return false; + const e = err as GrpcCallError; + return /\b404\b|does not exist|no such file/i.test(`${e.details ?? ""} ${e.message ?? ""}`); +}; + +type ProtoLoader = typeof import("@grpc/proto-loader"); +type Grpc = typeof import("@grpc/grpc-js"); + +interface GrpcDuplex { + write(msg: unknown): void; + end(): void; + cancel(): void; + on(event: string, cb: (arg: never) => void): void; +} + +type GrpcWritable = Pick; +type GrpcReadable = Pick; + +interface SandboxAgent { + Exec(): GrpcDuplex; + Upload(cb: (err: unknown, res?: { bytes_written?: string }) => void): GrpcWritable; + Download(req: unknown): GrpcReadable; + waitForReady(deadline: Date, cb: (err?: Error) => void): void; + close(): void; +} + +interface ExecOutputFrame { + output?: "stdout" | "stderr" | "exit"; + stdout?: Buffer; + stderr?: Buffer; + exit?: { exit_code?: number; hitl_rejected?: boolean; timed_out?: boolean }; +} + +interface DownloadFrame { + output?: "header" | "chunk" | "end"; + chunk?: Buffer; +} + +let grpcModules: Promise<[ProtoLoader, Grpc]> | null = null; + +function loadGrpc(): Promise<[ProtoLoader, Grpc]> { + grpcModules ??= Promise.all([ + import("@grpc/proto-loader").catch((e: unknown) => { + throw new Error(`@grpc/proto-loader is not installed: ${errMessage(e)}`); + }), + import("@grpc/grpc-js").catch((e: unknown) => { + throw new Error(`@grpc/grpc-js is not installed: ${errMessage(e)}`); + }), + ]) as Promise<[ProtoLoader, Grpc]>; + return grpcModules; +} + +async function dialSandboxAgent(localPort: number, readyTimeoutMs: number): Promise { + const [loader, grpc] = await loadGrpc(); + const definition = loader.loadSync(PROTO_PATH, { + keepCase: true, + longs: String, + enums: String, + defaults: true, + oneofs: true, + }); + const pkg = grpc.loadPackageDefinition(definition) as unknown as { + do: { + teams: { + hosted_agents: { + runtime: { + sandbox_agent: { + v1: { + SandboxAgentService: new ( + addr: string, + creds: ReturnType, + options: Record, + ) => SandboxAgent; + }; + }; + }; + }; + }; + }; + }; + const Service = pkg.do.teams.hosted_agents.runtime.sandbox_agent.v1.SandboxAgentService; + const agent = new Service(`127.0.0.1:${localPort}`, grpc.credentials.createInsecure(), { + "grpc.max_receive_message_length": MAX_EXEC_OUTPUT_BYTES, + "grpc.max_send_message_length": MAX_EXEC_OUTPUT_BYTES, + }); + try { + await new Promise((resolve, reject) => { + agent.waitForReady(new Date(Date.now() + readyTimeoutMs), (err?: Error) => (err ? reject(err) : resolve())); + }); + } catch (e) { + agent.close(); + throw e; + } + return agent; +} + +export function createSdkManagedAgentsClient(opts: SdkManagedAgentsClientOptions): ManagedAgentsClient { + const apiBaseUrl = (opts.apiBaseUrl ?? DEFAULT_API_BASE_URL).replace(/\/+$/, ""); + const template = opts.template ?? DEFAULT_TEMPLATE; + const maxCommandMs = opts.maxCommandMs ?? DEFAULT_MAX_COMMAND_MS; + const stallProbeMs = opts.stallProbeMs ?? STALL_PROBE_MS; + const egressAllow = opts.egressProxyUrl ? managedAgentsEgressAllow(opts.egressProxyUrl) : undefined; + + function send(method: string, path: string, body?: string, contentType?: string, signal?: AbortSignal) { + return fetch(`${apiBaseUrl}${path}`, { + method, + headers: { + authorization: `Bearer ${opts.apiToken}`, + accept: "application/json", + ...(contentType ? { "content-type": contentType } : {}), + }, + ...(body !== undefined ? { body } : {}), + signal: signal ?? AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }); + } + + function call( + method: string, + path: string, + body?: string, + contentType?: string, + timeoutMs = REQUEST_TIMEOUT_MS, + ): Promise { + const operation = (signal?: AbortSignal): Promise => send(method, path, body, contentType, signal); + return method === "GET" || method === "DELETE" + ? fetchWithRetry((signal) => operation(signal), "idempotent", { timeoutMs }) + : operation(AbortSignal.timeout(timeoutMs)); + } + + async function failure(action: string, sessionId: string, res: Response): Promise { + const raw = await res.text().catch(() => ""); + let message = raw.slice(0, 200); + try { + const parsed = JSON.parse(raw) as { error?: { message?: string } }; + if (parsed.error?.message) message = parsed.error.message; + } catch { + message = raw.slice(0, 200); + } + const detail = withRequestId(`http ${res.status} ${message}`, res.headers); + if (res.status === 404 || res.status === 410) return new ManagedAgentsSandboxGoneError(sessionId, detail); + return new Error(`do-managed-agents ${action}: ${detail}`); + } + + async function callJson( + method: string, + path: string, + sessionId: string, + body?: string, + contentType?: string, + timeoutMs = REQUEST_TIMEOUT_MS, + ): Promise { + const res = await call(method, path, body, contentType, timeoutMs); + if (!res.ok) throw await failure(`${method} ${path}`, sessionId, res); + if (res.status === 204) return undefined as T; + return (await res.json()) as T; + } + + const getInfo = async (sessionId: string): Promise => { + const body = await callJson<{ session?: WireSession }>("GET", `/v2/agents/sessions/${sessionId}`, sessionId); + if (!body.session) throw new ManagedAgentsSandboxGoneError(sessionId, "response carried no session"); + return toInfo(body.session); + }; + + async function awaitUsable(sessionId: string): Promise { + const deadline = Date.now() + READY_TIMEOUT_MS; + let resumed = false; + let strikes = 0; + const tolerate = (err: unknown): void => { + if (err instanceof ManagedAgentsSandboxGoneError) throw err; + if (++strikes > READY_TRANSIENT_LIMIT) throw err; + }; + for (;;) { + let info: ManagedAgentsSessionInfo | undefined; + try { + info = await getInfo(sessionId); + strikes = 0; + } catch (err) { + tolerate(err); + } + if (info) { + if (USABLE_STATES.has(info.state)) return info; + if (GONE_STATES.has(info.state)) + throw new ManagedAgentsSandboxGoneError(sessionId, info.errorMessage ?? `status ${info.state}`); + if (info.state === "paused" && !resumed) { + try { + await callJson("POST", `/v2/agents/sessions/${sessionId}/resume`, sessionId); + resumed = true; + strikes = 0; + } catch (err) { + tolerate(err); + } + } + } + if (Date.now() >= deadline) + throw new Error(`do-managed-agents session ${sessionId} did not become ready within ${READY_TIMEOUT_MS}ms`); + await new Promise((resolve) => setTimeout(resolve, READY_POLL_MS)); + } + } + + function buildSession(info: ManagedAgentsSessionInfo): ManagedAgentsSession { + const sessionId = info.sessionId; + let guest: Promise<{ tunnel: ManagedAgentsTunnel; agent: SandboxAgent }> | null = null; + let lastTunnel: ManagedAgentsTunnel | null = null; + + const connect = (): Promise<{ tunnel: ManagedAgentsTunnel; agent: SandboxAgent }> => + (guest ??= (async () => { + const tunnel = await openManagedAgentsTunnel({ + apiBaseUrl, + sessionId, + remotePort: SANDBOX_AGENT_PORT, + getToken: async () => opts.apiToken, + }); + lastTunnel = tunnel; + try { + const dial = dialSandboxAgent(tunnel.localPort, CONNECT_TIMEOUT_MS); + const abort = tunnel.whenFailed().then((detail) => { + throw new Error(detail); + }); + dial.catch(() => undefined); + abort.catch(() => undefined); + return { tunnel, agent: await Promise.race([dial, abort]) }; + } catch (e) { + const detail = tunnel.lastFailure() ?? errMessage(e); + await tunnel.close(); + throw new ManagedAgentsSandboxGoneError(sessionId, `guest unreachable over the port-forward: ${detail}`); + } + })().catch((e: unknown) => { + guest = null; + throw e; + })); + + async function disconnect(): Promise { + const pending = guest; + guest = null; + if (!pending) return; + await pending + .then(async ({ tunnel, agent }) => { + agent.close(); + await tunnel.close(); + }) + .catch(() => undefined); + } + + const guestDetail = (err: unknown): string => lastTunnel?.lastFailure() ?? errMessage(err); + + return { + sessionId, + sandboxId: info.sandboxId, + + async runCommand(command, runOpts): Promise { + const { agent } = await connect(); + const timeoutMs = Math.min(runOpts?.timeoutMs ?? maxCommandMs, maxCommandMs); + return new Promise((resolve, reject) => { + const stream = agent.Exec(); + const out: Buffer[] = []; + const errOut: Buffer[] = []; + let outBytes = 0; + let errBytes = 0; + let exit: { exit_code?: number; hitl_rejected?: boolean } | undefined; + let settled = false; + let quietSince = Date.now(); + const settle = (fn: () => void): void => { + if (settled) return; + settled = true; + clearInterval(watchdog); + fn(); + }; + + const watchdog = setInterval(() => { + if (settled || Date.now() - quietSince < stallProbeMs) return; + void getInfo(sessionId).then( + (live) => { + quietSince = Date.now(); + if (settled || USABLE_STATES.has(live.state)) return; + settle(() => { + void disconnect(); + reject( + GONE_STATES.has(live.state) + ? new ManagedAgentsSandboxGoneError(sessionId, `status ${live.state}`) + : new ManagedAgentsCommandLostError(sessionId, `the session became ${live.state} mid-command`), + ); + }); + }, + () => undefined, + ); + }, stallProbeMs); + watchdog.unref(); + + stream.on("data", ((frame: ExecOutputFrame) => { + quietSince = Date.now(); + if (frame.stdout && outBytes < MAX_EXEC_OUTPUT_BYTES) { + out.push(frame.stdout); + outBytes += frame.stdout.length; + } + if (frame.stderr && errBytes < MAX_EXEC_OUTPUT_BYTES) { + errOut.push(frame.stderr); + errBytes += frame.stderr.length; + } + if (frame.exit) exit = frame.exit; + }) as (arg: never) => void); + + stream.on("error", ((err: unknown) => { + settle(() => { + const detail = guestDetail(err); + void disconnect(); + reject( + isGuestGone(err) + ? new ManagedAgentsCommandLostError(sessionId, detail) + : new Error(`do-managed-agents exec failed: ${detail}`), + ); + }); + }) as (arg: never) => void); + + stream.on("end", (() => { + settle(() => { + if (exit?.hitl_rejected) { + reject(new ManagedAgentsHitlRejectedError(sessionId)); + return; + } + resolve({ + stdout: Buffer.concat(out).toString("utf8"), + stderr: Buffer.concat(errOut).toString("utf8"), + exitCode: exit?.exit_code ?? -1, + }); + }); + }) as (arg: never) => void); + + stream.write({ + start: { + argv: ["/bin/bash", "-c", command], + ...(runOpts?.workdir ? { workdir: runOpts.workdir } : {}), + ...(runOpts?.env ? { env: runOpts.env } : {}), + timeout_seconds: Math.ceil(timeoutMs / 1000), + }, + }); + stream.end(); + }); + }, + + async readFileBytes(absPath): Promise { + const { agent } = await connect(); + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + const stream = agent.Download({ path: absPath, as_archive: false, chunk_size_bytes: DOWNLOAD_CHUNK_BYTES }); + let settled = false; + const settle = (fn: () => void): void => { + if (settled) return; + settled = true; + fn(); + }; + stream.on("data", ((frame: DownloadFrame) => { + if (frame.chunk?.length) chunks.push(frame.chunk); + }) as (arg: never) => void); + stream.on("error", ((err: unknown) => { + settle(() => { + if (isMissingPath(err)) { + resolve(null); + return; + } + const detail = guestDetail(err); + void disconnect(); + reject( + grpcCode(err) === GRPC_UNAVAILABLE + ? new ManagedAgentsSandboxGoneError(sessionId, detail) + : new Error(`do-managed-agents download ${absPath} failed: ${detail}`), + ); + }); + }) as (arg: never) => void); + stream.on("end", (() => settle(() => resolve(new Uint8Array(Buffer.concat(chunks))))) as ( + arg: never, + ) => void); + }); + }, + + async writeFileBytes(absPath, data): Promise { + const { agent } = await connect(); + await new Promise((resolve, reject) => { + const stream = agent.Upload((err: unknown) => { + if (!err) { + resolve(); + return; + } + const detail = guestDetail(err); + void disconnect(); + reject( + isGuestGone(err) + ? new ManagedAgentsSandboxGoneError(sessionId, detail) + : new Error(`do-managed-agents upload ${absPath} failed: ${detail}`), + ); + }); + stream.write({ header: { path: absPath, mode: 0o644, is_archive: false } }); + for (let offset = 0; offset < data.length; offset += UPLOAD_CHUNK_BYTES) + stream.write({ chunk: Buffer.from(data.subarray(offset, offset + UPLOAD_CHUNK_BYTES)) }); + stream.write({ end: {} }); + stream.end(); + }); + }, + + async pause(): Promise { + await disconnect(); + await callJson("POST", `/v2/agents/sessions/${sessionId}/pause`, sessionId); + }, + + async resume(): Promise { + await callJson("POST", `/v2/agents/sessions/${sessionId}/resume`, sessionId); + }, + + async kill(): Promise { + await disconnect(); + await callJson("DELETE", `/v2/agents/sessions/${sessionId}`, sessionId); + }, + }; + } + + return { + nativePause: true, + + info: getInfo, + + async create(createOpts): Promise { + const manifest = managedAgentsManifest({ + name: createOpts.name, + template, + ...(opts.agent ? { agent: opts.agent } : {}), + ...(opts.sizeSlug ? { sizeSlug: opts.sizeSlug } : {}), + ...(opts.idleTimeoutSec ? { idleTimeoutSec: opts.idleTimeoutSec } : {}), + ...((createOpts.egressAllow ?? egressAllow) ? { egressAllow: createOpts.egressAllow ?? egressAllow } : {}), + }); + const body = await callJson<{ session?: WireSession }>( + "POST", + "/v2/agents/sessions", + createOpts.name, + manifest, + "application/x-yaml", + CREATE_TIMEOUT_MS, + ); + const created = body.session ? toSummary(body.session) : null; + if (!created?.sessionId) throw new Error("do-managed-agents create session: response carried no session id"); + return buildSession(await awaitUsable(created.sessionId)); + }, + + async connect(sessionId): Promise { + return buildSession(await awaitUsable(sessionId)); + }, + + async list(name): Promise { + const body = await callJson<{ sessions?: WireSession[] }>( + "GET", + `/v2/agents/sessions?name=${encodeURIComponent(name)}`, + name, + ); + return (body.sessions ?? []).map(toSummary).filter((s) => s.name === name && !GONE_STATES.has(s.state)); + }, + + async kill(sessionId): Promise { + await callJson("DELETE", `/v2/agents/sessions/${sessionId}`, sessionId); + }, + }; +} diff --git a/src/sandbox/managed-agents-sandbox-agent.proto b/src/sandbox/managed-agents-sandbox-agent.proto new file mode 100644 index 0000000000..cd9dd068e9 --- /dev/null +++ b/src/sandbox/managed-agents-sandbox-agent.proto @@ -0,0 +1,102 @@ +syntax = "proto3"; + +package do.teams.hosted_agents.runtime.sandbox_agent.v1; + +// The subset of the in-guest sandbox-agent gRPC surface this backend calls. +// Field numbers and names match the server; the RPCs and messages qm does not +// use are omitted rather than reproduced. +service SandboxAgentService { + rpc Exec(stream ExecRequest) returns (stream ExecOutput); + rpc Upload(stream UploadRequest) returns (UploadResponse); + rpc Download(DownloadRequest) returns (stream DownloadChunk); +} + +message ExecRequest { + oneof request { + ExecStart start = 1; + bytes stdin = 2; + ExecSignal signal = 3; + } +} + +message ExecStart { + // argv[0] is the program. No shell is spawned unless the caller asks for + // one, so qm passes ["/bin/bash", "-c", command]. + repeated string argv = 1; + string workdir = 2; + map env = 3; + // Hard timeout; 0 means none. Enforced in the guest with SIGTERM then SIGKILL. + int64 timeout_seconds = 4; +} + +message ExecSignal { + int32 signal = 1; +} + +message ExecOutput { + oneof output { + bytes stdout = 1; + bytes stderr = 2; + ExecExit exit = 3; + } +} + +message ExecExit { + int32 exit_code = 1; + int64 duration_ms = 2; + // Set when an in-guest approval check rejected the command; exit_code is -1. + bool hitl_rejected = 3; + bool timed_out = 4; +} + +message UploadRequest { + oneof request { + UploadHeader header = 1; + bytes chunk = 2; + UploadEnd end = 3; + } +} + +message UploadHeader { + // Absolute path inside the guest. Transfer is confined to the workspace root. + string path = 1; + // POSIX mode; zero takes the guest default. + uint32 mode = 2; + // True when the payload is a tar archive to extract under path. + bool is_archive = 3; +} + +message UploadEnd { + // Optional hex SHA-256 of the payload; the guest fails the upload on mismatch. + string sha256 = 1; +} + +message UploadResponse { + string path = 1; + int64 bytes_written = 2; +} + +message DownloadRequest { + string path = 1; + bool as_archive = 2; + // Read buffer size in bytes; zero takes the guest default. + uint32 chunk_size_bytes = 3; +} + +message DownloadChunk { + oneof output { + DownloadHeader header = 1; + bytes chunk = 2; + DownloadEnd end = 3; + } +} + +message DownloadHeader { + // -1 when unknown, as for a streaming archive. + int64 size_bytes = 1; + bool is_archive = 2; +} + +message DownloadEnd { + string sha256 = 1; +} diff --git a/src/sandbox/managed-agents-sandbox.ts b/src/sandbox/managed-agents-sandbox.ts new file mode 100644 index 0000000000..f40c7942cc --- /dev/null +++ b/src/sandbox/managed-agents-sandbox.ts @@ -0,0 +1,533 @@ +import { cleanupFailedProvision } from "./sandbox.ts"; +import { randomUUID } from "node:crypto"; +import type { WorkspaceLayer } from "../types.ts"; +import type { WorkspaceStore } from "../workspace/workspace-store.ts"; +import type { DurableMap } from "../persistence/durable-map.ts"; +import { createMemoryMap } from "../persistence/durable-map.ts"; +import { createKeyedQueue } from "../util/async.ts"; +import { swallowAs, errMessage } from "../util/errors.ts"; +import { shq } from "../util/shell.ts"; +import { nonInteractiveShellPrefix, DROPPED_PROXY_ENV, forceThroughProxyEnv } from "./sandbox-env.ts"; +import { createExecProcessSessions, type ExecProcessIo } from "./exec-process-session.ts"; +import { materializeRoLayers } from "./ro-layers.ts"; +import { createLayerToolInstaller } from "./layer-tool-install.ts"; +import type { LayerInstallFile } from "../deployment/load-layer.ts"; +import { + createBackendBlobStaging, + createExecExport, + createExecFileOps, + posixJoin, + type BlobStagingOptions, +} from "./exec-file-ops.ts"; +import { + ephemeralCredLinkScript, + ephemeralCredLinkPaths, + type CredentialPathSpec, +} from "../credentials/resident-paths.ts"; +import { killableScript, killScript } from "./exec-kill.ts"; +import { visibleNotInstalled, visibleTools } from "./sandbox.ts"; +import { sandboxScopeName } from "./exec-sandbox-base.ts"; +import { + ManagedAgentsCommandLostError, + ManagedAgentsSandboxGoneError, + type ManagedAgentsClient, + type ManagedAgentsSessionInfo, + type ManagedAgentsSession, +} from "./managed-agents-client.ts"; +import { + createHomeSnapshotOps, + createMemorySnapshotStore, + HOME_SNAPSHOT_PRUNE, + snapshotDue, + type HomeSnapshotStore, +} from "./home-snapshot.ts"; +import type { + AgentComputerProfile, + ComputerStatus, + ExecOptions, + ExecResult, + ProvisionOptions, + Sandbox, + SandboxHandle, + TeardownOptions, +} from "./sandbox.ts"; + +const HOME_DIR = "/workspace/home"; +const WORKSPACE_BASENAME = "workspace"; +const RO_LAYERS_TAR = ".ro-layers.tar"; +const RO_LAYERS_MANIFEST = ".ro-layers.manifest"; +const HOME_TAR = `${HOME_DIR}/.qm-home.tar`; + +const SNAPSHOT_PRUNE = HOME_SNAPSHOT_PRUNE; +const DEFAULT_SNAPSHOT_INTERVAL_MS = 5 * 60_000; + +export interface StoredManagedAgentsSandbox { + sessionId: string; + sandboxId: string; + preservationState?: "running" | "paused" | "pause_failed"; + preservationError?: string; + createdAtMs: number; + lastSnapshotMs?: number; + homeDirty?: boolean; +} + +export interface ManagedAgentsSandboxOptions extends BlobStagingOptions { + client: ManagedAgentsClient; + namePrefix?: string; + defaultTimeoutSec?: number; + snapshotIntervalMs?: number; + egressProxyUrl?: string; + extraTools?: string[]; + credentialPaths?: CredentialPathSpec[]; + layerToolFiles?: () => readonly LayerInstallFile[]; + store?: DurableMap; + snapshots?: HomeSnapshotStore; + onError?: (e: { category: string; code: string; message: string; scopeLabel?: string }) => void; +} + +export function createManagedAgentsSandbox(workspace: WorkspaceStore, opts: ManagedAgentsSandboxOptions): Sandbox { + const client = opts.client; + const prefix = opts.namePrefix ?? "qm"; + const defaultTimeoutSec = opts.defaultTimeoutSec ?? 600; + const snapshotIntervalMs = opts.snapshotIntervalMs ?? DEFAULT_SNAPSHOT_INTERVAL_MS; + const workspaceDir = `${HOME_DIR}/${WORKSPACE_BASENAME}`; + const store = opts.store ?? createMemoryMap(); + const snapshots = opts.snapshots ?? createMemorySnapshotStore(); + const provisionQueue = createKeyedQueue(); + + const sessionByName = new Map(); + const scopeByName = new Map(); + const scratchKeyByName = new Map(); + const activeScratch = new Map(); + + const reportError = (category: string, code: string, message: string, scopeLabel?: string): void => { + opts.onError?.({ category, code, message, ...(scopeLabel ? { scopeLabel } : {}) }); + }; + + const homeSnapshots = createHomeSnapshotOps({ + label: "do-managed-agents", + homeDir: HOME_DIR, + homeTarPath: HOME_TAR, + prunePaths: [...SNAPSHOT_PRUNE, ...ephemeralCredLinkPaths(opts.credentialPaths ?? []).map(({ rel }) => `./${rel}`)], + store: snapshots, + io: { + runCommand: (session, script, timeoutMs) => session.runCommand(script, { timeoutMs }), + readFileBytes: (session, abs) => session.readFileBytes(abs), + writeFileBytes: (session, abs, data) => session.writeFileBytes(abs, data), + }, + }); + + async function snapshotHome(scope: string, session: ManagedAgentsSession): Promise { + await homeSnapshots.snapshotHome(scope, session); + await store.merge(scope, { lastSnapshotMs: Date.now(), homeDirty: false }); + } + + const hydrateHome = (scope: string, session: ManagedAgentsSession): Promise => + homeSnapshots.hydrateHome(scope, session); + + const recordSession = (scope: string, session: ManagedAgentsSession, createdAtMs: number): Promise => + store.put(scope, { + sessionId: session.sessionId, + sandboxId: session.sandboxId, + createdAtMs, + preservationState: "running", + }); + + async function ensureSession( + scope: string, + name: string, + onStatus?: (text: string) => void, + ): Promise<{ session: ManagedAgentsSession; coldStart: boolean }> { + return provisionQueue(scope, async () => { + const cached = sessionByName.get(name); + if (cached) return { session: cached, coldStart: false }; + + const adopt = (session: ManagedAgentsSession): { session: ManagedAgentsSession; coldStart: boolean } => { + sessionByName.set(name, session); + return { session, coldStart: false }; + }; + + const stored = await store.get(scope); + if (stored) { + try { + onStatus?.("Resuming the sandbox…"); + const session = await client.connect(stored.sessionId); + await store.merge(scope, { preservationState: "running", sandboxId: session.sandboxId }); + return adopt(session); + } catch (err) { + if (!(err instanceof ManagedAgentsSandboxGoneError)) throw err; + } + } + + for (const summary of await client.list(name)) { + try { + const session = await client.connect(summary.sessionId); + await recordSession(scope, session, Date.now()); + return adopt(session); + } catch (err) { + if (!(err instanceof ManagedAgentsSandboxGoneError)) throw err; + } + } + + onStatus?.("Creating the sandbox…"); + const session = await client.create({ name }); + sessionByName.set(name, session); + await recordSession(scope, session, Date.now()); + let hydrated: boolean; + try { + hydrated = await hydrateHome(scope, session); + } catch (e) { + reportError("sandbox_hydrate", "hydrate_failed", errMessage(e), scope); + sessionByName.delete(name); + await client.kill(session.sessionId).catch(() => undefined); + throw new Error( + `do-managed-agents provision: home hydration failed (${errMessage(e)}); not risking the stored snapshot`, + { + cause: e, + }, + ); + } + return { session, coldStart: !hydrated }; + }); + } + + async function ensureScratch(key: string): Promise<{ name: string; coldStart: boolean }> { + const name = sandboxScopeName(`${prefix}-scratch`, key); + return provisionQueue(`scratch:${key}`, async () => { + scratchKeyByName.set(name, key); + const active = activeScratch.get(name) ?? 0; + if (active === 0 && !sessionByName.has(name)) { + sessionByName.set(name, await client.create({ name })); + } + activeScratch.set(name, active + 1); + return { name, coldStart: active === 0 }; + }); + } + + async function withSession(name: string, action: (session: ManagedAgentsSession) => Promise): Promise { + const scratchKey = scratchKeyByName.get(name); + if (scratchKey === undefined && !scopeByName.has(name)) throw new Error("sandbox handle has been released"); + + const reviveScratch = async (): Promise => { + const session = await client.create({ name }); + sessionByName.set(name, session); + return session; + }; + const first = + scratchKey !== undefined + ? { session: sessionByName.get(name) ?? (await reviveScratch()) } + : await ensureSession(scopeByName.get(name) ?? "default", name); + try { + return await action(first.session); + } catch (err) { + if (err instanceof ManagedAgentsCommandLostError) sessionByName.delete(name); + if (!(err instanceof ManagedAgentsSandboxGoneError)) throw err; + sessionByName.delete(name); + const second = + scratchKey !== undefined + ? { session: await reviveScratch() } + : await ensureSession(scopeByName.get(name) ?? "default", name); + return action(second.session); + } + } + + async function execRaw(name: string, script: string, timeoutSec: number): Promise { + return withSession(name, async (session) => { + const r = await session.runCommand(`timeout ${timeoutSec} sh -c ${shq(script)}`, { + timeoutMs: timeoutSec * 1000 + 30_000, + }); + return { stdout: r.stdout, stderr: r.stderr, code: r.exitCode, timedOut: r.exitCode === 124 }; + }); + } + + const profile: AgentComputerProfile = { + backend: "do-managed-agents", + writablePersistence: client.nativePause ? "provider_managed" : "snapshot_to_workspace", + processSessions: true, + parksOnTeardown: true, + egressEnforcement: opts.egressProxyUrl ? "domain" : "none", + spec: { + os: "Linux — DigitalOcean Managed Agents Firecracker microVM (provider pause preserves state; publish durable work to git or Files)", + runtimes: ["Node", "Python 3"], + get tools() { + return visibleTools(["git", "curl", "tar", "python3", "gh", ...(opts.extraTools ?? [])]); + }, + get notInstalled() { + return visibleNotInstalled( + ["jq", "rg", "unzip", "wget", "aws", "gcloud", "kubectl", "flyctl", "glab"], + opts.extraTools ?? [], + ); + }, + homeDir: HOME_DIR, + workdir: workspaceDir, + }, + }; + + const procIo: ExecProcessIo = { + async run(handle, command, execOpts): Promise { + const timeoutSec = execOpts?.timeoutMs ? Math.ceil(execOpts.timeoutMs / 1000) : defaultTimeoutSec; + return execRaw(handle.id, command, timeoutSec); + }, + }; + const procSessions = createExecProcessSessions(procIo); + + const writeAbsBytes = (name: string, absPath: string, data: Uint8Array): Promise => + withSession(name, (session) => session.writeFileBytes(absPath, data)); + const readAbsBytes = (name: string, absPath: string): Promise => + withSession(name, (session) => session.readFileBytes(absPath)); + const installLayerTools = opts.layerToolFiles ? createLayerToolInstaller(opts.layerToolFiles) : null; + + const execFileOps = createExecFileOps({ + label: "do-managed-agents", + exec: (id, script, t) => execRaw(id, script, t), + writeInline: (id, abs, data) => writeAbsBytes(id, abs, data), + }); + + const execExport = createExecExport({ + label: "do-managed-agents", + exec: (id, script, t) => execRaw(id, script, t), + readAbsBytes, + defaultHomeDir: HOME_DIR, + ephemeralCredentialPrefixes: ephemeralCredLinkPaths(opts.credentialPaths ?? []).map(({ rel }) => rel), + }); + + const blobStaging = createBackendBlobStaging("do-managed-agents", (id, script, t) => execRaw(id, script, t), opts); + + async function destroyStoredScope(scope: string): Promise { + const name = sandboxScopeName(prefix, scope); + const stored = await store.get(scope); + const cached = sessionByName.get(name); + const ids = new Set([stored?.sessionId, cached?.sessionId].filter((id): id is string => !!id)); + for (const id of ids) { + try { + if (cached && id === cached.sessionId) await cached.kill(); + else await client.kill(id); + } catch (error) { + if (!(error instanceof ManagedAgentsSandboxGoneError)) throw error; + } + } + await store.delete(scope); + sessionByName.delete(name); + scopeByName.delete(name); + } + + const sandbox: Sandbox = { + destroyScope(scopeId: string): Promise { + return provisionQueue(scopeId, () => destroyStoredScope(scopeId)); + }, + + profile, + startProcess: procSessions.startProcess, + readProcess: procSessions.readProcess, + writeStdin: procSessions.writeStdin, + signalProcess: procSessions.signalProcess, + listProcesses: procSessions.listProcesses, + ...execFileOps, + ...blobStaging, + + async provision(layers: WorkspaceLayer[], provOpts?: ProvisionOptions): Promise { + const scratch = provOpts?.scratch; + const writable = layers.find((l) => l.mode === "rw") ?? layers[0]; + const scope = writable?.scopeId ?? "default"; + let name: string; + let coldStart: boolean; + if (scratch) { + ({ name, coldStart } = await ensureScratch(scratch.key)); + } else { + name = sandboxScopeName(prefix, scope); + scopeByName.set(name, scope); + ({ coldStart } = await ensureSession(scope, name, provOpts?.onStatus)); + } + + const forceEgress = !!opts.egressProxyUrl && !!provOpts?.egressToken; + const turnEnv = Object.fromEntries( + Object.entries(provOpts?.env ?? {}).filter(([k]) => !DROPPED_PROXY_ENV.has(k)), + ); + const env = { + ...turnEnv, + ...(forceEgress ? forceThroughProxyEnv(opts.egressProxyUrl!, provOpts!.egressToken!) : {}), + }; + const handle: SandboxHandle = { + id: name, + rootDir: workspaceDir, + homeDir: HOME_DIR, + coldStart, + ...(scratch ? { scratch: true } : {}), + ...(Object.keys(env).length ? { env } : {}), + }; + + try { + const credLinks = scratch ? "" : ` && ${ephemeralCredLinkScript(HOME_DIR, opts.credentialPaths ?? [])}`; + const prep = await execRaw(name, `mkdir -p ${shq(workspaceDir)}${credLinks}`, 60); + if (prep.code !== 0) + throw new Error(`do-managed-agents provision prep failed: ${(prep.stderr || prep.stdout).slice(0, 200)}`); + + await materializeRoLayers( + workspace, + layers, + handle, + { + readFile: (h, rel) => sandbox.readFile(h, rel), + writeFileBytes: (h, rel, data) => sandbox.writeFileBytes(h, rel, data), + exec: (script, t) => execRaw(name, script, t), + }, + { manifest: RO_LAYERS_MANIFEST, tar: RO_LAYERS_TAR, label: "do-managed-agents" }, + ); + await installLayerTools?.({ + exec: (script, t) => execRaw(name, script, t), + writeAbs: (abs, data) => writeAbsBytes(name, abs, data), + }); + + return handle; + } catch (err) { + await cleanupFailedProvision(sandbox, handle, err); + throw err; + } + }, + + async run(handle, command, execOpts?: ExecOptions): Promise { + const timeoutSec = execOpts?.timeoutMs ? Math.ceil(execOpts.timeoutMs / 1000) : defaultTimeoutSec; + const exports = Object.entries(handle.env ?? {}) + .map(([k, v]) => `export ${k}=${shq(v)}`) + .join("; "); + const script = `${nonInteractiveShellPrefix()}${exports ? exports + "; " : ""}cd ${handle.rootDir} 2>/dev/null; ${command}`; + const signal = execOpts?.signal; + if (!signal) return execRaw(handle.id, script, timeoutSec); + const killUid = randomUUID(); + const fireKill = (): void => { + execRaw(handle.id, killScript(killUid), 15).catch( + swallowAs("managed-agents-sandbox: kill in-flight exec", undefined), + ); + }; + signal.throwIfAborted(); + const onAbort = (): void => fireKill(); + signal.addEventListener("abort", onAbort, { once: true }); + try { + return await execRaw(handle.id, killableScript(script, killUid), timeoutSec); + } finally { + signal.removeEventListener("abort", onAbort); + } + }, + + async writeFileBytes(handle, relPath, data): Promise { + await writeAbsBytes(handle.id, posixJoin(handle.rootDir, relPath), data); + }, + async writeFile(handle, relPath, data): Promise { + await sandbox.writeFileBytes(handle, relPath, Buffer.from(data, "utf8")); + }, + async readFileBytes(handle, relPath): Promise { + return readAbsBytes(handle.id, posixJoin(handle.rootDir, relPath)); + }, + async readFile(handle, relPath): Promise { + const bytes = await sandbox.readFileBytes(handle, relPath); + return bytes === null ? null : Buffer.from(bytes).toString("utf8"); + }, + + exportFiles: execExport.exportFiles, + + async persistHomeSnapshot(scopeId: string): Promise { + const name = sandboxScopeName(prefix, scopeId); + scopeByName.set(name, scopeId); + const { session } = await ensureSession(scopeId, name); + await snapshotHome(scopeId, session); + }, + + async computerStatus(scopeId: string): Promise { + const name = sandboxScopeName(prefix, scopeId); + const stored = await store.get(scopeId); + + if (!stored) return { machine: "no sandbox provisioned yet", provisioned: false, guestResponsive: false }; + const machine = `do-managed-agents sandbox ${stored.sandboxId || stored.sessionId}`; + const recovery = { + strategy: "workspace_snapshot" as const, + state: stored.preservationState, + ...(stored.preservationError ? { error: stored.preservationError } : {}), + ...(stored.lastSnapshotMs ? { checkpointAtMs: stored.lastSnapshotMs } : {}), + }; + try { + const info: ManagedAgentsSessionInfo = await client.info(stored.sessionId); + if (info.state === "paused") + return { + machine, + listed: info.state, + lifecycleState: "paused", + provisioned: true, + guestResponsive: false, + recovery: { ...recovery, state: "paused" }, + }; + scopeByName.set(name, scopeId); + let session = sessionByName.get(name); + if (!session) { + session = await client.connect(stored.sessionId); + sessionByName.set(name, session); + } + const r = await session.runCommand("echo responsive", { timeoutMs: 30_000 }); + return { + machine, + recovery, + provisioned: true, + guestResponsive: r.exitCode === 0 && /responsive/.test(r.stdout), + }; + } catch (e) { + return { + recovery, + machine: `${machine} (${errMessage(e).slice(0, 120)})`, + provisioned: !(e instanceof ManagedAgentsSandboxGoneError), + guestResponsive: false, + }; + } + }, + + async teardown(handle, tdOpts?: TeardownOptions): Promise { + if (handle.scratch) { + const key = scratchKeyByName.get(handle.id); + return provisionQueue(key ? `scratch:${key}` : handle.id, async () => { + const remaining = (activeScratch.get(handle.id) ?? 1) - 1; + if (remaining > 0) { + activeScratch.set(handle.id, remaining); + return; + } + activeScratch.delete(handle.id); + const session = sessionByName.get(handle.id); + if (session) { + if (tdOpts?.destroy) await session.kill(); + else await session.kill().catch(swallowAs("managed-agents-sandbox: scratch kill", undefined)); + } + sessionByName.delete(handle.id); + scratchKeyByName.delete(handle.id); + }); + } + if (tdOpts?.destroy && !scopeByName.has(handle.id)) return; + const scope = scopeByName.get(handle.id) ?? "default"; + return provisionQueue(scope, () => teardownScope(handle, scope, tdOpts)); + }, + }; + + async function teardownScope(handle: SandboxHandle, scope: string, tdOpts?: TeardownOptions): Promise { + const session = sessionByName.get(handle.id); + if (tdOpts?.destroy) return destroyStoredScope(scope); + if (!session) return; + + const stored = await store.get(scope); + if (!tdOpts?.homeUnchanged) await store.merge(scope, { homeDirty: true }); + if (snapshotDue(stored, tdOpts, snapshotIntervalMs)) { + try { + await snapshotHome(scope, session); + } catch (e) { + reportError("sandbox_snapshot", "teardown_snapshot_failed", errMessage(e), scope); + } + } + if (tdOpts?.keepWarm) return; + try { + await session.pause(); + await store.merge(scope, { preservationState: "paused", preservationError: undefined }); + sessionByName.delete(handle.id); + } catch (error) { + await store.merge(scope, { preservationState: "pause_failed", preservationError: errMessage(error) }); + reportError("sandbox_preservation", "pause_failed", errMessage(error), scope); + throw error; + } + } + + return sandbox; +} diff --git a/src/sandbox/managed-agents-tunnel.ts b/src/sandbox/managed-agents-tunnel.ts new file mode 100644 index 0000000000..a0557e7302 --- /dev/null +++ b/src/sandbox/managed-agents-tunnel.ts @@ -0,0 +1,151 @@ +import { createServer, type Server, type Socket } from "node:net"; +import { once } from "node:events"; +import type WebSocket from "ws"; + +export const SANDBOX_AGENT_PORT = 8443; + +const TUNNEL_CLOSE_SESSION_UNAVAILABLE = 4001; +const TUNNEL_CLOSE_GUEST_DIAL_FAILED = 4002; +const TUNNEL_CLOSE_UPSTREAM_ERROR = 4011; +const REJECTION_BODY_MAX = 300; +const PING_INTERVAL_MS = 30_000; + +export interface ManagedAgentsTunnel { + readonly localPort: number; + lastFailure(): string | null; + whenFailed(): Promise; + close(): Promise; +} + +export interface ManagedAgentsTunnelOptions { + apiBaseUrl: string; + sessionId: string; + remotePort: number; + getToken: () => Promise; + pingIntervalMs?: number; +} + +export function tunnelUrl(apiBaseUrl: string, sessionId: string, remotePort: number): string { + const url = new URL(apiBaseUrl); + if (url.protocol !== "http:" && url.protocol !== "https:") + throw new Error(`DO_AGENTS_API_BASE_URL must be http(s), got ${url.protocol}`); + url.protocol = url.protocol === "https:" ? "wss:" : "ws:"; + url.pathname = `${url.pathname.replace(/\/$/, "")}/v2/agents/sessions/${sessionId}/port-forward/${remotePort}`; + return url.toString(); +} + +type WebSocketCtor = typeof WebSocket; + +let wsModule: WebSocketCtor | null = null; +async function loadWebSocket(): Promise { + wsModule ??= (await import("ws")).default; + return wsModule; +} + +export async function openManagedAgentsTunnel(opts: ManagedAgentsTunnelOptions): Promise { + const WebSocketImpl = await loadWebSocket(); + const url = tunnelUrl(opts.apiBaseUrl, opts.sessionId, opts.remotePort); + const sockets = new Set(); + let failure: string | null = null; + let closed = false; + let announceFailure: (detail: string) => void = () => undefined; + const failed = new Promise((resolve) => { + announceFailure = resolve; + }); + const fail = (detail: string): void => { + failure = detail; + announceFailure(detail); + }; + + const bridge = (local: Socket): void => { + sockets.add(local); + local.once("close", () => sockets.delete(local)); + void opts + .getToken() + .then((token) => { + if (closed) { + local.destroy(); + return; + } + const ws = new WebSocketImpl(url, { headers: { Authorization: `Bearer ${token}` } }); + let rejected = false; + + ws.on("unexpected-response", (_req, res) => { + rejected = true; + let body = ""; + res.on("data", (chunk: Buffer) => { + body += chunk.toString("utf8"); + }); + res.on("end", () => { + fail(`${res.statusCode ?? 0} ${body.trim().slice(0, REJECTION_BODY_MAX)}`.trim()); + local.destroy(); + }); + }); + + ws.on("open", () => { + const heartbeat = setInterval(() => { + if (ws.readyState === WebSocketImpl.OPEN) ws.ping(); + }, opts.pingIntervalMs ?? PING_INTERVAL_MS); + heartbeat.unref(); + ws.once("close", () => clearInterval(heartbeat)); + local.on("data", (chunk: Buffer) => { + local.pause(); + ws.send(chunk, (err) => { + if (err) return; + local.resume(); + }); + }); + ws.on("message", (data: Buffer) => { + if (!local.write(data)) { + ws.pause(); + local.once("drain", () => ws.resume()); + } + }); + }); + + ws.on("close", (code: number, reason: Buffer) => { + if ( + code === TUNNEL_CLOSE_SESSION_UNAVAILABLE || + code === TUNNEL_CLOSE_GUEST_DIAL_FAILED || + code === TUNNEL_CLOSE_UPSTREAM_ERROR + ) + fail(`close ${code} ${reason.toString("utf8")}`.trim()); + local.destroy(); + }); + + ws.on("error", (err: Error) => { + if (!rejected) fail(err.message); + local.destroy(); + }); + + local.on("close", () => { + if (ws.readyState === WebSocketImpl.OPEN || ws.readyState === WebSocketImpl.CONNECTING) ws.close(); + }); + local.on("error", () => ws.close()); + }) + .catch((err: unknown) => { + fail(err instanceof Error ? err.message : String(err)); + local.destroy(); + }); + }; + + const server: Server = createServer(bridge); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + server.unref(); + const address = server.address(); + if (address === null || typeof address === "string") + throw new Error("Managed Agents tunnel failed to bind a local port"); + + return { + localPort: address.port, + lastFailure: () => failure, + whenFailed: () => failed, + close: async () => { + closed = true; + for (const socket of sockets) socket.destroy(); + sockets.clear(); + await new Promise((resolve) => server.close(() => resolve())); + }, + }; +} diff --git a/src/sandbox/sandbox-routing.ts b/src/sandbox/sandbox-routing.ts index 4960a78bd2..db73377801 100644 --- a/src/sandbox/sandbox-routing.ts +++ b/src/sandbox/sandbox-routing.ts @@ -17,7 +17,16 @@ import { } from "./sandbox.ts"; export type SandboxBackendName = - "sprites" | "aws" | "local" | "smolmachines" | "e2b" | "modal" | "porter" | "agent37" | "superserve"; + | "sprites" + | "aws" + | "local" + | "smolmachines" + | "e2b" + | "modal" + | "porter" + | "agent37" + | "superserve" + | "do-managed-agents"; export class NoDefaultSandboxError extends Error { constructor() { diff --git a/src/wiring.ts b/src/wiring.ts index 8fb402aba7..05c90f97a3 100644 --- a/src/wiring.ts +++ b/src/wiring.ts @@ -230,6 +230,8 @@ import { import { createSdkSuperserveClient } from "./sandbox/superserve-client.ts"; import { createE2bSandbox, type StoredE2bSandbox } from "./sandbox/e2b-sandbox.ts"; import { createSdkE2bClient } from "./sandbox/e2b-client.ts"; +import { createManagedAgentsSandbox, type StoredManagedAgentsSandbox } from "./sandbox/managed-agents-sandbox.ts"; +import { createSdkManagedAgentsClient } from "./sandbox/managed-agents-client.ts"; import { createS3SnapshotStore } from "./sandbox/home-snapshot.ts"; import { createModalSandbox, type StoredModalSandbox } from "./sandbox/modal-sandbox.ts"; import { createSdkModalClient } from "./sandbox/modal-client.ts"; @@ -932,6 +934,41 @@ export function buildApp( onError: sandboxOnError, }); }; + const managedAgentsBodies = artifactMap("do_managed_agents_sandbox_bodies"); + const buildManagedAgents = (): Sandbox => { + const managedAgents = config.managedAgentsSandbox; + if (!managedAgents.apiToken) throw new Error("SANDBOX_BACKEND=do-managed-agents requires DO_AGENTS_API_TOKEN"); + return createManagedAgentsSandbox(workspace, { + client: createSdkManagedAgentsClient({ + apiToken: managedAgents.apiToken, + ...(managedAgents.apiBaseUrl ? { apiBaseUrl: managedAgents.apiBaseUrl } : {}), + ...(managedAgents.template ? { template: managedAgents.template } : {}), + ...(managedAgents.sizeSlug ? { sizeSlug: managedAgents.sizeSlug } : {}), + ...(managedAgents.idleTimeoutSec ? { idleTimeoutSec: managedAgents.idleTimeoutSec } : {}), + ...(managedAgents.egressProxyUrl ? { egressProxyUrl: managedAgents.egressProxyUrl } : {}), + }), + ...(managedAgents.namePrefix ? { namePrefix: managedAgents.namePrefix } : {}), + ...(managedAgents.defaultTimeoutSec ? { defaultTimeoutSec: managedAgents.defaultTimeoutSec } : {}), + ...(managedAgents.snapshotIntervalSec !== undefined + ? { snapshotIntervalMs: managedAgents.snapshotIntervalSec * 1000 } + : {}), + ...(managedAgents.egressProxyUrl ? { egressProxyUrl: managedAgents.egressProxyUrl } : {}), + extraTools: deploymentLayer.advertisedTools, + credentialPaths: deploymentLayer.credentialPaths, + layerToolFiles: () => deploymentLayer.installFiles, + blobTransfer, + ...(config.signingSecret ? { signingSecret: config.signingSecret } : {}), + ...(config.capabilitySecret ? { capabilitySecret: config.capabilitySecret } : {}), + ...(config.apiBaseUrl ? { apiBaseUrl: config.apiBaseUrl } : {}), + store: managedAgentsBodies, + ...(managedAgents.snapshotS3Bucket + ? { + snapshots: createS3SnapshotStore({ bucket: managedAgents.snapshotS3Bucket, prefix: "managed-agents-home" }), + } + : {}), + onError: sandboxOnError, + }); + }; const buildModal = (): Sandbox => { const modal = config.modalSandbox; if (!modal.tokenId || !modal.tokenSecret) @@ -1081,6 +1118,7 @@ export function buildApp( porter: buildPorter, agent37: buildAgent37, superserve: buildSuperserve, + "do-managed-agents": buildManagedAgents, }; const enabledBackends = new Set(enabledSandboxBackends(config)); const sandboxBackends: Partial> = { @@ -1110,13 +1148,19 @@ export function buildApp( ...(await environments.list()).map((environment) => environment.id), ], legacySandboxes: async () => { - const [e2b, modal, aws, superserve] = await Promise.all([ + const [e2b, modal, aws, superserve, managedAgents] = await Promise.all([ e2bBodies.entries(), modalBodies.entries(), awsBodies.entries(), superserveBodies.entries(), + managedAgentsBodies.entries(), ]); return [ + ...managedAgents.map(([scopeId, body]) => ({ + scopeId, + backend: "do-managed-agents" as const, + machineId: body.sandboxId, + })), ...e2b.map(([scopeId, body]) => ({ scopeId, backend: "e2b" as const, machineId: body.sandboxId })), ...modal.map(([scopeId, body]) => ({ scopeId, backend: "modal" as const, machineId: body.sandboxId })), ...aws.map(([scopeId, body]) => ({ scopeId, backend: "aws" as const, machineId: body.microvmId })), diff --git a/test/live-slack-sandbox-providers.test.ts b/test/live-slack-sandbox-providers.test.ts index ea5634a729..9bf5081d0e 100644 --- a/test/live-slack-sandbox-providers.test.ts +++ b/test/live-slack-sandbox-providers.test.ts @@ -64,6 +64,7 @@ test("every provider has its own parallel execution scenario", () => { assert.deepEqual(sandboxProviders.toSorted(), [ "agent37", "aws", + "do-managed-agents", "e2b", "local", "modal", @@ -159,7 +160,7 @@ test("explicit provider selection retains strict coverage without treating defer "sandbox-execute-modal", ], ); - assert.equal(selectSandboxProviderScenarios("all").length, 9); + assert.equal(selectSandboxProviderScenarios("all").length, 10); assert.deepEqual(selectSandboxProviderScenarios(undefined), []); for (const value of ["", "sprites,", "typo", "sprites,sprites", "all,sprites", "constructor", "__proto__"]) assert.throws(() => selectSandboxProviderScenarios(value)); diff --git a/test/live-slack/scenarios-sandbox-providers.ts b/test/live-slack/scenarios-sandbox-providers.ts index a8675f071b..8e7183c907 100644 --- a/test/live-slack/scenarios-sandbox-providers.ts +++ b/test/live-slack/scenarios-sandbox-providers.ts @@ -14,6 +14,7 @@ const providerCoverage: Record = { porter: true, agent37: true, superserve: true, + "do-managed-agents": true, }; export const sandboxProviders = Object.keys(providerCoverage) as SandboxBackendName[]; From 9541689512e6c559980880c54d9b7507a42f355a Mon Sep 17 00:00:00 2001 From: Naveen Kurra Date: Mon, 5 Oct 2026 12:33:33 -0400 Subject: [PATCH 02/19] Test the Managed Agents backend against a fake control plane and guest The fake service answers the session REST surface and serves a real WebSocket port-forward bridged to a real gRPC sandbox-agent over a temporary directory, so the tests exercise the actual wire path rather than a stub of it: upgrade, bridge, gRPC exec, upload and download. Covers the cases that are easy to get wrong and expensive to debug in production. A port-forward the edge refuses reads as a retryable gone error carrying the rejection, not as a command that may have half run. A transient lifecycle failure while waiting for readiness is polled through, while one that never clears surfaces its own status rather than a readiness timeout. Pause and resume preserve the home, scratch sandboxes are reference counted and never persisted, and the egress allowlist is only claimed when a proxy is actually configured. --- test/managed-agents-client.test.ts | 261 ++++++++++++++++++ test/managed-agents-sandbox.test.ts | 222 +++++++++++++++ test/support/fake-managed-agents-service.ts | 291 ++++++++++++++++++++ test/support/fake-managed-agents.ts | 226 +++++++++++++++ 4 files changed, 1000 insertions(+) create mode 100644 test/managed-agents-client.test.ts create mode 100644 test/managed-agents-sandbox.test.ts create mode 100644 test/support/fake-managed-agents-service.ts create mode 100644 test/support/fake-managed-agents.ts diff --git a/test/managed-agents-client.test.ts b/test/managed-agents-client.test.ts new file mode 100644 index 0000000000..94f1406f10 --- /dev/null +++ b/test/managed-agents-client.test.ts @@ -0,0 +1,261 @@ +import { test, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { connect } from "node:net"; +import { once } from "node:events"; +import { join } from "node:path"; +import { + createSdkManagedAgentsClient, + managedAgentsEgressAllow, + managedAgentsManifest, + managedAgentsSessionState, + ManagedAgentsSandboxGoneError, + ManagedAgentsCommandLostError, + type ManagedAgentsClient, + type ManagedAgentsSession, +} from "../src/sandbox/managed-agents-client.ts"; +import { tunnelUrl, openManagedAgentsTunnel, SANDBOX_AGENT_PORT } from "../src/sandbox/managed-agents-tunnel.ts"; +import { startFakeManagedAgentsService, type FakeManagedAgentsService } from "./support/fake-managed-agents-service.ts"; + +let service: FakeManagedAgentsService; +let client: ManagedAgentsClient; +const opened: ManagedAgentsSession[] = []; + +const session = async (name: string): Promise => { + const created = await client.create({ name }); + opened.push(created); + return created; +}; + +before(async () => { + service = await startFakeManagedAgentsService(); + client = createSdkManagedAgentsClient({ + apiToken: service.token, + apiBaseUrl: service.apiBaseUrl, + template: "base", + }); +}); + +after(async () => { + await Promise.all(opened.map((s) => s.kill().catch(() => undefined))); + await service?.close(); +}); + +test("the port-forward url targets sandbox-agent over wss", () => { + assert.equal( + tunnelUrl("https://api.digitalocean.com", "sess-1", SANDBOX_AGENT_PORT), + "wss://api.digitalocean.com/v2/agents/sessions/sess-1/port-forward/8443", + ); + assert.equal( + tunnelUrl("http://127.0.0.1:9000/edge", "sess-2", SANDBOX_AGENT_PORT), + "ws://127.0.0.1:9000/edge/v2/agents/sessions/sess-2/port-forward/8443", + ); + assert.throws(() => tunnelUrl("ftp://example.com", "s", 8443), /must be http/); +}); + +test("the manifest asks for a bare sandbox and pre-allows bash so Managed Agents never gates a command", () => { + const yaml = managedAgentsManifest({ name: "qm-personal-tester", template: "base" }); + assert.match(yaml, /^name: "qm-personal-tester"$/m); + assert.match(yaml, /^agent: "none"$/m); + assert.match(yaml, /^template: "base"$/m); + assert.match(yaml, /tool: bash/); + assert.match(yaml, /action: allow/); + assert.doesNotMatch(yaml, /persistent_workspace/, "Managed Agents deprecated the key and warns when it is sent"); +}); + +test("the manifest omits the template so Managed Agents lands a bare session on its own base", () => { + const yaml = managedAgentsManifest({ name: "n", template: "" }); + assert.match(yaml, /^agent: "none"$/m); + assert.doesNotMatch(yaml, /^template:/m); +}); + +test("the manifest carries size, idle timeout and the egress allowlist only when configured", () => { + const bare = managedAgentsManifest({ name: "n", template: "base" }); + assert.doesNotMatch(bare, /size:|idle_timeout:|egress:/); + const full = managedAgentsManifest({ + name: "n", + template: "base", + sizeSlug: "mv-2vcpu-4gb", + idleTimeoutSec: 600, + egressAllow: ["proxy.example.com"], + }); + assert.match(full, /^size: "mv-2vcpu-4gb"$/m); + assert.match(full, /^idle_timeout: "600s"$/m); + assert.match(full, /^ {2}- "proxy\.example\.com"$/m); +}); + +test("the vendored proto is the client subset and carries nothing provider-internal", () => { + const proto = readFileSync(join(process.cwd(), "src/sandbox/managed-agents-sandbox-agent.proto"), "utf8"); + assert.match(proto, /service SandboxAgentService/); + for (const rpc of ["Exec", "Upload", "Download"]) assert.match(proto, new RegExp(`rpc ${rpc}\\(`)); + for (const unused of ["Shell", "ImportFromURL", "ExportToURL", "Health", "StartChildProcess", "ProxyPort"]) + assert.doesNotMatch(proto, new RegExp(`rpc ${unused}\\(`), `${unused} is unused and should not be vendored`); + for (const internal of [/do\/doge/, /dorpc/, /sandboxsvc/, /sandbox-service/, /microvm\.v1/, /MSANDBOX/, /169\.254/]) + assert.doesNotMatch(proto, internal, `${internal.source} is provider-internal`); +}); + +test("creating a session posts a yaml manifest and waits for ready", async () => { + const s = await session("qm-create"); + assert.ok(s.sessionId); + assert.equal(service.sessions().length, 1); + assert.match(service.manifests()[0] ?? "", /template: "base"/); + assert.equal(service.sessions()[0]?.template, "base"); +}); + +test("a command runs in the guest over the port-forward tunnel", async () => { + const s = await session("qm-exec"); + const result = await s.runCommand("echo hello-from-guest; echo oops >&2; exit 3"); + assert.equal(result.exitCode, 3); + assert.match(result.stdout, /hello-from-guest/); + assert.match(result.stderr, /oops/); + assert.ok(service.tunnelCount() > 0, "exec must travel through the tunnel, not the REST exec endpoint"); +}); + +test("a port-forward the edge refuses reads as gone, carrying the rejection, not as a half-run command", async () => { + const s = await session("qm-no-tunnel"); + service.rejectPortForward("team is not entitled to port-forward"); + const err = await s.runCommand("echo never-runs").then( + () => null, + (e: unknown) => e, + ); + assert.ok(err instanceof ManagedAgentsSandboxGoneError, `expected a retryable gone error, got ${String(err)}`); + assert.match(err.message, /403/); + assert.match(err.message, /not entitled to port-forward/); + assert.doesNotMatch(err.message, /may have partially executed/); + service.rejectPortForward(null); +}); + +test("a transient lifecycle failure while waiting for readiness is polled through, not surfaced", async () => { + const created = await session("qm-transient-ready"); + service.failNextLifecycleCalls(2, 403); + const reconnected = await client.connect(created.sessionId); + opened.push(reconnected); + assert.equal(reconnected.sessionId, created.sessionId); +}); + +test("a lifecycle failure that never clears is surfaced instead of polled to the deadline", async () => { + const created = await session("qm-stuck-ready"); + service.failNextLifecycleCalls(50, 403); + const err = await client.connect(created.sessionId).then( + () => null, + (e: unknown) => e, + ); + service.failNextLifecycleCalls(0, 403); + assert.ok(err instanceof Error, "expected the persistent failure to surface"); + assert.match(err.message, /403/); + assert.doesNotMatch(err.message, /did not become ready/); +}); + +test("command env and working directory reach the guest", async () => { + const s = await session("qm-env"); + const result = await s.runCommand("echo VAR=$MY_VAR", { env: { MY_VAR: "set-by-qm" } }); + assert.match(result.stdout, /VAR=set-by-qm/); +}); + +test("files round trip through the guest upload and download streams", async () => { + const s = await session("qm-files"); + const body = Buffer.from("managed agents file payload"); + await s.writeFileBytes("/home/user/notes/a.txt", new Uint8Array(body)); + const read = await s.readFileBytes("/home/user/notes/a.txt"); + assert.ok(read); + assert.equal(Buffer.from(read).toString("utf8"), "managed agents file payload"); +}); + +test("a missing file reads as null rather than an error", async () => { + const s = await session("qm-missing"); + assert.equal(await s.readFileBytes("/home/user/nope.txt"), null); +}); + +test("a session paused mid-command fails fast instead of stalling until the command timeout", async () => { + const watchful = createSdkManagedAgentsClient({ + apiToken: service.token, + apiBaseUrl: service.apiBaseUrl, + stallProbeMs: 50, + }); + const s = await watchful.create({ name: "qm-paused-midcommand" }); + opened.push(s); + const started = Date.now(); + const running = s.runCommand("sleep 30", { timeoutMs: 120_000 }); + await new Promise((resolve) => setTimeout(resolve, 150)); + service.setStatus("qm-paused-midcommand", "SESSION_STATUS_PAUSED"); + const err = await running.then( + () => null, + (e: unknown) => e, + ); + assert.ok(err instanceof ManagedAgentsCommandLostError, `expected a command-lost error, got ${String(err)}`); + assert.ok(Date.now() - started < 10_000, `expected a fast failure, took ${Date.now() - started}ms`); +}); + +test("the tunnel heartbeats so an edge idle timeout cannot kill a long silent command", async () => { + const created = await session("qm-heartbeat"); + const before = service.pingCount(); + const tunnel = await openManagedAgentsTunnel({ + apiBaseUrl: service.apiBaseUrl, + sessionId: created.sessionId, + remotePort: SANDBOX_AGENT_PORT, + getToken: async () => service.token, + pingIntervalMs: 20, + }); + try { + const probe = connect(tunnel.localPort, "127.0.0.1"); + await once(probe, "connect"); + await new Promise((resolve) => setTimeout(resolve, 400)); + assert.ok(service.pingCount() > before, `expected heartbeat pings, saw ${service.pingCount() - before}`); + probe.destroy(); + } finally { + await tunnel.close(); + } +}); + +test("a large file survives the chunked upload path", async () => { + const s = await session("qm-large"); + const body = Buffer.alloc(900 * 1024, "x"); + await s.writeFileBytes("/home/user/big.bin", new Uint8Array(body)); + const read = await s.readFileBytes("/home/user/big.bin"); + assert.equal(read?.length, body.length); +}); + +test("list resolves a session by name and skips destroyed rows", async () => { + const s = await session("qm-list"); + const found = await client.list("qm-list"); + assert.equal(found.length, 1); + assert.equal(found[0]?.sessionId, s.sessionId); + await s.kill(); + assert.deepEqual(await client.list("qm-list"), []); +}); + +test("connect resumes a paused session before handing it back", async () => { + const s = await session("qm-paused"); + await s.pause(); + assert.equal((await client.info(s.sessionId)).state, "paused"); + const again = await client.connect(s.sessionId); + assert.equal((await client.info(again.sessionId)).state, "ready"); + const result = await again.runCommand("echo awake"); + assert.match(result.stdout, /awake/); +}); + +test("a destroyed session reports a terminal state, and reconnecting to it fails fast", async () => { + const s = await session("qm-gone"); + await client.kill(s.sessionId); + assert.equal((await client.info(s.sessionId)).state, "destroyed"); + await assert.rejects(() => client.connect(s.sessionId), ManagedAgentsSandboxGoneError); +}); + +test("a session the control plane has never heard of reads as gone", async () => { + await assert.rejects(() => client.info("11111111-2222-3333-4444-555555555555"), ManagedAgentsSandboxGoneError); +}); + +test("a bad token is refused by the control plane", async () => { + const wrong = createSdkManagedAgentsClient({ apiToken: "nope", apiBaseUrl: service.apiBaseUrl, template: "base" }); + await assert.rejects(() => wrong.create({ name: "qm-unauth" }), /401|bad token/); +}); + +test("session status wire values map onto backend states", () => { + assert.equal(managedAgentsSessionState("SESSION_STATUS_READY"), "ready"); + assert.equal(managedAgentsSessionState("SESSION_STATUS_PAUSED"), "paused"); + assert.equal(managedAgentsSessionState("nonsense"), "unspecified"); +}); + +test("the egress allowlist carries just the proxy host", () => { + assert.deepEqual(managedAgentsEgressAllow("https://egress.example.com:443/path"), ["egress.example.com"]); +}); diff --git a/test/managed-agents-sandbox.test.ts b/test/managed-agents-sandbox.test.ts new file mode 100644 index 0000000000..be5d27a8a9 --- /dev/null +++ b/test/managed-agents-sandbox.test.ts @@ -0,0 +1,222 @@ +import { test, after, beforeEach } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pollProcess } from "../src/sandbox/process-poll.ts"; +import { createManagedAgentsSandbox, type StoredManagedAgentsSandbox } from "../src/sandbox/managed-agents-sandbox.ts"; +import { + managedAgentsEgressAllow, + managedAgentsSessionState, + ManagedAgentsHitlRejectedError, +} from "../src/sandbox/managed-agents-client.ts"; +import { sandboxScopeName } from "../src/sandbox/exec-sandbox-base.ts"; +import { createLocalWorkspaceStore } from "../src/workspace/workspace-store.ts"; +import { supportsProcessSessions } from "../src/sandbox/sandbox.ts"; +import { createMemoryMap, type DurableMap } from "../src/persistence/durable-map.ts"; +import { scopeId } from "../src/types.ts"; +import { installFakeManagedAgents, type FakeManagedAgents } from "./support/fake-managed-agents.ts"; +import type { Sandbox } from "../src/sandbox/sandbox.ts"; + +let fake: FakeManagedAgents; +let sandbox: Sandbox; +const scope = scopeId("personal", "tester"); +const layers = [{ scopeId: scope, mountPath: "/", mode: "rw" as const }]; +const scopeName = (): string => sandboxScopeName("qmt", scope); + +function make(extra: Record = {}): Sandbox { + return createManagedAgentsSandbox(createLocalWorkspaceStore(mkdtempSync(join(tmpdir(), "managed-agents-ws-"))), { + client: fake.client, + namePrefix: "qmt", + ...extra, + }); +} + +beforeEach(() => { + fake = installFakeManagedAgents(); + sandbox = make(); +}); + +after(() => fake?.cleanup()); + +test("provision runs commands with env and cwd", async () => { + const h = await sandbox.provision(layers, { env: { MY_VAR: "v1" } }); + assert.equal(h.coldStart, true); + const r = await sandbox.run(h, "pwd; echo VAR=$MY_VAR"); + assert.equal(r.code, 0); + assert.match(r.stdout, /workspace/); + assert.match(r.stdout, /VAR=v1/); +}); + +test("one session per scope is reused across provisions", async () => { + const first = await sandbox.provision(layers); + await sandbox.teardown(first, { keepWarm: true }); + const second = await sandbox.provision(layers); + assert.equal(second.coldStart, false); + assert.equal(fake.createdCount(scopeName()), 1); +}); + +test("the session is named after the scope so an operator can find it", async () => { + await sandbox.provision(layers); + const current = fake.current(scopeName()); + assert.ok(current, "expected a session registered under the scope name"); + assert.ok(scopeName().length <= 64, "Managed Agents caps session names at 64 characters"); + assert.doesNotMatch(scopeName(), /^[0-9a-f]{8}-[0-9a-f]{4}-/, "Managed Agents rejects UUID-shaped session names"); +}); + +test("teardown pauses the session and the next provision resumes it", async () => { + const h = await sandbox.provision(layers); + await sandbox.run(h, "echo hi > kept.txt"); + await sandbox.teardown(h); + assert.deepEqual(fake.pauseCalls().length, 1); + assert.equal(fake.current(scopeName())?.state, "paused"); + + const again = await sandbox.provision(layers); + assert.equal(fake.resumeCalls().length, 1); + const r = await sandbox.run(again, "cat kept.txt"); + assert.match(r.stdout, /hi/); + assert.equal(fake.createdCount(scopeName()), 1); +}); + +test("keepWarm teardown leaves the session running for background work", async () => { + const h = await sandbox.provision(layers); + await sandbox.teardown(h, { keepWarm: true }); + assert.equal(fake.pauseCalls().length, 0); + assert.equal(fake.current(scopeName())?.state, "ready"); +}); + +test("a lost session is re-adopted by name when the durable record is gone", async () => { + const store: DurableMap = createMemoryMap(); + sandbox = make({ store }); + await sandbox.provision(layers); + const before = fake.current(scopeName()); + await store.delete(scope); + + const again = await sandbox.provision(layers); + assert.equal(again.coldStart, false); + assert.equal(fake.createdCount(scopeName()), 1); + assert.equal(fake.current(scopeName())?.sessionId, before?.sessionId); +}); + +test("a destroyed session is replaced on the next provision", async () => { + await sandbox.provision(layers); + fake.destroy(scopeName()); + const again = await sandbox.provision(layers); + assert.ok(again.id); + assert.equal(fake.createdCount(scopeName()), 2); +}); + +test("destroying a scope disconnects the live guest rather than only deleting the session", async () => { + const h = await sandbox.provision(layers); + await sandbox.run(h, "true"); + const live = fake.current(scopeName()); + assert.ok(live, "expected a live session before destroying the scope"); + await sandbox.destroyScope!(scope); + assert.deepEqual(fake.guestDisconnects(), [live.sessionId]); + assert.equal(fake.current(scopeName()), null); +}); + +test("a command lost mid-flight is surfaced rather than silently retried", async () => { + const h = await sandbox.provision(layers); + fake.loseNextCommand(scopeName()); + await assert.rejects(() => sandbox.run(h, "echo nope"), /was lost while a command was running/); +}); + +test("a Managed Agents approval rejection names the permissions fix", async () => { + const h = await sandbox.provision(layers); + fake.rejectNextCommand(scopeName()); + await assert.rejects(() => sandbox.run(h, "echo nope"), ManagedAgentsHitlRejectedError); + await assert.rejects(() => { + fake.rejectNextCommand(scopeName()); + return sandbox.run(h, "echo nope"); + }, /permissions to allow bash/); +}); + +test("files round trip through the data plane", async () => { + const h = await sandbox.provision(layers); + await sandbox.writeFile(h, "notes/a.txt", "hello managed agents"); + assert.equal(await sandbox.readFile(h, "notes/a.txt"), "hello managed agents"); + assert.equal(await sandbox.readFile(h, "notes/missing.txt"), null); +}); + +test("destroy removes the session and its durable record", async () => { + const store: DurableMap = createMemoryMap(); + sandbox = make({ store }); + const h = await sandbox.provision(layers); + await sandbox.teardown(h, { destroy: true }); + assert.equal(await store.get(scope), null); + assert.equal(fake.current(scopeName()), null); +}); + +test("computerStatus reports a paused machine without waking it", async () => { + const h = await sandbox.provision(layers); + await sandbox.teardown(h); + const status = await sandbox.computerStatus!(scope); + assert.equal(status.provisioned, true); + assert.equal(status.guestResponsive, false); + assert.equal(status.lifecycleState, "paused"); + assert.equal(fake.resumeCalls().length, 0); +}); + +test("computerStatus probes a running machine", async () => { + await sandbox.provision(layers); + const status = await sandbox.computerStatus!(scope); + assert.equal(status.provisioned, true); + assert.equal(status.guestResponsive, true); +}); + +test("computerStatus on an unprovisioned scope reports nothing provisioned", async () => { + const status = await sandbox.computerStatus!(scope); + assert.equal(status.provisioned, false); + assert.equal(status.guestResponsive, false); +}); + +test("scratch sandboxes are reference counted and never persisted", async () => { + const store: DurableMap = createMemoryMap(); + sandbox = make({ store }); + const a = await sandbox.provision(layers, { scratch: { key: "k1" } }); + const b = await sandbox.provision(layers, { scratch: { key: "k1" } }); + assert.equal(a.id, b.id); + assert.equal(a.scratch, true); + await sandbox.teardown(a); + assert.ok(fake.current(a.id), "still held by the second handle"); + await sandbox.teardown(b); + assert.equal(fake.current(a.id), null); + assert.equal(await store.get(scope), null); +}); + +test("the profile advertises provider-managed persistence and parks on teardown", () => { + assert.equal(sandbox.profile.backend, "do-managed-agents"); + assert.equal(sandbox.profile.writablePersistence, "provider_managed"); + assert.equal(sandbox.profile.parksOnTeardown, true); + assert.equal(supportsProcessSessions(sandbox), true); +}); + +test("egress enforcement is only claimed when a proxy is configured", () => { + assert.equal(sandbox.profile.egressEnforcement, "none"); + assert.equal(make({ egressProxyUrl: "https://egress.example.com" }).profile.egressEnforcement, "domain"); +}); + +test("the egress allowlist carries just the proxy host", () => { + assert.deepEqual(managedAgentsEgressAllow("https://egress.example.com:443/path"), ["egress.example.com"]); + assert.deepEqual(managedAgentsEgressAllow("http://10.0.0.7:8080"), ["10.0.0.7"]); +}); + +test("session status wire values map onto backend states", () => { + assert.equal(managedAgentsSessionState("SESSION_STATUS_READY"), "ready"); + assert.equal(managedAgentsSessionState("SESSION_STATUS_PAUSED"), "paused"); + assert.equal(managedAgentsSessionState("SESSION_STATUS_DESTROYED"), "destroyed"); + assert.equal(managedAgentsSessionState("nonsense"), "unspecified"); + assert.equal(managedAgentsSessionState(undefined), "unspecified"); +}); + +test("process sessions stream output through the data plane exec path", async () => { + assert.ok(supportsProcessSessions(sandbox)); + if (!supportsProcessSessions(sandbox)) return; + const h = await sandbox.provision(layers); + const { processId } = await sandbox.startProcess(h, "echo one; echo two"); + const { output, status } = await pollProcess(sandbox, h, processId, { deadlineMs: 5_000, waitMs: 100 }); + assert.equal(status.state, "exited"); + assert.match(output, /one/); + assert.match(output, /two/); +}); diff --git a/test/support/fake-managed-agents-service.ts b/test/support/fake-managed-agents-service.ts new file mode 100644 index 0000000000..0f72382ee2 --- /dev/null +++ b/test/support/fake-managed-agents-service.ts @@ -0,0 +1,291 @@ +import { createServer as createHttpServer, type Server as HttpServer, type ServerResponse } from "node:http"; +import { connect, type Socket } from "node:net"; +import { once } from "node:events"; +import { fileURLToPath } from "node:url"; +import { randomUUID } from "node:crypto"; +import { spawn } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { WebSocketServer, type WebSocket } from "ws"; +import * as loader from "@grpc/proto-loader"; +import * as grpc from "@grpc/grpc-js"; + +const PROTO_PATH = fileURLToPath(new URL("../../src/sandbox/managed-agents-sandbox-agent.proto", import.meta.url)); + +interface SessionRow { + session_id: string; + sandbox_id: string; + name: string; + status: string; + template: string; + created_at: string; +} + +export interface FakeManagedAgentsService { + apiBaseUrl: string; + token: string; + sessions(): SessionRow[]; + manifests(): string[]; + execScripts(): string[]; + home(): string; + setStatus(name: string, status: string): void; + rejectPortForward(body: string | null): void; + failNextLifecycleCalls(count: number, status: number): void; + tunnelCount(): number; + pingCount(): number; + close(): Promise; +} + +function loadService(): grpc.ServiceDefinition { + const definition = loader.loadSync(PROTO_PATH, { + keepCase: true, + longs: String, + enums: String, + defaults: true, + oneofs: true, + }); + const pkg = grpc.loadPackageDefinition(definition) as unknown as { + do: { + teams: { + hosted_agents: { runtime: { sandbox_agent: { v1: { SandboxAgentService: grpc.ServiceClientConstructor } } } }; + }; + }; + }; + return pkg.do.teams.hosted_agents.runtime.sandbox_agent.v1.SandboxAgentService.service; +} + +export async function startFakeManagedAgentsService(): Promise { + const token = "dop_v1_test-token"; + const root = mkdtempSync(join(tmpdir(), "fake-managed-agents-guest-")); + const rows = new Map(); + const manifests: string[] = []; + const execScripts: string[] = []; + let tunnels = 0; + let pings = 0; + let lifecycleFailures = 0; + let lifecycleFailureStatus = 403; + + const guest = new grpc.Server(); + guest.addService(loadService(), { + Exec: (call: grpc.ServerDuplexStream, unknown>) => { + call.on("data", (frame: { start?: { argv?: string[]; workdir?: string; env?: Record } }) => { + if (!frame.start) return; + const argv = frame.start.argv ?? []; + const script = argv[argv.length - 1] ?? ""; + execScripts.push(script); + try { + const cwd = (frame.start.workdir || root).replace(/^\/home\/user/, root); + mkdirSync(cwd, { recursive: true }); + const child = spawn("sh", ["-c", script.replace(/\/home\/user/g, root)], { + cwd, + env: { ...process.env, ...frame.start.env, HOME: root }, + }); + child.stdout.on("data", (chunk: Buffer) => call.write({ stdout: chunk })); + child.stderr.on("data", (chunk: Buffer) => call.write({ stderr: chunk })); + child.on("close", (code: number | null) => { + call.write({ exit: { exit_code: code ?? -1, duration_ms: 1, hitl_rejected: false } }); + call.end(); + }); + child.on("error", (e: Error) => { + call.write({ stderr: Buffer.from(e.message) }); + call.write({ exit: { exit_code: 127, duration_ms: 0, hitl_rejected: false } }); + call.end(); + }); + } catch (e) { + call.write({ stderr: Buffer.from(e instanceof Error ? e.message : String(e)) }); + call.write({ exit: { exit_code: 127, duration_ms: 0, hitl_rejected: false } }); + call.end(); + } + }); + }, + Upload: (call: grpc.ServerReadableStream, unknown>, cb: grpc.sendUnaryData) => { + let path = ""; + const chunks: Buffer[] = []; + call.on("data", (frame: { header?: { path?: string }; chunk?: Buffer }) => { + if (frame.header?.path) path = frame.header.path; + if (frame.chunk?.length) chunks.push(frame.chunk); + }); + call.on("end", () => { + const hostPath = path.replace(/^\/home\/user/, root); + mkdirSync(dirname(hostPath), { recursive: true }); + const body = Buffer.concat(chunks); + writeFileSync(hostPath, body); + cb(null, { path, bytes_written: String(body.length) }); + }); + }, + Download: (call: grpc.ServerWritableStream<{ path?: string }, unknown>) => { + const hostPath = (call.request.path ?? "").replace(/^\/home\/user/, root); + let body: Buffer; + try { + body = readFileSync(hostPath); + } catch { + call.emit("error", { + code: grpc.status.INTERNAL, + details: `envd download: download status 404: {"code":404,"message":"path '${call.request.path ?? ""}' does not exist"}`, + }); + return; + } + call.write({ header: { size_bytes: String(body.length), is_archive: false } }); + call.write({ chunk: body }); + call.write({ end: { sha256: "" } }); + call.end(); + }, + }); + + const guestPort = await new Promise((resolve, reject) => { + guest.bindAsync("127.0.0.1:0", grpc.ServerCredentials.createInsecure(), (err, port) => + err ? reject(err) : resolve(port), + ); + }); + + const byName = (name: string): SessionRow | undefined => + [...rows.values()].find((r) => r.name === name && r.status !== "SESSION_STATUS_DESTROYED"); + + const json = (res: ServerResponse, code: number, body: unknown): void => { + res.writeHead(code, { "content-type": "application/json" }); + res.end(JSON.stringify(body)); + }; + + const http: HttpServer = createHttpServer((req, res) => { + if (req.headers.authorization !== `Bearer ${token}`) { + json(res, 401, { error: { code: "unauthorized", message: "bad token" } }); + return; + } + const url = new URL(req.url ?? "/", "http://localhost"); + const parts = url.pathname.split("/").filter(Boolean); + const id = parts[3] ?? ""; + const action = parts[4] ?? ""; + + if (req.method === "POST" && url.pathname === "/v2/agents/sessions") { + let body = ""; + req.on("data", (c: Buffer) => (body += c.toString("utf8"))); + req.on("end", () => { + manifests.push(body); + const name = /^name:\s*"?([^"\n]+)"?/m.exec(body)?.[1] ?? "unnamed"; + const row: SessionRow = { + session_id: randomUUID(), + sandbox_id: `sbx-${rows.size + 1}`, + name, + status: "SESSION_STATUS_READY", + template: /^template:\s*"?([^"\n]+)"?/m.exec(body)?.[1] ?? "", + created_at: new Date().toISOString(), + }; + rows.set(row.session_id, row); + json(res, 201, { session: row }); + }); + return; + } + if (req.method === "GET" && url.pathname === "/v2/agents/sessions") { + const name = url.searchParams.get("name"); + const all = [...rows.values()].filter((r) => !name || r.name === name); + json(res, 200, { sessions: all, next_page_token: "" }); + return; + } + if (lifecycleFailures > 0) { + lifecycleFailures -= 1; + json(res, lifecycleFailureStatus, { error: { code: "forbidden", message: "transient" } }); + return; + } + if (req.method === "GET" && parts.length === 4) { + const row = rows.get(id); + if (!row) { + json(res, 404, { error: { code: "not_found", message: "no such session" } }); + return; + } + json(res, 200, { session: row }); + return; + } + if (req.method === "DELETE" && parts.length === 4) { + const row = rows.get(id); + if (row) row.status = "SESSION_STATUS_DESTROYED"; + res.writeHead(204).end(); + return; + } + if (req.method === "POST" && (action === "pause" || action === "resume")) { + const row = rows.get(id); + if (!row) { + json(res, 404, { error: { code: "not_found", message: "no such session" } }); + return; + } + row.status = action === "pause" ? "SESSION_STATUS_PAUSED" : "SESSION_STATUS_READY"; + res.writeHead(204).end(); + return; + } + json(res, 404, { error: { code: "not_found", message: url.pathname } }); + }); + + const wss = new WebSocketServer({ noServer: true }); + let portForwardRejection: string | null = null; + http.on("upgrade", (req, socket, head) => { + const url = new URL(req.url ?? "/", "http://localhost"); + const match = /^\/v2\/agents\/sessions\/([^/]+)\/port-forward\/(\d+)$/.exec(url.pathname); + if (req.headers.authorization !== `Bearer ${token}` || !match) { + socket.destroy(); + return; + } + if (portForwardRejection !== null) { + socket.end( + `HTTP/1.1 403 Forbidden\r\nContent-Length: ${Buffer.byteLength(portForwardRejection)}\r\nConnection: close\r\n\r\n${portForwardRejection}`, + ); + return; + } + const row = rows.get(match[1] ?? ""); + if (!row || row.status === "SESSION_STATUS_DESTROYED") { + socket.destroy(); + return; + } + wss.handleUpgrade(req, socket, head, (ws: WebSocket) => { + tunnels += 1; + ws.on("ping", () => { + pings += 1; + }); + const upstream: Socket = connect(guestPort, "127.0.0.1"); + ws.on("message", (data: Buffer) => upstream.write(data)); + upstream.on("data", (chunk: Buffer) => ws.send(chunk)); + const shutdown = (): void => { + upstream.destroy(); + if (ws.readyState === ws.OPEN) ws.close(); + }; + ws.on("close", shutdown); + ws.on("error", shutdown); + upstream.on("close", shutdown); + upstream.on("error", shutdown); + }); + }); + + http.listen(0, "127.0.0.1"); + await once(http, "listening"); + const address = http.address(); + if (address === null || typeof address === "string") throw new Error("fake managed agents service failed to bind"); + + return { + apiBaseUrl: `http://127.0.0.1:${address.port}`, + token, + sessions: () => [...rows.values()], + manifests: () => [...manifests], + execScripts: () => [...execScripts], + home: () => root, + setStatus: (name, status) => { + const row = byName(name); + if (row) row.status = status; + }, + rejectPortForward: (body) => { + portForwardRejection = body; + }, + failNextLifecycleCalls: (count, status) => { + lifecycleFailures = count; + lifecycleFailureStatus = status; + }, + tunnelCount: () => tunnels, + pingCount: () => pings, + close: async () => { + for (const ws of wss.clients) ws.terminate(); + wss.close(); + http.closeAllConnections(); + await new Promise((resolve) => http.close(() => resolve())); + guest.forceShutdown(); + rmSync(root, { recursive: true, force: true }); + }, + }; +} diff --git a/test/support/fake-managed-agents.ts b/test/support/fake-managed-agents.ts new file mode 100644 index 0000000000..2eb9e57a16 --- /dev/null +++ b/test/support/fake-managed-agents.ts @@ -0,0 +1,226 @@ +import { spawnSync } from "node:child_process"; +import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { + ManagedAgentsCommandLostError, + ManagedAgentsHitlRejectedError, + ManagedAgentsSandboxGoneError, + type ManagedAgentsClient, + type ManagedAgentsCommandResult, + type ManagedAgentsSession, + type ManagedAgentsSessionInfo, + type ManagedAgentsSessionState, + type ManagedAgentsSessionSummary, +} from "../../src/sandbox/managed-agents-client.ts"; +import { mkdtempSync } from "node:fs"; + +const GUEST_HOME_DIR = "/workspace/home"; + +interface FakeRecord { + sessionId: string; + sandboxId: string; + name: string; + state: ManagedAgentsSessionState; + home: string; + createdAt: number; + loseNextCommand: boolean; + rejectNextCommand: boolean; +} + +export interface FakeManagedAgents { + client: ManagedAgentsClient; + current(name: string): { sessionId: string; sandboxId: string; state: ManagedAgentsSessionState } | null; + createdCount(name: string): number; + homeDir(name: string): string; + pause(name: string): void; + destroy(name: string): void; + loseNextCommand(name: string): void; + rejectNextCommand(name: string): void; + execScripts(): string[]; + pauseCalls(): string[]; + resumeCalls(): string[]; + guestDisconnects(): string[]; + cleanup(): void; +} + +export function installFakeManagedAgents(): FakeManagedAgents { + const root = mkdtempSync(join(tmpdir(), "fake-managed-agents-")); + const records = new Map(); + const execScripts: string[] = []; + const pauseCalls: string[] = []; + const resumeCalls: string[] = []; + const guestDisconnects: string[] = []; + let nextId = 1; + let clock = 0; + + const gone = (state: ManagedAgentsSessionState): boolean => + state === "destroyed" || state === "destroying" || state === "failed"; + + const byName = (name: string): FakeRecord | undefined => { + const all = [...records.values()].filter((r) => r.name === name).sort((a, b) => b.createdAt - a.createdAt); + return all.find((r) => !gone(r.state)) ?? all[0]; + }; + + const need = (name: string): FakeRecord => { + const r = byName(name); + if (!r) throw new Error(`fake-managed-agents: no session named ${name}`); + return r; + }; + + const remap = (r: FakeRecord, script: string): string => { + const homeRe = r.home.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const remapPath = new RegExp(`${homeRe}/tmp/|${homeRe}(?![A-Za-z0-9._-])|/tmp/`, "g"); + return ( + `export HOME=${JSON.stringify(r.home)}; ` + + script + .replace(/\btimeout \d+ /g, "") + .replaceAll(GUEST_HOME_DIR, r.home) + .replace(remapPath, (mm) => (mm.startsWith(r.home) ? mm : `${r.home}/tmp/`)) + ); + }; + + const alive = (r: FakeRecord): void => { + if (gone(r.state)) throw new ManagedAgentsSandboxGoneError(r.sessionId, `status ${r.state}`); + if (r.state === "paused") r.state = "ready"; + }; + + const info = (r: FakeRecord): ManagedAgentsSessionInfo => ({ + sessionId: r.sessionId, + sandboxId: r.sandboxId, + name: r.name, + state: r.state, + createdAtMs: r.createdAt, + }); + + const session = (r: FakeRecord): ManagedAgentsSession => ({ + sessionId: r.sessionId, + sandboxId: r.sandboxId, + async runCommand(command): Promise { + alive(r); + if (r.loseNextCommand) { + r.loseNextCommand = false; + r.state = "destroyed"; + throw new ManagedAgentsCommandLostError(r.sessionId, "sandbox is not running anymore"); + } + if (r.rejectNextCommand) { + r.rejectNextCommand = false; + throw new ManagedAgentsHitlRejectedError(r.sessionId); + } + execScripts.push(command); + mkdirSync(join(r.home, "tmp"), { recursive: true }); + const spawned = spawnSync("sh", ["-c", remap(r, command)], { + encoding: "buffer", + maxBuffer: 128 * 1024 * 1024, + env: { ...process.env, COPYFILE_DISABLE: "1" }, + }); + return { + stdout: (spawned.stdout ?? Buffer.alloc(0)).toString("utf8"), + stderr: (spawned.stderr ?? Buffer.alloc(0)).toString("utf8"), + exitCode: spawned.status ?? (spawned.signal ? 137 : -1), + }; + }, + async readFileBytes(absPath): Promise { + alive(r); + const hostPath = absPath.startsWith(GUEST_HOME_DIR) ? r.home + absPath.slice(GUEST_HOME_DIR.length) : absPath; + if (!existsSync(hostPath)) return null; + return new Uint8Array(readFileSync(hostPath)); + }, + async writeFileBytes(absPath, data): Promise { + alive(r); + const hostPath = absPath.startsWith(GUEST_HOME_DIR) ? r.home + absPath.slice(GUEST_HOME_DIR.length) : absPath; + mkdirSync(dirname(hostPath), { recursive: true }); + writeFileSync(hostPath, Buffer.from(data)); + }, + async pause(): Promise { + if (gone(r.state)) throw new ManagedAgentsSandboxGoneError(r.sessionId, `status ${r.state}`); + guestDisconnects.push(r.sessionId); + pauseCalls.push(r.sessionId); + r.state = "paused"; + }, + async resume(): Promise { + if (gone(r.state)) throw new ManagedAgentsSandboxGoneError(r.sessionId, `status ${r.state}`); + resumeCalls.push(r.sessionId); + r.state = "ready"; + }, + async kill(): Promise { + guestDisconnects.push(r.sessionId); + r.state = "destroyed"; + rmSync(r.home, { recursive: true, force: true }); + }, + }); + + const client: ManagedAgentsClient = { + nativePause: true, + async info(sessionId): Promise { + const r = records.get(sessionId); + if (!r || gone(r.state)) throw new ManagedAgentsSandboxGoneError(sessionId, "session was not found"); + return info(r); + }, + async create(opts): Promise { + const id = `sess-${nextId++}`; + const r: FakeRecord = { + sessionId: id, + sandboxId: `sbx-${id}`, + name: opts.name, + state: "ready", + home: join(root, id), + createdAt: ++clock, + loseNextCommand: false, + rejectNextCommand: false, + }; + mkdirSync(r.home, { recursive: true }); + records.set(id, r); + return session(r); + }, + async connect(sessionId): Promise { + const r = records.get(sessionId); + if (!r || gone(r.state)) throw new ManagedAgentsSandboxGoneError(sessionId, "session was not found"); + if (r.state === "paused") { + resumeCalls.push(r.sessionId); + r.state = "ready"; + } + return session(r); + }, + async list(name): Promise { + return [...records.values()] + .filter((r) => !gone(r.state) && r.name === name) + .map((r) => ({ sessionId: r.sessionId, sandboxId: r.sandboxId, name: r.name, state: r.state })); + }, + async kill(sessionId): Promise { + const r = records.get(sessionId); + if (!r) return; + r.state = "destroyed"; + rmSync(r.home, { recursive: true, force: true }); + }, + }; + + return { + client, + current: (name) => { + const r = byName(name); + return r && !gone(r.state) ? { sessionId: r.sessionId, sandboxId: r.sandboxId, state: r.state } : null; + }, + createdCount: (name) => [...records.values()].filter((r) => r.name === name).length, + homeDir: (name) => need(name).home, + pause: (name) => { + need(name).state = "paused"; + }, + destroy: (name) => { + const r = need(name); + r.state = "destroyed"; + rmSync(r.home, { recursive: true, force: true }); + }, + loseNextCommand: (name) => { + need(name).loseNextCommand = true; + }, + rejectNextCommand: (name) => { + need(name).rejectNextCommand = true; + }, + execScripts: () => [...execScripts], + pauseCalls: () => [...pauseCalls], + resumeCalls: () => [...resumeCalls], + guestDisconnects: () => [...guestDisconnects], + cleanup: () => rmSync(root, { recursive: true, force: true }), + }; +} From dff5ca948b5ebbe18b3da8c114f4e1acf8862a4e Mon Sep 17 00:00:00 2001 From: Naveen Kurra Date: Mon, 5 Oct 2026 12:33:33 -0400 Subject: [PATCH 03/19] Document the Managed Agents sandbox backend docs/managed-agents.md covers what the backend provides, the two planes it talks to, and each operation: create, exec, upload and download, pause and resume, and delete. Pause and resume are both automatic and there is no manual control for either, which is worth stating plainly because the sandbox tool offers no verb for them. The gotchas are the things that cost real debugging time. Deleting a conversation does not delete the microVM, because this backend implements no deep-idle reaping. PUBLIC_API_URL has to be reachable from inside the guest, since microVMs call back into core for connector credentials and file callbacks, and a deployment behind localhost runs shell commands fine while failing every cron and send. The bare base carries no jq, rg or unzip, all of which prompts reach for by name. The ADR records the two asks on the provider and why the vendored proto is a stopgap rather than a choice. --- README.md | 1 + adrs/managed-agents-sandbox-backend.md | 96 ++++++++++ docs/managed-agents.md | 255 +++++++++++++++++++++++++ docs/sandbox-preservation.md | 2 + 4 files changed, 354 insertions(+) create mode 100644 adrs/managed-agents-sandbox-backend.md create mode 100644 docs/managed-agents.md diff --git a/README.md b/README.md index 80b54cab83..d42902d6d2 100644 --- a/README.md +++ b/README.md @@ -242,6 +242,7 @@ upstream source history to merge. - [`docs/principal-links.md`](./docs/principal-links.md) — one person, several sign-ins: linking principals - [`docs/porter.md`](./docs/porter.md) — running qm on Porter - [`docs/superserve.md`](./docs/superserve.md) — using Superserve for agent sandboxes +- [`docs/managed-agents.md`](./docs/managed-agents.md) — using DigitalOcean Managed Agents microVMs for agent sandboxes - [`.env.example`](./.env.example) — every knob, documented in place - [`docs/swarms.md`](./docs/swarms.md) — durable agent pools, scoped messages, and blank Modal workers - [`docs/model-gateway.md`](./docs/model-gateway.md) — discover and route models through a gateway diff --git a/adrs/managed-agents-sandbox-backend.md b/adrs/managed-agents-sandbox-backend.md new file mode 100644 index 0000000000..a6362ea3b1 --- /dev/null +++ b/adrs/managed-agents-sandbox-backend.md @@ -0,0 +1,96 @@ +# Managed Agents as an agent-computer backend + +Naveen, from the Managed Agents team at DigitalOcean. Managed Agents (Managed Agent Runtime Stack) hosts coding agents in DO-managed Firecracker microVMs. We looked at what it would take for qm to run on it and the answer is the same one Agent37 arrived at: the part of qm that wants hosting is the agent computer, so this is another sandbox backend, next to E2B, Modal, Sprites, smolmachines, Porter, AWS, Superserve and Agent37. + +The reason this is worth doing at all is that Managed Agents is a managed-loop product by default — it boots Claude Code or OpenCode _inside_ the microVM, next to a runtime process that translates agent events back to our control plane and an event stream that `doctl agents attach` tails. That is not what qm wants, because qm already has a loop. But our platform now admits a session with no agent in it at all: a manifest that says `agent: none` lands on Managed Agents's own bare base template, with no managed agent, no event-translation runtime, and no model credential. The caller drives the microVM itself over exec, workspace transfer and port-forward. That mode exists for customers who want Managed Agents purely for code execution — evaluations, CI-style fan-out, arbitrary compute — and qm lands in the same seat. Nothing about qm's orchestrator, harness router or tool surface has to change. + +```mermaid +flowchart LR + subgraph MANAGED["Managed Agents default: managed loop"] + M1["microVM"] --> M2["Claude Code / OpenCode"] --> M3["event-translation runtime"] --> M4["event stream"] --> M5["doctl agents attach"] + end + + subgraph INVERTED["agent: none -- what qm uses"] + Q1["qm core
orchestrator + vendored harness"] --> Q2["the loop stays here"] + Q2 --> Q3["microVM
bare sandbox, no agent"] + end +``` + +The lifecycle lines up closely enough with the E2B backend that this is modeled on that pair of files rather than on a new shape. One Managed Agents session per qm scope, created on first use, named with the scope name so an operator can find it with `doctl agents list`, paused at teardown, deleted when qm destroys the scope. Our pause keeps the microVM's disk and memory, which is what `provider_managed` persistence already means in your profile. + +Everything qm needs is on one host with one credential. Session lifecycle is our control plane, which external callers reach as REST through the DO API edge, which validates a DO IAM token and forwards the authenticated team and user as headers — the same path doctl takes. So that half is plain HTTP+JSON with a token, and the team identity comes from the token rather than from configuration. + +Exec and file transfer take a different route to the same host. Every microVM runs `sandbox-agent`, a gRPC server on guest port 8443 that our own control plane dials for exec, transfer and readiness. qm reaches it the way `doctl agents port-forward` does: a bearer-authenticated WebSocket to `/v2/agents/sessions/{id}/port-forward/8443`, which the control plane bridges to the guest port and qm exposes as a local TCP listener for a gRPC client. 8443 is explicitly allowed by our tunnel port policy, the guest listener is plaintext h2c with no client certificate, and we register the port-forward route without a request deadline on purpose. So a long command is bounded by qm's own `SANDBOX_TIMEOUT_SEC` and nothing else. + +Worth saying why qm does not use `POST /v2/agents/sessions/{id}/sandbox/exec`, since we built it and it looks like the obvious fit. It terminates in the control plane and buffers the result, so it clamps to four minutes and one mebibyte per stream and carries no per-command environment. Those bounds are right for `doctl agents exec` and wrong for an agent turn. + +Here is a turn end to end. Lifecycle goes through the edge as REST; exec and files go through the tunnel to the guest. + +```mermaid +sequenceDiagram + participant O as qm orchestrator + participant B as managed-agents-sandbox.ts + participant E as DO API edge + participant H as control plane + participant T as port-forward tunnel + participant V as microVM
sandbox-agent :8443 + + Note over O,B: provision + O->>B: provision layers for scope + B->>E: POST /v2/agents/sessions
agents.yaml: name, agent: none, permissions, egress + Note right of E: validates DO IAM token,
forwards team + user headers + E->>H: CreateSession + H->>V: provision the microVM
internal, not qm's concern + H-->>B: session_id, PROVISIONING + B->>E: GET /v2/agents/sessions/{id} + E-->>B: READY + B->>T: wss /port-forward/8443
Authorization: Bearer + T->>V: bridge to guest :8443 + B->>V: Upload
ro layers + layer tool files + B-->>O: handle + + Note over O,B: run + O->>B: run command + B->>V: Exec stream over the tunnel
argv, env, workdir, timeout + V-->>B: stdout, stderr, exit + B-->>O: ExecResult + V->>O: self-API callback with capability token
needs core's URL on the egress allowlist + + Note over O,B: teardown + O->>B: teardown + alt keepWarm for background work + B->>B: leave the session running + else park the scope + B->>V: Download
portable tar home snapshot + B->>E: POST /v2/agents/sessions/{id}/pause + else destroy the scope + B->>E: DELETE /v2/agents/sessions/{id} + end +``` + +What the PR adds + +- `SANDBOX_BACKEND=do-managed-agents`, with `DO_AGENTS_API_TOKEN` required — a DO IAM token, which is also where the team identity comes from, and the only credential the backend needs. Optional knobs: `DO_AGENTS_API_BASE_URL`, `DO_AGENTS_TEMPLATE`, `DO_AGENTS_NAME_PREFIX`, `DO_AGENTS_SIZE_SLUG`, `DO_AGENTS_IDLE_TIMEOUT_SEC`, `DO_AGENTS_EGRESS_PROXY_URL`, `DO_AGENTS_SNAPSHOT_INTERVAL_SEC`, `DO_AGENTS_SNAPSHOT_S3_BUCKET`, and the shared `SANDBOX_TIMEOUT_SEC`. +- `src/sandbox/managed-agents-tunnel.ts`, the port-forward client: one WebSocket per TCP connection, bytes piped both ways with one chunk in flight per direction so a slow consumer cannot make core buffer the stream. The local listener is unreferenced, so an abandoned tunnel can never hold core's event loop open. +- `src/sandbox/managed-agents-client.ts`, holding both transports behind one interface the way `e2b-client.ts` wraps the E2B SDK: REST for `/v2/agents/sessions{,/{id},/{id}/pause,/{id}/resume}`, and `SandboxAgentService` stubs for `Exec`, `Upload` and `Download` over the tunnel. Our 404s, terminal statuses and gRPC `UNAVAILABLE` translate to the same two error classes the E2B client uses — one for a sandbox that was already gone before a command started, which the backend reconnects and retries, and one for a sandbox lost mid-command, which is reported rather than retried because the command may have partially run. The session body is an agents.yaml manifest in its flat form, which is the shape we are standardising on. +- `src/sandbox/managed-agents-sandbox-agent.proto`, a vendored copy of our guest proto with the internal annotations stripped, loaded at runtime through `@grpc/proto-loader`. No build step and no generated code in the tree. +- `src/sandbox/managed-agents-sandbox.ts`. Provision creates or resumes the session and waits for it to become usable. Run is `Exec` over the tunnel. Files are `Upload` and `Download` rather than base64 through exec, since we have real transfer RPCs. Process sessions, read-only layers, layer tool install, home snapshots and blob staging come from the shared exec helpers unchanged. +- Scope recovery without local state. Session names are team-unique among non-terminal sessions and `ListSessions` takes a `?name=` filter, so the backend can re-adopt a running sandbox after losing its durable record, the way the E2B backend re-adopts by sandbox metadata. `sandboxScopeName` already produces names that fit our 64-character, not-UUID-shaped rule. +- Profile: `writablePersistence: "provider_managed"`, `processSessions: true`, `parksOnTeardown: true`, and `egressEnforcement: "domain"`. That last one is real rather than aspirational — a Managed Agents sandbox gets NAT egress with a per-session host allowlist, and the manifest's `permissions.network` block narrows within it. +- A `permissions` block on the session that sets `defaultAction: allow` and allows `bash`. Our managed exec path is approval-gated by default, which means every command would wait on a human decision through our approval router. qm runs its own approval gates at its own tool boundary and fires many commands per turn, so two gating systems in series would just deadlock. qm's posture stays authoritative; ours gets out of the way. This is deliberate and we would rather state it here than have it discovered later. + +What it does not do + +- No provider checkpoints, though not for the reason you might expect. We do expose session checkpoints publicly — `POST /v2/agents/sessions/{id}/checkpoints` is synchronous and returns a terminal checkpoint — along with list, rollback and fork. This backend simply does not use them yet, so it gets native pause and falls back to your portable tar home snapshot for recovery, under `DO_AGENTS_SNAPSHOT_INTERVAL_SEC` and `DO_AGENTS_SNAPSHOT_S3_BUCKET`. Moving to native checkpoints, the way the Sprites backend reports `provider_snapshot`, is a clean follow-up. The one thing to check first is that our checkpoints bind machine state to the event-log cursor and refuse during an active turn, both of which are managed-loop concepts that need verifying against a session that has no turns. +- No idle reaping from qm's side. Our control plane owns the idle timeout, set per session with `DO_AGENTS_IDLE_TIMEOUT_SEC` and bounded by tenant policy. Paused sessions still count against team quota, which is worth knowing before pointing a large deployment at us. +- No image plumbing. `agent: none` lands on our own bare base template, so `DO_AGENTS_TEMPLATE` is an override rather than a requirement; set it only to name a template already provisioned for the team, because an unprovisioned name fails session create. Building a qm-flavoured template, so the layer tool install step finds its files already present, is a follow-up and not in this PR. +- No new CLI target. `qm init` still deploys core to Docker, Fly or AWS. This only changes where the computers live. + +Three things we need from you, or at least need to agree on + +- Reachability runs both ways. qm needs to reach `api.digitalocean.com` and nothing else, which is the easy direction. But qm's sandboxes also call _back_ into core's self-API with their capability tokens, the way the E2B backend needs a public `PUBLIC_API_URL`. That means core's public URL has to be on the session's egress allowlist. For a qm deployment inside DO this is fine; for one on Fly or AWS it means our allowlist has to carry an arbitrary external host, which we should confirm is acceptable to your operators and ours. +- We should agree on what the bare base carries. `agent: none` removed the prerequisite that used to sit here: no template has to be provisioned for your team, and we verified the guest is genuinely bare — no agent CLI on `PATH`, no event-translation runtime, no agent supervisor, just `sandbox-agent`, `envd`, `otelcol` and s6 supervision. `bash`, `git`, `curl`, `tar`, Node, Python, `make` and `gcc` are present; `jq`, `rg` and `unzip` are not, and those three are reached for by name in agent prompts. The profile declares them not installed so the model does not try, but either we add the small tools to the base or qm deployments want a template of their own, and we should decide which before this carries real traffic. + +- The vendored proto should not have to exist, and we would like to delete it. It is here because our public API has no streaming exec: the REST endpoint buffers in the control plane, so it clamps to four minutes and one mebibyte per stream and carries no per-command environment, and the only public transport without those limits is the raw port-forward. So qm ends up speaking our guest contract, which leaves one wire with two owners in two repos — change a field and qm breaks with no version bump and no type error, just a decode mismatch in someone's sandbox. Note that every other backend in this tree consumes a published provider SDK and vendors nothing; this is the one exception. We are asking internally to expose streaming exec at the public edge with per-command environment and no request deadline, which the guest already supports. When that lands, `managed-agents-sandbox-agent.proto`, `managed-agents-tunnel.ts` and the two gRPC dependencies all go away and this backend talks to a published, versioned endpoint. Until then we would rather vendor the three RPCs we call, visibly and with a test pinning the subset, than quietly depend on an unpublished contract. + +Tests run against an in-process control plane, a real gRPC `sandbox-agent` and a WebSocket tunnel between them, so the transport is exercised end to end and CI needs no DO account and no credentials. We would rather this live upstream than in a fork, and we will keep it green as the sandbox contract moves. diff --git a/docs/managed-agents.md b/docs/managed-agents.md new file mode 100644 index 0000000000..170247f180 --- /dev/null +++ b/docs/managed-agents.md @@ -0,0 +1,255 @@ +# Managed Agents agent sandboxes + +`SANDBOX_BACKEND=do-managed-agents` runs each QM scope in a DigitalOcean Managed Agents microVM — a +Firecracker guest with a persistent home, hypervisor-level isolation between +scopes, and provider-managed pause and resume. + +Managed Agents is normally a managed-agent product: you hand it a manifest naming a +coding agent and it runs that agent's loop inside the guest for you. QM does +not want that. The manifest says `agent: none`, which Managed Agents admits as a bare +sandbox — its own base image, no managed agent, no in-guest event-translation +runtime, no model credential. QM keeps its own agent loop in core and drives +the guest as a computer, exactly as it does for every other backend. See +[the design record](../adrs/managed-agents-sandbox-backend.md) for why. + +## What it provides + +A microVM per scope, not per turn. Each person and each room gets its own +guest, isolated at the hypervisor rather than by container namespaces. Teardown +pauses rather than destroys, so files, installed packages, and clones survive +between conversations and the next turn resumes the same machine. + +Persistence is the provider's, not ours. Managed Agents preserves the guest's disk across +pause and resume on its own, so the backend reports `provider_managed` +persistence and does not need to tar the home up after every turn. The S3 +snapshot path exists only to survive losing the session entirely. + +One credential, one endpoint. A DigitalOcean IAM token authenticates both +halves of the integration, and there are no client certificates, internal +hostnames, or VPC prerequisites to arrange. + +## Architecture + +Two planes, one token. Lifecycle is ordinary REST. Commands and files travel a +WebSocket the control plane bridges into the guest. + +```mermaid +flowchart LR + subgraph core["QM core"] + orch["agent loop"] + cl["managed-agents-client"] + tun["managed-agents-tunnel
local TCP listener"] + end + + subgraph cp["DigitalOcean control plane
api.digitalocean.com"] + rest["/v2/agents/sessions
REST lifecycle"] + pf["/port-forward/8443
WebSocket bridge"] + end + + subgraph vm["Managed Agents microVM — agent: none"] + sa["sandbox-agent
gRPC :8443"] + proc["bash, git, node, python3
/workspace/home"] + end + + orch --> cl + cl -->|"create, get, pause, resume, delete"| rest + cl --> tun + tun -->|"wss + bearer token"| pf + pf -->|"guest TCP :8443"| sa + sa --> proc + rest -.->|"provisions"| vm +``` + +The tunnel is the part worth understanding. Every Managed Agents microVM runs +`sandbox-agent`, a gRPC server on guest port 8443 that Managed Agents's own control plane +uses for exec and file transfer. QM reaches it the same way +`doctl agents port-forward` does: it opens a WebSocket to +`/v2/agents/sessions/{id}/port-forward/8443`, which the control plane bridges to +the guest port, and exposes that as a local TCP listener a gRPC client dials. +There is no direct network path from core to the microVM; the control plane +proxies bytes. + +QM deliberately does not use the REST `POST /v2/agents/sessions/{id}/sandbox/exec`. +That endpoint buffers output in the control plane, so it clamps commands to four +minutes and one mebibyte per stream and carries no per-command environment. The +port-forward route has no request deadline, so a command is bounded only by +`SANDBOX_TIMEOUT_SEC`. + +## Operations + +### Create + +Nothing is provisioned when core boots, and a turn that needs no shell never +touches Managed Agents. The first execute-family tool call provisions, which takes about +ten seconds end to end: post the manifest, poll until the session reports ready, +open the tunnel, run the command. + +Every later turn for the same scope reconnects in well under a second, because +the session name is derived from the scope. The backend tries, in order, the +session id it has stored, then whatever Managed Agents lists under that name, and only +creates when both come up empty. A paused session is resumed rather than +replaced. + +With `EAGER_PROVISION` at its default, a scope that has used tools before starts +provisioning at the top of every turn instead of waiting for the model to ask, +which hides the cold start. Set `EAGER_PROVISION=0` for the strictly on-demand +path. + +### Exec + +Commands run as `/bin/bash -c` over a bidirectional gRPC stream, carrying +per-command environment, working directory, and a timeout. Output streams back +as it is produced, capped at 16 MiB per stream — write larger results to files +and read them instead. + +Background process sessions, `startProcess` and friends, are built on the same +stream, so long-running servers behave as they do on other backends. + +### Upload and download + +File transfer uses the guest agent's `Upload` and `Download` RPCs over the same +tunnel, chunked at 256 KiB. A download of a missing file reads as `null` rather +than raising. + +`sandbox-agent` confines transfer to the guest workspace, so a scope's home +lives at `/workspace/home` rather than the usual `/home/user`. That also places +it inside the volume Managed Agents preserves, so the home survives pause and resume on +its own. + +### Pause and resume + +Both are automatic, and neither is a verb a user or the model can call. + +A scope pauses when its turn tears down. If a home snapshot is due the backend +takes it first, then calls `POST /v2/agents/sessions/{id}/pause` and closes the +tunnel. A pause that fails is not silent: the scope is recorded as +`pause_failed` with the provider's error, which surfaces in the computer status +and in error reporting. Two cases skip the pause — an internal `keepWarm` +teardown, used when core knows another turn is imminent, and a teardown that +destroys instead. + +A scope resumes the next time anything needs the guest. Provisioning polls the +session, and on finding it paused issues `POST /resume` once and keeps polling +until it reports usable, up to five minutes. So the resume is a side effect of +the next command rather than a step anybody triggers; the only visible evidence +is that the first command of a conversation takes a little longer. + +There is no manual pause or resume from QM. The `sandbox` tool's verbs are +`list`, `create`, `set_default`, `status`, and `retire` — and for this backend +`restart` is not offered either, because `do-managed-agents` implements no `restartComputer`. +What you can do is **observe** and **destroy**: `status` reports +`lifecycleState: "paused"` along with the snapshot state and any +`pause_failed` error, and `retire` deletes outright. If you want a scope to stop +costing anything, retire it rather than looking for a pause button. + +### Delete + +Teardown at the end of a turn **pauses**. Actual deletion happens in four places +and no others: + +- An explicit retire, through the agent's `sandbox` tool or the admin API. +- Scratch sandboxes, which are always killed rather than paused. +- A failed home hydration, which kills the session rather than risk overwriting + a good snapshot with a half-populated home. +- Swarm workers, which are retired when the swarm finishes. + +## Gotchas + +**Deleting a conversation does not delete the microVM.** Nothing in QM ties +scope deletion to sandbox deletion, and the `do-managed-agents` backend implements no deep +idle reaper, so core's sweeper never reclaims one. A paused microVM stays +paused, indefinitely, until somebody retires it or Managed Agents's own idle policy +reclaims it. Budget for that: set `DO_AGENTS_IDLE_TIMEOUT_SEC` so the provider has a +backstop, and retire scopes you are finished with. + +**`PUBLIC_API_URL` has to be reachable from inside the guest.** Reachability +runs both ways: core calls the control plane, and microVMs call back into core's +self-API for the connector SDK and file callbacks. A deployment behind +`localhost` serves shell commands fine but cannot complete a cron, a send, or a +credential fetch. + +**The bare base is smaller than agents assume.** It carries `bash`, `git`, +`curl`, `tar`, `node`, `python3`, `make`, and `gcc`. It does not carry `jq`, +`rg`, or `unzip`, all of which prompts reach for by name. The profile reports +them as not installed so the model does not try; naming your own template in +`DO_AGENTS_TEMPLATE` is how you add them. + +**Set `DO_AGENTS_SNAPSHOT_S3_BUCKET`.** It is how a scope's home survives losing its +session. Managed Agents exposes session checkpoints, but QM does not use them yet, so +without the bucket a destroyed or reclaimed session takes the home with it. See +[sandbox recovery](sandbox-preservation.md) for capture and hydration rules. + +**Sessions are created with `bash` pre-allowed.** Managed Agents has its own +human-in-the-loop gate on command execution, enforced in the guest. Leaving it +on would hold every command behind an approval nobody is watching, so the +manifest allows `bash` outright and QM's own approval gates stay authoritative. + +**Name your deployments apart.** Session names are derived from the scope and +`DO_AGENTS_NAME_PREFIX`, which is also how an operator finds a scope's microVM in the +Managed Agents console without consulting core's database. Two deployments sharing a prefix +on one Managed Agents team will adopt each other's sandboxes. Managed Agents caps names at 64 +characters and rejects UUID-shaped ones. + +## Configure + +```sh +SANDBOX_BACKEND=do-managed-agents +DO_AGENTS_API_TOKEN=dop_v1_your-digitalocean-iam-token +DO_AGENTS_NAME_PREFIX=my-qm +DO_AGENTS_SNAPSHOT_S3_BUCKET=my-qm-sandbox-homes +``` + +Store the token as a core service secret. Production requires `DATABASE_URL` +for session records and provisioning locks across instances. + +| Variable | Default | Purpose | +| --------------------------------- | ------------------------------ | ----------------------------------------------------------------- | +| `DO_AGENTS_API_TOKEN` | Required | DigitalOcean IAM token, held by core. | +| `DO_AGENTS_API_BASE_URL` | `https://api.digitalocean.com` | Control-plane endpoint override; any path prefix is preserved. | +| `DO_AGENTS_TEMPLATE` | Provider bare base | Template override; unset means the bare base `agent: none` picks. | +| `DO_AGENTS_NAME_PREFIX` | `qm` | Namespace for scope discovery. | +| `DO_AGENTS_SIZE_SLUG` | Provider default | microVM size. | +| `DO_AGENTS_IDLE_TIMEOUT_SEC` | Provider default | Provider-side idle reclaim backstop. | +| `DO_AGENTS_EGRESS_PROXY_URL` | Unset | Proxy host to allow; everything else is denied. | +| `DO_AGENTS_SNAPSHOT_S3_BUCKET` | Unset | Portable home tars. Without it the backend reports no recovery. | +| `DO_AGENTS_SNAPSHOT_INTERVAL_SEC` | `300` | Snapshot throttle. | +| `SANDBOX_TIMEOUT_SEC` | `600` | Default command deadline in seconds. | + +Without `DO_AGENTS_EGRESS_PROXY_URL`, sandboxes have unrestricted egress and the +profile reports no enforcement. With it, sessions are created denying all +outbound traffic except the proxy host, so `HTTPS_PROXY` cannot be bypassed from +inside the microVM. + +## Local development + +```sh +export DO_AGENTS_API_TOKEN=dop_v1_... +bash scripts/dev-instance.sh up --surface web --sandbox do-managed-agents +``` + +Two things differ from the other backends. `dev-instance` reads provider secrets +from your shell or `~/.config/qm/dev.env`, not from the worktree `.env`, so the +token has to be exported. And a live slot reloads from its persisted boot spec, +so changing `--sandbox` on a running slot keeps the old backend — take it down +first. + +## Tests + +Backend tests run without an account. They stand up an in-process control plane, +a real gRPC `sandbox-agent`, and a WebSocket tunnel between them, so the +transport is exercised end to end: + +```sh +node --test test/managed-agents-client.test.ts test/managed-agents-sandbox.test.ts +``` + +Two smokes exercise a real account. The first provisions a microVM and runs +commands, files, and a pause cycle through the `Sandbox` interface; set +`DO_AGENTS_SMOKE_LONG=1` to add a 250-second command that outlives the REST exec +clamp. The second drives a live model turn against a running core and asserts +the reply carries output that could only have come from the guest. + +```sh +DO_AGENTS_API_TOKEN=dop_v1_... npm run smoke:managed-agents-sandbox +CORE_SIGNING_SECRET=... PORTAL_IDENTITY_SECRET=... npm run smoke:managed-agents-turn +``` diff --git a/docs/sandbox-preservation.md b/docs/sandbox-preservation.md index c8517560df..630752d762 100644 --- a/docs/sandbox-preservation.md +++ b/docs/sandbox-preservation.md @@ -34,3 +34,5 @@ Modal machines are created with a one-core, 2 GiB reservation (`MODAL_CPUS`, `MO When `MODAL_EGRESS_PROXY_URL` is set, machines are created with Modal's outbound allowlist limited to the proxy host (a domain allowlist for an https proxy on port 443, a CIDR allowlist for an IP address; other proxy URLs are rejected at startup because Modal domain allowlists pass only TLS on 443), and the profile reports `egressEnforcement: "domain"`. Command execution, file transfer, and checkpoints travel over Modal's control plane and are unaffected. Without the setting, core warns that Modal sandboxes run fail-open. Turn environment reaches Modal commands through the exec `env` parameter rather than inlined `export` statements, exec deadlines are rounded up to whole seconds with a single thirty-second grace beyond the in-guest `timeout`, and a command longer than 64 KiB is written to `/tmp` through the filesystem API and executed from there. + +Managed Agents sessions name a sandbox template explicitly, which yields a microVM carrying no managed agent loop, and the disk persists across the provider's pause and resume. Used-turn teardown snapshots the home to `DO_AGENTS_SNAPSHOT_S3_BUCKET` when the home changed, throttled by `DO_AGENTS_SNAPSHOT_INTERVAL_SEC`, and then pauses the session; a pause failure is recorded as `pause_failed` and the running session is retained for retry. Status reports `recovery.strategy: "workspace_snapshot"`. Managed Agents does expose session checkpoints on its public surface, but core does not capture them yet, so the portable tar is the only recovery path: a replacement session for a scope hydrates from it, and a failed hydration kills the replacement rather than adopting a blank home over a saved one. Session lifecycle travels over the Managed Agents REST surface with a DigitalOcean IAM token; commands and file transfer travel over a bearer-authenticated WebSocket port-forward to the guest's `sandbox-agent` gRPC server, which has no request deadline, so command length is bounded by `SANDBOX_TIMEOUT_SEC` rather than by the control plane. Sessions are created with a permissions block that allows `bash` outright, because Managed Agents would otherwise hold every command at its own approval gate behind core's. See [Managed Agents sandboxes](managed-agents.md) for configuration. From 306f05a85d2f71beadfad0496300374d5204474f Mon Sep 17 00:00:00 2001 From: Josh France Date: Tue, 6 Oct 2026 18:39:08 +0000 Subject: [PATCH 04/19] Poll Managed Agents readiness every 250ms for the first 10s Resume completes in about a second, but the fixed 2s poll added most of a second interval to every warm turn. Poll quickly at first and fall back to 2s so slow resumes don't hammer the API. --- src/sandbox/managed-agents-client.ts | 9 +++++++-- test/managed-agents-client.test.ts | 10 ++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/sandbox/managed-agents-client.ts b/src/sandbox/managed-agents-client.ts index ef35565e67..4546bbc569 100644 --- a/src/sandbox/managed-agents-client.ts +++ b/src/sandbox/managed-agents-client.ts @@ -103,6 +103,8 @@ const CREATE_TIMEOUT_MS = 120_000; const REQUEST_TIMEOUT_MS = 30_000; const READY_TIMEOUT_MS = 300_000; const READY_POLL_MS = 2_000; +const READY_FAST_POLL_MS = 250; +const READY_FAST_WINDOW_MS = 10_000; const STALL_PROBE_MS = 45_000; const READY_TRANSIENT_LIMIT = 5; const UPLOAD_CHUNK_BYTES = 256 * 1024; @@ -374,7 +376,8 @@ export function createSdkManagedAgentsClient(opts: SdkManagedAgentsClientOptions }; async function awaitUsable(sessionId: string): Promise { - const deadline = Date.now() + READY_TIMEOUT_MS; + const started = Date.now(); + const deadline = started + READY_TIMEOUT_MS; let resumed = false; let strikes = 0; const tolerate = (err: unknown): void => { @@ -405,7 +408,9 @@ export function createSdkManagedAgentsClient(opts: SdkManagedAgentsClientOptions } if (Date.now() >= deadline) throw new Error(`do-managed-agents session ${sessionId} did not become ready within ${READY_TIMEOUT_MS}ms`); - await new Promise((resolve) => setTimeout(resolve, READY_POLL_MS)); + await new Promise((resolve) => + setTimeout(resolve, Date.now() - started < READY_FAST_WINDOW_MS ? READY_FAST_POLL_MS : READY_POLL_MS), + ); } } diff --git a/test/managed-agents-client.test.ts b/test/managed-agents-client.test.ts index 94f1406f10..4c8336e9fb 100644 --- a/test/managed-agents-client.test.ts +++ b/test/managed-agents-client.test.ts @@ -166,6 +166,16 @@ test("a missing file reads as null rather than an error", async () => { assert.equal(await s.readFileBytes("/home/user/nope.txt"), null); }); +test("resuming a paused session does not wait out a fixed poll interval", async () => { + const created = await session("qm-resume-latency"); + service.setStatus("qm-resume-latency", "SESSION_STATUS_PAUSED"); + const started = Date.now(); + const back = await client.connect(created.sessionId); + opened.push(back); + assert.equal(back.sessionId, created.sessionId); + assert.ok(Date.now() - started < 1_000, `resume took ${Date.now() - started}ms`); +}); + test("a session paused mid-command fails fast instead of stalling until the command timeout", async () => { const watchful = createSdkManagedAgentsClient({ apiToken: service.token, From 15e71bf0379eec590478da826b9c383a49e4227b Mon Sep 17 00:00:00 2001 From: Josh France Date: Wed, 7 Oct 2026 18:22:09 +0000 Subject: [PATCH 05/19] Drop personal names from the Managed Agents ADR and smoke script --- adrs/managed-agents-sandbox-backend.md | 2 +- scripts/managed-agents-turn-smoke.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/adrs/managed-agents-sandbox-backend.md b/adrs/managed-agents-sandbox-backend.md index a6362ea3b1..fdeaf128cc 100644 --- a/adrs/managed-agents-sandbox-backend.md +++ b/adrs/managed-agents-sandbox-backend.md @@ -1,6 +1,6 @@ # Managed Agents as an agent-computer backend -Naveen, from the Managed Agents team at DigitalOcean. Managed Agents (Managed Agent Runtime Stack) hosts coding agents in DO-managed Firecracker microVMs. We looked at what it would take for qm to run on it and the answer is the same one Agent37 arrived at: the part of qm that wants hosting is the agent computer, so this is another sandbox backend, next to E2B, Modal, Sprites, smolmachines, Porter, AWS, Superserve and Agent37. +This was written by the Managed Agents team at DigitalOcean. Managed Agents (Managed Agent Runtime Stack) hosts coding agents in DO-managed Firecracker microVMs. We looked at what it would take for qm to run on it and the answer is the same one Agent37 arrived at: the part of qm that wants hosting is the agent computer, so this is another sandbox backend, next to E2B, Modal, Sprites, smolmachines, Porter, AWS, Superserve and Agent37. The reason this is worth doing at all is that Managed Agents is a managed-loop product by default — it boots Claude Code or OpenCode _inside_ the microVM, next to a runtime process that translates agent events back to our control plane and an event stream that `doctl agents attach` tails. That is not what qm wants, because qm already has a loop. But our platform now admits a session with no agent in it at all: a manifest that says `agent: none` lands on Managed Agents's own bare base template, with no managed agent, no event-translation runtime, and no model credential. The caller drives the microVM itself over exec, workspace transfer and port-forward. That mode exists for customers who want Managed Agents purely for code execution — evaluations, CI-style fan-out, arbitrary compute — and qm lands in the same seat. Nothing about qm's orchestrator, harness router or tool surface has to change. diff --git a/scripts/managed-agents-turn-smoke.ts b/scripts/managed-agents-turn-smoke.ts index 8be7ba7ade..396717de94 100644 --- a/scripts/managed-agents-turn-smoke.ts +++ b/scripts/managed-agents-turn-smoke.ts @@ -3,7 +3,7 @@ import { randomUUID } from "node:crypto"; import { stagingApiHeaders } from "../src/deployment/postdeploy-smoke.ts"; const core = process.env.CORE_URL ?? "http://localhost:8081"; -const principal = process.env.PRINCIPAL ?? "naveenkurra"; +const principal = process.env.PRINCIPAL ?? "managed-agents-smoke"; const sourceSecret = process.env.CORE_SIGNING_SECRET ?? ""; const portalSecret = process.env.PORTAL_IDENTITY_SECRET ?? ""; From 1e2410b2870355f7fa39e67c3e9cb6403aaf04a7 Mon Sep 17 00:00:00 2001 From: Josh France Date: Wed, 7 Oct 2026 20:49:23 +0000 Subject: [PATCH 06/19] Add a QM sandbox template for Managed Agents deploy/do-managed-agents/Dockerfile is the E2B image plus ripgrep with HOME at the adapter's home, registered on the agentless sandbox base. With DO_AGENTS_TEMPLATE set, the profile advertises jq, rg, unzip and wget as installed. docs/managed-agents.md covers building it. --- adrs/managed-agents-sandbox-backend.md | 4 +-- deploy/do-managed-agents/Dockerfile | 18 +++++++++++++ docs/managed-agents.md | 36 +++++++++++++++++++++----- src/sandbox/managed-agents-sandbox.ts | 2 ++ src/wiring.ts | 8 ++++-- test/managed-agents-sandbox.test.ts | 15 ++++++++++- 6 files changed, 72 insertions(+), 11 deletions(-) create mode 100644 deploy/do-managed-agents/Dockerfile diff --git a/adrs/managed-agents-sandbox-backend.md b/adrs/managed-agents-sandbox-backend.md index fdeaf128cc..c41ad7218c 100644 --- a/adrs/managed-agents-sandbox-backend.md +++ b/adrs/managed-agents-sandbox-backend.md @@ -83,13 +83,13 @@ What it does not do - No provider checkpoints, though not for the reason you might expect. We do expose session checkpoints publicly — `POST /v2/agents/sessions/{id}/checkpoints` is synchronous and returns a terminal checkpoint — along with list, rollback and fork. This backend simply does not use them yet, so it gets native pause and falls back to your portable tar home snapshot for recovery, under `DO_AGENTS_SNAPSHOT_INTERVAL_SEC` and `DO_AGENTS_SNAPSHOT_S3_BUCKET`. Moving to native checkpoints, the way the Sprites backend reports `provider_snapshot`, is a clean follow-up. The one thing to check first is that our checkpoints bind machine state to the event-log cursor and refuse during an active turn, both of which are managed-loop concepts that need verifying against a session that has no turns. - No idle reaping from qm's side. Our control plane owns the idle timeout, set per session with `DO_AGENTS_IDLE_TIMEOUT_SEC` and bounded by tenant policy. Paused sessions still count against team quota, which is worth knowing before pointing a large deployment at us. -- No image plumbing. `agent: none` lands on our own bare base template, so `DO_AGENTS_TEMPLATE` is an override rather than a requirement; set it only to name a template already provisioned for the team, because an unprovisioned name fails session create. Building a qm-flavoured template, so the layer tool install step finds its files already present, is a follow-up and not in this PR. +- A QM template, optional. `agent: none` lands on our own bare base template, so `DO_AGENTS_TEMPLATE` is an override rather than a requirement; an unprovisioned name fails session create. `deploy/do-managed-agents/Dockerfile` is the QM image (the E2B image plus `ripgrep`, `HOME=/workspace/home`), registered as a team template on the agentless `sandbox` base; `docs/managed-agents.md` has the build steps. Baking layer tool files into it is a follow-up. - No new CLI target. `qm init` still deploys core to Docker, Fly or AWS. This only changes where the computers live. Three things we need from you, or at least need to agree on - Reachability runs both ways. qm needs to reach `api.digitalocean.com` and nothing else, which is the easy direction. But qm's sandboxes also call _back_ into core's self-API with their capability tokens, the way the E2B backend needs a public `PUBLIC_API_URL`. That means core's public URL has to be on the session's egress allowlist. For a qm deployment inside DO this is fine; for one on Fly or AWS it means our allowlist has to carry an arbitrary external host, which we should confirm is acceptable to your operators and ours. -- We should agree on what the bare base carries. `agent: none` removed the prerequisite that used to sit here: no template has to be provisioned for your team, and we verified the guest is genuinely bare — no agent CLI on `PATH`, no event-translation runtime, no agent supervisor, just `sandbox-agent`, `envd`, `otelcol` and s6 supervision. `bash`, `git`, `curl`, `tar`, Node, Python, `make` and `gcc` are present; `jq`, `rg` and `unzip` are not, and those three are reached for by name in agent prompts. The profile declares them not installed so the model does not try, but either we add the small tools to the base or qm deployments want a template of their own, and we should decide which before this carries real traffic. +- We should agree on what the bare base carries. `agent: none` removed the prerequisite that used to sit here: no template has to be provisioned for your team, and we verified the guest is genuinely bare — no agent CLI on `PATH`, no event-translation runtime, no agent supervisor, just `sandbox-agent`, `envd`, `otelcol` and s6 supervision. `bash`, `git`, `curl`, `tar`, Node, Python, `make` and `gcc` are present; `jq`, `rg` and `unzip` are not, and those three are reached for by name in agent prompts. The profile declares them not installed so the model does not try. The bare base also leaves `HOME` at `/root`. The QM template covers both for deployments that register it; adding the small tools to the base would still help deployments that do not. - The vendored proto should not have to exist, and we would like to delete it. It is here because our public API has no streaming exec: the REST endpoint buffers in the control plane, so it clamps to four minutes and one mebibyte per stream and carries no per-command environment, and the only public transport without those limits is the raw port-forward. So qm ends up speaking our guest contract, which leaves one wire with two owners in two repos — change a field and qm breaks with no version bump and no type error, just a decode mismatch in someone's sandbox. Note that every other backend in this tree consumes a published provider SDK and vendors nothing; this is the one exception. We are asking internally to expose streaming exec at the public edge with per-command environment and no request deadline, which the guest already supports. When that lands, `managed-agents-sandbox-agent.proto`, `managed-agents-tunnel.ts` and the two gRPC dependencies all go away and this backend talks to a published, versioned endpoint. Until then we would rather vendor the three RPCs we call, visibly and with a test pinning the subset, than quietly depend on an unpublished contract. diff --git a/deploy/do-managed-agents/Dockerfile b/deploy/do-managed-agents/Dockerfile new file mode 100644 index 0000000000..d997df523e --- /dev/null +++ b/deploy/do-managed-agents/Dockerfile @@ -0,0 +1,18 @@ +# QM sandbox image for DigitalOcean Managed Agents (Harness Runtime BYOT). +# Registered on the agentless `sandbox` base; see docs/managed-agents.md. +FROM node:24-slim@sha256:6f7b03f7c2c8e2e784dcf9295400527b9b1270fd37b7e9a7285cf83b6951452d + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates git curl wget jq tar xz-utils unzip zip ripgrep \ + python3 python3-pip python3-venv \ + openssh-client gnupg less vim-tiny procps \ + && rm -rf /var/lib/apt/lists/* \ + && node --version && npm --version + +# The platform owns /workspace and ignores USER, so the image creates no user or +# workspace; it only points HOME at the home the adapter manages. Clear node's +# REPL entrypoint so the session process stays up. +ENV HOME=/workspace/home +ENTRYPOINT ["sleep", "infinity"] +CMD [] diff --git a/docs/managed-agents.md b/docs/managed-agents.md index 170247f180..b86a70539a 100644 --- a/docs/managed-agents.md +++ b/docs/managed-agents.md @@ -168,11 +168,14 @@ self-API for the connector SDK and file callbacks. A deployment behind `localhost` serves shell commands fine but cannot complete a cron, a send, or a credential fetch. -**The bare base is smaller than agents assume.** It carries `bash`, `git`, -`curl`, `tar`, `node`, `python3`, `make`, and `gcc`. It does not carry `jq`, -`rg`, or `unzip`, all of which prompts reach for by name. The profile reports -them as not installed so the model does not try; naming your own template in -`DO_AGENTS_TEMPLATE` is how you add them. +**Use the QM template.** The bare base `agent: none` boots is Ubuntu with Node 20, +`bash`, `git`, `curl`, `tar`, `python3`, `make`, and `gcc`, but no `jq`, `rg`, `unzip`, or +`wget`, and it leaves `HOME` at `/root` instead of the home the adapter manages. Register +`deploy/do-managed-agents/Dockerfile` as a team template and name it in +`DO_AGENTS_TEMPLATE`; it is the same Node 24 image as the E2B template plus `ripgrep`, +with `HOME` set to `/workspace/home`. When `DO_AGENTS_TEMPLATE` is set the profile +advertises `jq`, `rg`, `unzip`, and `wget` as installed, so a custom template should +carry them too. See [Build the QM template](#build-the-qm-template). **Set `DO_AGENTS_SNAPSHOT_S3_BUCKET`.** It is how a scope's home survives losing its session. Managed Agents exposes session checkpoints, but QM does not use them yet, so @@ -206,7 +209,7 @@ for session records and provisioning locks across instances. | --------------------------------- | ------------------------------ | ----------------------------------------------------------------- | | `DO_AGENTS_API_TOKEN` | Required | DigitalOcean IAM token, held by core. | | `DO_AGENTS_API_BASE_URL` | `https://api.digitalocean.com` | Control-plane endpoint override; any path prefix is preserved. | -| `DO_AGENTS_TEMPLATE` | Provider bare base | Template override; unset means the bare base `agent: none` picks. | +| `DO_AGENTS_TEMPLATE` | Provider bare base | Team template to boot. Unset boots the bare base; use QM template | | `DO_AGENTS_NAME_PREFIX` | `qm` | Namespace for scope discovery. | | `DO_AGENTS_SIZE_SLUG` | Provider default | microVM size. | | `DO_AGENTS_IDLE_TIMEOUT_SEC` | Provider default | Provider-side idle reclaim backstop. | @@ -253,3 +256,24 @@ the reply carries output that could only have come from the guest. DO_AGENTS_API_TOKEN=dop_v1_... npm run smoke:managed-agents-sandbox CORE_SIGNING_SECRET=... PORTAL_IDENTITY_SECRET=... npm run smoke:managed-agents-turn ``` + +## Build the QM template + +Managed Agents builds a team template from an image in your DigitalOcean Container +Registry, rebased onto a platform base. QM uses the `sandbox` base, which carries no +agent. With Docker and an authenticated `doctl`: + +```sh +REG=registry.digitalocean.com/ +docker build --platform linux/amd64 -t $REG/qm-sandbox:v1 deploy/do-managed-agents +doctl registry login && docker push $REG/qm-sandbox:v1 +doctl harness-runtime template create --name qm-sandbox --base-template sandbox \ + --source-oci-ref $REG/qm-sandbox:v1 +doctl harness-runtime template list-builds qm-sandbox # wait for SUCCEEDED +``` + +Then set `DO_AGENTS_TEMPLATE=qm-sandbox`. To ship an image change, push a new tag and run +`doctl harness-runtime template update qm-sandbox --source-oci-ref $REG/qm-sandbox:v2`. +Templates do not pick up platform fixes on their own, so rebuild after Managed Agents +updates as well. Existing sessions keep the template they were created from; a scope only +boots the new one after its session is destroyed. diff --git a/src/sandbox/managed-agents-sandbox.ts b/src/sandbox/managed-agents-sandbox.ts index f40c7942cc..292cf0da3b 100644 --- a/src/sandbox/managed-agents-sandbox.ts +++ b/src/sandbox/managed-agents-sandbox.ts @@ -61,6 +61,8 @@ const HOME_TAR = `${HOME_DIR}/.qm-home.tar`; const SNAPSHOT_PRUNE = HOME_SNAPSHOT_PRUNE; const DEFAULT_SNAPSHOT_INTERVAL_MS = 5 * 60_000; +export const QM_TEMPLATE_TOOLS = ["jq", "rg", "unzip", "wget"]; + export interface StoredManagedAgentsSandbox { sessionId: string; sandboxId: string; diff --git a/src/wiring.ts b/src/wiring.ts index 05c90f97a3..ea5ef257f5 100644 --- a/src/wiring.ts +++ b/src/wiring.ts @@ -230,7 +230,11 @@ import { import { createSdkSuperserveClient } from "./sandbox/superserve-client.ts"; import { createE2bSandbox, type StoredE2bSandbox } from "./sandbox/e2b-sandbox.ts"; import { createSdkE2bClient } from "./sandbox/e2b-client.ts"; -import { createManagedAgentsSandbox, type StoredManagedAgentsSandbox } from "./sandbox/managed-agents-sandbox.ts"; +import { + createManagedAgentsSandbox, + QM_TEMPLATE_TOOLS, + type StoredManagedAgentsSandbox, +} from "./sandbox/managed-agents-sandbox.ts"; import { createSdkManagedAgentsClient } from "./sandbox/managed-agents-client.ts"; import { createS3SnapshotStore } from "./sandbox/home-snapshot.ts"; import { createModalSandbox, type StoredModalSandbox } from "./sandbox/modal-sandbox.ts"; @@ -953,7 +957,7 @@ export function buildApp( ? { snapshotIntervalMs: managedAgents.snapshotIntervalSec * 1000 } : {}), ...(managedAgents.egressProxyUrl ? { egressProxyUrl: managedAgents.egressProxyUrl } : {}), - extraTools: deploymentLayer.advertisedTools, + extraTools: [...(managedAgents.template ? QM_TEMPLATE_TOOLS : []), ...deploymentLayer.advertisedTools], credentialPaths: deploymentLayer.credentialPaths, layerToolFiles: () => deploymentLayer.installFiles, blobTransfer, diff --git a/test/managed-agents-sandbox.test.ts b/test/managed-agents-sandbox.test.ts index be5d27a8a9..7fff72f764 100644 --- a/test/managed-agents-sandbox.test.ts +++ b/test/managed-agents-sandbox.test.ts @@ -4,7 +4,11 @@ import { mkdtempSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { pollProcess } from "../src/sandbox/process-poll.ts"; -import { createManagedAgentsSandbox, type StoredManagedAgentsSandbox } from "../src/sandbox/managed-agents-sandbox.ts"; +import { + createManagedAgentsSandbox, + QM_TEMPLATE_TOOLS, + type StoredManagedAgentsSandbox, +} from "../src/sandbox/managed-agents-sandbox.ts"; import { managedAgentsEgressAllow, managedAgentsSessionState, @@ -192,6 +196,15 @@ test("the profile advertises provider-managed persistence and parks on teardown" assert.equal(supportsProcessSessions(sandbox), true); }); +test("the QM template's tools are advertised as installed, not missing", () => { + assert.ok(sandbox.profile.spec?.notInstalled?.includes("jq")); + const spec = make({ extraTools: QM_TEMPLATE_TOOLS }).profile.spec; + for (const tool of QM_TEMPLATE_TOOLS) { + assert.ok(spec?.tools?.includes(tool), tool); + assert.ok(!spec?.notInstalled?.includes(tool), tool); + } +}); + test("egress enforcement is only claimed when a proxy is configured", () => { assert.equal(sandbox.profile.egressEnforcement, "none"); assert.equal(make({ egressProxyUrl: "https://egress.example.com" }).profile.egressEnforcement, "domain"); From c24b67f8fa90baf64d20090296c14f1971ea968c Mon Sep 17 00:00:00 2001 From: Josh France Date: Thu, 8 Oct 2026 14:06:24 +0000 Subject: [PATCH 07/19] Remove the Managed Agents setup doc --- README.md | 1 - adrs/managed-agents-sandbox-backend.md | 2 +- deploy/do-managed-agents/Dockerfile | 2 +- docs/managed-agents.md | 279 ------------------------- docs/sandbox-preservation.md | 2 +- 5 files changed, 3 insertions(+), 283 deletions(-) delete mode 100644 docs/managed-agents.md diff --git a/README.md b/README.md index d42902d6d2..80b54cab83 100644 --- a/README.md +++ b/README.md @@ -242,7 +242,6 @@ upstream source history to merge. - [`docs/principal-links.md`](./docs/principal-links.md) — one person, several sign-ins: linking principals - [`docs/porter.md`](./docs/porter.md) — running qm on Porter - [`docs/superserve.md`](./docs/superserve.md) — using Superserve for agent sandboxes -- [`docs/managed-agents.md`](./docs/managed-agents.md) — using DigitalOcean Managed Agents microVMs for agent sandboxes - [`.env.example`](./.env.example) — every knob, documented in place - [`docs/swarms.md`](./docs/swarms.md) — durable agent pools, scoped messages, and blank Modal workers - [`docs/model-gateway.md`](./docs/model-gateway.md) — discover and route models through a gateway diff --git a/adrs/managed-agents-sandbox-backend.md b/adrs/managed-agents-sandbox-backend.md index c41ad7218c..68eebc9c52 100644 --- a/adrs/managed-agents-sandbox-backend.md +++ b/adrs/managed-agents-sandbox-backend.md @@ -83,7 +83,7 @@ What it does not do - No provider checkpoints, though not for the reason you might expect. We do expose session checkpoints publicly — `POST /v2/agents/sessions/{id}/checkpoints` is synchronous and returns a terminal checkpoint — along with list, rollback and fork. This backend simply does not use them yet, so it gets native pause and falls back to your portable tar home snapshot for recovery, under `DO_AGENTS_SNAPSHOT_INTERVAL_SEC` and `DO_AGENTS_SNAPSHOT_S3_BUCKET`. Moving to native checkpoints, the way the Sprites backend reports `provider_snapshot`, is a clean follow-up. The one thing to check first is that our checkpoints bind machine state to the event-log cursor and refuse during an active turn, both of which are managed-loop concepts that need verifying against a session that has no turns. - No idle reaping from qm's side. Our control plane owns the idle timeout, set per session with `DO_AGENTS_IDLE_TIMEOUT_SEC` and bounded by tenant policy. Paused sessions still count against team quota, which is worth knowing before pointing a large deployment at us. -- A QM template, optional. `agent: none` lands on our own bare base template, so `DO_AGENTS_TEMPLATE` is an override rather than a requirement; an unprovisioned name fails session create. `deploy/do-managed-agents/Dockerfile` is the QM image (the E2B image plus `ripgrep`, `HOME=/workspace/home`), registered as a team template on the agentless `sandbox` base; `docs/managed-agents.md` has the build steps. Baking layer tool files into it is a follow-up. +- A QM template, optional. `agent: none` lands on our own bare base template, so `DO_AGENTS_TEMPLATE` is an override rather than a requirement; an unprovisioned name fails session create. `deploy/do-managed-agents/Dockerfile` is the QM image (the E2B image plus `ripgrep`, `HOME=/workspace/home`), registered as a team template on the agentless `sandbox` base. Baking layer tool files into it is a follow-up. - No new CLI target. `qm init` still deploys core to Docker, Fly or AWS. This only changes where the computers live. Three things we need from you, or at least need to agree on diff --git a/deploy/do-managed-agents/Dockerfile b/deploy/do-managed-agents/Dockerfile index d997df523e..57826db552 100644 --- a/deploy/do-managed-agents/Dockerfile +++ b/deploy/do-managed-agents/Dockerfile @@ -1,5 +1,5 @@ # QM sandbox image for DigitalOcean Managed Agents (Harness Runtime BYOT). -# Registered on the agentless `sandbox` base; see docs/managed-agents.md. +# Registered on the agentless `sandbox` base. FROM node:24-slim@sha256:6f7b03f7c2c8e2e784dcf9295400527b9b1270fd37b7e9a7285cf83b6951452d ENV DEBIAN_FRONTEND=noninteractive diff --git a/docs/managed-agents.md b/docs/managed-agents.md deleted file mode 100644 index b86a70539a..0000000000 --- a/docs/managed-agents.md +++ /dev/null @@ -1,279 +0,0 @@ -# Managed Agents agent sandboxes - -`SANDBOX_BACKEND=do-managed-agents` runs each QM scope in a DigitalOcean Managed Agents microVM — a -Firecracker guest with a persistent home, hypervisor-level isolation between -scopes, and provider-managed pause and resume. - -Managed Agents is normally a managed-agent product: you hand it a manifest naming a -coding agent and it runs that agent's loop inside the guest for you. QM does -not want that. The manifest says `agent: none`, which Managed Agents admits as a bare -sandbox — its own base image, no managed agent, no in-guest event-translation -runtime, no model credential. QM keeps its own agent loop in core and drives -the guest as a computer, exactly as it does for every other backend. See -[the design record](../adrs/managed-agents-sandbox-backend.md) for why. - -## What it provides - -A microVM per scope, not per turn. Each person and each room gets its own -guest, isolated at the hypervisor rather than by container namespaces. Teardown -pauses rather than destroys, so files, installed packages, and clones survive -between conversations and the next turn resumes the same machine. - -Persistence is the provider's, not ours. Managed Agents preserves the guest's disk across -pause and resume on its own, so the backend reports `provider_managed` -persistence and does not need to tar the home up after every turn. The S3 -snapshot path exists only to survive losing the session entirely. - -One credential, one endpoint. A DigitalOcean IAM token authenticates both -halves of the integration, and there are no client certificates, internal -hostnames, or VPC prerequisites to arrange. - -## Architecture - -Two planes, one token. Lifecycle is ordinary REST. Commands and files travel a -WebSocket the control plane bridges into the guest. - -```mermaid -flowchart LR - subgraph core["QM core"] - orch["agent loop"] - cl["managed-agents-client"] - tun["managed-agents-tunnel
local TCP listener"] - end - - subgraph cp["DigitalOcean control plane
api.digitalocean.com"] - rest["/v2/agents/sessions
REST lifecycle"] - pf["/port-forward/8443
WebSocket bridge"] - end - - subgraph vm["Managed Agents microVM — agent: none"] - sa["sandbox-agent
gRPC :8443"] - proc["bash, git, node, python3
/workspace/home"] - end - - orch --> cl - cl -->|"create, get, pause, resume, delete"| rest - cl --> tun - tun -->|"wss + bearer token"| pf - pf -->|"guest TCP :8443"| sa - sa --> proc - rest -.->|"provisions"| vm -``` - -The tunnel is the part worth understanding. Every Managed Agents microVM runs -`sandbox-agent`, a gRPC server on guest port 8443 that Managed Agents's own control plane -uses for exec and file transfer. QM reaches it the same way -`doctl agents port-forward` does: it opens a WebSocket to -`/v2/agents/sessions/{id}/port-forward/8443`, which the control plane bridges to -the guest port, and exposes that as a local TCP listener a gRPC client dials. -There is no direct network path from core to the microVM; the control plane -proxies bytes. - -QM deliberately does not use the REST `POST /v2/agents/sessions/{id}/sandbox/exec`. -That endpoint buffers output in the control plane, so it clamps commands to four -minutes and one mebibyte per stream and carries no per-command environment. The -port-forward route has no request deadline, so a command is bounded only by -`SANDBOX_TIMEOUT_SEC`. - -## Operations - -### Create - -Nothing is provisioned when core boots, and a turn that needs no shell never -touches Managed Agents. The first execute-family tool call provisions, which takes about -ten seconds end to end: post the manifest, poll until the session reports ready, -open the tunnel, run the command. - -Every later turn for the same scope reconnects in well under a second, because -the session name is derived from the scope. The backend tries, in order, the -session id it has stored, then whatever Managed Agents lists under that name, and only -creates when both come up empty. A paused session is resumed rather than -replaced. - -With `EAGER_PROVISION` at its default, a scope that has used tools before starts -provisioning at the top of every turn instead of waiting for the model to ask, -which hides the cold start. Set `EAGER_PROVISION=0` for the strictly on-demand -path. - -### Exec - -Commands run as `/bin/bash -c` over a bidirectional gRPC stream, carrying -per-command environment, working directory, and a timeout. Output streams back -as it is produced, capped at 16 MiB per stream — write larger results to files -and read them instead. - -Background process sessions, `startProcess` and friends, are built on the same -stream, so long-running servers behave as they do on other backends. - -### Upload and download - -File transfer uses the guest agent's `Upload` and `Download` RPCs over the same -tunnel, chunked at 256 KiB. A download of a missing file reads as `null` rather -than raising. - -`sandbox-agent` confines transfer to the guest workspace, so a scope's home -lives at `/workspace/home` rather than the usual `/home/user`. That also places -it inside the volume Managed Agents preserves, so the home survives pause and resume on -its own. - -### Pause and resume - -Both are automatic, and neither is a verb a user or the model can call. - -A scope pauses when its turn tears down. If a home snapshot is due the backend -takes it first, then calls `POST /v2/agents/sessions/{id}/pause` and closes the -tunnel. A pause that fails is not silent: the scope is recorded as -`pause_failed` with the provider's error, which surfaces in the computer status -and in error reporting. Two cases skip the pause — an internal `keepWarm` -teardown, used when core knows another turn is imminent, and a teardown that -destroys instead. - -A scope resumes the next time anything needs the guest. Provisioning polls the -session, and on finding it paused issues `POST /resume` once and keeps polling -until it reports usable, up to five minutes. So the resume is a side effect of -the next command rather than a step anybody triggers; the only visible evidence -is that the first command of a conversation takes a little longer. - -There is no manual pause or resume from QM. The `sandbox` tool's verbs are -`list`, `create`, `set_default`, `status`, and `retire` — and for this backend -`restart` is not offered either, because `do-managed-agents` implements no `restartComputer`. -What you can do is **observe** and **destroy**: `status` reports -`lifecycleState: "paused"` along with the snapshot state and any -`pause_failed` error, and `retire` deletes outright. If you want a scope to stop -costing anything, retire it rather than looking for a pause button. - -### Delete - -Teardown at the end of a turn **pauses**. Actual deletion happens in four places -and no others: - -- An explicit retire, through the agent's `sandbox` tool or the admin API. -- Scratch sandboxes, which are always killed rather than paused. -- A failed home hydration, which kills the session rather than risk overwriting - a good snapshot with a half-populated home. -- Swarm workers, which are retired when the swarm finishes. - -## Gotchas - -**Deleting a conversation does not delete the microVM.** Nothing in QM ties -scope deletion to sandbox deletion, and the `do-managed-agents` backend implements no deep -idle reaper, so core's sweeper never reclaims one. A paused microVM stays -paused, indefinitely, until somebody retires it or Managed Agents's own idle policy -reclaims it. Budget for that: set `DO_AGENTS_IDLE_TIMEOUT_SEC` so the provider has a -backstop, and retire scopes you are finished with. - -**`PUBLIC_API_URL` has to be reachable from inside the guest.** Reachability -runs both ways: core calls the control plane, and microVMs call back into core's -self-API for the connector SDK and file callbacks. A deployment behind -`localhost` serves shell commands fine but cannot complete a cron, a send, or a -credential fetch. - -**Use the QM template.** The bare base `agent: none` boots is Ubuntu with Node 20, -`bash`, `git`, `curl`, `tar`, `python3`, `make`, and `gcc`, but no `jq`, `rg`, `unzip`, or -`wget`, and it leaves `HOME` at `/root` instead of the home the adapter manages. Register -`deploy/do-managed-agents/Dockerfile` as a team template and name it in -`DO_AGENTS_TEMPLATE`; it is the same Node 24 image as the E2B template plus `ripgrep`, -with `HOME` set to `/workspace/home`. When `DO_AGENTS_TEMPLATE` is set the profile -advertises `jq`, `rg`, `unzip`, and `wget` as installed, so a custom template should -carry them too. See [Build the QM template](#build-the-qm-template). - -**Set `DO_AGENTS_SNAPSHOT_S3_BUCKET`.** It is how a scope's home survives losing its -session. Managed Agents exposes session checkpoints, but QM does not use them yet, so -without the bucket a destroyed or reclaimed session takes the home with it. See -[sandbox recovery](sandbox-preservation.md) for capture and hydration rules. - -**Sessions are created with `bash` pre-allowed.** Managed Agents has its own -human-in-the-loop gate on command execution, enforced in the guest. Leaving it -on would hold every command behind an approval nobody is watching, so the -manifest allows `bash` outright and QM's own approval gates stay authoritative. - -**Name your deployments apart.** Session names are derived from the scope and -`DO_AGENTS_NAME_PREFIX`, which is also how an operator finds a scope's microVM in the -Managed Agents console without consulting core's database. Two deployments sharing a prefix -on one Managed Agents team will adopt each other's sandboxes. Managed Agents caps names at 64 -characters and rejects UUID-shaped ones. - -## Configure - -```sh -SANDBOX_BACKEND=do-managed-agents -DO_AGENTS_API_TOKEN=dop_v1_your-digitalocean-iam-token -DO_AGENTS_NAME_PREFIX=my-qm -DO_AGENTS_SNAPSHOT_S3_BUCKET=my-qm-sandbox-homes -``` - -Store the token as a core service secret. Production requires `DATABASE_URL` -for session records and provisioning locks across instances. - -| Variable | Default | Purpose | -| --------------------------------- | ------------------------------ | ----------------------------------------------------------------- | -| `DO_AGENTS_API_TOKEN` | Required | DigitalOcean IAM token, held by core. | -| `DO_AGENTS_API_BASE_URL` | `https://api.digitalocean.com` | Control-plane endpoint override; any path prefix is preserved. | -| `DO_AGENTS_TEMPLATE` | Provider bare base | Team template to boot. Unset boots the bare base; use QM template | -| `DO_AGENTS_NAME_PREFIX` | `qm` | Namespace for scope discovery. | -| `DO_AGENTS_SIZE_SLUG` | Provider default | microVM size. | -| `DO_AGENTS_IDLE_TIMEOUT_SEC` | Provider default | Provider-side idle reclaim backstop. | -| `DO_AGENTS_EGRESS_PROXY_URL` | Unset | Proxy host to allow; everything else is denied. | -| `DO_AGENTS_SNAPSHOT_S3_BUCKET` | Unset | Portable home tars. Without it the backend reports no recovery. | -| `DO_AGENTS_SNAPSHOT_INTERVAL_SEC` | `300` | Snapshot throttle. | -| `SANDBOX_TIMEOUT_SEC` | `600` | Default command deadline in seconds. | - -Without `DO_AGENTS_EGRESS_PROXY_URL`, sandboxes have unrestricted egress and the -profile reports no enforcement. With it, sessions are created denying all -outbound traffic except the proxy host, so `HTTPS_PROXY` cannot be bypassed from -inside the microVM. - -## Local development - -```sh -export DO_AGENTS_API_TOKEN=dop_v1_... -bash scripts/dev-instance.sh up --surface web --sandbox do-managed-agents -``` - -Two things differ from the other backends. `dev-instance` reads provider secrets -from your shell or `~/.config/qm/dev.env`, not from the worktree `.env`, so the -token has to be exported. And a live slot reloads from its persisted boot spec, -so changing `--sandbox` on a running slot keeps the old backend — take it down -first. - -## Tests - -Backend tests run without an account. They stand up an in-process control plane, -a real gRPC `sandbox-agent`, and a WebSocket tunnel between them, so the -transport is exercised end to end: - -```sh -node --test test/managed-agents-client.test.ts test/managed-agents-sandbox.test.ts -``` - -Two smokes exercise a real account. The first provisions a microVM and runs -commands, files, and a pause cycle through the `Sandbox` interface; set -`DO_AGENTS_SMOKE_LONG=1` to add a 250-second command that outlives the REST exec -clamp. The second drives a live model turn against a running core and asserts -the reply carries output that could only have come from the guest. - -```sh -DO_AGENTS_API_TOKEN=dop_v1_... npm run smoke:managed-agents-sandbox -CORE_SIGNING_SECRET=... PORTAL_IDENTITY_SECRET=... npm run smoke:managed-agents-turn -``` - -## Build the QM template - -Managed Agents builds a team template from an image in your DigitalOcean Container -Registry, rebased onto a platform base. QM uses the `sandbox` base, which carries no -agent. With Docker and an authenticated `doctl`: - -```sh -REG=registry.digitalocean.com/ -docker build --platform linux/amd64 -t $REG/qm-sandbox:v1 deploy/do-managed-agents -doctl registry login && docker push $REG/qm-sandbox:v1 -doctl harness-runtime template create --name qm-sandbox --base-template sandbox \ - --source-oci-ref $REG/qm-sandbox:v1 -doctl harness-runtime template list-builds qm-sandbox # wait for SUCCEEDED -``` - -Then set `DO_AGENTS_TEMPLATE=qm-sandbox`. To ship an image change, push a new tag and run -`doctl harness-runtime template update qm-sandbox --source-oci-ref $REG/qm-sandbox:v2`. -Templates do not pick up platform fixes on their own, so rebuild after Managed Agents -updates as well. Existing sessions keep the template they were created from; a scope only -boots the new one after its session is destroyed. diff --git a/docs/sandbox-preservation.md b/docs/sandbox-preservation.md index 630752d762..bfea57b03e 100644 --- a/docs/sandbox-preservation.md +++ b/docs/sandbox-preservation.md @@ -35,4 +35,4 @@ When `MODAL_EGRESS_PROXY_URL` is set, machines are created with Modal's outbound Turn environment reaches Modal commands through the exec `env` parameter rather than inlined `export` statements, exec deadlines are rounded up to whole seconds with a single thirty-second grace beyond the in-guest `timeout`, and a command longer than 64 KiB is written to `/tmp` through the filesystem API and executed from there. -Managed Agents sessions name a sandbox template explicitly, which yields a microVM carrying no managed agent loop, and the disk persists across the provider's pause and resume. Used-turn teardown snapshots the home to `DO_AGENTS_SNAPSHOT_S3_BUCKET` when the home changed, throttled by `DO_AGENTS_SNAPSHOT_INTERVAL_SEC`, and then pauses the session; a pause failure is recorded as `pause_failed` and the running session is retained for retry. Status reports `recovery.strategy: "workspace_snapshot"`. Managed Agents does expose session checkpoints on its public surface, but core does not capture them yet, so the portable tar is the only recovery path: a replacement session for a scope hydrates from it, and a failed hydration kills the replacement rather than adopting a blank home over a saved one. Session lifecycle travels over the Managed Agents REST surface with a DigitalOcean IAM token; commands and file transfer travel over a bearer-authenticated WebSocket port-forward to the guest's `sandbox-agent` gRPC server, which has no request deadline, so command length is bounded by `SANDBOX_TIMEOUT_SEC` rather than by the control plane. Sessions are created with a permissions block that allows `bash` outright, because Managed Agents would otherwise hold every command at its own approval gate behind core's. See [Managed Agents sandboxes](managed-agents.md) for configuration. +Managed Agents sessions name a sandbox template explicitly, which yields a microVM carrying no managed agent loop, and the disk persists across the provider's pause and resume. Used-turn teardown snapshots the home to `DO_AGENTS_SNAPSHOT_S3_BUCKET` when the home changed, throttled by `DO_AGENTS_SNAPSHOT_INTERVAL_SEC`, and then pauses the session; a pause failure is recorded as `pause_failed` and the running session is retained for retry. Status reports `recovery.strategy: "workspace_snapshot"`. Managed Agents does expose session checkpoints on its public surface, but core does not capture them yet, so the portable tar is the only recovery path: a replacement session for a scope hydrates from it, and a failed hydration kills the replacement rather than adopting a blank home over a saved one. Session lifecycle travels over the Managed Agents REST surface with a DigitalOcean IAM token; commands and file transfer travel over a bearer-authenticated WebSocket port-forward to the guest's `sandbox-agent` gRPC server, which has no request deadline, so command length is bounded by `SANDBOX_TIMEOUT_SEC` rather than by the control plane. Sessions are created with a permissions block that allows `bash` outright, because Managed Agents would otherwise hold every command at its own approval gate behind core's. From a1f3dbb80d36f58fc14930ffa7d2792967980c12 Mon Sep 17 00:00:00 2001 From: Josh France Date: Thu, 8 Oct 2026 20:19:17 +0000 Subject: [PATCH 08/19] Describe the qm-sandbox template accurately in the ADR --- adrs/managed-agents-sandbox-backend.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adrs/managed-agents-sandbox-backend.md b/adrs/managed-agents-sandbox-backend.md index 68eebc9c52..10e52bdc8b 100644 --- a/adrs/managed-agents-sandbox-backend.md +++ b/adrs/managed-agents-sandbox-backend.md @@ -83,7 +83,7 @@ What it does not do - No provider checkpoints, though not for the reason you might expect. We do expose session checkpoints publicly — `POST /v2/agents/sessions/{id}/checkpoints` is synchronous and returns a terminal checkpoint — along with list, rollback and fork. This backend simply does not use them yet, so it gets native pause and falls back to your portable tar home snapshot for recovery, under `DO_AGENTS_SNAPSHOT_INTERVAL_SEC` and `DO_AGENTS_SNAPSHOT_S3_BUCKET`. Moving to native checkpoints, the way the Sprites backend reports `provider_snapshot`, is a clean follow-up. The one thing to check first is that our checkpoints bind machine state to the event-log cursor and refuse during an active turn, both of which are managed-loop concepts that need verifying against a session that has no turns. - No idle reaping from qm's side. Our control plane owns the idle timeout, set per session with `DO_AGENTS_IDLE_TIMEOUT_SEC` and bounded by tenant policy. Paused sessions still count against team quota, which is worth knowing before pointing a large deployment at us. -- A QM template, optional. `agent: none` lands on our own bare base template, so `DO_AGENTS_TEMPLATE` is an override rather than a requirement; an unprovisioned name fails session create. `deploy/do-managed-agents/Dockerfile` is the QM image (the E2B image plus `ripgrep`, `HOME=/workspace/home`), registered as a team template on the agentless `sandbox` base. Baking layer tool files into it is a follow-up. +- A QM template, optional. `agent: none` lands on our own bare base template, so `DO_AGENTS_TEMPLATE` is an override rather than a requirement; an unprovisioned name fails session create. `deploy/do-managed-agents/Dockerfile` defines it: `node:24-slim` plus the tools agent prompts reach for by name (`jq`, `rg`, `unzip`, `wget`, alongside git, curl and Python), with `HOME=/workspace/home`. It is registered as the `qm-sandbox` team template on the agentless `sandbox` base, and qm advertises those extra tools in the sandbox profile only when `DO_AGENTS_TEMPLATE` is set. Baking layer tool files into it is a follow-up. - No new CLI target. `qm init` still deploys core to Docker, Fly or AWS. This only changes where the computers live. Three things we need from you, or at least need to agree on From 64e8eec9b29f965cc474e72a043f98d7c036345b Mon Sep 17 00:00:00 2001 From: Josh France Date: Thu, 8 Oct 2026 20:31:56 +0000 Subject: [PATCH 09/19] Trim the ADR to the design --- adrs/managed-agents-sandbox-backend.md | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/adrs/managed-agents-sandbox-backend.md b/adrs/managed-agents-sandbox-backend.md index 10e52bdc8b..4158dedfdc 100644 --- a/adrs/managed-agents-sandbox-backend.md +++ b/adrs/managed-agents-sandbox-backend.md @@ -22,8 +22,6 @@ Everything qm needs is on one host with one credential. Session lifecycle is our Exec and file transfer take a different route to the same host. Every microVM runs `sandbox-agent`, a gRPC server on guest port 8443 that our own control plane dials for exec, transfer and readiness. qm reaches it the way `doctl agents port-forward` does: a bearer-authenticated WebSocket to `/v2/agents/sessions/{id}/port-forward/8443`, which the control plane bridges to the guest port and qm exposes as a local TCP listener for a gRPC client. 8443 is explicitly allowed by our tunnel port policy, the guest listener is plaintext h2c with no client certificate, and we register the port-forward route without a request deadline on purpose. So a long command is bounded by qm's own `SANDBOX_TIMEOUT_SEC` and nothing else. -Worth saying why qm does not use `POST /v2/agents/sessions/{id}/sandbox/exec`, since we built it and it looks like the obvious fit. It terminates in the control plane and buffers the result, so it clamps to four minutes and one mebibyte per stream and carries no per-command environment. Those bounds are right for `doctl agents exec` and wrong for an agent turn. - Here is a turn end to end. Lifecycle goes through the edge as REST; exec and files go through the tunnel to the guest. ```mermaid @@ -81,16 +79,6 @@ What the PR adds What it does not do -- No provider checkpoints, though not for the reason you might expect. We do expose session checkpoints publicly — `POST /v2/agents/sessions/{id}/checkpoints` is synchronous and returns a terminal checkpoint — along with list, rollback and fork. This backend simply does not use them yet, so it gets native pause and falls back to your portable tar home snapshot for recovery, under `DO_AGENTS_SNAPSHOT_INTERVAL_SEC` and `DO_AGENTS_SNAPSHOT_S3_BUCKET`. Moving to native checkpoints, the way the Sprites backend reports `provider_snapshot`, is a clean follow-up. The one thing to check first is that our checkpoints bind machine state to the event-log cursor and refuse during an active turn, both of which are managed-loop concepts that need verifying against a session that has no turns. - No idle reaping from qm's side. Our control plane owns the idle timeout, set per session with `DO_AGENTS_IDLE_TIMEOUT_SEC` and bounded by tenant policy. Paused sessions still count against team quota, which is worth knowing before pointing a large deployment at us. - A QM template, optional. `agent: none` lands on our own bare base template, so `DO_AGENTS_TEMPLATE` is an override rather than a requirement; an unprovisioned name fails session create. `deploy/do-managed-agents/Dockerfile` defines it: `node:24-slim` plus the tools agent prompts reach for by name (`jq`, `rg`, `unzip`, `wget`, alongside git, curl and Python), with `HOME=/workspace/home`. It is registered as the `qm-sandbox` team template on the agentless `sandbox` base, and qm advertises those extra tools in the sandbox profile only when `DO_AGENTS_TEMPLATE` is set. Baking layer tool files into it is a follow-up. - No new CLI target. `qm init` still deploys core to Docker, Fly or AWS. This only changes where the computers live. - -Three things we need from you, or at least need to agree on - -- Reachability runs both ways. qm needs to reach `api.digitalocean.com` and nothing else, which is the easy direction. But qm's sandboxes also call _back_ into core's self-API with their capability tokens, the way the E2B backend needs a public `PUBLIC_API_URL`. That means core's public URL has to be on the session's egress allowlist. For a qm deployment inside DO this is fine; for one on Fly or AWS it means our allowlist has to carry an arbitrary external host, which we should confirm is acceptable to your operators and ours. -- We should agree on what the bare base carries. `agent: none` removed the prerequisite that used to sit here: no template has to be provisioned for your team, and we verified the guest is genuinely bare — no agent CLI on `PATH`, no event-translation runtime, no agent supervisor, just `sandbox-agent`, `envd`, `otelcol` and s6 supervision. `bash`, `git`, `curl`, `tar`, Node, Python, `make` and `gcc` are present; `jq`, `rg` and `unzip` are not, and those three are reached for by name in agent prompts. The profile declares them not installed so the model does not try. The bare base also leaves `HOME` at `/root`. The QM template covers both for deployments that register it; adding the small tools to the base would still help deployments that do not. - -- The vendored proto should not have to exist, and we would like to delete it. It is here because our public API has no streaming exec: the REST endpoint buffers in the control plane, so it clamps to four minutes and one mebibyte per stream and carries no per-command environment, and the only public transport without those limits is the raw port-forward. So qm ends up speaking our guest contract, which leaves one wire with two owners in two repos — change a field and qm breaks with no version bump and no type error, just a decode mismatch in someone's sandbox. Note that every other backend in this tree consumes a published provider SDK and vendors nothing; this is the one exception. We are asking internally to expose streaming exec at the public edge with per-command environment and no request deadline, which the guest already supports. When that lands, `managed-agents-sandbox-agent.proto`, `managed-agents-tunnel.ts` and the two gRPC dependencies all go away and this backend talks to a published, versioned endpoint. Until then we would rather vendor the three RPCs we call, visibly and with a test pinning the subset, than quietly depend on an unpublished contract. - -Tests run against an in-process control plane, a real gRPC `sandbox-agent` and a WebSocket tunnel between them, so the transport is exercised end to end and CI needs no DO account and no credentials. We would rather this live upstream than in a fork, and we will keep it green as the sandbox contract moves. From b806885bc872419d2f2cf123e73ddbbec040139e Mon Sep 17 00:00:00 2001 From: rvarma Date: Fri, 9 Oct 2026 11:26:20 +0530 Subject: [PATCH 10/19] Capture a Managed Agents checkpoint on teardown. The portable home tar survives a destroyed session, but it is not the provider's machine-state checkpoint. Call CreateCheckpoint while the session is still running so the live microVM is captured before pause. --- adrs/managed-agents-sandbox-backend.md | 3 +- docs/sandbox-preservation.md | 2 +- src/sandbox/managed-agents-client.ts | 50 +++++++++++++++++++ src/sandbox/managed-agents-sandbox.ts | 33 ++++++++++++- test/managed-agents-client.test.ts | 15 ++++++ test/managed-agents-sandbox.test.ts | 54 +++++++++++++++++++++ test/support/fake-managed-agents-service.ts | 44 +++++++++++++++++ test/support/fake-managed-agents.ts | 28 +++++++++++ 8 files changed, 225 insertions(+), 4 deletions(-) diff --git a/adrs/managed-agents-sandbox-backend.md b/adrs/managed-agents-sandbox-backend.md index 4158dedfdc..d783446f3d 100644 --- a/adrs/managed-agents-sandbox-backend.md +++ b/adrs/managed-agents-sandbox-backend.md @@ -59,6 +59,7 @@ sequenceDiagram alt keepWarm for background work B->>B: leave the session running else park the scope + B->>E: POST /v2/agents/sessions/{id}/checkpoints B->>V: Download
portable tar home snapshot B->>E: POST /v2/agents/sessions/{id}/pause else destroy the scope @@ -70,7 +71,7 @@ What the PR adds - `SANDBOX_BACKEND=do-managed-agents`, with `DO_AGENTS_API_TOKEN` required — a DO IAM token, which is also where the team identity comes from, and the only credential the backend needs. Optional knobs: `DO_AGENTS_API_BASE_URL`, `DO_AGENTS_TEMPLATE`, `DO_AGENTS_NAME_PREFIX`, `DO_AGENTS_SIZE_SLUG`, `DO_AGENTS_IDLE_TIMEOUT_SEC`, `DO_AGENTS_EGRESS_PROXY_URL`, `DO_AGENTS_SNAPSHOT_INTERVAL_SEC`, `DO_AGENTS_SNAPSHOT_S3_BUCKET`, and the shared `SANDBOX_TIMEOUT_SEC`. - `src/sandbox/managed-agents-tunnel.ts`, the port-forward client: one WebSocket per TCP connection, bytes piped both ways with one chunk in flight per direction so a slow consumer cannot make core buffer the stream. The local listener is unreferenced, so an abandoned tunnel can never hold core's event loop open. -- `src/sandbox/managed-agents-client.ts`, holding both transports behind one interface the way `e2b-client.ts` wraps the E2B SDK: REST for `/v2/agents/sessions{,/{id},/{id}/pause,/{id}/resume}`, and `SandboxAgentService` stubs for `Exec`, `Upload` and `Download` over the tunnel. Our 404s, terminal statuses and gRPC `UNAVAILABLE` translate to the same two error classes the E2B client uses — one for a sandbox that was already gone before a command started, which the backend reconnects and retries, and one for a sandbox lost mid-command, which is reported rather than retried because the command may have partially run. The session body is an agents.yaml manifest in its flat form, which is the shape we are standardising on. +- `src/sandbox/managed-agents-client.ts`, holding both transports behind one interface the way `e2b-client.ts` wraps the E2B SDK: REST for `/v2/agents/sessions{,/{id},/{id}/pause,/{id}/resume,/{id}/checkpoints}`, and `SandboxAgentService` stubs for `Exec`, `Upload` and `Download` over the tunnel. Our 404s, terminal statuses and gRPC `UNAVAILABLE` translate to the same two error classes the E2B client uses — one for a sandbox that was already gone before a command started, which the backend reconnects and retries, and one for a sandbox lost mid-command, which is reported rather than retried because the command may have partially run. The session body is an agents.yaml manifest in its flat form, which is the shape we are standardising on. - `src/sandbox/managed-agents-sandbox-agent.proto`, a vendored copy of our guest proto with the internal annotations stripped, loaded at runtime through `@grpc/proto-loader`. No build step and no generated code in the tree. - `src/sandbox/managed-agents-sandbox.ts`. Provision creates or resumes the session and waits for it to become usable. Run is `Exec` over the tunnel. Files are `Upload` and `Download` rather than base64 through exec, since we have real transfer RPCs. Process sessions, read-only layers, layer tool install, home snapshots and blob staging come from the shared exec helpers unchanged. - Scope recovery without local state. Session names are team-unique among non-terminal sessions and `ListSessions` takes a `?name=` filter, so the backend can re-adopt a running sandbox after losing its durable record, the way the E2B backend re-adopts by sandbox metadata. `sandboxScopeName` already produces names that fit our 64-character, not-UUID-shaped rule. diff --git a/docs/sandbox-preservation.md b/docs/sandbox-preservation.md index bfea57b03e..a563a03d4c 100644 --- a/docs/sandbox-preservation.md +++ b/docs/sandbox-preservation.md @@ -35,4 +35,4 @@ When `MODAL_EGRESS_PROXY_URL` is set, machines are created with Modal's outbound Turn environment reaches Modal commands through the exec `env` parameter rather than inlined `export` statements, exec deadlines are rounded up to whole seconds with a single thirty-second grace beyond the in-guest `timeout`, and a command longer than 64 KiB is written to `/tmp` through the filesystem API and executed from there. -Managed Agents sessions name a sandbox template explicitly, which yields a microVM carrying no managed agent loop, and the disk persists across the provider's pause and resume. Used-turn teardown snapshots the home to `DO_AGENTS_SNAPSHOT_S3_BUCKET` when the home changed, throttled by `DO_AGENTS_SNAPSHOT_INTERVAL_SEC`, and then pauses the session; a pause failure is recorded as `pause_failed` and the running session is retained for retry. Status reports `recovery.strategy: "workspace_snapshot"`. Managed Agents does expose session checkpoints on its public surface, but core does not capture them yet, so the portable tar is the only recovery path: a replacement session for a scope hydrates from it, and a failed hydration kills the replacement rather than adopting a blank home over a saved one. Session lifecycle travels over the Managed Agents REST surface with a DigitalOcean IAM token; commands and file transfer travel over a bearer-authenticated WebSocket port-forward to the guest's `sandbox-agent` gRPC server, which has no request deadline, so command length is bounded by `SANDBOX_TIMEOUT_SEC` rather than by the control plane. Sessions are created with a permissions block that allows `bash` outright, because Managed Agents would otherwise hold every command at its own approval gate behind core's. +Managed Agents sessions name a sandbox template explicitly, which yields a microVM carrying no managed agent loop, and the disk persists across the provider's pause and resume. Used-turn teardown calls `POST /v2/agents/sessions/{id}/checkpoints` when the home changed, throttled by `DO_AGENTS_SNAPSHOT_INTERVAL_SEC`, then snapshots the home to `DO_AGENTS_SNAPSHOT_S3_BUCKET` and pauses the session. The checkpoint is the provider's machine-state capture of the live session (`CreateCheckpoint` blocks until the capture is `READY`); only the newest is kept, and the previous one is deleted after the new capture succeeds. A capture failure is recorded on `recovery.error` and the pause still proceeds. A pause failure is recorded as `pause_failed` and the running session is retained for retry. Status reports `recovery.strategy: "workspace_snapshot"` plus `recovery.checkpointId`. A checkpoint is bound to its session, so a replacement session for a destroyed or reclaimed scope still hydrates from the portable tar, and a failed hydration kills the replacement rather than adopting a blank home over a saved one. Session lifecycle travels over the Managed Agents REST surface with a DigitalOcean IAM token; commands and file transfer travel over a bearer-authenticated WebSocket port-forward to the guest's `sandbox-agent` gRPC server, which has no request deadline, so command length is bounded by `SANDBOX_TIMEOUT_SEC` rather than by the control plane. Sessions are created with a permissions block that allows `bash` outright, because Managed Agents would otherwise hold every command at its own approval gate behind core's. diff --git a/src/sandbox/managed-agents-client.ts b/src/sandbox/managed-agents-client.ts index 4546bbc569..e1446d2a7f 100644 --- a/src/sandbox/managed-agents-client.ts +++ b/src/sandbox/managed-agents-client.ts @@ -33,12 +33,20 @@ export interface ManagedAgentsSessionInfo extends ManagedAgentsSessionSummary { template?: string; } +export interface ManagedAgentsCheckpoint { + checkpointId: string; + status: string; + createdAtMs?: number; +} + export interface ManagedAgentsSession { readonly sessionId: string; readonly sandboxId: string; runCommand(command: string, opts?: ManagedAgentsRunOpts): Promise; readFileBytes(absPath: string): Promise; writeFileBytes(absPath: string, data: Uint8Array): Promise; + createCheckpoint(label?: string): Promise; + deleteCheckpoint(checkpointId: string): Promise; pause(): Promise; resume(): Promise; kill(): Promise; @@ -100,6 +108,7 @@ const DEFAULT_TEMPLATE = ""; const DEFAULT_AGENT = "none"; const DEFAULT_MAX_COMMAND_MS = 3600_000; const CREATE_TIMEOUT_MS = 120_000; +const CHECKPOINT_TIMEOUT_MS = 5 * 60_000; const REQUEST_TIMEOUT_MS = 30_000; const READY_TIMEOUT_MS = 300_000; const READY_POLL_MS = 2_000; @@ -177,6 +186,13 @@ interface WireSession { created_at?: string; } +interface WireCheckpoint { + checkpoint_id?: string; + status?: string; + error_message?: string; + created_at?: string; +} + const toSummary = (s: WireSession): ManagedAgentsSessionSummary => ({ sessionId: s.session_id ?? s.id ?? "", sandboxId: s.sandbox_id ?? "", @@ -613,6 +629,40 @@ export function createSdkManagedAgentsClient(opts: SdkManagedAgentsClientOptions }); }, + async createCheckpoint(label?: string): Promise { + const body = await callJson<{ checkpoint?: WireCheckpoint }>( + "POST", + `/v2/agents/sessions/${sessionId}/checkpoints`, + sessionId, + JSON.stringify(label ? { label } : {}), + "application/json", + CHECKPOINT_TIMEOUT_MS, + ); + const captured = body.checkpoint; + if (!captured?.checkpoint_id) + throw new Error(`do-managed-agents create checkpoint ${sessionId}: response carried no checkpoint id`); + if (captured.status !== "READY") { + const detail = captured.error_message ? `: ${captured.error_message}` : ""; + throw new Error( + `do-managed-agents create checkpoint ${sessionId}: status ${captured.status ?? "missing"}${detail}`, + ); + } + const createdAtMs = captured.created_at ? Date.parse(captured.created_at) : Number.NaN; + return { + checkpointId: captured.checkpoint_id, + status: captured.status, + ...(Number.isFinite(createdAtMs) ? { createdAtMs } : {}), + }; + }, + + async deleteCheckpoint(checkpointId: string): Promise { + await callJson( + "DELETE", + `/v2/agents/sessions/${sessionId}/checkpoints/${encodeURIComponent(checkpointId)}`, + sessionId, + ); + }, + async pause(): Promise { await disconnect(); await callJson("POST", `/v2/agents/sessions/${sessionId}/pause`, sessionId); diff --git a/src/sandbox/managed-agents-sandbox.ts b/src/sandbox/managed-agents-sandbox.ts index 292cf0da3b..b9e53013b9 100644 --- a/src/sandbox/managed-agents-sandbox.ts +++ b/src/sandbox/managed-agents-sandbox.ts @@ -60,6 +60,7 @@ const HOME_TAR = `${HOME_DIR}/.qm-home.tar`; const SNAPSHOT_PRUNE = HOME_SNAPSHOT_PRUNE; const DEFAULT_SNAPSHOT_INTERVAL_MS = 5 * 60_000; +const CHECKPOINT_LABEL = "qm teardown"; export const QM_TEMPLATE_TOOLS = ["jq", "rg", "unzip", "wget"]; @@ -71,6 +72,9 @@ export interface StoredManagedAgentsSandbox { createdAtMs: number; lastSnapshotMs?: number; homeDirty?: boolean; + checkpointId?: string; + checkpointAtMs?: number; + checkpointError?: string; } export interface ManagedAgentsSandboxOptions extends BlobStagingOptions { @@ -124,6 +128,22 @@ export function createManagedAgentsSandbox(workspace: WorkspaceStore, opts: Mana await store.merge(scope, { lastSnapshotMs: Date.now(), homeDirty: false }); } + async function captureCheckpoint(scope: string, session: ManagedAgentsSession): Promise { + const previous = (await store.get(scope))?.checkpointId; + const captured = await session.createCheckpoint(CHECKPOINT_LABEL); + await store.merge(scope, { + checkpointId: captured.checkpointId, + checkpointAtMs: captured.createdAtMs ?? Date.now(), + checkpointError: undefined, + }); + if (!previous || previous === captured.checkpointId) return; + try { + await session.deleteCheckpoint(previous); + } catch (e) { + reportError("sandbox_snapshot", "checkpoint_delete_failed", errMessage(e), scope); + } + } + const hydrateHome = (scope: string, session: ManagedAgentsSession): Promise => homeSnapshots.hydrateHome(scope, session); @@ -440,11 +460,14 @@ export function createManagedAgentsSandbox(workspace: WorkspaceStore, opts: Mana if (!stored) return { machine: "no sandbox provisioned yet", provisioned: false, guestResponsive: false }; const machine = `do-managed-agents sandbox ${stored.sandboxId || stored.sessionId}`; + const checkpointAtMs = stored.checkpointAtMs ?? stored.lastSnapshotMs; + const recoveryError = stored.preservationError ?? stored.checkpointError; const recovery = { strategy: "workspace_snapshot" as const, state: stored.preservationState, - ...(stored.preservationError ? { error: stored.preservationError } : {}), - ...(stored.lastSnapshotMs ? { checkpointAtMs: stored.lastSnapshotMs } : {}), + ...(recoveryError ? { error: recoveryError } : {}), + ...(stored.checkpointId ? { checkpointId: stored.checkpointId } : {}), + ...(checkpointAtMs ? { checkpointAtMs } : {}), }; try { const info: ManagedAgentsSessionInfo = await client.info(stored.sessionId); @@ -513,6 +536,12 @@ export function createManagedAgentsSandbox(workspace: WorkspaceStore, opts: Mana const stored = await store.get(scope); if (!tdOpts?.homeUnchanged) await store.merge(scope, { homeDirty: true }); if (snapshotDue(stored, tdOpts, snapshotIntervalMs)) { + try { + await captureCheckpoint(scope, session); + } catch (e) { + await store.merge(scope, { checkpointError: errMessage(e) }); + reportError("sandbox_snapshot", "checkpoint_failed", errMessage(e), scope); + } try { await snapshotHome(scope, session); } catch (e) { diff --git a/test/managed-agents-client.test.ts b/test/managed-agents-client.test.ts index 4c8336e9fb..d537acb93f 100644 --- a/test/managed-agents-client.test.ts +++ b/test/managed-agents-client.test.ts @@ -234,6 +234,21 @@ test("list resolves a session by name and skips destroyed rows", async () => { assert.deepEqual(await client.list("qm-list"), []); }); +test("create checkpoint posts to the session checkpoint route and returns the ready id", async () => { + const s = await session("qm-checkpoint"); + const captured = await s.createCheckpoint("qm teardown"); + assert.match(captured.checkpointId, /^cp_/); + assert.equal(captured.status, "READY"); + assert.ok(captured.createdAtMs); + const stored = service.checkpoints(s.sessionId); + assert.equal(stored.length, 1); + assert.equal(stored[0]?.checkpoint_id, captured.checkpointId); + assert.equal(stored[0]?.label, "qm teardown"); + assert.equal(stored[0]?.status, "READY"); + await s.deleteCheckpoint(captured.checkpointId); + assert.equal(service.checkpoints(s.sessionId).length, 0); +}); + test("connect resumes a paused session before handing it back", async () => { const s = await session("qm-paused"); await s.pause(); diff --git a/test/managed-agents-sandbox.test.ts b/test/managed-agents-sandbox.test.ts index 7fff72f764..f8dd30a27c 100644 --- a/test/managed-agents-sandbox.test.ts +++ b/test/managed-agents-sandbox.test.ts @@ -68,6 +68,60 @@ test("the session is named after the scope so an operator can find it", async () assert.doesNotMatch(scopeName(), /^[0-9a-f]{8}-[0-9a-f]{4}-/, "Managed Agents rejects UUID-shaped session names"); }); +test("teardown captures a provider checkpoint before pausing", async () => { + const store: DurableMap = createMemoryMap(); + sandbox = make({ store }); + const h = await sandbox.provision(layers); + await sandbox.run(h, "echo hi > kept.txt"); + await sandbox.teardown(h); + const calls = fake.checkpointCalls(); + assert.equal(calls.length, 1); + assert.equal(calls[0]?.label, "qm teardown"); + assert.equal(calls[0]?.sessionId, fake.current(scopeName())?.sessionId); + const stored = await store.get(scope); + assert.equal(stored?.checkpointId, calls[0]?.checkpointId); + const status = await sandbox.computerStatus!(scope); + assert.equal(status.recovery?.checkpointId, stored?.checkpointId); + assert.equal(status.lifecycleState, "paused"); + assert.equal(fake.pauseCalls().length, 1); +}); + +test("a checkpoint failure is recorded and the session still pauses", async () => { + const store: DurableMap = createMemoryMap(); + sandbox = make({ store }); + const h = await sandbox.provision(layers); + fake.failNextCheckpoint(scopeName()); + await sandbox.teardown(h); + assert.equal(fake.checkpointCalls().length, 0); + assert.equal(fake.pauseCalls().length, 1); + assert.equal(fake.current(scopeName())?.state, "paused"); + const status = await sandbox.computerStatus!(scope); + assert.match(status.recovery?.error ?? "", /create checkpoint/); +}); + +test("checkpoint capture is throttled across a pause and resume", async () => { + const h = await sandbox.provision(layers); + await sandbox.teardown(h); + const again = await sandbox.provision(layers); + await sandbox.teardown(again); + assert.equal(fake.checkpointCalls().length, 1); +}); + +test("a newer checkpoint replaces the previous one", async () => { + const store: DurableMap = createMemoryMap(); + sandbox = make({ store, snapshotIntervalMs: -1 }); + const h = await sandbox.provision(layers); + await sandbox.teardown(h); + const first = (await store.get(scope))?.checkpointId; + const again = await sandbox.provision(layers); + await sandbox.teardown(again); + const second = (await store.get(scope))?.checkpointId; + assert.ok(first); + assert.ok(second); + assert.notEqual(first, second); + assert.deepEqual(fake.deletedCheckpoints(), [first]); +}); + test("teardown pauses the session and the next provision resumes it", async () => { const h = await sandbox.provision(layers); await sandbox.run(h, "echo hi > kept.txt"); diff --git a/test/support/fake-managed-agents-service.ts b/test/support/fake-managed-agents-service.ts index 0f72382ee2..ec3b799850 100644 --- a/test/support/fake-managed-agents-service.ts +++ b/test/support/fake-managed-agents-service.ts @@ -29,6 +29,7 @@ export interface FakeManagedAgentsService { manifests(): string[]; execScripts(): string[]; home(): string; + checkpoints(sessionId: string): Array<{ checkpoint_id: string; label: string; status: string }>; setStatus(name: string, status: string): void; rejectPortForward(body: string | null): void; failNextLifecycleCalls(count: number, status: number): void; @@ -59,6 +60,7 @@ export async function startFakeManagedAgentsService(): Promise(); + const checkpoints = new Map>(); const manifests: string[] = []; const execScripts: string[] = []; let tunnels = 0; @@ -202,6 +204,47 @@ export async function startFakeManagedAgentsService(): Promise (body += c.toString("utf8"))); + req.on("end", () => { + const row = rows.get(id); + if (!row || row.status === "SESSION_STATUS_DESTROYED") { + json(res, 404, { error: { code: "not_found", message: "no such session" } }); + return; + } + let label = ""; + try { + label = (JSON.parse(body) as { label?: string }).label ?? ""; + } catch { + label = ""; + } + const checkpoint = { + checkpoint_id: `cp_${randomUUID().slice(0, 8)}`, + session_id: row.session_id, + status: "READY", + kind: "explicit", + label, + created_at: new Date().toISOString(), + size_bytes: 1, + }; + const list = checkpoints.get(row.session_id) ?? []; + list.unshift(checkpoint); + checkpoints.set(row.session_id, list); + json(res, 200, { checkpoint }); + }); + return; + } + if (req.method === "DELETE" && action === "checkpoints" && parts.length === 6) { + const checkpointId = decodeURIComponent(parts[5] ?? ""); + const list = checkpoints.get(id) ?? []; + checkpoints.set( + id, + list.filter((c) => c.checkpoint_id !== checkpointId), + ); + json(res, 200, {}); + return; + } if (req.method === "POST" && (action === "pause" || action === "resume")) { const row = rows.get(id); if (!row) { @@ -266,6 +309,7 @@ export async function startFakeManagedAgentsService(): Promise [...manifests], execScripts: () => [...execScripts], home: () => root, + checkpoints: (sessionId) => [...(checkpoints.get(sessionId) ?? [])], setStatus: (name, status) => { const row = byName(name); if (row) row.status = status; diff --git a/test/support/fake-managed-agents.ts b/test/support/fake-managed-agents.ts index 2eb9e57a16..826f260817 100644 --- a/test/support/fake-managed-agents.ts +++ b/test/support/fake-managed-agents.ts @@ -8,6 +8,7 @@ import { ManagedAgentsSandboxGoneError, type ManagedAgentsClient, type ManagedAgentsCommandResult, + type ManagedAgentsCheckpoint, type ManagedAgentsSession, type ManagedAgentsSessionInfo, type ManagedAgentsSessionState, @@ -26,6 +27,7 @@ interface FakeRecord { createdAt: number; loseNextCommand: boolean; rejectNextCommand: boolean; + failNextCheckpoint: boolean; } export interface FakeManagedAgents { @@ -41,6 +43,9 @@ export interface FakeManagedAgents { pauseCalls(): string[]; resumeCalls(): string[]; guestDisconnects(): string[]; + checkpointCalls(): Array<{ sessionId: string; checkpointId: string; label?: string }>; + deletedCheckpoints(): string[]; + failNextCheckpoint(name: string): void; cleanup(): void; } @@ -51,7 +56,10 @@ export function installFakeManagedAgents(): FakeManagedAgents { const pauseCalls: string[] = []; const resumeCalls: string[] = []; const guestDisconnects: string[] = []; + const checkpointCalls: Array<{ sessionId: string; checkpointId: string; label?: string }> = []; + const deletedCheckpoints: string[] = []; let nextId = 1; + let nextCheckpoint = 1; let clock = 0; const gone = (state: ManagedAgentsSessionState): boolean => @@ -132,6 +140,20 @@ export function installFakeManagedAgents(): FakeManagedAgents { mkdirSync(dirname(hostPath), { recursive: true }); writeFileSync(hostPath, Buffer.from(data)); }, + async createCheckpoint(label?: string): Promise { + if (gone(r.state)) throw new ManagedAgentsSandboxGoneError(r.sessionId, `status ${r.state}`); + if (r.failNextCheckpoint) { + r.failNextCheckpoint = false; + throw new Error("do-managed-agents create checkpoint: http 500 capture failed"); + } + const checkpointId = `cp-${nextCheckpoint++}`; + checkpointCalls.push({ sessionId: r.sessionId, checkpointId, ...(label ? { label } : {}) }); + return { checkpointId, status: "READY", createdAtMs: Date.now() }; + }, + async deleteCheckpoint(checkpointId: string): Promise { + if (gone(r.state)) throw new ManagedAgentsSandboxGoneError(r.sessionId, `status ${r.state}`); + deletedCheckpoints.push(checkpointId); + }, async pause(): Promise { if (gone(r.state)) throw new ManagedAgentsSandboxGoneError(r.sessionId, `status ${r.state}`); guestDisconnects.push(r.sessionId); @@ -168,6 +190,7 @@ export function installFakeManagedAgents(): FakeManagedAgents { createdAt: ++clock, loseNextCommand: false, rejectNextCommand: false, + failNextCheckpoint: false, }; mkdirSync(r.home, { recursive: true }); records.set(id, r); @@ -217,6 +240,11 @@ export function installFakeManagedAgents(): FakeManagedAgents { rejectNextCommand: (name) => { need(name).rejectNextCommand = true; }, + checkpointCalls: () => [...checkpointCalls], + deletedCheckpoints: () => [...deletedCheckpoints], + failNextCheckpoint: (name) => { + need(name).failNextCheckpoint = true; + }, execScripts: () => [...execScripts], pauseCalls: () => [...pauseCalls], resumeCalls: () => [...resumeCalls], From f072b15934c55fdb2517e0f89e0c4341b4e811eb Mon Sep 17 00:00:00 2001 From: rvarma Date: Fri, 9 Oct 2026 11:34:45 +0530 Subject: [PATCH 11/19] Retry checkpoint deletes and throttle captures apart from home tars. A failed delete used to drop the only reference to the previous checkpoint, and a failed home tar cleared the capture throttle because both shared lastSnapshotMs. --- docs/sandbox-preservation.md | 2 +- src/sandbox/managed-agents-sandbox.ts | 44 ++++++++++++++++++++++----- test/managed-agents-sandbox.test.ts | 43 ++++++++++++++++++++++++++ test/support/fake-managed-agents.ts | 10 ++++++ 4 files changed, 90 insertions(+), 9 deletions(-) diff --git a/docs/sandbox-preservation.md b/docs/sandbox-preservation.md index a563a03d4c..fde62d2557 100644 --- a/docs/sandbox-preservation.md +++ b/docs/sandbox-preservation.md @@ -35,4 +35,4 @@ When `MODAL_EGRESS_PROXY_URL` is set, machines are created with Modal's outbound Turn environment reaches Modal commands through the exec `env` parameter rather than inlined `export` statements, exec deadlines are rounded up to whole seconds with a single thirty-second grace beyond the in-guest `timeout`, and a command longer than 64 KiB is written to `/tmp` through the filesystem API and executed from there. -Managed Agents sessions name a sandbox template explicitly, which yields a microVM carrying no managed agent loop, and the disk persists across the provider's pause and resume. Used-turn teardown calls `POST /v2/agents/sessions/{id}/checkpoints` when the home changed, throttled by `DO_AGENTS_SNAPSHOT_INTERVAL_SEC`, then snapshots the home to `DO_AGENTS_SNAPSHOT_S3_BUCKET` and pauses the session. The checkpoint is the provider's machine-state capture of the live session (`CreateCheckpoint` blocks until the capture is `READY`); only the newest is kept, and the previous one is deleted after the new capture succeeds. A capture failure is recorded on `recovery.error` and the pause still proceeds. A pause failure is recorded as `pause_failed` and the running session is retained for retry. Status reports `recovery.strategy: "workspace_snapshot"` plus `recovery.checkpointId`. A checkpoint is bound to its session, so a replacement session for a destroyed or reclaimed scope still hydrates from the portable tar, and a failed hydration kills the replacement rather than adopting a blank home over a saved one. Session lifecycle travels over the Managed Agents REST surface with a DigitalOcean IAM token; commands and file transfer travel over a bearer-authenticated WebSocket port-forward to the guest's `sandbox-agent` gRPC server, which has no request deadline, so command length is bounded by `SANDBOX_TIMEOUT_SEC` rather than by the control plane. Sessions are created with a permissions block that allows `bash` outright, because Managed Agents would otherwise hold every command at its own approval gate behind core's. +Managed Agents sessions name a sandbox template explicitly, which yields a microVM carrying no managed agent loop, and the disk persists across the provider's pause and resume. Used-turn teardown calls `POST /v2/agents/sessions/{id}/checkpoints` when the home changed, throttled by `DO_AGENTS_SNAPSHOT_INTERVAL_SEC` from the last successful capture, then snapshots the home to `DO_AGENTS_SNAPSHOT_S3_BUCKET` on that same interval measured from the last successful tar. A failed tar is retried without taking another checkpoint. The checkpoint is the provider's machine-state capture of the live session (`CreateCheckpoint` blocks until the capture is `READY`); only the newest is kept, and the previous one is deleted after the new capture succeeds. A delete that fails stays recorded and is retried on a later teardown. A capture failure is recorded on `recovery.error` and the pause still proceeds. A pause failure is recorded as `pause_failed` and the running session is retained for retry. Status reports `recovery.strategy: "workspace_snapshot"` plus `recovery.checkpointId`. A checkpoint is bound to its session, so a replacement session for a destroyed or reclaimed scope still hydrates from the portable tar, and a failed hydration kills the replacement rather than adopting a blank home over a saved one. Session lifecycle travels over the Managed Agents REST surface with a DigitalOcean IAM token; commands and file transfer travel over a bearer-authenticated WebSocket port-forward to the guest's `sandbox-agent` gRPC server, which has no request deadline, so command length is bounded by `SANDBOX_TIMEOUT_SEC` rather than by the control plane. Sessions are created with a permissions block that allows `bash` outright, because Managed Agents would otherwise hold every command at its own approval gate behind core's. diff --git a/src/sandbox/managed-agents-sandbox.ts b/src/sandbox/managed-agents-sandbox.ts index b9e53013b9..a339f7a497 100644 --- a/src/sandbox/managed-agents-sandbox.ts +++ b/src/sandbox/managed-agents-sandbox.ts @@ -75,6 +75,7 @@ export interface StoredManagedAgentsSandbox { checkpointId?: string; checkpointAtMs?: number; checkpointError?: string; + pendingCheckpointDeletes?: string[]; } export interface ManagedAgentsSandboxOptions extends BlobStagingOptions { @@ -128,20 +129,43 @@ export function createManagedAgentsSandbox(workspace: WorkspaceStore, opts: Mana await store.merge(scope, { lastSnapshotMs: Date.now(), homeDirty: false }); } + async function pendingDeletes(scope: string): Promise { + return (await store.get(scope))?.pendingCheckpointDeletes ?? []; + } + + async function forgetCheckpoint(scope: string, session: ManagedAgentsSession, checkpointId: string): Promise { + try { + await session.deleteCheckpoint(checkpointId); + } catch (e) { + reportError("sandbox_snapshot", "checkpoint_delete_failed", errMessage(e), scope); + return; + } + const pending = await pendingDeletes(scope); + await store.merge(scope, { pendingCheckpointDeletes: pending.filter((id) => id !== checkpointId) }); + } + + async function retryPendingCheckpointDeletes(scope: string, session: ManagedAgentsSession): Promise { + const current = (await store.get(scope))?.checkpointId; + for (const checkpointId of await pendingDeletes(scope)) { + if (checkpointId === current) continue; + await forgetCheckpoint(scope, session, checkpointId); + } + } + async function captureCheckpoint(scope: string, session: ManagedAgentsSession): Promise { - const previous = (await store.get(scope))?.checkpointId; + const stored = await store.get(scope); + const previous = stored?.checkpointId; const captured = await session.createCheckpoint(CHECKPOINT_LABEL); + const pending = stored?.pendingCheckpointDeletes ?? []; + const queued = + previous && previous !== captured.checkpointId && !pending.includes(previous) ? [...pending, previous] : pending; await store.merge(scope, { checkpointId: captured.checkpointId, checkpointAtMs: captured.createdAtMs ?? Date.now(), checkpointError: undefined, + pendingCheckpointDeletes: queued, }); - if (!previous || previous === captured.checkpointId) return; - try { - await session.deleteCheckpoint(previous); - } catch (e) { - reportError("sandbox_snapshot", "checkpoint_delete_failed", errMessage(e), scope); - } + if (previous && previous !== captured.checkpointId) await forgetCheckpoint(scope, session, previous); } const hydrateHome = (scope: string, session: ManagedAgentsSession): Promise => @@ -535,13 +559,17 @@ export function createManagedAgentsSandbox(workspace: WorkspaceStore, opts: Mana const stored = await store.get(scope); if (!tdOpts?.homeUnchanged) await store.merge(scope, { homeDirty: true }); - if (snapshotDue(stored, tdOpts, snapshotIntervalMs)) { + await retryPendingCheckpointDeletes(scope, session); + const homeDirty = stored?.homeDirty; + if (snapshotDue({ lastSnapshotMs: stored?.checkpointAtMs, homeDirty }, tdOpts, snapshotIntervalMs)) { try { await captureCheckpoint(scope, session); } catch (e) { await store.merge(scope, { checkpointError: errMessage(e) }); reportError("sandbox_snapshot", "checkpoint_failed", errMessage(e), scope); } + } + if (snapshotDue({ lastSnapshotMs: stored?.lastSnapshotMs, homeDirty }, tdOpts, snapshotIntervalMs)) { try { await snapshotHome(scope, session); } catch (e) { diff --git a/test/managed-agents-sandbox.test.ts b/test/managed-agents-sandbox.test.ts index f8dd30a27c..eb2bf6db22 100644 --- a/test/managed-agents-sandbox.test.ts +++ b/test/managed-agents-sandbox.test.ts @@ -16,6 +16,7 @@ import { } from "../src/sandbox/managed-agents-client.ts"; import { sandboxScopeName } from "../src/sandbox/exec-sandbox-base.ts"; import { createLocalWorkspaceStore } from "../src/workspace/workspace-store.ts"; +import type { HomeSnapshotStore } from "../src/sandbox/home-snapshot.ts"; import { supportsProcessSessions } from "../src/sandbox/sandbox.ts"; import { createMemoryMap, type DurableMap } from "../src/persistence/durable-map.ts"; import { scopeId } from "../src/types.ts"; @@ -120,6 +121,48 @@ test("a newer checkpoint replaces the previous one", async () => { assert.ok(second); assert.notEqual(first, second); assert.deepEqual(fake.deletedCheckpoints(), [first]); + assert.deepEqual((await store.get(scope))?.pendingCheckpointDeletes ?? [], []); +}); + +test("a failed checkpoint delete is remembered and retried without another capture", async () => { + const store: DurableMap = createMemoryMap(); + const fast = make({ store, snapshotIntervalMs: -1 }); + const h = await fast.provision(layers); + await fast.teardown(h); + const first = (await store.get(scope))?.checkpointId; + assert.ok(first); + fake.failNextCheckpointDelete(scopeName()); + const again = await fast.provision(layers); + await fast.teardown(again); + assert.deepEqual((await store.get(scope))?.pendingCheckpointDeletes, [first]); + assert.deepEqual(fake.deletedCheckpoints(), []); + + const slow = make({ store }); + const resumed = await slow.provision(layers); + const captures = fake.checkpointCalls().length; + await slow.teardown(resumed, { homeUnchanged: true }); + assert.equal(fake.checkpointCalls().length, captures); + assert.deepEqual(fake.deletedCheckpoints(), [first]); + assert.deepEqual((await store.get(scope))?.pendingCheckpointDeletes ?? [], []); +}); + +test("a failed home snapshot retries without recapturing the checkpoint", async () => { + const uploads: number[] = []; + const snapshots: HomeSnapshotStore = { + open: async () => null, + put: async () => undefined, + createUpload: async () => { + uploads.push(1); + throw new Error("snapshot upload failed"); + }, + }; + sandbox = make({ snapshots }); + const h = await sandbox.provision(layers); + await sandbox.teardown(h); + const again = await sandbox.provision(layers); + await sandbox.teardown(again); + assert.equal(fake.checkpointCalls().length, 1); + assert.equal(uploads.length, 2); }); test("teardown pauses the session and the next provision resumes it", async () => { diff --git a/test/support/fake-managed-agents.ts b/test/support/fake-managed-agents.ts index 826f260817..a3ad3b0fd4 100644 --- a/test/support/fake-managed-agents.ts +++ b/test/support/fake-managed-agents.ts @@ -28,6 +28,7 @@ interface FakeRecord { loseNextCommand: boolean; rejectNextCommand: boolean; failNextCheckpoint: boolean; + failNextCheckpointDelete: boolean; } export interface FakeManagedAgents { @@ -46,6 +47,7 @@ export interface FakeManagedAgents { checkpointCalls(): Array<{ sessionId: string; checkpointId: string; label?: string }>; deletedCheckpoints(): string[]; failNextCheckpoint(name: string): void; + failNextCheckpointDelete(name: string): void; cleanup(): void; } @@ -152,6 +154,10 @@ export function installFakeManagedAgents(): FakeManagedAgents { }, async deleteCheckpoint(checkpointId: string): Promise { if (gone(r.state)) throw new ManagedAgentsSandboxGoneError(r.sessionId, `status ${r.state}`); + if (r.failNextCheckpointDelete) { + r.failNextCheckpointDelete = false; + throw new Error("do-managed-agents delete checkpoint: http 500 delete failed"); + } deletedCheckpoints.push(checkpointId); }, async pause(): Promise { @@ -191,6 +197,7 @@ export function installFakeManagedAgents(): FakeManagedAgents { loseNextCommand: false, rejectNextCommand: false, failNextCheckpoint: false, + failNextCheckpointDelete: false, }; mkdirSync(r.home, { recursive: true }); records.set(id, r); @@ -245,6 +252,9 @@ export function installFakeManagedAgents(): FakeManagedAgents { failNextCheckpoint: (name) => { need(name).failNextCheckpoint = true; }, + failNextCheckpointDelete: (name) => { + need(name).failNextCheckpointDelete = true; + }, execScripts: () => [...execScripts], pauseCalls: () => [...pauseCalls], resumeCalls: () => [...resumeCalls], From fe378adc34751c107cb0fba00ae5fbe3f7966100 Mon Sep 17 00:00:00 2001 From: rvarma Date: Fri, 9 Oct 2026 12:40:03 +0530 Subject: [PATCH 12/19] Roll a wedged Managed Agents computer back to its last checkpoint. Restart calls the harness rollback API so the same session boots a new microVM from the stored capture. --- adrs/managed-agents-sandbox-backend.md | 7 ++- docs/sandbox-preservation.md | 2 +- src/sandbox/managed-agents-client.ts | 29 +++++++++ src/sandbox/managed-agents-sandbox.ts | 32 ++++++++++ test/managed-agents-client.test.ts | 15 +++++ test/managed-agents-sandbox.test.ts | 65 +++++++++++++++++++++ test/support/fake-managed-agents-service.ts | 13 +++++ test/support/fake-managed-agents.ts | 25 ++++++++ 8 files changed, 186 insertions(+), 2 deletions(-) diff --git a/adrs/managed-agents-sandbox-backend.md b/adrs/managed-agents-sandbox-backend.md index d783446f3d..5de8ef4d80 100644 --- a/adrs/managed-agents-sandbox-backend.md +++ b/adrs/managed-agents-sandbox-backend.md @@ -65,13 +65,18 @@ sequenceDiagram else destroy the scope B->>E: DELETE /v2/agents/sessions/{id} end + + Note over O,B: restart a wedged computer + O->>B: restart + B->>E: POST /v2/agents/sessions/{id}/checkpoints/{checkpoint_id}/rollback + E-->>B: same session_id, new sandbox_id ``` What the PR adds - `SANDBOX_BACKEND=do-managed-agents`, with `DO_AGENTS_API_TOKEN` required — a DO IAM token, which is also where the team identity comes from, and the only credential the backend needs. Optional knobs: `DO_AGENTS_API_BASE_URL`, `DO_AGENTS_TEMPLATE`, `DO_AGENTS_NAME_PREFIX`, `DO_AGENTS_SIZE_SLUG`, `DO_AGENTS_IDLE_TIMEOUT_SEC`, `DO_AGENTS_EGRESS_PROXY_URL`, `DO_AGENTS_SNAPSHOT_INTERVAL_SEC`, `DO_AGENTS_SNAPSHOT_S3_BUCKET`, and the shared `SANDBOX_TIMEOUT_SEC`. - `src/sandbox/managed-agents-tunnel.ts`, the port-forward client: one WebSocket per TCP connection, bytes piped both ways with one chunk in flight per direction so a slow consumer cannot make core buffer the stream. The local listener is unreferenced, so an abandoned tunnel can never hold core's event loop open. -- `src/sandbox/managed-agents-client.ts`, holding both transports behind one interface the way `e2b-client.ts` wraps the E2B SDK: REST for `/v2/agents/sessions{,/{id},/{id}/pause,/{id}/resume,/{id}/checkpoints}`, and `SandboxAgentService` stubs for `Exec`, `Upload` and `Download` over the tunnel. Our 404s, terminal statuses and gRPC `UNAVAILABLE` translate to the same two error classes the E2B client uses — one for a sandbox that was already gone before a command started, which the backend reconnects and retries, and one for a sandbox lost mid-command, which is reported rather than retried because the command may have partially run. The session body is an agents.yaml manifest in its flat form, which is the shape we are standardising on. +- `src/sandbox/managed-agents-client.ts`, holding both transports behind one interface the way `e2b-client.ts` wraps the E2B SDK: REST for `/v2/agents/sessions{,/{id},/{id}/pause,/{id}/resume,/{id}/checkpoints,/{id}/checkpoints/{checkpoint_id}/rollback}`, and `SandboxAgentService` stubs for `Exec`, `Upload` and `Download` over the tunnel. Our 404s, terminal statuses and gRPC `UNAVAILABLE` translate to the same two error classes the E2B client uses — one for a sandbox that was already gone before a command started, which the backend reconnects and retries, and one for a sandbox lost mid-command, which is reported rather than retried because the command may have partially run. The session body is an agents.yaml manifest in its flat form, which is the shape we are standardising on. - `src/sandbox/managed-agents-sandbox-agent.proto`, a vendored copy of our guest proto with the internal annotations stripped, loaded at runtime through `@grpc/proto-loader`. No build step and no generated code in the tree. - `src/sandbox/managed-agents-sandbox.ts`. Provision creates or resumes the session and waits for it to become usable. Run is `Exec` over the tunnel. Files are `Upload` and `Download` rather than base64 through exec, since we have real transfer RPCs. Process sessions, read-only layers, layer tool install, home snapshots and blob staging come from the shared exec helpers unchanged. - Scope recovery without local state. Session names are team-unique among non-terminal sessions and `ListSessions` takes a `?name=` filter, so the backend can re-adopt a running sandbox after losing its durable record, the way the E2B backend re-adopts by sandbox metadata. `sandboxScopeName` already produces names that fit our 64-character, not-UUID-shaped rule. diff --git a/docs/sandbox-preservation.md b/docs/sandbox-preservation.md index fde62d2557..749b945ba8 100644 --- a/docs/sandbox-preservation.md +++ b/docs/sandbox-preservation.md @@ -35,4 +35,4 @@ When `MODAL_EGRESS_PROXY_URL` is set, machines are created with Modal's outbound Turn environment reaches Modal commands through the exec `env` parameter rather than inlined `export` statements, exec deadlines are rounded up to whole seconds with a single thirty-second grace beyond the in-guest `timeout`, and a command longer than 64 KiB is written to `/tmp` through the filesystem API and executed from there. -Managed Agents sessions name a sandbox template explicitly, which yields a microVM carrying no managed agent loop, and the disk persists across the provider's pause and resume. Used-turn teardown calls `POST /v2/agents/sessions/{id}/checkpoints` when the home changed, throttled by `DO_AGENTS_SNAPSHOT_INTERVAL_SEC` from the last successful capture, then snapshots the home to `DO_AGENTS_SNAPSHOT_S3_BUCKET` on that same interval measured from the last successful tar. A failed tar is retried without taking another checkpoint. The checkpoint is the provider's machine-state capture of the live session (`CreateCheckpoint` blocks until the capture is `READY`); only the newest is kept, and the previous one is deleted after the new capture succeeds. A delete that fails stays recorded and is retried on a later teardown. A capture failure is recorded on `recovery.error` and the pause still proceeds. A pause failure is recorded as `pause_failed` and the running session is retained for retry. Status reports `recovery.strategy: "workspace_snapshot"` plus `recovery.checkpointId`. A checkpoint is bound to its session, so a replacement session for a destroyed or reclaimed scope still hydrates from the portable tar, and a failed hydration kills the replacement rather than adopting a blank home over a saved one. Session lifecycle travels over the Managed Agents REST surface with a DigitalOcean IAM token; commands and file transfer travel over a bearer-authenticated WebSocket port-forward to the guest's `sandbox-agent` gRPC server, which has no request deadline, so command length is bounded by `SANDBOX_TIMEOUT_SEC` rather than by the control plane. Sessions are created with a permissions block that allows `bash` outright, because Managed Agents would otherwise hold every command at its own approval gate behind core's. +Managed Agents sessions name a sandbox template explicitly, which yields a microVM carrying no managed agent loop, and the disk persists across the provider's pause and resume. Used-turn teardown calls `POST /v2/agents/sessions/{id}/checkpoints` when the home changed, throttled by `DO_AGENTS_SNAPSHOT_INTERVAL_SEC` from the last successful capture, then snapshots the home to `DO_AGENTS_SNAPSHOT_S3_BUCKET` on that same interval measured from the last successful tar. A failed tar is retried without taking another checkpoint. The checkpoint is the provider's machine-state capture of the live session (`CreateCheckpoint` blocks until the capture is `READY`); only the newest is kept, and the previous one is deleted after the new capture succeeds. A delete that fails stays recorded and is retried on a later teardown. A capture failure is recorded on `recovery.error` and the pause still proceeds. A pause failure is recorded as `pause_failed` and the running session is retained for retry. Status reports `recovery.strategy: "workspace_snapshot"` plus `recovery.checkpointId`. Restart of a wedged computer calls `POST /v2/agents/sessions/{id}/checkpoints/{checkpoint_id}/rollback` with that stored checkpoint: the session id stays, the response's sandbox id replaces the stored one, and any open port-forward is closed so the next command dials the restored microVM. A paused session is rolled back directly, without being resumed first. Restart with no checkpoint fails and leaves the session where it is. A checkpoint is bound to its session, so a replacement session for a destroyed or reclaimed scope still hydrates from the portable tar, and a failed hydration kills the replacement rather than adopting a blank home over a saved one. Session lifecycle travels over the Managed Agents REST surface with a DigitalOcean IAM token; commands and file transfer travel over a bearer-authenticated WebSocket port-forward to the guest's `sandbox-agent` gRPC server, which has no request deadline, so command length is bounded by `SANDBOX_TIMEOUT_SEC` rather than by the control plane. Sessions are created with a permissions block that allows `bash` outright, because Managed Agents would otherwise hold every command at its own approval gate behind core's. diff --git a/src/sandbox/managed-agents-client.ts b/src/sandbox/managed-agents-client.ts index e1446d2a7f..97b1447a3a 100644 --- a/src/sandbox/managed-agents-client.ts +++ b/src/sandbox/managed-agents-client.ts @@ -47,6 +47,7 @@ export interface ManagedAgentsSession { writeFileBytes(absPath: string, data: Uint8Array): Promise; createCheckpoint(label?: string): Promise; deleteCheckpoint(checkpointId: string): Promise; + close(): Promise; pause(): Promise; resume(): Promise; kill(): Promise; @@ -88,6 +89,7 @@ export interface ManagedAgentsClient { create(opts: ManagedAgentsCreateOpts): Promise; connect(sessionId: string): Promise; list(name: string): Promise; + rollback(sessionId: string, checkpointId: string): Promise; kill(sessionId: string): Promise; } @@ -111,6 +113,7 @@ const CREATE_TIMEOUT_MS = 120_000; const CHECKPOINT_TIMEOUT_MS = 5 * 60_000; const REQUEST_TIMEOUT_MS = 30_000; const READY_TIMEOUT_MS = 300_000; +const ROLLBACK_TIMEOUT_MS = READY_TIMEOUT_MS; const READY_POLL_MS = 2_000; const READY_FAST_POLL_MS = 250; const READY_FAST_WINDOW_MS = 10_000; @@ -430,6 +433,24 @@ export function createSdkManagedAgentsClient(opts: SdkManagedAgentsClientOptions } } + async function rollbackToCheckpoint(sessionId: string, checkpointId: string): Promise { + const body = await callJson<{ session?: WireSession }>( + "POST", + `/v2/agents/sessions/${sessionId}/checkpoints/${encodeURIComponent(checkpointId)}/rollback`, + sessionId, + undefined, + undefined, + ROLLBACK_TIMEOUT_MS, + ); + if (!body.session) throw new Error(`do-managed-agents rollback ${sessionId}: response carried no session`); + const restored = toInfo(body.session); + if (!restored.sandboxId) + throw new Error(`do-managed-agents rollback ${sessionId}: response carried no sandbox id`); + if (GONE_STATES.has(restored.state)) + throw new ManagedAgentsSandboxGoneError(sessionId, restored.errorMessage ?? `status ${restored.state}`); + return restored; + } + function buildSession(info: ManagedAgentsSessionInfo): ManagedAgentsSession { const sessionId = info.sessionId; let guest: Promise<{ tunnel: ManagedAgentsTunnel; agent: SandboxAgent }> | null = null; @@ -663,6 +684,10 @@ export function createSdkManagedAgentsClient(opts: SdkManagedAgentsClientOptions ); }, + async close(): Promise { + await disconnect(); + }, + async pause(): Promise { await disconnect(); await callJson("POST", `/v2/agents/sessions/${sessionId}/pause`, sessionId); @@ -710,6 +735,10 @@ export function createSdkManagedAgentsClient(opts: SdkManagedAgentsClientOptions return buildSession(await awaitUsable(sessionId)); }, + async rollback(sessionId, checkpointId): Promise { + return rollbackToCheckpoint(sessionId, checkpointId); + }, + async list(name): Promise { const body = await callJson<{ sessions?: WireSession[] }>( "GET", diff --git a/src/sandbox/managed-agents-sandbox.ts b/src/sandbox/managed-agents-sandbox.ts index a339f7a497..2d8150b11a 100644 --- a/src/sandbox/managed-agents-sandbox.ts +++ b/src/sandbox/managed-agents-sandbox.ts @@ -478,6 +478,38 @@ export function createManagedAgentsSandbox(workspace: WorkspaceStore, opts: Mana await snapshotHome(scopeId, session); }, + async restartComputer(scopeId: string): Promise { + const name = sandboxScopeName(prefix, scopeId); + scopeByName.set(name, scopeId); + return provisionQueue(scopeId, async () => { + const stored = await store.get(scopeId); + if (!stored?.sessionId) throw new Error("do-managed-agents restart: no sandbox provisioned yet"); + const checkpointId = stored.checkpointId; + if (!checkpointId) throw new Error("do-managed-agents restart: no checkpoint to restore"); + const cached = sessionByName.get(name); + sessionByName.delete(name); + if (cached) await cached.close(); + let restored: ManagedAgentsSessionInfo; + try { + restored = await client.rollback(stored.sessionId, checkpointId); + } catch (e) { + reportError("agent_computer", "checkpoint_rollback_failed", errMessage(e), scopeId); + throw e; + } + await store.merge(scopeId, { + sandboxId: restored.sandboxId, + preservationState: "running", + preservationError: undefined, + }); + reportError( + "agent_computer", + "checkpoint_restored", + `session ${stored.sessionId}: restored checkpoint ${checkpointId} onto sandbox ${restored.sandboxId}`, + scopeId, + ); + }); + }, + async computerStatus(scopeId: string): Promise { const name = sandboxScopeName(prefix, scopeId); const stored = await store.get(scopeId); diff --git a/test/managed-agents-client.test.ts b/test/managed-agents-client.test.ts index d537acb93f..d8cfa47489 100644 --- a/test/managed-agents-client.test.ts +++ b/test/managed-agents-client.test.ts @@ -249,6 +249,21 @@ test("create checkpoint posts to the session checkpoint route and returns the re assert.equal(service.checkpoints(s.sessionId).length, 0); }); +test("rollback rewinds the same session onto a new sandbox without resuming it first", async () => { + const s = await session("qm-rollback"); + const captured = await s.createCheckpoint("qm teardown"); + const previousSandbox = s.sandboxId; + await s.pause(); + assert.equal((await client.info(s.sessionId)).state, "paused"); + const restored = await client.rollback(s.sessionId, captured.checkpointId); + assert.equal(restored.sessionId, s.sessionId); + assert.notEqual(restored.sandboxId, previousSandbox); + assert.equal(restored.state, "ready"); + assert.equal((await client.info(s.sessionId)).sandboxId, restored.sandboxId); + assert.equal(service.checkpoints(s.sessionId).length, 1); + await assert.rejects(() => client.rollback(s.sessionId, "missing"), /404|no such checkpoint/); +}); + test("connect resumes a paused session before handing it back", async () => { const s = await session("qm-paused"); await s.pause(); diff --git a/test/managed-agents-sandbox.test.ts b/test/managed-agents-sandbox.test.ts index eb2bf6db22..d651e7474a 100644 --- a/test/managed-agents-sandbox.test.ts +++ b/test/managed-agents-sandbox.test.ts @@ -87,6 +87,71 @@ test("teardown captures a provider checkpoint before pausing", async () => { assert.equal(fake.pauseCalls().length, 1); }); +test("restart rolls a paused session back to its checkpoint without resuming the old microVM", async () => { + const store: DurableMap = createMemoryMap(); + const restored: string[] = []; + sandbox = make({ + store, + onError: (e: { code: string }) => { + if (e.code === "checkpoint_restored") restored.push(e.code); + }, + }); + const h = await sandbox.provision(layers); + await sandbox.teardown(h); + const before = await store.get(scope); + assert.ok(before?.checkpointId); + assert.equal(before.preservationState, "paused"); + const resumes = fake.resumeCalls().length; + const disconnects = fake.guestDisconnects().length; + await sandbox.restartComputer!(scope); + const after = await store.get(scope); + assert.equal(after?.sessionId, before.sessionId); + assert.notEqual(after?.sandboxId, before.sandboxId); + assert.equal(after?.preservationState, "running"); + assert.equal(after?.checkpointId, before.checkpointId); + assert.deepEqual(fake.rollbackCalls(), [{ sessionId: before.sessionId, checkpointId: before.checkpointId }]); + assert.equal(fake.resumeCalls().length, resumes); + assert.equal(fake.guestDisconnects().length, disconnects); + assert.equal(fake.createdCount(scopeName()), 1); + assert.deepEqual(restored, ["checkpoint_restored"]); + const again = await sandbox.provision(layers); + assert.equal(again.coldStart, false); + assert.equal(fake.current(scopeName())?.sandboxId, after?.sandboxId); +}); + +test("restart drops a warm guest's port-forward before rolling back", async () => { + const store: DurableMap = createMemoryMap(); + sandbox = make({ store }); + const h = await sandbox.provision(layers); + await sandbox.teardown(h, { keepWarm: true }); + const disconnects = fake.guestDisconnects().length; + await sandbox.restartComputer!(scope); + assert.equal(fake.guestDisconnects().length, disconnects + 1); + assert.equal(fake.rollbackCalls().length, 1); + assert.equal((await store.get(scope))?.preservationState, "running"); +}); + +test("restart without a checkpoint does not call rollback", async () => { + await sandbox.provision(layers); + await assert.rejects(() => sandbox.restartComputer!(scope), /no checkpoint to restore/); + assert.equal(fake.rollbackCalls().length, 0); +}); + +test("a failed rollback keeps the stored sandbox and the same session", async () => { + const store: DurableMap = createMemoryMap(); + sandbox = make({ store }); + const h = await sandbox.provision(layers); + await sandbox.teardown(h); + const before = await store.get(scope); + fake.failNextRollback(scopeName()); + await assert.rejects(() => sandbox.restartComputer!(scope), /rollback/); + assert.equal((await store.get(scope))?.sandboxId, before?.sandboxId); + assert.equal(fake.rollbackCalls().length, 0); + const again = await sandbox.provision(layers); + assert.equal(again.coldStart, false); + assert.equal(fake.createdCount(scopeName()), 1); +}); + test("a checkpoint failure is recorded and the session still pauses", async () => { const store: DurableMap = createMemoryMap(); sandbox = make({ store }); diff --git a/test/support/fake-managed-agents-service.ts b/test/support/fake-managed-agents-service.ts index ec3b799850..64294298a7 100644 --- a/test/support/fake-managed-agents-service.ts +++ b/test/support/fake-managed-agents-service.ts @@ -235,6 +235,19 @@ export async function startFakeManagedAgentsService(): Promise c.checkpoint_id === checkpointId)) { + json(res, 404, { error: { code: "not_found", message: "no such checkpoint" } }); + return; + } + row.sandbox_id = `sbx-restored-${randomUUID().slice(0, 8)}`; + row.status = "SESSION_STATUS_READY"; + json(res, 200, { session: row }); + return; + } if (req.method === "DELETE" && action === "checkpoints" && parts.length === 6) { const checkpointId = decodeURIComponent(parts[5] ?? ""); const list = checkpoints.get(id) ?? []; diff --git a/test/support/fake-managed-agents.ts b/test/support/fake-managed-agents.ts index a3ad3b0fd4..39754201da 100644 --- a/test/support/fake-managed-agents.ts +++ b/test/support/fake-managed-agents.ts @@ -29,6 +29,7 @@ interface FakeRecord { rejectNextCommand: boolean; failNextCheckpoint: boolean; failNextCheckpointDelete: boolean; + failNextRollback: boolean; } export interface FakeManagedAgents { @@ -46,8 +47,10 @@ export interface FakeManagedAgents { guestDisconnects(): string[]; checkpointCalls(): Array<{ sessionId: string; checkpointId: string; label?: string }>; deletedCheckpoints(): string[]; + rollbackCalls(): Array<{ sessionId: string; checkpointId: string }>; failNextCheckpoint(name: string): void; failNextCheckpointDelete(name: string): void; + failNextRollback(name: string): void; cleanup(): void; } @@ -60,7 +63,9 @@ export function installFakeManagedAgents(): FakeManagedAgents { const guestDisconnects: string[] = []; const checkpointCalls: Array<{ sessionId: string; checkpointId: string; label?: string }> = []; const deletedCheckpoints: string[] = []; + const rollbackCalls: Array<{ sessionId: string; checkpointId: string }> = []; let nextId = 1; + let nextSandbox = 1; let nextCheckpoint = 1; let clock = 0; @@ -160,6 +165,9 @@ export function installFakeManagedAgents(): FakeManagedAgents { } deletedCheckpoints.push(checkpointId); }, + async close(): Promise { + guestDisconnects.push(r.sessionId); + }, async pause(): Promise { if (gone(r.state)) throw new ManagedAgentsSandboxGoneError(r.sessionId, `status ${r.state}`); guestDisconnects.push(r.sessionId); @@ -198,6 +206,7 @@ export function installFakeManagedAgents(): FakeManagedAgents { rejectNextCommand: false, failNextCheckpoint: false, failNextCheckpointDelete: false, + failNextRollback: false, }; mkdirSync(r.home, { recursive: true }); records.set(id, r); @@ -217,6 +226,18 @@ export function installFakeManagedAgents(): FakeManagedAgents { .filter((r) => !gone(r.state) && r.name === name) .map((r) => ({ sessionId: r.sessionId, sandboxId: r.sandboxId, name: r.name, state: r.state })); }, + async rollback(sessionId, checkpointId): Promise { + const r = records.get(sessionId); + if (!r || gone(r.state)) throw new ManagedAgentsSandboxGoneError(sessionId, "session was not found"); + if (r.failNextRollback) { + r.failNextRollback = false; + throw new Error("do-managed-agents rollback: http 409 checkpoint is not ready"); + } + rollbackCalls.push({ sessionId, checkpointId }); + r.sandboxId = `sbx-restored-${nextSandbox++}`; + r.state = "ready"; + return info(r); + }, async kill(sessionId): Promise { const r = records.get(sessionId); if (!r) return; @@ -255,6 +276,10 @@ export function installFakeManagedAgents(): FakeManagedAgents { failNextCheckpointDelete: (name) => { need(name).failNextCheckpointDelete = true; }, + rollbackCalls: () => [...rollbackCalls], + failNextRollback: (name) => { + need(name).failNextRollback = true; + }, execScripts: () => [...execScripts], pauseCalls: () => [...pauseCalls], resumeCalls: () => [...resumeCalls], From da8678b71bff10a96f26878ad5028f13808f32ef Mon Sep 17 00:00:00 2001 From: Josh France Date: Fri, 9 Oct 2026 18:48:06 +0000 Subject: [PATCH 13/19] Fix lint and formatting in checkpoint changes --- src/sandbox/managed-agents-client.ts | 3 +-- test/support/fake-managed-agents-service.ts | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/sandbox/managed-agents-client.ts b/src/sandbox/managed-agents-client.ts index 97b1447a3a..db182465b5 100644 --- a/src/sandbox/managed-agents-client.ts +++ b/src/sandbox/managed-agents-client.ts @@ -444,8 +444,7 @@ export function createSdkManagedAgentsClient(opts: SdkManagedAgentsClientOptions ); if (!body.session) throw new Error(`do-managed-agents rollback ${sessionId}: response carried no session`); const restored = toInfo(body.session); - if (!restored.sandboxId) - throw new Error(`do-managed-agents rollback ${sessionId}: response carried no sandbox id`); + if (!restored.sandboxId) throw new Error(`do-managed-agents rollback ${sessionId}: response carried no sandbox id`); if (GONE_STATES.has(restored.state)) throw new ManagedAgentsSandboxGoneError(sessionId, restored.errorMessage ?? `status ${restored.state}`); return restored; diff --git a/test/support/fake-managed-agents-service.ts b/test/support/fake-managed-agents-service.ts index 64294298a7..71cb864cd8 100644 --- a/test/support/fake-managed-agents-service.ts +++ b/test/support/fake-managed-agents-service.ts @@ -213,7 +213,7 @@ export async function startFakeManagedAgentsService(): Promise Date: Sat, 10 Oct 2026 00:51:01 +0000 Subject: [PATCH 14/19] Return clean timeouts from Managed Agents exec Wrap the command so timeout -k force-kills stragglers and the exit code comes back directly; treat 124 and 137 as timed out. --- src/sandbox/managed-agents-sandbox.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/sandbox/managed-agents-sandbox.ts b/src/sandbox/managed-agents-sandbox.ts index 2d8150b11a..977aef8828 100644 --- a/src/sandbox/managed-agents-sandbox.ts +++ b/src/sandbox/managed-agents-sandbox.ts @@ -279,10 +279,15 @@ export function createManagedAgentsSandbox(workspace: WorkspaceStore, opts: Mana async function execRaw(name: string, script: string, timeoutSec: number): Promise { return withSession(name, async (session) => { - const r = await session.runCommand(`timeout ${timeoutSec} sh -c ${shq(script)}`, { + const r = await session.runCommand(`sh -c ${shq(`timeout -k 5 ${timeoutSec} sh -c ${shq(script)}; exit $?`)}`, { timeoutMs: timeoutSec * 1000 + 30_000, }); - return { stdout: r.stdout, stderr: r.stderr, code: r.exitCode, timedOut: r.exitCode === 124 }; + return { + stdout: r.stdout, + stderr: r.stderr, + code: r.exitCode, + timedOut: r.exitCode === 124 || r.exitCode === 137, + }; }); } From 83cc70e3e1278665a5c1cf96072b48db70fb7a49 Mon Sep 17 00:00:00 2001 From: Josh France Date: Sat, 10 Oct 2026 00:58:00 +0000 Subject: [PATCH 15/19] Keep Managed Agents errors and profile honest Report swallowed kill, disconnect and probe failures; fail a command stream that ends without an exit status; decide missing files from gRPC NOT_FOUND or a test -e probe instead of matching error text; list gh as not installed; drop the backend from the legacy sandbox import; strip comments from the template and vendored proto. --- deploy/do-managed-agents/Dockerfile | 5 - src/sandbox/managed-agents-client.ts | 96 ++++++++++--------- .../managed-agents-sandbox-agent.proto | 22 ++--- src/sandbox/managed-agents-sandbox.ts | 10 +- src/wiring.ts | 8 +- 5 files changed, 67 insertions(+), 74 deletions(-) diff --git a/deploy/do-managed-agents/Dockerfile b/deploy/do-managed-agents/Dockerfile index 57826db552..fd913e088c 100644 --- a/deploy/do-managed-agents/Dockerfile +++ b/deploy/do-managed-agents/Dockerfile @@ -1,5 +1,3 @@ -# QM sandbox image for DigitalOcean Managed Agents (Harness Runtime BYOT). -# Registered on the agentless `sandbox` base. FROM node:24-slim@sha256:6f7b03f7c2c8e2e784dcf9295400527b9b1270fd37b7e9a7285cf83b6951452d ENV DEBIAN_FRONTEND=noninteractive @@ -10,9 +8,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* \ && node --version && npm --version -# The platform owns /workspace and ignores USER, so the image creates no user or -# workspace; it only points HOME at the home the adapter manages. Clear node's -# REPL entrypoint so the session process stays up. ENV HOME=/workspace/home ENTRYPOINT ["sleep", "infinity"] CMD [] diff --git a/src/sandbox/managed-agents-client.ts b/src/sandbox/managed-agents-client.ts index db182465b5..7430b75117 100644 --- a/src/sandbox/managed-agents-client.ts +++ b/src/sandbox/managed-agents-client.ts @@ -1,6 +1,6 @@ import { fileURLToPath } from "node:url"; import { fetchWithRetry } from "../util/async.ts"; -import { errMessage, withRequestId } from "../util/errors.ts"; +import { errMessage, swallowAs, withRequestId } from "../util/errors.ts"; import { openManagedAgentsTunnel, SANDBOX_AGENT_PORT, type ManagedAgentsTunnel } from "./managed-agents-tunnel.ts"; export interface ManagedAgentsCommandResult { @@ -228,14 +228,6 @@ const isGuestGone = (err: unknown): boolean => { return code === GRPC_NOT_FOUND || code === GRPC_UNAVAILABLE; }; -const isMissingPath = (err: unknown): boolean => { - const code = grpcCode(err); - if (code === GRPC_NOT_FOUND) return true; - if (code !== GRPC_INTERNAL) return false; - const e = err as GrpcCallError; - return /\b404\b|does not exist|no such file/i.test(`${e.details ?? ""} ${e.message ?? ""}`); -}; - type ProtoLoader = typeof import("@grpc/proto-loader"); type Grpc = typeof import("@grpc/grpc-js"); @@ -361,7 +353,7 @@ export function createSdkManagedAgentsClient(opts: SdkManagedAgentsClientOptions } async function failure(action: string, sessionId: string, res: Response): Promise { - const raw = await res.text().catch(() => ""); + const raw = await res.text().catch(swallowAs(`do-managed-agents ${action} ${sessionId}: read error body`, "")); let message = raw.slice(0, 200); try { const parsed = JSON.parse(raw) as { error?: { message?: string } }; @@ -491,7 +483,40 @@ export function createSdkManagedAgentsClient(opts: SdkManagedAgentsClientOptions agent.close(); await tunnel.close(); }) - .catch(() => undefined); + .catch(swallowAs(`do-managed-agents ${sessionId}: disconnect`, undefined)); + } + + async function download(absPath: string): Promise { + const { agent } = await connect(); + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + const stream = agent.Download({ path: absPath, as_archive: false, chunk_size_bytes: DOWNLOAD_CHUNK_BYTES }); + let settled = false; + const settle = (fn: () => void): void => { + if (settled) return; + settled = true; + fn(); + }; + stream.on("data", ((frame: DownloadFrame) => { + if (frame.chunk?.length) chunks.push(frame.chunk); + }) as (arg: never) => void); + stream.on("error", ((err: unknown) => { + settle(() => { + if (grpcCode(err) === GRPC_NOT_FOUND) { + resolve(null); + return; + } + const detail = guestDetail(err); + void disconnect(); + reject( + grpcCode(err) === GRPC_UNAVAILABLE + ? new ManagedAgentsSandboxGoneError(sessionId, detail) + : new Error(`do-managed-agents download ${absPath} failed: ${detail}`, { cause: err }), + ); + }); + }) as (arg: never) => void); + stream.on("end", (() => settle(() => resolve(new Uint8Array(Buffer.concat(chunks))))) as (arg: never) => void); + }); } const guestDetail = (err: unknown): string => lastTunnel?.lastFailure() ?? errMessage(err); @@ -534,7 +559,7 @@ export function createSdkManagedAgentsClient(opts: SdkManagedAgentsClientOptions ); }); }, - () => undefined, + swallowAs(`do-managed-agents ${sessionId}: stall probe`, undefined), ); }, stallProbeMs); watchdog.unref(); @@ -570,10 +595,15 @@ export function createSdkManagedAgentsClient(opts: SdkManagedAgentsClientOptions reject(new ManagedAgentsHitlRejectedError(sessionId)); return; } + if (exit?.exit_code === undefined) { + void disconnect(); + reject(new ManagedAgentsCommandLostError(sessionId, "the command stream ended without an exit status")); + return; + } resolve({ stdout: Buffer.concat(out).toString("utf8"), stderr: Buffer.concat(errOut).toString("utf8"), - exitCode: exit?.exit_code ?? -1, + exitCode: exit.exit_code, }); }); }) as (arg: never) => void); @@ -591,38 +621,14 @@ export function createSdkManagedAgentsClient(opts: SdkManagedAgentsClientOptions }, async readFileBytes(absPath): Promise { - const { agent } = await connect(); - return new Promise((resolve, reject) => { - const chunks: Buffer[] = []; - const stream = agent.Download({ path: absPath, as_archive: false, chunk_size_bytes: DOWNLOAD_CHUNK_BYTES }); - let settled = false; - const settle = (fn: () => void): void => { - if (settled) return; - settled = true; - fn(); - }; - stream.on("data", ((frame: DownloadFrame) => { - if (frame.chunk?.length) chunks.push(frame.chunk); - }) as (arg: never) => void); - stream.on("error", ((err: unknown) => { - settle(() => { - if (isMissingPath(err)) { - resolve(null); - return; - } - const detail = guestDetail(err); - void disconnect(); - reject( - grpcCode(err) === GRPC_UNAVAILABLE - ? new ManagedAgentsSandboxGoneError(sessionId, detail) - : new Error(`do-managed-agents download ${absPath} failed: ${detail}`), - ); - }); - }) as (arg: never) => void); - stream.on("end", (() => settle(() => resolve(new Uint8Array(Buffer.concat(chunks))))) as ( - arg: never, - ) => void); - }); + try { + return await download(absPath); + } catch (err) { + if (grpcCode((err as { cause?: unknown }).cause) !== GRPC_INTERNAL) throw err; + const probe = await this.runCommand(`test -e '${absPath.replaceAll("'", "'\\''")}'`, { timeoutMs: 30_000 }); + if (probe.exitCode === 1) return null; + throw err; + } }, async writeFileBytes(absPath, data): Promise { diff --git a/src/sandbox/managed-agents-sandbox-agent.proto b/src/sandbox/managed-agents-sandbox-agent.proto index cd9dd068e9..59f5c8dda8 100644 --- a/src/sandbox/managed-agents-sandbox-agent.proto +++ b/src/sandbox/managed-agents-sandbox-agent.proto @@ -2,9 +2,6 @@ syntax = "proto3"; package do.teams.hosted_agents.runtime.sandbox_agent.v1; -// The subset of the in-guest sandbox-agent gRPC surface this backend calls. -// Field numbers and names match the server; the RPCs and messages qm does not -// use are omitted rather than reproduced. service SandboxAgentService { rpc Exec(stream ExecRequest) returns (stream ExecOutput); rpc Upload(stream UploadRequest) returns (UploadResponse); @@ -20,12 +17,11 @@ message ExecRequest { } message ExecStart { - // argv[0] is the program. No shell is spawned unless the caller asks for - // one, so qm passes ["/bin/bash", "-c", command]. + repeated string argv = 1; string workdir = 2; map env = 3; - // Hard timeout; 0 means none. Enforced in the guest with SIGTERM then SIGKILL. + int64 timeout_seconds = 4; } @@ -44,7 +40,7 @@ message ExecOutput { message ExecExit { int32 exit_code = 1; int64 duration_ms = 2; - // Set when an in-guest approval check rejected the command; exit_code is -1. + bool hitl_rejected = 3; bool timed_out = 4; } @@ -58,16 +54,16 @@ message UploadRequest { } message UploadHeader { - // Absolute path inside the guest. Transfer is confined to the workspace root. + string path = 1; - // POSIX mode; zero takes the guest default. + uint32 mode = 2; - // True when the payload is a tar archive to extract under path. + bool is_archive = 3; } message UploadEnd { - // Optional hex SHA-256 of the payload; the guest fails the upload on mismatch. + string sha256 = 1; } @@ -79,7 +75,7 @@ message UploadResponse { message DownloadRequest { string path = 1; bool as_archive = 2; - // Read buffer size in bytes; zero takes the guest default. + uint32 chunk_size_bytes = 3; } @@ -92,7 +88,7 @@ message DownloadChunk { } message DownloadHeader { - // -1 when unknown, as for a streaming archive. + int64 size_bytes = 1; bool is_archive = 2; } diff --git a/src/sandbox/managed-agents-sandbox.ts b/src/sandbox/managed-agents-sandbox.ts index 977aef8828..0ebdf9afcf 100644 --- a/src/sandbox/managed-agents-sandbox.ts +++ b/src/sandbox/managed-agents-sandbox.ts @@ -225,7 +225,9 @@ export function createManagedAgentsSandbox(workspace: WorkspaceStore, opts: Mana } catch (e) { reportError("sandbox_hydrate", "hydrate_failed", errMessage(e), scope); sessionByName.delete(name); - await client.kill(session.sessionId).catch(() => undefined); + await client.kill(session.sessionId).catch((err: unknown) => { + reportError("sandbox_hydrate", "orphan_kill_failed", errMessage(err), scope); + }); throw new Error( `do-managed-agents provision: home hydration failed (${errMessage(e)}); not risking the stored snapshot`, { @@ -301,11 +303,11 @@ export function createManagedAgentsSandbox(workspace: WorkspaceStore, opts: Mana os: "Linux — DigitalOcean Managed Agents Firecracker microVM (provider pause preserves state; publish durable work to git or Files)", runtimes: ["Node", "Python 3"], get tools() { - return visibleTools(["git", "curl", "tar", "python3", "gh", ...(opts.extraTools ?? [])]); + return visibleTools(["git", "curl", "tar", "python3", ...(opts.extraTools ?? [])]); }, get notInstalled() { return visibleNotInstalled( - ["jq", "rg", "unzip", "wget", "aws", "gcloud", "kubectl", "flyctl", "glab"], + ["jq", "rg", "unzip", "wget", "gh", "aws", "gcloud", "kubectl", "flyctl", "glab"], opts.extraTools ?? [], ); }, @@ -552,7 +554,7 @@ export function createManagedAgentsSandbox(workspace: WorkspaceStore, opts: Mana machine, recovery, provisioned: true, - guestResponsive: r.exitCode === 0 && /responsive/.test(r.stdout), + guestResponsive: r.exitCode === 0 && r.stdout.trim() === "responsive", }; } catch (e) { return { diff --git a/src/wiring.ts b/src/wiring.ts index ea5ef257f5..e3ea370e6f 100644 --- a/src/wiring.ts +++ b/src/wiring.ts @@ -1152,19 +1152,13 @@ export function buildApp( ...(await environments.list()).map((environment) => environment.id), ], legacySandboxes: async () => { - const [e2b, modal, aws, superserve, managedAgents] = await Promise.all([ + const [e2b, modal, aws, superserve] = await Promise.all([ e2bBodies.entries(), modalBodies.entries(), awsBodies.entries(), superserveBodies.entries(), - managedAgentsBodies.entries(), ]); return [ - ...managedAgents.map(([scopeId, body]) => ({ - scopeId, - backend: "do-managed-agents" as const, - machineId: body.sandboxId, - })), ...e2b.map(([scopeId, body]) => ({ scopeId, backend: "e2b" as const, machineId: body.sandboxId })), ...modal.map(([scopeId, body]) => ({ scopeId, backend: "modal" as const, machineId: body.sandboxId })), ...aws.map(([scopeId, body]) => ({ scopeId, backend: "aws" as const, machineId: body.microvmId })), From fc47338e2d3d800e1e529e3fd0be10ea86da985d Mon Sep 17 00:00:00 2001 From: Josh France Date: Sat, 10 Oct 2026 23:50:43 +0000 Subject: [PATCH 16/19] Add a script to register the qm-sandbox template on your own team --- deploy/do-managed-agents/register-template.sh | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100755 deploy/do-managed-agents/register-template.sh diff --git a/deploy/do-managed-agents/register-template.sh b/deploy/do-managed-agents/register-template.sh new file mode 100755 index 0000000000..3c6bf4c973 --- /dev/null +++ b/deploy/do-managed-agents/register-template.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Build the QM sandbox image and register it as a Managed Agents template +# on your own DigitalOcean team. Run once per team, and again after the +# Dockerfile changes. +# +# Needs: docker (buildx), doctl authenticated with a token that has +# registry and Managed Agents access, and an existing container registry. +# +# Usage: deploy/do-managed-agents/register-template.sh [tag] +# DO_REGISTRY registry name (default: first registry on the account) +# TEMPLATE_NAME template name (default: qm-sandbox) +set -euo pipefail + +TAG="${1:-$(date +%Y%m%d%H%M%S)}" +TEMPLATE_NAME="${TEMPLATE_NAME:-qm-sandbox}" +DIR="$(cd "$(dirname "$0")" && pwd)" + +for bin in docker doctl jq; do + command -v "$bin" >/dev/null || { echo "missing: $bin" >&2; exit 1; } +done + +DO_REGISTRY="${DO_REGISTRY:-$(doctl registry get -o json | jq -r 'if type=="array" then .[0].name else .name end // empty')}" +[ -n "$DO_REGISTRY" ] || { echo "no container registry found; create one or set DO_REGISTRY" >&2; exit 1; } +IMAGE="registry.digitalocean.com/$DO_REGISTRY/$TEMPLATE_NAME:$TAG" + +echo "building $IMAGE" +docker build --platform linux/amd64 -t "$IMAGE" "$DIR" +doctl registry login >/dev/null +docker push "$IMAGE" + +if doctl harness-runtime template get "$TEMPLATE_NAME" >/dev/null 2>&1; then + echo "updating template $TEMPLATE_NAME" + doctl harness-runtime template update "$TEMPLATE_NAME" --source-oci-ref "$IMAGE" >/dev/null +else + echo "creating template $TEMPLATE_NAME" + doctl harness-runtime template create --name "$TEMPLATE_NAME" \ + --base-template sandbox --source-oci-ref "$IMAGE" >/dev/null +fi + +echo "waiting for the template build" +for _ in $(seq 1 60); do + status="$(doctl harness-runtime template list-builds "$TEMPLATE_NAME" -o json \ + | jq -r 'sort_by(.created_at) | last | .status')" + case "$status" in + SUCCEEDED) echo "ready. Set DO_AGENTS_TEMPLATE=$TEMPLATE_NAME in your QM config."; exit 0 ;; + FAILED) echo "template build failed" >&2; exit 1 ;; + esac + sleep 20 +done +echo "timed out waiting for the template build (last status: $status)" >&2 +exit 1 From 3baa257ee469252aee058984f9784ad023a515c7 Mon Sep 17 00:00:00 2001 From: Josh France Date: Sat, 10 Oct 2026 23:58:28 +0000 Subject: [PATCH 17/19] Align managed-agents docs with the code --- adrs/managed-agents-sandbox-backend.md | 6 +++--- docs/sandbox-preservation.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/adrs/managed-agents-sandbox-backend.md b/adrs/managed-agents-sandbox-backend.md index 5de8ef4d80..2cfc9d4ddd 100644 --- a/adrs/managed-agents-sandbox-backend.md +++ b/adrs/managed-agents-sandbox-backend.md @@ -60,7 +60,7 @@ sequenceDiagram B->>B: leave the session running else park the scope B->>E: POST /v2/agents/sessions/{id}/checkpoints - B->>V: Download
portable tar home snapshot + B->>V: Download
portable tar home snapshot
(durable only with a snapshot bucket) B->>E: POST /v2/agents/sessions/{id}/pause else destroy the scope B->>E: DELETE /v2/agents/sessions/{id} @@ -80,11 +80,11 @@ What the PR adds - `src/sandbox/managed-agents-sandbox-agent.proto`, a vendored copy of our guest proto with the internal annotations stripped, loaded at runtime through `@grpc/proto-loader`. No build step and no generated code in the tree. - `src/sandbox/managed-agents-sandbox.ts`. Provision creates or resumes the session and waits for it to become usable. Run is `Exec` over the tunnel. Files are `Upload` and `Download` rather than base64 through exec, since we have real transfer RPCs. Process sessions, read-only layers, layer tool install, home snapshots and blob staging come from the shared exec helpers unchanged. - Scope recovery without local state. Session names are team-unique among non-terminal sessions and `ListSessions` takes a `?name=` filter, so the backend can re-adopt a running sandbox after losing its durable record, the way the E2B backend re-adopts by sandbox metadata. `sandboxScopeName` already produces names that fit our 64-character, not-UUID-shaped rule. -- Profile: `writablePersistence: "provider_managed"`, `processSessions: true`, `parksOnTeardown: true`, and `egressEnforcement: "domain"`. That last one is real rather than aspirational — a Managed Agents sandbox gets NAT egress with a per-session host allowlist, and the manifest's `permissions.network` block narrows within it. +- Profile: `writablePersistence: "provider_managed"`, `processSessions: true`, `parksOnTeardown: true`, and `egressEnforcement: "domain"` when `DO_AGENTS_EGRESS_PROXY_URL` is set (the proxy host becomes the session's egress allowlist), `"none"` otherwise. The domain mode is real rather than aspirational — a Managed Agents sandbox gets NAT egress with a per-session host allowlist. - A `permissions` block on the session that sets `defaultAction: allow` and allows `bash`. Our managed exec path is approval-gated by default, which means every command would wait on a human decision through our approval router. qm runs its own approval gates at its own tool boundary and fires many commands per turn, so two gating systems in series would just deadlock. qm's posture stays authoritative; ours gets out of the way. This is deliberate and we would rather state it here than have it discovered later. What it does not do - No idle reaping from qm's side. Our control plane owns the idle timeout, set per session with `DO_AGENTS_IDLE_TIMEOUT_SEC` and bounded by tenant policy. Paused sessions still count against team quota, which is worth knowing before pointing a large deployment at us. -- A QM template, optional. `agent: none` lands on our own bare base template, so `DO_AGENTS_TEMPLATE` is an override rather than a requirement; an unprovisioned name fails session create. `deploy/do-managed-agents/Dockerfile` defines it: `node:24-slim` plus the tools agent prompts reach for by name (`jq`, `rg`, `unzip`, `wget`, alongside git, curl and Python), with `HOME=/workspace/home`. It is registered as the `qm-sandbox` team template on the agentless `sandbox` base, and qm advertises those extra tools in the sandbox profile only when `DO_AGENTS_TEMPLATE` is set. Baking layer tool files into it is a follow-up. +- A QM template, optional. `agent: none` lands on our own bare base template, so `DO_AGENTS_TEMPLATE` is an override rather than a requirement; an unprovisioned name fails session create. `deploy/do-managed-agents/Dockerfile` defines it: `node:24-slim` plus the tools agent prompts reach for by name (`jq`, `rg`, `unzip`, `wget`, alongside git, curl and Python), with `HOME=/workspace/home`. `deploy/do-managed-agents/register-template.sh` builds it and registers it as a team template (default name `qm-sandbox`) on the agentless `sandbox` base, once per DO team. qm advertises those extra tools in the sandbox profile only when `DO_AGENTS_TEMPLATE` is set. Baking layer tool files into it is a follow-up. - No new CLI target. `qm init` still deploys core to Docker, Fly or AWS. This only changes where the computers live. diff --git a/docs/sandbox-preservation.md b/docs/sandbox-preservation.md index 749b945ba8..e7c36793be 100644 --- a/docs/sandbox-preservation.md +++ b/docs/sandbox-preservation.md @@ -35,4 +35,4 @@ When `MODAL_EGRESS_PROXY_URL` is set, machines are created with Modal's outbound Turn environment reaches Modal commands through the exec `env` parameter rather than inlined `export` statements, exec deadlines are rounded up to whole seconds with a single thirty-second grace beyond the in-guest `timeout`, and a command longer than 64 KiB is written to `/tmp` through the filesystem API and executed from there. -Managed Agents sessions name a sandbox template explicitly, which yields a microVM carrying no managed agent loop, and the disk persists across the provider's pause and resume. Used-turn teardown calls `POST /v2/agents/sessions/{id}/checkpoints` when the home changed, throttled by `DO_AGENTS_SNAPSHOT_INTERVAL_SEC` from the last successful capture, then snapshots the home to `DO_AGENTS_SNAPSHOT_S3_BUCKET` on that same interval measured from the last successful tar. A failed tar is retried without taking another checkpoint. The checkpoint is the provider's machine-state capture of the live session (`CreateCheckpoint` blocks until the capture is `READY`); only the newest is kept, and the previous one is deleted after the new capture succeeds. A delete that fails stays recorded and is retried on a later teardown. A capture failure is recorded on `recovery.error` and the pause still proceeds. A pause failure is recorded as `pause_failed` and the running session is retained for retry. Status reports `recovery.strategy: "workspace_snapshot"` plus `recovery.checkpointId`. Restart of a wedged computer calls `POST /v2/agents/sessions/{id}/checkpoints/{checkpoint_id}/rollback` with that stored checkpoint: the session id stays, the response's sandbox id replaces the stored one, and any open port-forward is closed so the next command dials the restored microVM. A paused session is rolled back directly, without being resumed first. Restart with no checkpoint fails and leaves the session where it is. A checkpoint is bound to its session, so a replacement session for a destroyed or reclaimed scope still hydrates from the portable tar, and a failed hydration kills the replacement rather than adopting a blank home over a saved one. Session lifecycle travels over the Managed Agents REST surface with a DigitalOcean IAM token; commands and file transfer travel over a bearer-authenticated WebSocket port-forward to the guest's `sandbox-agent` gRPC server, which has no request deadline, so command length is bounded by `SANDBOX_TIMEOUT_SEC` rather than by the control plane. Sessions are created with a permissions block that allows `bash` outright, because Managed Agents would otherwise hold every command at its own approval gate behind core's. +Managed Agents sessions name a sandbox template explicitly, which yields a microVM carrying no managed agent loop, and the disk persists across the provider's pause and resume. Used-turn teardown calls `POST /v2/agents/sessions/{id}/checkpoints` when the home changed, throttled by `DO_AGENTS_SNAPSHOT_INTERVAL_SEC` from the last successful capture, then snapshots the home as a tar on that same interval measured from the last successful tar. The tar goes to `DO_AGENTS_SNAPSHOT_S3_BUCKET` when it is set; without it the tar is held in core's memory and does not survive a core restart. A failed tar is retried without taking another checkpoint. The checkpoint is the provider's machine-state capture of the live session (`CreateCheckpoint` blocks until the capture is `READY`); only the newest is kept, and the previous one is deleted after the new capture succeeds. A delete that fails stays recorded and is retried on a later teardown. A capture failure is recorded on `recovery.error` and the pause still proceeds. A pause failure is recorded as `pause_failed` and the running session is retained for retry. Status reports `recovery.strategy: "workspace_snapshot"` plus `recovery.checkpointId`. Restart of a wedged computer calls `POST /v2/agents/sessions/{id}/checkpoints/{checkpoint_id}/rollback` with that stored checkpoint: the session id stays, the response's sandbox id replaces the stored one, and any open port-forward is closed so the next command dials the restored microVM. A paused session is rolled back directly, without being resumed first. Restart with no checkpoint fails and leaves the session where it is. A checkpoint is bound to its session, so a replacement session for a destroyed or reclaimed scope still hydrates from the portable tar when one exists, and a failed hydration kills the replacement rather than adopting a blank home over a saved one. Session lifecycle travels over the Managed Agents REST surface with a DigitalOcean IAM token; commands and file transfer travel over a bearer-authenticated WebSocket port-forward to the guest's `sandbox-agent` gRPC server, which has no request deadline, so command length is bounded by `SANDBOX_TIMEOUT_SEC` rather than by the control plane. Sessions are created with a permissions block that allows `bash` outright, because Managed Agents would otherwise hold every command at its own approval gate behind core's. From e9b2c78733cddc7d3ec545c1411b259261367536 Mon Sep 17 00:00:00 2001 From: Josh France Date: Sun, 11 Oct 2026 02:24:14 +0000 Subject: [PATCH 18/19] Replace the Managed Agents ADR with a setup guide --- README.md | 1 + adrs/managed-agents-sandbox-backend.md | 90 -------------------------- 2 files changed, 1 insertion(+), 90 deletions(-) delete mode 100644 adrs/managed-agents-sandbox-backend.md diff --git a/README.md b/README.md index 80b54cab83..b363e179e8 100644 --- a/README.md +++ b/README.md @@ -242,6 +242,7 @@ upstream source history to merge. - [`docs/principal-links.md`](./docs/principal-links.md) — one person, several sign-ins: linking principals - [`docs/porter.md`](./docs/porter.md) — running qm on Porter - [`docs/superserve.md`](./docs/superserve.md) — using Superserve for agent sandboxes +- [`docs/do-managed-agents.md`](./docs/do-managed-agents.md) — using DigitalOcean Managed Agents for agent sandboxes - [`.env.example`](./.env.example) — every knob, documented in place - [`docs/swarms.md`](./docs/swarms.md) — durable agent pools, scoped messages, and blank Modal workers - [`docs/model-gateway.md`](./docs/model-gateway.md) — discover and route models through a gateway diff --git a/adrs/managed-agents-sandbox-backend.md b/adrs/managed-agents-sandbox-backend.md deleted file mode 100644 index 2cfc9d4ddd..0000000000 --- a/adrs/managed-agents-sandbox-backend.md +++ /dev/null @@ -1,90 +0,0 @@ -# Managed Agents as an agent-computer backend - -This was written by the Managed Agents team at DigitalOcean. Managed Agents (Managed Agent Runtime Stack) hosts coding agents in DO-managed Firecracker microVMs. We looked at what it would take for qm to run on it and the answer is the same one Agent37 arrived at: the part of qm that wants hosting is the agent computer, so this is another sandbox backend, next to E2B, Modal, Sprites, smolmachines, Porter, AWS, Superserve and Agent37. - -The reason this is worth doing at all is that Managed Agents is a managed-loop product by default — it boots Claude Code or OpenCode _inside_ the microVM, next to a runtime process that translates agent events back to our control plane and an event stream that `doctl agents attach` tails. That is not what qm wants, because qm already has a loop. But our platform now admits a session with no agent in it at all: a manifest that says `agent: none` lands on Managed Agents's own bare base template, with no managed agent, no event-translation runtime, and no model credential. The caller drives the microVM itself over exec, workspace transfer and port-forward. That mode exists for customers who want Managed Agents purely for code execution — evaluations, CI-style fan-out, arbitrary compute — and qm lands in the same seat. Nothing about qm's orchestrator, harness router or tool surface has to change. - -```mermaid -flowchart LR - subgraph MANAGED["Managed Agents default: managed loop"] - M1["microVM"] --> M2["Claude Code / OpenCode"] --> M3["event-translation runtime"] --> M4["event stream"] --> M5["doctl agents attach"] - end - - subgraph INVERTED["agent: none -- what qm uses"] - Q1["qm core
orchestrator + vendored harness"] --> Q2["the loop stays here"] - Q2 --> Q3["microVM
bare sandbox, no agent"] - end -``` - -The lifecycle lines up closely enough with the E2B backend that this is modeled on that pair of files rather than on a new shape. One Managed Agents session per qm scope, created on first use, named with the scope name so an operator can find it with `doctl agents list`, paused at teardown, deleted when qm destroys the scope. Our pause keeps the microVM's disk and memory, which is what `provider_managed` persistence already means in your profile. - -Everything qm needs is on one host with one credential. Session lifecycle is our control plane, which external callers reach as REST through the DO API edge, which validates a DO IAM token and forwards the authenticated team and user as headers — the same path doctl takes. So that half is plain HTTP+JSON with a token, and the team identity comes from the token rather than from configuration. - -Exec and file transfer take a different route to the same host. Every microVM runs `sandbox-agent`, a gRPC server on guest port 8443 that our own control plane dials for exec, transfer and readiness. qm reaches it the way `doctl agents port-forward` does: a bearer-authenticated WebSocket to `/v2/agents/sessions/{id}/port-forward/8443`, which the control plane bridges to the guest port and qm exposes as a local TCP listener for a gRPC client. 8443 is explicitly allowed by our tunnel port policy, the guest listener is plaintext h2c with no client certificate, and we register the port-forward route without a request deadline on purpose. So a long command is bounded by qm's own `SANDBOX_TIMEOUT_SEC` and nothing else. - -Here is a turn end to end. Lifecycle goes through the edge as REST; exec and files go through the tunnel to the guest. - -```mermaid -sequenceDiagram - participant O as qm orchestrator - participant B as managed-agents-sandbox.ts - participant E as DO API edge - participant H as control plane - participant T as port-forward tunnel - participant V as microVM
sandbox-agent :8443 - - Note over O,B: provision - O->>B: provision layers for scope - B->>E: POST /v2/agents/sessions
agents.yaml: name, agent: none, permissions, egress - Note right of E: validates DO IAM token,
forwards team + user headers - E->>H: CreateSession - H->>V: provision the microVM
internal, not qm's concern - H-->>B: session_id, PROVISIONING - B->>E: GET /v2/agents/sessions/{id} - E-->>B: READY - B->>T: wss /port-forward/8443
Authorization: Bearer - T->>V: bridge to guest :8443 - B->>V: Upload
ro layers + layer tool files - B-->>O: handle - - Note over O,B: run - O->>B: run command - B->>V: Exec stream over the tunnel
argv, env, workdir, timeout - V-->>B: stdout, stderr, exit - B-->>O: ExecResult - V->>O: self-API callback with capability token
needs core's URL on the egress allowlist - - Note over O,B: teardown - O->>B: teardown - alt keepWarm for background work - B->>B: leave the session running - else park the scope - B->>E: POST /v2/agents/sessions/{id}/checkpoints - B->>V: Download
portable tar home snapshot
(durable only with a snapshot bucket) - B->>E: POST /v2/agents/sessions/{id}/pause - else destroy the scope - B->>E: DELETE /v2/agents/sessions/{id} - end - - Note over O,B: restart a wedged computer - O->>B: restart - B->>E: POST /v2/agents/sessions/{id}/checkpoints/{checkpoint_id}/rollback - E-->>B: same session_id, new sandbox_id -``` - -What the PR adds - -- `SANDBOX_BACKEND=do-managed-agents`, with `DO_AGENTS_API_TOKEN` required — a DO IAM token, which is also where the team identity comes from, and the only credential the backend needs. Optional knobs: `DO_AGENTS_API_BASE_URL`, `DO_AGENTS_TEMPLATE`, `DO_AGENTS_NAME_PREFIX`, `DO_AGENTS_SIZE_SLUG`, `DO_AGENTS_IDLE_TIMEOUT_SEC`, `DO_AGENTS_EGRESS_PROXY_URL`, `DO_AGENTS_SNAPSHOT_INTERVAL_SEC`, `DO_AGENTS_SNAPSHOT_S3_BUCKET`, and the shared `SANDBOX_TIMEOUT_SEC`. -- `src/sandbox/managed-agents-tunnel.ts`, the port-forward client: one WebSocket per TCP connection, bytes piped both ways with one chunk in flight per direction so a slow consumer cannot make core buffer the stream. The local listener is unreferenced, so an abandoned tunnel can never hold core's event loop open. -- `src/sandbox/managed-agents-client.ts`, holding both transports behind one interface the way `e2b-client.ts` wraps the E2B SDK: REST for `/v2/agents/sessions{,/{id},/{id}/pause,/{id}/resume,/{id}/checkpoints,/{id}/checkpoints/{checkpoint_id}/rollback}`, and `SandboxAgentService` stubs for `Exec`, `Upload` and `Download` over the tunnel. Our 404s, terminal statuses and gRPC `UNAVAILABLE` translate to the same two error classes the E2B client uses — one for a sandbox that was already gone before a command started, which the backend reconnects and retries, and one for a sandbox lost mid-command, which is reported rather than retried because the command may have partially run. The session body is an agents.yaml manifest in its flat form, which is the shape we are standardising on. -- `src/sandbox/managed-agents-sandbox-agent.proto`, a vendored copy of our guest proto with the internal annotations stripped, loaded at runtime through `@grpc/proto-loader`. No build step and no generated code in the tree. -- `src/sandbox/managed-agents-sandbox.ts`. Provision creates or resumes the session and waits for it to become usable. Run is `Exec` over the tunnel. Files are `Upload` and `Download` rather than base64 through exec, since we have real transfer RPCs. Process sessions, read-only layers, layer tool install, home snapshots and blob staging come from the shared exec helpers unchanged. -- Scope recovery without local state. Session names are team-unique among non-terminal sessions and `ListSessions` takes a `?name=` filter, so the backend can re-adopt a running sandbox after losing its durable record, the way the E2B backend re-adopts by sandbox metadata. `sandboxScopeName` already produces names that fit our 64-character, not-UUID-shaped rule. -- Profile: `writablePersistence: "provider_managed"`, `processSessions: true`, `parksOnTeardown: true`, and `egressEnforcement: "domain"` when `DO_AGENTS_EGRESS_PROXY_URL` is set (the proxy host becomes the session's egress allowlist), `"none"` otherwise. The domain mode is real rather than aspirational — a Managed Agents sandbox gets NAT egress with a per-session host allowlist. -- A `permissions` block on the session that sets `defaultAction: allow` and allows `bash`. Our managed exec path is approval-gated by default, which means every command would wait on a human decision through our approval router. qm runs its own approval gates at its own tool boundary and fires many commands per turn, so two gating systems in series would just deadlock. qm's posture stays authoritative; ours gets out of the way. This is deliberate and we would rather state it here than have it discovered later. - -What it does not do - -- No idle reaping from qm's side. Our control plane owns the idle timeout, set per session with `DO_AGENTS_IDLE_TIMEOUT_SEC` and bounded by tenant policy. Paused sessions still count against team quota, which is worth knowing before pointing a large deployment at us. -- A QM template, optional. `agent: none` lands on our own bare base template, so `DO_AGENTS_TEMPLATE` is an override rather than a requirement; an unprovisioned name fails session create. `deploy/do-managed-agents/Dockerfile` defines it: `node:24-slim` plus the tools agent prompts reach for by name (`jq`, `rg`, `unzip`, `wget`, alongside git, curl and Python), with `HOME=/workspace/home`. `deploy/do-managed-agents/register-template.sh` builds it and registers it as a team template (default name `qm-sandbox`) on the agentless `sandbox` base, once per DO team. qm advertises those extra tools in the sandbox profile only when `DO_AGENTS_TEMPLATE` is set. Baking layer tool files into it is a follow-up. -- No new CLI target. `qm init` still deploys core to Docker, Fly or AWS. This only changes where the computers live. From 71635d4d15c64640e9816c6f11bd92f4ff5f2051 Mon Sep 17 00:00:00 2001 From: Josh France Date: Sun, 11 Oct 2026 02:30:13 +0000 Subject: [PATCH 19/19] Add the Managed Agents setup guide, unlinked from the README --- README.md | 1 - docs/do-managed-agents.md | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 docs/do-managed-agents.md diff --git a/README.md b/README.md index b363e179e8..80b54cab83 100644 --- a/README.md +++ b/README.md @@ -242,7 +242,6 @@ upstream source history to merge. - [`docs/principal-links.md`](./docs/principal-links.md) — one person, several sign-ins: linking principals - [`docs/porter.md`](./docs/porter.md) — running qm on Porter - [`docs/superserve.md`](./docs/superserve.md) — using Superserve for agent sandboxes -- [`docs/do-managed-agents.md`](./docs/do-managed-agents.md) — using DigitalOcean Managed Agents for agent sandboxes - [`.env.example`](./.env.example) — every knob, documented in place - [`docs/swarms.md`](./docs/swarms.md) — durable agent pools, scoped messages, and blank Modal workers - [`docs/model-gateway.md`](./docs/model-gateway.md) — discover and route models through a gateway diff --git a/docs/do-managed-agents.md b/docs/do-managed-agents.md new file mode 100644 index 0000000000..31e2b5eb36 --- /dev/null +++ b/docs/do-managed-agents.md @@ -0,0 +1,39 @@ +# DigitalOcean Managed Agents sandboxes + +The `do-managed-agents` backend runs each QM scope on a DigitalOcean Managed Agents microVM. QM keeps its own agent loop; sessions start with `agent: none`, so DigitalOcean supplies the machine and nothing else. + +## Set it up + +1. Create a DigitalOcean API token with Managed Agents access. +2. Point core at it: + + ```sh + SANDBOX_BACKEND=do-managed-agents + DO_AGENTS_API_TOKEN=your-token + ``` + +That's enough to run. Sessions land on DigitalOcean's bare base image. + +## Optional + +**A QM template.** For an image with the tools agents usually reach for (Node, git, jq, ripgrep, Python), register one on your team. You need Docker, an authenticated `doctl`, and a container registry: + +```sh +deploy/do-managed-agents/register-template.sh +DO_AGENTS_TEMPLATE=qm-sandbox +``` + +Templates belong to a team, so each team registers its own. Rerun the script after changing the Dockerfile. + +**A snapshot bucket.** When a scope parks, QM takes a DigitalOcean checkpoint, which lets it roll a wedged computer back in place. A checkpoint is tied to its session, though, so if the session is deleted the checkpoint goes with it. Set `DO_AGENTS_SNAPSHOT_S3_BUCKET` to also keep a copy of the home directory that survives that. Without it, the copy lives in core's memory and is lost when core restarts. + +**An egress proxy.** Set `DO_AGENTS_EGRESS_PROXY_URL` and the proxy host becomes the session's only allowed destination. + +Other knobs: `DO_AGENTS_SIZE_SLUG`, `DO_AGENTS_IDLE_TIMEOUT_SEC`, `DO_AGENTS_NAME_PREFIX` (use a distinct one per deployment sharing a team), `DO_AGENTS_SNAPSHOT_INTERVAL_SEC`, and `DO_AGENTS_API_BASE_URL`. + +## Good to know + +- Commands and file transfers go over a port-forward to the microVM rather than DigitalOcean's REST exec, which caps output and runtime. Long commands are bounded only by `SANDBOX_TIMEOUT_SEC`. +- QM doesn't reap idle machines itself. DigitalOcean's idle timeout does, and paused sessions still count against team quota. +- Sessions allow `bash` outright, since QM already runs its own approval gates. +- `scripts/managed-agents-sandbox-smoke.ts` checks a token and template end to end before you point a deployment at them.