diff --git a/cli/src/secrets.ts b/cli/src/secrets.ts index c02dc5c7de..6560efe630 100644 --- a/cli/src/secrets.ts +++ b/cli/src/secrets.ts @@ -170,6 +170,14 @@ export const FIRST_PARTY_SECRET_SPECS: readonly SecretSpec[] = [ required: { when: { kind: "sandbox-backend", backend: "e2b" } }, description: "E2B API key used by the e2b sandbox backend (from e2b.dev dashboard).", }, + { + name: "DO_AGENTS_API_TOKEN", + service: "core", + required: { when: { kind: "sandbox-backend", backend: "do-managed-agents" } }, + description: + "DigitalOcean IAM token for the DigitalOcean Managed Agents sandbox backend. Carries the team identity that owns the sandbox sessions, and authenticates both the session REST calls and the port-forward tunnel that carries exec and file transfer.", + generate: "doctl auth init # then create a personal access token at cloud.digitalocean.com/account/api/tokens", + }, { name: "MODAL_TOKEN_ID", service: "core", diff --git a/deploy/do-managed-agents/Dockerfile b/deploy/do-managed-agents/Dockerfile new file mode 100644 index 0000000000..fd913e088c --- /dev/null +++ b/deploy/do-managed-agents/Dockerfile @@ -0,0 +1,13 @@ +FROM node:24-slim@sha256:6f7b03f7c2c8e2e784dcf9295400527b9b1270fd37b7e9a7285cf83b6951452d + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates git curl wget jq tar xz-utils unzip zip ripgrep \ + python3 python3-pip python3-venv \ + openssh-client gnupg less vim-tiny procps \ + && rm -rf /var/lib/apt/lists/* \ + && node --version && npm --version + +ENV HOME=/workspace/home +ENTRYPOINT ["sleep", "infinity"] +CMD [] diff --git a/deploy/do-managed-agents/register-template.sh b/deploy/do-managed-agents/register-template.sh new file mode 100755 index 0000000000..3c6bf4c973 --- /dev/null +++ b/deploy/do-managed-agents/register-template.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Build the QM sandbox image and register it as a Managed Agents template +# on your own DigitalOcean team. Run once per team, and again after the +# Dockerfile changes. +# +# Needs: docker (buildx), doctl authenticated with a token that has +# registry and Managed Agents access, and an existing container registry. +# +# Usage: deploy/do-managed-agents/register-template.sh [tag] +# DO_REGISTRY registry name (default: first registry on the account) +# TEMPLATE_NAME template name (default: qm-sandbox) +set -euo pipefail + +TAG="${1:-$(date +%Y%m%d%H%M%S)}" +TEMPLATE_NAME="${TEMPLATE_NAME:-qm-sandbox}" +DIR="$(cd "$(dirname "$0")" && pwd)" + +for bin in docker doctl jq; do + command -v "$bin" >/dev/null || { echo "missing: $bin" >&2; exit 1; } +done + +DO_REGISTRY="${DO_REGISTRY:-$(doctl registry get -o json | jq -r 'if type=="array" then .[0].name else .name end // empty')}" +[ -n "$DO_REGISTRY" ] || { echo "no container registry found; create one or set DO_REGISTRY" >&2; exit 1; } +IMAGE="registry.digitalocean.com/$DO_REGISTRY/$TEMPLATE_NAME:$TAG" + +echo "building $IMAGE" +docker build --platform linux/amd64 -t "$IMAGE" "$DIR" +doctl registry login >/dev/null +docker push "$IMAGE" + +if doctl harness-runtime template get "$TEMPLATE_NAME" >/dev/null 2>&1; then + echo "updating template $TEMPLATE_NAME" + doctl harness-runtime template update "$TEMPLATE_NAME" --source-oci-ref "$IMAGE" >/dev/null +else + echo "creating template $TEMPLATE_NAME" + doctl harness-runtime template create --name "$TEMPLATE_NAME" \ + --base-template sandbox --source-oci-ref "$IMAGE" >/dev/null +fi + +echo "waiting for the template build" +for _ in $(seq 1 60); do + status="$(doctl harness-runtime template list-builds "$TEMPLATE_NAME" -o json \ + | jq -r 'sort_by(.created_at) | last | .status')" + case "$status" in + SUCCEEDED) echo "ready. Set DO_AGENTS_TEMPLATE=$TEMPLATE_NAME in your QM config."; exit 0 ;; + FAILED) echo "template build failed" >&2; exit 1 ;; + esac + sleep 20 +done +echo "timed out waiting for the template build (last status: $status)" >&2 +exit 1 diff --git a/docs/do-managed-agents.md b/docs/do-managed-agents.md new file mode 100644 index 0000000000..31e2b5eb36 --- /dev/null +++ b/docs/do-managed-agents.md @@ -0,0 +1,39 @@ +# DigitalOcean Managed Agents sandboxes + +The `do-managed-agents` backend runs each QM scope on a DigitalOcean Managed Agents microVM. QM keeps its own agent loop; sessions start with `agent: none`, so DigitalOcean supplies the machine and nothing else. + +## Set it up + +1. Create a DigitalOcean API token with Managed Agents access. +2. Point core at it: + + ```sh + SANDBOX_BACKEND=do-managed-agents + DO_AGENTS_API_TOKEN=your-token + ``` + +That's enough to run. Sessions land on DigitalOcean's bare base image. + +## Optional + +**A QM template.** For an image with the tools agents usually reach for (Node, git, jq, ripgrep, Python), register one on your team. You need Docker, an authenticated `doctl`, and a container registry: + +```sh +deploy/do-managed-agents/register-template.sh +DO_AGENTS_TEMPLATE=qm-sandbox +``` + +Templates belong to a team, so each team registers its own. Rerun the script after changing the Dockerfile. + +**A snapshot bucket.** When a scope parks, QM takes a DigitalOcean checkpoint, which lets it roll a wedged computer back in place. A checkpoint is tied to its session, though, so if the session is deleted the checkpoint goes with it. Set `DO_AGENTS_SNAPSHOT_S3_BUCKET` to also keep a copy of the home directory that survives that. Without it, the copy lives in core's memory and is lost when core restarts. + +**An egress proxy.** Set `DO_AGENTS_EGRESS_PROXY_URL` and the proxy host becomes the session's only allowed destination. + +Other knobs: `DO_AGENTS_SIZE_SLUG`, `DO_AGENTS_IDLE_TIMEOUT_SEC`, `DO_AGENTS_NAME_PREFIX` (use a distinct one per deployment sharing a team), `DO_AGENTS_SNAPSHOT_INTERVAL_SEC`, and `DO_AGENTS_API_BASE_URL`. + +## Good to know + +- Commands and file transfers go over a port-forward to the microVM rather than DigitalOcean's REST exec, which caps output and runtime. Long commands are bounded only by `SANDBOX_TIMEOUT_SEC`. +- QM doesn't reap idle machines itself. DigitalOcean's idle timeout does, and paused sessions still count against team quota. +- Sessions allow `bash` outright, since QM already runs its own approval gates. +- `scripts/managed-agents-sandbox-smoke.ts` checks a token and template end to end before you point a deployment at them. diff --git a/docs/sandbox-preservation.md b/docs/sandbox-preservation.md index c8517560df..e7c36793be 100644 --- a/docs/sandbox-preservation.md +++ b/docs/sandbox-preservation.md @@ -34,3 +34,5 @@ Modal machines are created with a one-core, 2 GiB reservation (`MODAL_CPUS`, `MO When `MODAL_EGRESS_PROXY_URL` is set, machines are created with Modal's outbound allowlist limited to the proxy host (a domain allowlist for an https proxy on port 443, a CIDR allowlist for an IP address; other proxy URLs are rejected at startup because Modal domain allowlists pass only TLS on 443), and the profile reports `egressEnforcement: "domain"`. Command execution, file transfer, and checkpoints travel over Modal's control plane and are unaffected. Without the setting, core warns that Modal sandboxes run fail-open. Turn environment reaches Modal commands through the exec `env` parameter rather than inlined `export` statements, exec deadlines are rounded up to whole seconds with a single thirty-second grace beyond the in-guest `timeout`, and a command longer than 64 KiB is written to `/tmp` through the filesystem API and executed from there. + +Managed Agents sessions name a sandbox template explicitly, which yields a microVM carrying no managed agent loop, and the disk persists across the provider's pause and resume. Used-turn teardown calls `POST /v2/agents/sessions/{id}/checkpoints` when the home changed, throttled by `DO_AGENTS_SNAPSHOT_INTERVAL_SEC` from the last successful capture, then snapshots the home as a tar on that same interval measured from the last successful tar. The tar goes to `DO_AGENTS_SNAPSHOT_S3_BUCKET` when it is set; without it the tar is held in core's memory and does not survive a core restart. A failed tar is retried without taking another checkpoint. The checkpoint is the provider's machine-state capture of the live session (`CreateCheckpoint` blocks until the capture is `READY`); only the newest is kept, and the previous one is deleted after the new capture succeeds. A delete that fails stays recorded and is retried on a later teardown. A capture failure is recorded on `recovery.error` and the pause still proceeds. A pause failure is recorded as `pause_failed` and the running session is retained for retry. Status reports `recovery.strategy: "workspace_snapshot"` plus `recovery.checkpointId`. Restart of a wedged computer calls `POST /v2/agents/sessions/{id}/checkpoints/{checkpoint_id}/rollback` with that stored checkpoint: the session id stays, the response's sandbox id replaces the stored one, and any open port-forward is closed so the next command dials the restored microVM. A paused session is rolled back directly, without being resumed first. Restart with no checkpoint fails and leaves the session where it is. A checkpoint is bound to its session, so a replacement session for a destroyed or reclaimed scope still hydrates from the portable tar when one exists, and a failed hydration kills the replacement rather than adopting a blank home over a saved one. Session lifecycle travels over the Managed Agents REST surface with a DigitalOcean IAM token; commands and file transfer travel over a bearer-authenticated WebSocket port-forward to the guest's `sandbox-agent` gRPC server, which has no request deadline, so command length is bounded by `SANDBOX_TIMEOUT_SEC` rather than by the control plane. Sessions are created with a permissions block that allows `bash` outright, because Managed Agents would otherwise hold every command at its own approval gate behind core's. diff --git a/package-lock.json b/package-lock.json index 6683c716fd..5a60ebb9db 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,6 +21,8 @@ "@earendil-works/pi-ai": "0.82.0", "@earendil-works/pi-coding-agent": "https://github.com/yc-software/qm/releases/download/vendored-pi-coding-agent-0.82.0-security.5/earendil-works-pi-coding-agent-0.82.0-qm-security.5.tgz", "@fly/sprites": "0.2.3", + "@grpc/grpc-js": "^1.14.5", + "@grpc/proto-loader": "^0.8.1", "@openai/codex": "0.156.1", "@opencode-ai/plugin": "1.18.31", "@opencode-ai/sdk": "1.18.31", @@ -45,6 +47,7 @@ "socks-proxy-agent": "10.1.0", "tar-stream": "^3.2.0", "typebox": "^1.1.38", + "ws": "^8.21.1", "zod": "4.4.3" }, "devDependencies": { @@ -53,6 +56,7 @@ "@types/node": "^24.0.0", "@types/pg": "^8.11.10", "@types/tar-stream": "^3.1.4", + "@types/ws": "^8.18.1", "@yc-software/qm": "file:./cli", "esbuild": "0.28.1", "eslint": "^10.4.1", diff --git a/package.json b/package.json index 61e22feb32..04f900dca9 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,8 @@ "smoke:service-cred": "node scripts/service-credential-livetest.ts", "smoke:aws-sandbox": "node scripts/aws-sandbox-smoke.ts", "smoke:local-sandbox": "node scripts/local-sandbox-smoke.ts", + "smoke:managed-agents-sandbox": "node --env-file-if-exists=.env scripts/managed-agents-sandbox-smoke.ts", + "smoke:managed-agents-turn": "node --env-file-if-exists=.env scripts/managed-agents-turn-smoke.ts", "build:modal-image": "node scripts/modal-build-image.ts", "build:e2b-template": "node scripts/e2b-build-template.ts", "build:aws-image": "node scripts/aws-build-sandbox-image.ts", @@ -77,6 +79,8 @@ "@earendil-works/pi-ai": "0.82.0", "@earendil-works/pi-coding-agent": "https://github.com/yc-software/qm/releases/download/vendored-pi-coding-agent-0.82.0-security.5/earendil-works-pi-coding-agent-0.82.0-qm-security.5.tgz", "@fly/sprites": "0.2.3", + "@grpc/grpc-js": "^1.14.5", + "@grpc/proto-loader": "^0.8.1", "@openai/codex": "0.156.1", "@opencode-ai/plugin": "1.18.31", "@opencode-ai/sdk": "1.18.31", @@ -101,6 +105,7 @@ "socks-proxy-agent": "10.1.0", "tar-stream": "^3.2.0", "typebox": "^1.1.38", + "ws": "^8.21.1", "zod": "4.4.3" }, "overrides": { @@ -126,6 +131,7 @@ "@types/node": "^24.0.0", "@types/pg": "^8.11.10", "@types/tar-stream": "^3.1.4", + "@types/ws": "^8.18.1", "@yc-software/qm": "file:./cli", "devbar.sh": "1.1.0", "esbuild": "0.28.1", diff --git a/scripts/dev/cli.ts b/scripts/dev/cli.ts index a0436ab0fd..f5614ade4d 100644 --- a/scripts/dev/cli.ts +++ b/scripts/dev/cli.ts @@ -671,7 +671,7 @@ async function main(): Promise { }); default: console.error( - "usage: dev [up|down|status|restart|canary|logs|doctor] [--json] [--force] [--rotate] [--strict] [--sandbox local|sprites|smolmachines|e2b|porter|agent37|superserve|auto] [--surface web|slack|both] [--no-slack] [--no-watch] [--org id] [--fix]", + "usage: dev [up|down|status|restart|canary|logs|doctor] [--json] [--force] [--rotate] [--strict] [--sandbox local|sprites|smolmachines|e2b|porter|agent37|superserve|do-managed-agents|auto] [--surface web|slack|both] [--no-slack] [--no-watch] [--org id] [--fix]", ); return EXIT.usage; } diff --git a/scripts/dev/lib/sandbox.ts b/scripts/dev/lib/sandbox.ts index 9d48432748..3dc0b35122 100644 --- a/scripts/dev/lib/sandbox.ts +++ b/scripts/dev/lib/sandbox.ts @@ -190,6 +190,44 @@ export async function resolveSandbox(opts: { return { backend: "superserve", env, detail: `Superserve (template ${template})`, publicApiUrl: apiUrl, warnings }; } + if (backend === "do-managed-agents") { + const managedAgentsToken = opts.baseEnv.DO_AGENTS_API_TOKEN?.trim(); + if (!managedAgentsToken) + throw new Error( + "--sandbox do-managed-agents requires DO_AGENTS_API_TOKEN in the environment (a DigitalOcean API token for a team with Managed Agents access)", + ); + let managedAgentsApiUrl = opts.baseEnv.PUBLIC_API_URL || null; + if (!managedAgentsApiUrl) { + managedAgentsApiUrl = await startQuickTunnel(opts.corePort, opts.lock, opts.log); + if (!managedAgentsApiUrl) + warnings.push( + "cloudflared tunnel didn't come up -- agent self-API (crons/sends) won't be reachable from the sandbox", + ); + } + const env: Record = { + SANDBOX_BACKEND: "do-managed-agents", + DO_AGENTS_API_TOKEN: managedAgentsToken, + DO_AGENTS_NAME_PREFIX: opts.baseEnv.DO_AGENTS_NAME_PREFIX || "qmdev", + }; + if (opts.baseEnv.DO_AGENTS_API_BASE_URL) env.DO_AGENTS_API_BASE_URL = opts.baseEnv.DO_AGENTS_API_BASE_URL; + if (opts.baseEnv.DO_AGENTS_TEMPLATE) env.DO_AGENTS_TEMPLATE = opts.baseEnv.DO_AGENTS_TEMPLATE; + if (opts.baseEnv.DO_AGENTS_SIZE_SLUG) env.DO_AGENTS_SIZE_SLUG = opts.baseEnv.DO_AGENTS_SIZE_SLUG; + if (opts.baseEnv.DO_AGENTS_EGRESS_PROXY_URL) + env.DO_AGENTS_EGRESS_PROXY_URL = opts.baseEnv.DO_AGENTS_EGRESS_PROXY_URL; + else + warnings.push( + "DO_AGENTS_EGRESS_PROXY_URL unset -- do-managed-agents sandbox runs with NO egress enforcement; set it to QA the forced-proxy path", + ); + if (managedAgentsApiUrl) env.PUBLIC_API_URL = managedAgentsApiUrl; + return { + backend: "do-managed-agents", + env, + detail: `Managed Agents (${opts.baseEnv.DO_AGENTS_API_BASE_URL || "api.digitalocean.com"})`, + publicApiUrl: managedAgentsApiUrl, + warnings, + }; + } + if (backend === "porter") { const porterToken = opts.baseEnv.PORTER_DEPLOY_API_TOKEN; const projectId = opts.baseEnv.PORTER_DEPLOY_PROJECT_ID; diff --git a/scripts/dev/lib/types.ts b/scripts/dev/lib/types.ts index f1491f7832..f96294a41d 100644 --- a/scripts/dev/lib/types.ts +++ b/scripts/dev/lib/types.ts @@ -104,7 +104,8 @@ export interface StatusReport { children: Record; } -export type DevSandboxBackend = "local" | "sprites" | "smolmachines" | "e2b" | "porter" | "agent37" | "superserve"; +export type DevSandboxBackend = + "local" | "sprites" | "smolmachines" | "e2b" | "porter" | "agent37" | "superserve" | "do-managed-agents"; export type DevSandboxChoice = DevSandboxBackend | "auto"; export interface BootSpec { diff --git a/scripts/managed-agents-sandbox-smoke.ts b/scripts/managed-agents-sandbox-smoke.ts new file mode 100644 index 0000000000..96ba31e6f1 --- /dev/null +++ b/scripts/managed-agents-sandbox-smoke.ts @@ -0,0 +1,122 @@ +#!/usr/bin/env node +import assert from "node:assert/strict"; +import { randomBytes } from "node:crypto"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createManagedAgentsSandbox } from "../src/sandbox/managed-agents-sandbox.ts"; +import { createSdkManagedAgentsClient } from "../src/sandbox/managed-agents-client.ts"; +import { createLocalWorkspaceStore } from "../src/workspace/workspace-store.ts"; +import { scopeId } from "../src/types.ts"; + +const apiToken = process.env.DO_AGENTS_API_TOKEN; +const apiBaseUrl = process.env.DO_AGENTS_API_BASE_URL; +const template = process.env.DO_AGENTS_TEMPLATE ?? ""; +const agent = process.env.DO_AGENTS_AGENT; +const keep = process.env.DO_AGENTS_SMOKE_KEEP === "1"; + +const log = (...a: unknown[]) => console.log("[managed-agents-smoke]", ...a); + +async function main(): Promise { + if (!apiToken) throw new Error("DO_AGENTS_API_TOKEN is required"); + + const client = createSdkManagedAgentsClient({ + apiToken, + ...(apiBaseUrl ? { apiBaseUrl } : {}), + ...(agent ? { agent } : {}), + ...(template ? { template } : {}), + }); + + const prefix = `qms${randomBytes(3).toString("hex")}`; + const sandbox = createManagedAgentsSandbox( + createLocalWorkspaceStore(mkdtempSync(join(tmpdir(), "managed-agents-smoke-ws-"))), + { + client, + namePrefix: prefix, + }, + ); + const scope = scopeId("personal", "smoke"); + const layers = [{ scopeId: scope, mountPath: "/", mode: "rw" as const }]; + + log(`provisioning a qm agent computer (prefix ${prefix})...`); + const handle = await sandbox.provision(layers, { env: { QM_SMOKE: "1" } }); + log("provisioned", handle.id, "coldStart", handle.coldStart); + + try { + log("running a command through the qm Sandbox interface..."); + const r = await sandbox.run(handle, "pwd; echo QM_SMOKE=$QM_SMOKE; whoami"); + assert.equal(r.code, 0, `run exited ${r.code}: ${r.stderr}`); + log("stdout:", JSON.stringify(r.stdout)); + assert.match(r.stdout, /QM_SMOKE=1/); + + log("writing and reading a workspace file through qm..."); + await sandbox.writeFile(handle, "notes/hello.txt", "written by qm"); + assert.equal(await sandbox.readFile(handle, "notes/hello.txt"), "written by qm"); + log("file round-trip OK"); + + log("checking the toolchain qm expects..."); + const binaryName = (entry: string) => entry.trim().split(/\s+/)[0] ?? ""; + const declaredPresent = (sandbox.profile.spec?.tools ?? []).map(binaryName).filter(Boolean); + const declaredAbsent = (sandbox.profile.spec?.notInstalled ?? []).map(binaryName).filter(Boolean); + const plumbingNeeds = ["sh", "timeout", "tar", "base64", "sha256sum", "setsid", "mkfifo", "cut"]; + const probe = [...new Set([...plumbingNeeds, ...declaredPresent, ...declaredAbsent])]; + const tools = await sandbox.run( + handle, + `for t in ${probe.join(" ")}; do printf '%s=%s ' $t $(command -v $t >/dev/null 2>&1 && echo yes || echo NO); done; echo`, + ); + log(tools.stdout.trim()); + const onPath = new Set( + tools.stdout + .trim() + .split(/\s+/) + .flatMap((pair) => { + const [name, state] = pair.split("="); + return name && state === "yes" ? [name] : []; + }), + ); + for (const t of plumbingNeeds) + assert.ok(onPath.has(t), `qm's own file and process plumbing needs ${t}, which the guest does not have`); + for (const t of declaredPresent) + assert.ok(onPath.has(t), `the computer profile advertises ${t}, which the guest does not have`); + for (const t of declaredAbsent) + assert.ok(!onPath.has(t), `the computer profile says ${t} is absent, but the guest has it`); + + log("checking the guest carries no managed coding agent..."); + const bare = await sandbox.run( + handle, + "for t in codex claude claude-code opencode cursor hermes; do command -v $t >/dev/null && echo AGENT_ON_PATH=$t; done; echo ---; ps -eo comm= | sort -u | tr '\\n' ' '", + ); + log("guest:", bare.stdout.trim()); + assert.doesNotMatch(bare.stdout, /AGENT_ON_PATH=/, "a bare sandbox must carry no agent CLI"); + + log("checking the workspace survives a pause and resume..."); + await sandbox.run(handle, "echo persisted > survives-pause.txt"); + await sandbox.teardown(handle, {}); + const resumed = await sandbox.provision(layers, { env: { QM_SMOKE: "1" } }); + const after = await sandbox.run(resumed, "cat survives-pause.txt"); + assert.match(after.stdout, /persisted/, "the workspace did not survive a pause and resume"); + log("workspace intact after pause and resume"); + + if (process.env.DO_AGENTS_SMOKE_LONG === "1") { + log("checking a command survives past the 4-minute REST exec clamp..."); + const long = await sandbox.run(resumed, "sleep 250; echo past-the-clamp", { timeoutMs: 400_000 }); + assert.equal(long.code, 0, `long run exited ${long.code}: ${long.stderr}`); + assert.match(long.stdout, /past-the-clamp/); + log("250s command completed over the tunnel"); + } + + log("ALL CHECKS PASSED"); + } finally { + if (keep) { + log("DO_AGENTS_SMOKE_KEEP=1 — leaving the sandbox alive"); + } else { + log("tearing down and destroying the scope..."); + await sandbox.teardown(handle, { destroy: true }).catch((e: unknown) => log("teardown failed:", e)); + } + } +} + +main().catch((e: unknown) => { + console.error("[managed-agents-smoke] FAILED", e); + process.exit(1); +}); diff --git a/scripts/managed-agents-turn-smoke.ts b/scripts/managed-agents-turn-smoke.ts new file mode 100644 index 0000000000..396717de94 --- /dev/null +++ b/scripts/managed-agents-turn-smoke.ts @@ -0,0 +1,52 @@ +#!/usr/bin/env node +import { randomUUID } from "node:crypto"; +import { stagingApiHeaders } from "../src/deployment/postdeploy-smoke.ts"; + +const core = process.env.CORE_URL ?? "http://localhost:8081"; +const principal = process.env.PRINCIPAL ?? "managed-agents-smoke"; +const sourceSecret = process.env.CORE_SIGNING_SECRET ?? ""; +const portalSecret = process.env.PORTAL_IDENTITY_SECRET ?? ""; + +const log = (...a: unknown[]) => console.log("[managed-agents-turn]", ...a); + +async function main(): Promise { + if (!sourceSecret || !portalSecret) throw new Error("CORE_SIGNING_SECRET and PORTAL_IDENTITY_SECRET are required"); + + const nonce = randomUUID(); + const body = JSON.stringify({ + surface: "web", + actor: { externalId: principal }, + conversation: { kind: "dm", threadRef: `web:${principal}:managed-agents-turn-${nonce}` }, + text: "Use your execute tool to run exactly this shell command in your agent computer: `echo DO_AGENTS_TURN_OK; uname -sr; pwd; whoami`. Then reply with the exact stdout you got back and nothing else.", + origin: { kind: "human" }, + addressed: true, + skipMemory: true, + idempotencyKey: nonce, + }); + + const path = "/v1/turns"; + const headers = await stagingApiHeaders(principal, sourceSecret, portalSecret, "POST", path, body); + + log(`POST ${core}${path} (this provisions a Managed Agents microVM on the first execute)...`); + const started = Date.now(); + const res = await fetch(`${core}${path}`, { + method: "POST", + headers: { ...headers, "content-type": "application/json" }, + body, + signal: AbortSignal.timeout(600_000), + }); + const text = await res.text(); + log(`HTTP ${res.status} in ${Math.round((Date.now() - started) / 1000)}s`); + if (!res.ok) throw new Error(`turn failed: ${text.slice(0, 2000)}`); + + const turn = JSON.parse(text) as { status?: string; sessionId?: string; reply?: string }; + log("status:", turn.status, "session:", turn.sessionId); + log("reply:", JSON.stringify(turn.reply)); + if (!turn.reply?.includes("DO_AGENTS_TURN_OK")) throw new Error("the model reply does not contain DO_AGENTS_TURN_OK"); + log("REAL QM TURN EXECUTED IN A Managed Agents SANDBOX"); +} + +main().catch((e: unknown) => { + console.error("[managed-agents-turn] FAILED", e); + process.exit(1); +}); diff --git a/src/config.ts b/src/config.ts index df35cd61df..b55f0203b9 100644 --- a/src/config.ts +++ b/src/config.ts @@ -3,6 +3,7 @@ import { isStrongSigningSecret } from "./auth/source-auth.ts"; import { parseSandboxCapabilityTtlMs } from "./auth/capability-token.ts"; import { parseScopeId } from "./types.ts"; import type { LegacySandboxScopeDefaults } from "./sandbox/sandbox-resource-upgrade.ts"; +import type { SandboxBackendName } from "./sandbox/sandbox-routing.ts"; import { existsSync, readdirSync } from "node:fs"; import { parseProviderBaseUrl, @@ -73,9 +74,8 @@ export interface Config { securityPosture: SecurityPosture; sharingPosture: SharingPosture; legacySandboxScopeDefaults?: LegacySandboxScopeDefaults; - sandboxBackend: "aws" | "local" | "sprites" | "smolmachines" | "e2b" | "modal" | "porter" | "agent37" | "superserve"; - sandboxSecondaryBackend?: - "aws" | "local" | "sprites" | "smolmachines" | "e2b" | "modal" | "porter" | "agent37" | "superserve"; + sandboxBackend: SandboxBackendName; + sandboxSecondaryBackend?: SandboxBackendName; deployProvider: "docker" | "aws" | "fly" | "porter"; egressServiceHosts?: string[]; brandingDefault?: OrgBranding; @@ -205,6 +205,7 @@ export interface Config { agent37Sandbox: Agent37SandboxEnv; superserveSandbox: SuperserveSandboxEnv; e2bSandbox: E2bSandboxEnv; + managedAgentsSandbox: ManagedAgentsSandboxEnv; modalSandbox: ModalSandboxEnv; porterSandbox: PorterSandboxEnv; porterDeploy: PorterDeployEnv; @@ -431,6 +432,40 @@ function e2bSandboxEnv(env: NodeJS.ProcessEnv): E2bSandboxEnv { }; } +interface ManagedAgentsSandboxEnv { + apiToken?: string; + apiBaseUrl?: string; + template?: string; + namePrefix?: string; + sizeSlug?: string; + idleTimeoutSec?: number; + egressProxyUrl?: string; + snapshotS3Bucket?: string; + snapshotIntervalSec?: number; + defaultTimeoutSec?: number; +} + +function managedAgentsSandboxEnv(env: NodeJS.ProcessEnv): ManagedAgentsSandboxEnv { + return { + ...(env.DO_AGENTS_API_TOKEN ? { apiToken: env.DO_AGENTS_API_TOKEN } : {}), + ...(env.DO_AGENTS_API_BASE_URL ? { apiBaseUrl: env.DO_AGENTS_API_BASE_URL } : {}), + ...(env.DO_AGENTS_TEMPLATE ? { template: env.DO_AGENTS_TEMPLATE } : {}), + ...(env.DO_AGENTS_NAME_PREFIX ? { namePrefix: env.DO_AGENTS_NAME_PREFIX } : {}), + ...(env.DO_AGENTS_SIZE_SLUG ? { sizeSlug: env.DO_AGENTS_SIZE_SLUG } : {}), + ...(numEnvStrict("DO_AGENTS_IDLE_TIMEOUT_SEC", env.DO_AGENTS_IDLE_TIMEOUT_SEC) !== undefined + ? { idleTimeoutSec: numEnvStrict("DO_AGENTS_IDLE_TIMEOUT_SEC", env.DO_AGENTS_IDLE_TIMEOUT_SEC) } + : {}), + ...(env.DO_AGENTS_EGRESS_PROXY_URL ? { egressProxyUrl: env.DO_AGENTS_EGRESS_PROXY_URL } : {}), + ...(env.DO_AGENTS_SNAPSHOT_S3_BUCKET ? { snapshotS3Bucket: env.DO_AGENTS_SNAPSHOT_S3_BUCKET } : {}), + ...(numEnvStrict("DO_AGENTS_SNAPSHOT_INTERVAL_SEC", env.DO_AGENTS_SNAPSHOT_INTERVAL_SEC) !== undefined + ? { snapshotIntervalSec: numEnvStrict("DO_AGENTS_SNAPSHOT_INTERVAL_SEC", env.DO_AGENTS_SNAPSHOT_INTERVAL_SEC) } + : {}), + ...(numEnvStrict("SANDBOX_TIMEOUT_SEC", env.SANDBOX_TIMEOUT_SEC) !== undefined + ? { defaultTimeoutSec: numEnvStrict("SANDBOX_TIMEOUT_SEC", env.SANDBOX_TIMEOUT_SEC) } + : {}), + }; +} + interface ModalSandboxEnv { nativeSnapshotsEnabled?: boolean; nativeSnapshotIntervalSec?: number; @@ -999,6 +1034,7 @@ export function enabledSandboxBackends(config: Config): Array b porter: (env) => sandboxBackendSelected(env, "porter"), agent37: (env) => sandboxBackendSelected(env, "agent37"), superserve: (env) => sandboxBackendSelected(env, "superserve"), + "do-managed-agents": (env) => sandboxBackendSelected(env, "do-managed-agents"), "porter-deploy": (env) => env.DEPLOY_PROVIDER === "porter", "fly-shared-deploy": (env) => env.DEPLOY_PROVIDER === "fly" && Boolean(env.FLY_DEPLOY_SHARED_APP_NAME?.trim()), "fly-deploy": (env) => env.DEPLOY_PROVIDER === "fly", diff --git a/src/sandbox/managed-agents-client.ts b/src/sandbox/managed-agents-client.ts new file mode 100644 index 0000000000..7430b75117 --- /dev/null +++ b/src/sandbox/managed-agents-client.ts @@ -0,0 +1,760 @@ +import { fileURLToPath } from "node:url"; +import { fetchWithRetry } from "../util/async.ts"; +import { errMessage, swallowAs, withRequestId } from "../util/errors.ts"; +import { openManagedAgentsTunnel, SANDBOX_AGENT_PORT, type ManagedAgentsTunnel } from "./managed-agents-tunnel.ts"; + +export interface ManagedAgentsCommandResult { + stdout: string; + stderr: string; + exitCode: number; + hitlRejected?: boolean; +} + +interface ManagedAgentsRunOpts { + timeoutMs?: number; + workdir?: string; + env?: Record; +} + +export type ManagedAgentsSessionState = + "unspecified" | "provisioning" | "ready" | "detached" | "destroying" | "destroyed" | "failed" | "paused"; + +export interface ManagedAgentsSessionSummary { + sessionId: string; + sandboxId: string; + name: string; + state: ManagedAgentsSessionState; +} + +export interface ManagedAgentsSessionInfo extends ManagedAgentsSessionSummary { + createdAtMs?: number; + errorMessage?: string; + sizeSlug?: string; + template?: string; +} + +export interface ManagedAgentsCheckpoint { + checkpointId: string; + status: string; + createdAtMs?: number; +} + +export interface ManagedAgentsSession { + readonly sessionId: string; + readonly sandboxId: string; + runCommand(command: string, opts?: ManagedAgentsRunOpts): Promise; + readFileBytes(absPath: string): Promise; + writeFileBytes(absPath: string, data: Uint8Array): Promise; + createCheckpoint(label?: string): Promise; + deleteCheckpoint(checkpointId: string): Promise; + close(): Promise; + pause(): Promise; + resume(): Promise; + kill(): Promise; +} + +export class ManagedAgentsSandboxGoneError extends Error { + constructor(sessionId: string, detail: string) { + super(`do-managed-agents session ${sessionId} is gone: ${detail}`); + this.name = "ManagedAgentsSandboxGoneError"; + } +} + +export class ManagedAgentsCommandLostError extends Error { + constructor(sessionId: string, detail: string) { + super( + `do-managed-agents session ${sessionId} was lost while a command was running (${detail}); the command may have partially executed and was not retried`, + ); + this.name = "ManagedAgentsCommandLostError"; + } +} + +export class ManagedAgentsHitlRejectedError extends Error { + constructor(sessionId: string) { + super( + `do-managed-agents rejected a command in session ${sessionId} at its own approval gate; set the session permissions to allow bash so qm's approval gates stay authoritative`, + ); + this.name = "ManagedAgentsHitlRejectedError"; + } +} + +interface ManagedAgentsCreateOpts { + name: string; + egressAllow?: string[]; +} + +export interface ManagedAgentsClient { + readonly nativePause?: boolean; + info(sessionId: string): Promise; + create(opts: ManagedAgentsCreateOpts): Promise; + connect(sessionId: string): Promise; + list(name: string): Promise; + rollback(sessionId: string, checkpointId: string): Promise; + kill(sessionId: string): Promise; +} + +export interface SdkManagedAgentsClientOptions { + apiToken: string; + apiBaseUrl?: string; + template?: string; + agent?: string; + sizeSlug?: string; + idleTimeoutSec?: number; + maxCommandMs?: number; + stallProbeMs?: number; + egressProxyUrl?: string; +} + +const DEFAULT_API_BASE_URL = "https://api.digitalocean.com"; +const DEFAULT_TEMPLATE = ""; +const DEFAULT_AGENT = "none"; +const DEFAULT_MAX_COMMAND_MS = 3600_000; +const CREATE_TIMEOUT_MS = 120_000; +const CHECKPOINT_TIMEOUT_MS = 5 * 60_000; +const REQUEST_TIMEOUT_MS = 30_000; +const READY_TIMEOUT_MS = 300_000; +const ROLLBACK_TIMEOUT_MS = READY_TIMEOUT_MS; +const READY_POLL_MS = 2_000; +const READY_FAST_POLL_MS = 250; +const READY_FAST_WINDOW_MS = 10_000; +const STALL_PROBE_MS = 45_000; +const READY_TRANSIENT_LIMIT = 5; +const UPLOAD_CHUNK_BYTES = 256 * 1024; +const DOWNLOAD_CHUNK_BYTES = 256 * 1024; +const MAX_EXEC_OUTPUT_BYTES = 16 * 1024 * 1024; +const CONNECT_TIMEOUT_MS = 30_000; +const GRPC_NOT_FOUND = 5; +const GRPC_INTERNAL = 13; +const GRPC_UNAVAILABLE = 14; + +const PROTO_PATH = fileURLToPath(new URL("./managed-agents-sandbox-agent.proto", import.meta.url)); + +const STATE_BY_WIRE: Record = { + SESSION_STATUS_UNSPECIFIED: "unspecified", + SESSION_STATUS_PROVISIONING: "provisioning", + SESSION_STATUS_READY: "ready", + SESSION_STATUS_DETACHED: "detached", + SESSION_STATUS_DESTROYING: "destroying", + SESSION_STATUS_DESTROYED: "destroyed", + SESSION_STATUS_FAILED: "failed", + SESSION_STATUS_PAUSED: "paused", +}; + +const GONE_STATES: ReadonlySet = new Set([ + "destroying", + "destroyed", + "failed", +]); + +const USABLE_STATES: ReadonlySet = new Set(["ready", "detached"]); + +export const managedAgentsSessionState = (wire: string | undefined): ManagedAgentsSessionState => + STATE_BY_WIRE[wire ?? ""] ?? "unspecified"; + +export function managedAgentsEgressAllow(egressProxyUrl: string): string[] { + const host = new URL(egressProxyUrl).hostname.replace(/^\[|\]$/g, ""); + if (!host) throw new Error(`DO_AGENTS_EGRESS_PROXY_URL ${egressProxyUrl} has no host to put on the egress allowlist`); + return [host]; +} + +export function managedAgentsManifest(opts: { + name: string; + template: string; + agent?: string; + sizeSlug?: string; + idleTimeoutSec?: number; + egressAllow?: readonly string[]; +}): string { + const lines = [`name: ${JSON.stringify(opts.name)}`, `agent: ${JSON.stringify(opts.agent ?? DEFAULT_AGENT)}`]; + if (opts.template) lines.push(`template: ${JSON.stringify(opts.template)}`); + if (opts.sizeSlug) lines.push(`size: ${JSON.stringify(opts.sizeSlug)}`); + if (opts.idleTimeoutSec) lines.push(`idle_timeout: ${JSON.stringify(`${opts.idleTimeoutSec}s`)}`); + if (opts.egressAllow?.length) { + lines.push("egress:"); + for (const host of opts.egressAllow) lines.push(` - ${JSON.stringify(host)}`); + } + lines.push("permissions:", " default: allow", " rules:", " - tool: bash", " action: allow"); + return `${lines.join("\n")}\n`; +} + +interface WireSession { + id?: string; + session_id?: string; + sandbox_id?: string; + name?: string; + status?: string; + template?: string; + size_slug?: string; + error_message?: string; + created_at?: string; +} + +interface WireCheckpoint { + checkpoint_id?: string; + status?: string; + error_message?: string; + created_at?: string; +} + +const toSummary = (s: WireSession): ManagedAgentsSessionSummary => ({ + sessionId: s.session_id ?? s.id ?? "", + sandboxId: s.sandbox_id ?? "", + name: s.name ?? "", + state: managedAgentsSessionState(s.status), +}); + +const toInfo = (s: WireSession): ManagedAgentsSessionInfo => { + const createdAtMs = s.created_at ? Date.parse(s.created_at) : Number.NaN; + return { + ...toSummary(s), + ...(Number.isFinite(createdAtMs) ? { createdAtMs } : {}), + ...(s.error_message ? { errorMessage: s.error_message } : {}), + ...(s.size_slug ? { sizeSlug: s.size_slug } : {}), + ...(s.template ? { template: s.template } : {}), + }; +}; + +interface GrpcCallError { + code?: number; + details?: string; + message?: string; +} + +const grpcCode = (err: unknown): number | undefined => + typeof err === "object" && err !== null ? (err as GrpcCallError).code : undefined; + +const isGuestGone = (err: unknown): boolean => { + const code = grpcCode(err); + return code === GRPC_NOT_FOUND || code === GRPC_UNAVAILABLE; +}; + +type ProtoLoader = typeof import("@grpc/proto-loader"); +type Grpc = typeof import("@grpc/grpc-js"); + +interface GrpcDuplex { + write(msg: unknown): void; + end(): void; + cancel(): void; + on(event: string, cb: (arg: never) => void): void; +} + +type GrpcWritable = Pick; +type GrpcReadable = Pick; + +interface SandboxAgent { + Exec(): GrpcDuplex; + Upload(cb: (err: unknown, res?: { bytes_written?: string }) => void): GrpcWritable; + Download(req: unknown): GrpcReadable; + waitForReady(deadline: Date, cb: (err?: Error) => void): void; + close(): void; +} + +interface ExecOutputFrame { + output?: "stdout" | "stderr" | "exit"; + stdout?: Buffer; + stderr?: Buffer; + exit?: { exit_code?: number; hitl_rejected?: boolean; timed_out?: boolean }; +} + +interface DownloadFrame { + output?: "header" | "chunk" | "end"; + chunk?: Buffer; +} + +let grpcModules: Promise<[ProtoLoader, Grpc]> | null = null; + +function loadGrpc(): Promise<[ProtoLoader, Grpc]> { + grpcModules ??= Promise.all([ + import("@grpc/proto-loader").catch((e: unknown) => { + throw new Error(`@grpc/proto-loader is not installed: ${errMessage(e)}`); + }), + import("@grpc/grpc-js").catch((e: unknown) => { + throw new Error(`@grpc/grpc-js is not installed: ${errMessage(e)}`); + }), + ]) as Promise<[ProtoLoader, Grpc]>; + return grpcModules; +} + +async function dialSandboxAgent(localPort: number, readyTimeoutMs: number): Promise { + const [loader, grpc] = await loadGrpc(); + const definition = loader.loadSync(PROTO_PATH, { + keepCase: true, + longs: String, + enums: String, + defaults: true, + oneofs: true, + }); + const pkg = grpc.loadPackageDefinition(definition) as unknown as { + do: { + teams: { + hosted_agents: { + runtime: { + sandbox_agent: { + v1: { + SandboxAgentService: new ( + addr: string, + creds: ReturnType, + options: Record, + ) => SandboxAgent; + }; + }; + }; + }; + }; + }; + }; + const Service = pkg.do.teams.hosted_agents.runtime.sandbox_agent.v1.SandboxAgentService; + const agent = new Service(`127.0.0.1:${localPort}`, grpc.credentials.createInsecure(), { + "grpc.max_receive_message_length": MAX_EXEC_OUTPUT_BYTES, + "grpc.max_send_message_length": MAX_EXEC_OUTPUT_BYTES, + }); + try { + await new Promise((resolve, reject) => { + agent.waitForReady(new Date(Date.now() + readyTimeoutMs), (err?: Error) => (err ? reject(err) : resolve())); + }); + } catch (e) { + agent.close(); + throw e; + } + return agent; +} + +export function createSdkManagedAgentsClient(opts: SdkManagedAgentsClientOptions): ManagedAgentsClient { + const apiBaseUrl = (opts.apiBaseUrl ?? DEFAULT_API_BASE_URL).replace(/\/+$/, ""); + const template = opts.template ?? DEFAULT_TEMPLATE; + const maxCommandMs = opts.maxCommandMs ?? DEFAULT_MAX_COMMAND_MS; + const stallProbeMs = opts.stallProbeMs ?? STALL_PROBE_MS; + const egressAllow = opts.egressProxyUrl ? managedAgentsEgressAllow(opts.egressProxyUrl) : undefined; + + function send(method: string, path: string, body?: string, contentType?: string, signal?: AbortSignal) { + return fetch(`${apiBaseUrl}${path}`, { + method, + headers: { + authorization: `Bearer ${opts.apiToken}`, + accept: "application/json", + ...(contentType ? { "content-type": contentType } : {}), + }, + ...(body !== undefined ? { body } : {}), + signal: signal ?? AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }); + } + + function call( + method: string, + path: string, + body?: string, + contentType?: string, + timeoutMs = REQUEST_TIMEOUT_MS, + ): Promise { + const operation = (signal?: AbortSignal): Promise => send(method, path, body, contentType, signal); + return method === "GET" || method === "DELETE" + ? fetchWithRetry((signal) => operation(signal), "idempotent", { timeoutMs }) + : operation(AbortSignal.timeout(timeoutMs)); + } + + async function failure(action: string, sessionId: string, res: Response): Promise { + const raw = await res.text().catch(swallowAs(`do-managed-agents ${action} ${sessionId}: read error body`, "")); + let message = raw.slice(0, 200); + try { + const parsed = JSON.parse(raw) as { error?: { message?: string } }; + if (parsed.error?.message) message = parsed.error.message; + } catch { + message = raw.slice(0, 200); + } + const detail = withRequestId(`http ${res.status} ${message}`, res.headers); + if (res.status === 404 || res.status === 410) return new ManagedAgentsSandboxGoneError(sessionId, detail); + return new Error(`do-managed-agents ${action}: ${detail}`); + } + + async function callJson( + method: string, + path: string, + sessionId: string, + body?: string, + contentType?: string, + timeoutMs = REQUEST_TIMEOUT_MS, + ): Promise { + const res = await call(method, path, body, contentType, timeoutMs); + if (!res.ok) throw await failure(`${method} ${path}`, sessionId, res); + if (res.status === 204) return undefined as T; + return (await res.json()) as T; + } + + const getInfo = async (sessionId: string): Promise => { + const body = await callJson<{ session?: WireSession }>("GET", `/v2/agents/sessions/${sessionId}`, sessionId); + if (!body.session) throw new ManagedAgentsSandboxGoneError(sessionId, "response carried no session"); + return toInfo(body.session); + }; + + async function awaitUsable(sessionId: string): Promise { + const started = Date.now(); + const deadline = started + READY_TIMEOUT_MS; + let resumed = false; + let strikes = 0; + const tolerate = (err: unknown): void => { + if (err instanceof ManagedAgentsSandboxGoneError) throw err; + if (++strikes > READY_TRANSIENT_LIMIT) throw err; + }; + for (;;) { + let info: ManagedAgentsSessionInfo | undefined; + try { + info = await getInfo(sessionId); + strikes = 0; + } catch (err) { + tolerate(err); + } + if (info) { + if (USABLE_STATES.has(info.state)) return info; + if (GONE_STATES.has(info.state)) + throw new ManagedAgentsSandboxGoneError(sessionId, info.errorMessage ?? `status ${info.state}`); + if (info.state === "paused" && !resumed) { + try { + await callJson("POST", `/v2/agents/sessions/${sessionId}/resume`, sessionId); + resumed = true; + strikes = 0; + } catch (err) { + tolerate(err); + } + } + } + if (Date.now() >= deadline) + throw new Error(`do-managed-agents session ${sessionId} did not become ready within ${READY_TIMEOUT_MS}ms`); + await new Promise((resolve) => + setTimeout(resolve, Date.now() - started < READY_FAST_WINDOW_MS ? READY_FAST_POLL_MS : READY_POLL_MS), + ); + } + } + + async function rollbackToCheckpoint(sessionId: string, checkpointId: string): Promise { + const body = await callJson<{ session?: WireSession }>( + "POST", + `/v2/agents/sessions/${sessionId}/checkpoints/${encodeURIComponent(checkpointId)}/rollback`, + sessionId, + undefined, + undefined, + ROLLBACK_TIMEOUT_MS, + ); + if (!body.session) throw new Error(`do-managed-agents rollback ${sessionId}: response carried no session`); + const restored = toInfo(body.session); + if (!restored.sandboxId) throw new Error(`do-managed-agents rollback ${sessionId}: response carried no sandbox id`); + if (GONE_STATES.has(restored.state)) + throw new ManagedAgentsSandboxGoneError(sessionId, restored.errorMessage ?? `status ${restored.state}`); + return restored; + } + + function buildSession(info: ManagedAgentsSessionInfo): ManagedAgentsSession { + const sessionId = info.sessionId; + let guest: Promise<{ tunnel: ManagedAgentsTunnel; agent: SandboxAgent }> | null = null; + let lastTunnel: ManagedAgentsTunnel | null = null; + + const connect = (): Promise<{ tunnel: ManagedAgentsTunnel; agent: SandboxAgent }> => + (guest ??= (async () => { + const tunnel = await openManagedAgentsTunnel({ + apiBaseUrl, + sessionId, + remotePort: SANDBOX_AGENT_PORT, + getToken: async () => opts.apiToken, + }); + lastTunnel = tunnel; + try { + const dial = dialSandboxAgent(tunnel.localPort, CONNECT_TIMEOUT_MS); + const abort = tunnel.whenFailed().then((detail) => { + throw new Error(detail); + }); + dial.catch(() => undefined); + abort.catch(() => undefined); + return { tunnel, agent: await Promise.race([dial, abort]) }; + } catch (e) { + const detail = tunnel.lastFailure() ?? errMessage(e); + await tunnel.close(); + throw new ManagedAgentsSandboxGoneError(sessionId, `guest unreachable over the port-forward: ${detail}`); + } + })().catch((e: unknown) => { + guest = null; + throw e; + })); + + async function disconnect(): Promise { + const pending = guest; + guest = null; + if (!pending) return; + await pending + .then(async ({ tunnel, agent }) => { + agent.close(); + await tunnel.close(); + }) + .catch(swallowAs(`do-managed-agents ${sessionId}: disconnect`, undefined)); + } + + async function download(absPath: string): Promise { + const { agent } = await connect(); + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + const stream = agent.Download({ path: absPath, as_archive: false, chunk_size_bytes: DOWNLOAD_CHUNK_BYTES }); + let settled = false; + const settle = (fn: () => void): void => { + if (settled) return; + settled = true; + fn(); + }; + stream.on("data", ((frame: DownloadFrame) => { + if (frame.chunk?.length) chunks.push(frame.chunk); + }) as (arg: never) => void); + stream.on("error", ((err: unknown) => { + settle(() => { + if (grpcCode(err) === GRPC_NOT_FOUND) { + resolve(null); + return; + } + const detail = guestDetail(err); + void disconnect(); + reject( + grpcCode(err) === GRPC_UNAVAILABLE + ? new ManagedAgentsSandboxGoneError(sessionId, detail) + : new Error(`do-managed-agents download ${absPath} failed: ${detail}`, { cause: err }), + ); + }); + }) as (arg: never) => void); + stream.on("end", (() => settle(() => resolve(new Uint8Array(Buffer.concat(chunks))))) as (arg: never) => void); + }); + } + + const guestDetail = (err: unknown): string => lastTunnel?.lastFailure() ?? errMessage(err); + + return { + sessionId, + sandboxId: info.sandboxId, + + async runCommand(command, runOpts): Promise { + const { agent } = await connect(); + const timeoutMs = Math.min(runOpts?.timeoutMs ?? maxCommandMs, maxCommandMs); + return new Promise((resolve, reject) => { + const stream = agent.Exec(); + const out: Buffer[] = []; + const errOut: Buffer[] = []; + let outBytes = 0; + let errBytes = 0; + let exit: { exit_code?: number; hitl_rejected?: boolean } | undefined; + let settled = false; + let quietSince = Date.now(); + const settle = (fn: () => void): void => { + if (settled) return; + settled = true; + clearInterval(watchdog); + fn(); + }; + + const watchdog = setInterval(() => { + if (settled || Date.now() - quietSince < stallProbeMs) return; + void getInfo(sessionId).then( + (live) => { + quietSince = Date.now(); + if (settled || USABLE_STATES.has(live.state)) return; + settle(() => { + void disconnect(); + reject( + GONE_STATES.has(live.state) + ? new ManagedAgentsSandboxGoneError(sessionId, `status ${live.state}`) + : new ManagedAgentsCommandLostError(sessionId, `the session became ${live.state} mid-command`), + ); + }); + }, + swallowAs(`do-managed-agents ${sessionId}: stall probe`, undefined), + ); + }, stallProbeMs); + watchdog.unref(); + + stream.on("data", ((frame: ExecOutputFrame) => { + quietSince = Date.now(); + if (frame.stdout && outBytes < MAX_EXEC_OUTPUT_BYTES) { + out.push(frame.stdout); + outBytes += frame.stdout.length; + } + if (frame.stderr && errBytes < MAX_EXEC_OUTPUT_BYTES) { + errOut.push(frame.stderr); + errBytes += frame.stderr.length; + } + if (frame.exit) exit = frame.exit; + }) as (arg: never) => void); + + stream.on("error", ((err: unknown) => { + settle(() => { + const detail = guestDetail(err); + void disconnect(); + reject( + isGuestGone(err) + ? new ManagedAgentsCommandLostError(sessionId, detail) + : new Error(`do-managed-agents exec failed: ${detail}`), + ); + }); + }) as (arg: never) => void); + + stream.on("end", (() => { + settle(() => { + if (exit?.hitl_rejected) { + reject(new ManagedAgentsHitlRejectedError(sessionId)); + return; + } + if (exit?.exit_code === undefined) { + void disconnect(); + reject(new ManagedAgentsCommandLostError(sessionId, "the command stream ended without an exit status")); + return; + } + resolve({ + stdout: Buffer.concat(out).toString("utf8"), + stderr: Buffer.concat(errOut).toString("utf8"), + exitCode: exit.exit_code, + }); + }); + }) as (arg: never) => void); + + stream.write({ + start: { + argv: ["/bin/bash", "-c", command], + ...(runOpts?.workdir ? { workdir: runOpts.workdir } : {}), + ...(runOpts?.env ? { env: runOpts.env } : {}), + timeout_seconds: Math.ceil(timeoutMs / 1000), + }, + }); + stream.end(); + }); + }, + + async readFileBytes(absPath): Promise { + try { + return await download(absPath); + } catch (err) { + if (grpcCode((err as { cause?: unknown }).cause) !== GRPC_INTERNAL) throw err; + const probe = await this.runCommand(`test -e '${absPath.replaceAll("'", "'\\''")}'`, { timeoutMs: 30_000 }); + if (probe.exitCode === 1) return null; + throw err; + } + }, + + async writeFileBytes(absPath, data): Promise { + const { agent } = await connect(); + await new Promise((resolve, reject) => { + const stream = agent.Upload((err: unknown) => { + if (!err) { + resolve(); + return; + } + const detail = guestDetail(err); + void disconnect(); + reject( + isGuestGone(err) + ? new ManagedAgentsSandboxGoneError(sessionId, detail) + : new Error(`do-managed-agents upload ${absPath} failed: ${detail}`), + ); + }); + stream.write({ header: { path: absPath, mode: 0o644, is_archive: false } }); + for (let offset = 0; offset < data.length; offset += UPLOAD_CHUNK_BYTES) + stream.write({ chunk: Buffer.from(data.subarray(offset, offset + UPLOAD_CHUNK_BYTES)) }); + stream.write({ end: {} }); + stream.end(); + }); + }, + + async createCheckpoint(label?: string): Promise { + const body = await callJson<{ checkpoint?: WireCheckpoint }>( + "POST", + `/v2/agents/sessions/${sessionId}/checkpoints`, + sessionId, + JSON.stringify(label ? { label } : {}), + "application/json", + CHECKPOINT_TIMEOUT_MS, + ); + const captured = body.checkpoint; + if (!captured?.checkpoint_id) + throw new Error(`do-managed-agents create checkpoint ${sessionId}: response carried no checkpoint id`); + if (captured.status !== "READY") { + const detail = captured.error_message ? `: ${captured.error_message}` : ""; + throw new Error( + `do-managed-agents create checkpoint ${sessionId}: status ${captured.status ?? "missing"}${detail}`, + ); + } + const createdAtMs = captured.created_at ? Date.parse(captured.created_at) : Number.NaN; + return { + checkpointId: captured.checkpoint_id, + status: captured.status, + ...(Number.isFinite(createdAtMs) ? { createdAtMs } : {}), + }; + }, + + async deleteCheckpoint(checkpointId: string): Promise { + await callJson( + "DELETE", + `/v2/agents/sessions/${sessionId}/checkpoints/${encodeURIComponent(checkpointId)}`, + sessionId, + ); + }, + + async close(): Promise { + await disconnect(); + }, + + async pause(): Promise { + await disconnect(); + await callJson("POST", `/v2/agents/sessions/${sessionId}/pause`, sessionId); + }, + + async resume(): Promise { + await callJson("POST", `/v2/agents/sessions/${sessionId}/resume`, sessionId); + }, + + async kill(): Promise { + await disconnect(); + await callJson("DELETE", `/v2/agents/sessions/${sessionId}`, sessionId); + }, + }; + } + + return { + nativePause: true, + + info: getInfo, + + async create(createOpts): Promise { + const manifest = managedAgentsManifest({ + name: createOpts.name, + template, + ...(opts.agent ? { agent: opts.agent } : {}), + ...(opts.sizeSlug ? { sizeSlug: opts.sizeSlug } : {}), + ...(opts.idleTimeoutSec ? { idleTimeoutSec: opts.idleTimeoutSec } : {}), + ...((createOpts.egressAllow ?? egressAllow) ? { egressAllow: createOpts.egressAllow ?? egressAllow } : {}), + }); + const body = await callJson<{ session?: WireSession }>( + "POST", + "/v2/agents/sessions", + createOpts.name, + manifest, + "application/x-yaml", + CREATE_TIMEOUT_MS, + ); + const created = body.session ? toSummary(body.session) : null; + if (!created?.sessionId) throw new Error("do-managed-agents create session: response carried no session id"); + return buildSession(await awaitUsable(created.sessionId)); + }, + + async connect(sessionId): Promise { + return buildSession(await awaitUsable(sessionId)); + }, + + async rollback(sessionId, checkpointId): Promise { + return rollbackToCheckpoint(sessionId, checkpointId); + }, + + async list(name): Promise { + const body = await callJson<{ sessions?: WireSession[] }>( + "GET", + `/v2/agents/sessions?name=${encodeURIComponent(name)}`, + name, + ); + return (body.sessions ?? []).map(toSummary).filter((s) => s.name === name && !GONE_STATES.has(s.state)); + }, + + async kill(sessionId): Promise { + await callJson("DELETE", `/v2/agents/sessions/${sessionId}`, sessionId); + }, + }; +} diff --git a/src/sandbox/managed-agents-sandbox-agent.proto b/src/sandbox/managed-agents-sandbox-agent.proto new file mode 100644 index 0000000000..59f5c8dda8 --- /dev/null +++ b/src/sandbox/managed-agents-sandbox-agent.proto @@ -0,0 +1,98 @@ +syntax = "proto3"; + +package do.teams.hosted_agents.runtime.sandbox_agent.v1; + +service SandboxAgentService { + rpc Exec(stream ExecRequest) returns (stream ExecOutput); + rpc Upload(stream UploadRequest) returns (UploadResponse); + rpc Download(DownloadRequest) returns (stream DownloadChunk); +} + +message ExecRequest { + oneof request { + ExecStart start = 1; + bytes stdin = 2; + ExecSignal signal = 3; + } +} + +message ExecStart { + + repeated string argv = 1; + string workdir = 2; + map env = 3; + + int64 timeout_seconds = 4; +} + +message ExecSignal { + int32 signal = 1; +} + +message ExecOutput { + oneof output { + bytes stdout = 1; + bytes stderr = 2; + ExecExit exit = 3; + } +} + +message ExecExit { + int32 exit_code = 1; + int64 duration_ms = 2; + + bool hitl_rejected = 3; + bool timed_out = 4; +} + +message UploadRequest { + oneof request { + UploadHeader header = 1; + bytes chunk = 2; + UploadEnd end = 3; + } +} + +message UploadHeader { + + string path = 1; + + uint32 mode = 2; + + bool is_archive = 3; +} + +message UploadEnd { + + string sha256 = 1; +} + +message UploadResponse { + string path = 1; + int64 bytes_written = 2; +} + +message DownloadRequest { + string path = 1; + bool as_archive = 2; + + uint32 chunk_size_bytes = 3; +} + +message DownloadChunk { + oneof output { + DownloadHeader header = 1; + bytes chunk = 2; + DownloadEnd end = 3; + } +} + +message DownloadHeader { + + int64 size_bytes = 1; + bool is_archive = 2; +} + +message DownloadEnd { + string sha256 = 1; +} diff --git a/src/sandbox/managed-agents-sandbox.ts b/src/sandbox/managed-agents-sandbox.ts new file mode 100644 index 0000000000..0ebdf9afcf --- /dev/null +++ b/src/sandbox/managed-agents-sandbox.ts @@ -0,0 +1,631 @@ +import { cleanupFailedProvision } from "./sandbox.ts"; +import { randomUUID } from "node:crypto"; +import type { WorkspaceLayer } from "../types.ts"; +import type { WorkspaceStore } from "../workspace/workspace-store.ts"; +import type { DurableMap } from "../persistence/durable-map.ts"; +import { createMemoryMap } from "../persistence/durable-map.ts"; +import { createKeyedQueue } from "../util/async.ts"; +import { swallowAs, errMessage } from "../util/errors.ts"; +import { shq } from "../util/shell.ts"; +import { nonInteractiveShellPrefix, DROPPED_PROXY_ENV, forceThroughProxyEnv } from "./sandbox-env.ts"; +import { createExecProcessSessions, type ExecProcessIo } from "./exec-process-session.ts"; +import { materializeRoLayers } from "./ro-layers.ts"; +import { createLayerToolInstaller } from "./layer-tool-install.ts"; +import type { LayerInstallFile } from "../deployment/load-layer.ts"; +import { + createBackendBlobStaging, + createExecExport, + createExecFileOps, + posixJoin, + type BlobStagingOptions, +} from "./exec-file-ops.ts"; +import { + ephemeralCredLinkScript, + ephemeralCredLinkPaths, + type CredentialPathSpec, +} from "../credentials/resident-paths.ts"; +import { killableScript, killScript } from "./exec-kill.ts"; +import { visibleNotInstalled, visibleTools } from "./sandbox.ts"; +import { sandboxScopeName } from "./exec-sandbox-base.ts"; +import { + ManagedAgentsCommandLostError, + ManagedAgentsSandboxGoneError, + type ManagedAgentsClient, + type ManagedAgentsSessionInfo, + type ManagedAgentsSession, +} from "./managed-agents-client.ts"; +import { + createHomeSnapshotOps, + createMemorySnapshotStore, + HOME_SNAPSHOT_PRUNE, + snapshotDue, + type HomeSnapshotStore, +} from "./home-snapshot.ts"; +import type { + AgentComputerProfile, + ComputerStatus, + ExecOptions, + ExecResult, + ProvisionOptions, + Sandbox, + SandboxHandle, + TeardownOptions, +} from "./sandbox.ts"; + +const HOME_DIR = "/workspace/home"; +const WORKSPACE_BASENAME = "workspace"; +const RO_LAYERS_TAR = ".ro-layers.tar"; +const RO_LAYERS_MANIFEST = ".ro-layers.manifest"; +const HOME_TAR = `${HOME_DIR}/.qm-home.tar`; + +const SNAPSHOT_PRUNE = HOME_SNAPSHOT_PRUNE; +const DEFAULT_SNAPSHOT_INTERVAL_MS = 5 * 60_000; +const CHECKPOINT_LABEL = "qm teardown"; + +export const QM_TEMPLATE_TOOLS = ["jq", "rg", "unzip", "wget"]; + +export interface StoredManagedAgentsSandbox { + sessionId: string; + sandboxId: string; + preservationState?: "running" | "paused" | "pause_failed"; + preservationError?: string; + createdAtMs: number; + lastSnapshotMs?: number; + homeDirty?: boolean; + checkpointId?: string; + checkpointAtMs?: number; + checkpointError?: string; + pendingCheckpointDeletes?: string[]; +} + +export interface ManagedAgentsSandboxOptions extends BlobStagingOptions { + client: ManagedAgentsClient; + namePrefix?: string; + defaultTimeoutSec?: number; + snapshotIntervalMs?: number; + egressProxyUrl?: string; + extraTools?: string[]; + credentialPaths?: CredentialPathSpec[]; + layerToolFiles?: () => readonly LayerInstallFile[]; + store?: DurableMap; + snapshots?: HomeSnapshotStore; + onError?: (e: { category: string; code: string; message: string; scopeLabel?: string }) => void; +} + +export function createManagedAgentsSandbox(workspace: WorkspaceStore, opts: ManagedAgentsSandboxOptions): Sandbox { + const client = opts.client; + const prefix = opts.namePrefix ?? "qm"; + const defaultTimeoutSec = opts.defaultTimeoutSec ?? 600; + const snapshotIntervalMs = opts.snapshotIntervalMs ?? DEFAULT_SNAPSHOT_INTERVAL_MS; + const workspaceDir = `${HOME_DIR}/${WORKSPACE_BASENAME}`; + const store = opts.store ?? createMemoryMap(); + const snapshots = opts.snapshots ?? createMemorySnapshotStore(); + const provisionQueue = createKeyedQueue(); + + const sessionByName = new Map(); + const scopeByName = new Map(); + const scratchKeyByName = new Map(); + const activeScratch = new Map(); + + const reportError = (category: string, code: string, message: string, scopeLabel?: string): void => { + opts.onError?.({ category, code, message, ...(scopeLabel ? { scopeLabel } : {}) }); + }; + + const homeSnapshots = createHomeSnapshotOps({ + label: "do-managed-agents", + homeDir: HOME_DIR, + homeTarPath: HOME_TAR, + prunePaths: [...SNAPSHOT_PRUNE, ...ephemeralCredLinkPaths(opts.credentialPaths ?? []).map(({ rel }) => `./${rel}`)], + store: snapshots, + io: { + runCommand: (session, script, timeoutMs) => session.runCommand(script, { timeoutMs }), + readFileBytes: (session, abs) => session.readFileBytes(abs), + writeFileBytes: (session, abs, data) => session.writeFileBytes(abs, data), + }, + }); + + async function snapshotHome(scope: string, session: ManagedAgentsSession): Promise { + await homeSnapshots.snapshotHome(scope, session); + await store.merge(scope, { lastSnapshotMs: Date.now(), homeDirty: false }); + } + + async function pendingDeletes(scope: string): Promise { + return (await store.get(scope))?.pendingCheckpointDeletes ?? []; + } + + async function forgetCheckpoint(scope: string, session: ManagedAgentsSession, checkpointId: string): Promise { + try { + await session.deleteCheckpoint(checkpointId); + } catch (e) { + reportError("sandbox_snapshot", "checkpoint_delete_failed", errMessage(e), scope); + return; + } + const pending = await pendingDeletes(scope); + await store.merge(scope, { pendingCheckpointDeletes: pending.filter((id) => id !== checkpointId) }); + } + + async function retryPendingCheckpointDeletes(scope: string, session: ManagedAgentsSession): Promise { + const current = (await store.get(scope))?.checkpointId; + for (const checkpointId of await pendingDeletes(scope)) { + if (checkpointId === current) continue; + await forgetCheckpoint(scope, session, checkpointId); + } + } + + async function captureCheckpoint(scope: string, session: ManagedAgentsSession): Promise { + const stored = await store.get(scope); + const previous = stored?.checkpointId; + const captured = await session.createCheckpoint(CHECKPOINT_LABEL); + const pending = stored?.pendingCheckpointDeletes ?? []; + const queued = + previous && previous !== captured.checkpointId && !pending.includes(previous) ? [...pending, previous] : pending; + await store.merge(scope, { + checkpointId: captured.checkpointId, + checkpointAtMs: captured.createdAtMs ?? Date.now(), + checkpointError: undefined, + pendingCheckpointDeletes: queued, + }); + if (previous && previous !== captured.checkpointId) await forgetCheckpoint(scope, session, previous); + } + + const hydrateHome = (scope: string, session: ManagedAgentsSession): Promise => + homeSnapshots.hydrateHome(scope, session); + + const recordSession = (scope: string, session: ManagedAgentsSession, createdAtMs: number): Promise => + store.put(scope, { + sessionId: session.sessionId, + sandboxId: session.sandboxId, + createdAtMs, + preservationState: "running", + }); + + async function ensureSession( + scope: string, + name: string, + onStatus?: (text: string) => void, + ): Promise<{ session: ManagedAgentsSession; coldStart: boolean }> { + return provisionQueue(scope, async () => { + const cached = sessionByName.get(name); + if (cached) return { session: cached, coldStart: false }; + + const adopt = (session: ManagedAgentsSession): { session: ManagedAgentsSession; coldStart: boolean } => { + sessionByName.set(name, session); + return { session, coldStart: false }; + }; + + const stored = await store.get(scope); + if (stored) { + try { + onStatus?.("Resuming the sandbox…"); + const session = await client.connect(stored.sessionId); + await store.merge(scope, { preservationState: "running", sandboxId: session.sandboxId }); + return adopt(session); + } catch (err) { + if (!(err instanceof ManagedAgentsSandboxGoneError)) throw err; + } + } + + for (const summary of await client.list(name)) { + try { + const session = await client.connect(summary.sessionId); + await recordSession(scope, session, Date.now()); + return adopt(session); + } catch (err) { + if (!(err instanceof ManagedAgentsSandboxGoneError)) throw err; + } + } + + onStatus?.("Creating the sandbox…"); + const session = await client.create({ name }); + sessionByName.set(name, session); + await recordSession(scope, session, Date.now()); + let hydrated: boolean; + try { + hydrated = await hydrateHome(scope, session); + } catch (e) { + reportError("sandbox_hydrate", "hydrate_failed", errMessage(e), scope); + sessionByName.delete(name); + await client.kill(session.sessionId).catch((err: unknown) => { + reportError("sandbox_hydrate", "orphan_kill_failed", errMessage(err), scope); + }); + throw new Error( + `do-managed-agents provision: home hydration failed (${errMessage(e)}); not risking the stored snapshot`, + { + cause: e, + }, + ); + } + return { session, coldStart: !hydrated }; + }); + } + + async function ensureScratch(key: string): Promise<{ name: string; coldStart: boolean }> { + const name = sandboxScopeName(`${prefix}-scratch`, key); + return provisionQueue(`scratch:${key}`, async () => { + scratchKeyByName.set(name, key); + const active = activeScratch.get(name) ?? 0; + if (active === 0 && !sessionByName.has(name)) { + sessionByName.set(name, await client.create({ name })); + } + activeScratch.set(name, active + 1); + return { name, coldStart: active === 0 }; + }); + } + + async function withSession(name: string, action: (session: ManagedAgentsSession) => Promise): Promise { + const scratchKey = scratchKeyByName.get(name); + if (scratchKey === undefined && !scopeByName.has(name)) throw new Error("sandbox handle has been released"); + + const reviveScratch = async (): Promise => { + const session = await client.create({ name }); + sessionByName.set(name, session); + return session; + }; + const first = + scratchKey !== undefined + ? { session: sessionByName.get(name) ?? (await reviveScratch()) } + : await ensureSession(scopeByName.get(name) ?? "default", name); + try { + return await action(first.session); + } catch (err) { + if (err instanceof ManagedAgentsCommandLostError) sessionByName.delete(name); + if (!(err instanceof ManagedAgentsSandboxGoneError)) throw err; + sessionByName.delete(name); + const second = + scratchKey !== undefined + ? { session: await reviveScratch() } + : await ensureSession(scopeByName.get(name) ?? "default", name); + return action(second.session); + } + } + + async function execRaw(name: string, script: string, timeoutSec: number): Promise { + return withSession(name, async (session) => { + const r = await session.runCommand(`sh -c ${shq(`timeout -k 5 ${timeoutSec} sh -c ${shq(script)}; exit $?`)}`, { + timeoutMs: timeoutSec * 1000 + 30_000, + }); + return { + stdout: r.stdout, + stderr: r.stderr, + code: r.exitCode, + timedOut: r.exitCode === 124 || r.exitCode === 137, + }; + }); + } + + const profile: AgentComputerProfile = { + backend: "do-managed-agents", + writablePersistence: client.nativePause ? "provider_managed" : "snapshot_to_workspace", + processSessions: true, + parksOnTeardown: true, + egressEnforcement: opts.egressProxyUrl ? "domain" : "none", + spec: { + os: "Linux — DigitalOcean Managed Agents Firecracker microVM (provider pause preserves state; publish durable work to git or Files)", + runtimes: ["Node", "Python 3"], + get tools() { + return visibleTools(["git", "curl", "tar", "python3", ...(opts.extraTools ?? [])]); + }, + get notInstalled() { + return visibleNotInstalled( + ["jq", "rg", "unzip", "wget", "gh", "aws", "gcloud", "kubectl", "flyctl", "glab"], + opts.extraTools ?? [], + ); + }, + homeDir: HOME_DIR, + workdir: workspaceDir, + }, + }; + + const procIo: ExecProcessIo = { + async run(handle, command, execOpts): Promise { + const timeoutSec = execOpts?.timeoutMs ? Math.ceil(execOpts.timeoutMs / 1000) : defaultTimeoutSec; + return execRaw(handle.id, command, timeoutSec); + }, + }; + const procSessions = createExecProcessSessions(procIo); + + const writeAbsBytes = (name: string, absPath: string, data: Uint8Array): Promise => + withSession(name, (session) => session.writeFileBytes(absPath, data)); + const readAbsBytes = (name: string, absPath: string): Promise => + withSession(name, (session) => session.readFileBytes(absPath)); + const installLayerTools = opts.layerToolFiles ? createLayerToolInstaller(opts.layerToolFiles) : null; + + const execFileOps = createExecFileOps({ + label: "do-managed-agents", + exec: (id, script, t) => execRaw(id, script, t), + writeInline: (id, abs, data) => writeAbsBytes(id, abs, data), + }); + + const execExport = createExecExport({ + label: "do-managed-agents", + exec: (id, script, t) => execRaw(id, script, t), + readAbsBytes, + defaultHomeDir: HOME_DIR, + ephemeralCredentialPrefixes: ephemeralCredLinkPaths(opts.credentialPaths ?? []).map(({ rel }) => rel), + }); + + const blobStaging = createBackendBlobStaging("do-managed-agents", (id, script, t) => execRaw(id, script, t), opts); + + async function destroyStoredScope(scope: string): Promise { + const name = sandboxScopeName(prefix, scope); + const stored = await store.get(scope); + const cached = sessionByName.get(name); + const ids = new Set([stored?.sessionId, cached?.sessionId].filter((id): id is string => !!id)); + for (const id of ids) { + try { + if (cached && id === cached.sessionId) await cached.kill(); + else await client.kill(id); + } catch (error) { + if (!(error instanceof ManagedAgentsSandboxGoneError)) throw error; + } + } + await store.delete(scope); + sessionByName.delete(name); + scopeByName.delete(name); + } + + const sandbox: Sandbox = { + destroyScope(scopeId: string): Promise { + return provisionQueue(scopeId, () => destroyStoredScope(scopeId)); + }, + + profile, + startProcess: procSessions.startProcess, + readProcess: procSessions.readProcess, + writeStdin: procSessions.writeStdin, + signalProcess: procSessions.signalProcess, + listProcesses: procSessions.listProcesses, + ...execFileOps, + ...blobStaging, + + async provision(layers: WorkspaceLayer[], provOpts?: ProvisionOptions): Promise { + const scratch = provOpts?.scratch; + const writable = layers.find((l) => l.mode === "rw") ?? layers[0]; + const scope = writable?.scopeId ?? "default"; + let name: string; + let coldStart: boolean; + if (scratch) { + ({ name, coldStart } = await ensureScratch(scratch.key)); + } else { + name = sandboxScopeName(prefix, scope); + scopeByName.set(name, scope); + ({ coldStart } = await ensureSession(scope, name, provOpts?.onStatus)); + } + + const forceEgress = !!opts.egressProxyUrl && !!provOpts?.egressToken; + const turnEnv = Object.fromEntries( + Object.entries(provOpts?.env ?? {}).filter(([k]) => !DROPPED_PROXY_ENV.has(k)), + ); + const env = { + ...turnEnv, + ...(forceEgress ? forceThroughProxyEnv(opts.egressProxyUrl!, provOpts!.egressToken!) : {}), + }; + const handle: SandboxHandle = { + id: name, + rootDir: workspaceDir, + homeDir: HOME_DIR, + coldStart, + ...(scratch ? { scratch: true } : {}), + ...(Object.keys(env).length ? { env } : {}), + }; + + try { + const credLinks = scratch ? "" : ` && ${ephemeralCredLinkScript(HOME_DIR, opts.credentialPaths ?? [])}`; + const prep = await execRaw(name, `mkdir -p ${shq(workspaceDir)}${credLinks}`, 60); + if (prep.code !== 0) + throw new Error(`do-managed-agents provision prep failed: ${(prep.stderr || prep.stdout).slice(0, 200)}`); + + await materializeRoLayers( + workspace, + layers, + handle, + { + readFile: (h, rel) => sandbox.readFile(h, rel), + writeFileBytes: (h, rel, data) => sandbox.writeFileBytes(h, rel, data), + exec: (script, t) => execRaw(name, script, t), + }, + { manifest: RO_LAYERS_MANIFEST, tar: RO_LAYERS_TAR, label: "do-managed-agents" }, + ); + await installLayerTools?.({ + exec: (script, t) => execRaw(name, script, t), + writeAbs: (abs, data) => writeAbsBytes(name, abs, data), + }); + + return handle; + } catch (err) { + await cleanupFailedProvision(sandbox, handle, err); + throw err; + } + }, + + async run(handle, command, execOpts?: ExecOptions): Promise { + const timeoutSec = execOpts?.timeoutMs ? Math.ceil(execOpts.timeoutMs / 1000) : defaultTimeoutSec; + const exports = Object.entries(handle.env ?? {}) + .map(([k, v]) => `export ${k}=${shq(v)}`) + .join("; "); + const script = `${nonInteractiveShellPrefix()}${exports ? exports + "; " : ""}cd ${handle.rootDir} 2>/dev/null; ${command}`; + const signal = execOpts?.signal; + if (!signal) return execRaw(handle.id, script, timeoutSec); + const killUid = randomUUID(); + const fireKill = (): void => { + execRaw(handle.id, killScript(killUid), 15).catch( + swallowAs("managed-agents-sandbox: kill in-flight exec", undefined), + ); + }; + signal.throwIfAborted(); + const onAbort = (): void => fireKill(); + signal.addEventListener("abort", onAbort, { once: true }); + try { + return await execRaw(handle.id, killableScript(script, killUid), timeoutSec); + } finally { + signal.removeEventListener("abort", onAbort); + } + }, + + async writeFileBytes(handle, relPath, data): Promise { + await writeAbsBytes(handle.id, posixJoin(handle.rootDir, relPath), data); + }, + async writeFile(handle, relPath, data): Promise { + await sandbox.writeFileBytes(handle, relPath, Buffer.from(data, "utf8")); + }, + async readFileBytes(handle, relPath): Promise { + return readAbsBytes(handle.id, posixJoin(handle.rootDir, relPath)); + }, + async readFile(handle, relPath): Promise { + const bytes = await sandbox.readFileBytes(handle, relPath); + return bytes === null ? null : Buffer.from(bytes).toString("utf8"); + }, + + exportFiles: execExport.exportFiles, + + async persistHomeSnapshot(scopeId: string): Promise { + const name = sandboxScopeName(prefix, scopeId); + scopeByName.set(name, scopeId); + const { session } = await ensureSession(scopeId, name); + await snapshotHome(scopeId, session); + }, + + async restartComputer(scopeId: string): Promise { + const name = sandboxScopeName(prefix, scopeId); + scopeByName.set(name, scopeId); + return provisionQueue(scopeId, async () => { + const stored = await store.get(scopeId); + if (!stored?.sessionId) throw new Error("do-managed-agents restart: no sandbox provisioned yet"); + const checkpointId = stored.checkpointId; + if (!checkpointId) throw new Error("do-managed-agents restart: no checkpoint to restore"); + const cached = sessionByName.get(name); + sessionByName.delete(name); + if (cached) await cached.close(); + let restored: ManagedAgentsSessionInfo; + try { + restored = await client.rollback(stored.sessionId, checkpointId); + } catch (e) { + reportError("agent_computer", "checkpoint_rollback_failed", errMessage(e), scopeId); + throw e; + } + await store.merge(scopeId, { + sandboxId: restored.sandboxId, + preservationState: "running", + preservationError: undefined, + }); + reportError( + "agent_computer", + "checkpoint_restored", + `session ${stored.sessionId}: restored checkpoint ${checkpointId} onto sandbox ${restored.sandboxId}`, + scopeId, + ); + }); + }, + + async computerStatus(scopeId: string): Promise { + const name = sandboxScopeName(prefix, scopeId); + const stored = await store.get(scopeId); + + if (!stored) return { machine: "no sandbox provisioned yet", provisioned: false, guestResponsive: false }; + const machine = `do-managed-agents sandbox ${stored.sandboxId || stored.sessionId}`; + const checkpointAtMs = stored.checkpointAtMs ?? stored.lastSnapshotMs; + const recoveryError = stored.preservationError ?? stored.checkpointError; + const recovery = { + strategy: "workspace_snapshot" as const, + state: stored.preservationState, + ...(recoveryError ? { error: recoveryError } : {}), + ...(stored.checkpointId ? { checkpointId: stored.checkpointId } : {}), + ...(checkpointAtMs ? { checkpointAtMs } : {}), + }; + try { + const info: ManagedAgentsSessionInfo = await client.info(stored.sessionId); + if (info.state === "paused") + return { + machine, + listed: info.state, + lifecycleState: "paused", + provisioned: true, + guestResponsive: false, + recovery: { ...recovery, state: "paused" }, + }; + scopeByName.set(name, scopeId); + let session = sessionByName.get(name); + if (!session) { + session = await client.connect(stored.sessionId); + sessionByName.set(name, session); + } + const r = await session.runCommand("echo responsive", { timeoutMs: 30_000 }); + return { + machine, + recovery, + provisioned: true, + guestResponsive: r.exitCode === 0 && r.stdout.trim() === "responsive", + }; + } catch (e) { + return { + recovery, + machine: `${machine} (${errMessage(e).slice(0, 120)})`, + provisioned: !(e instanceof ManagedAgentsSandboxGoneError), + guestResponsive: false, + }; + } + }, + + async teardown(handle, tdOpts?: TeardownOptions): Promise { + if (handle.scratch) { + const key = scratchKeyByName.get(handle.id); + return provisionQueue(key ? `scratch:${key}` : handle.id, async () => { + const remaining = (activeScratch.get(handle.id) ?? 1) - 1; + if (remaining > 0) { + activeScratch.set(handle.id, remaining); + return; + } + activeScratch.delete(handle.id); + const session = sessionByName.get(handle.id); + if (session) { + if (tdOpts?.destroy) await session.kill(); + else await session.kill().catch(swallowAs("managed-agents-sandbox: scratch kill", undefined)); + } + sessionByName.delete(handle.id); + scratchKeyByName.delete(handle.id); + }); + } + if (tdOpts?.destroy && !scopeByName.has(handle.id)) return; + const scope = scopeByName.get(handle.id) ?? "default"; + return provisionQueue(scope, () => teardownScope(handle, scope, tdOpts)); + }, + }; + + async function teardownScope(handle: SandboxHandle, scope: string, tdOpts?: TeardownOptions): Promise { + const session = sessionByName.get(handle.id); + if (tdOpts?.destroy) return destroyStoredScope(scope); + if (!session) return; + + const stored = await store.get(scope); + if (!tdOpts?.homeUnchanged) await store.merge(scope, { homeDirty: true }); + await retryPendingCheckpointDeletes(scope, session); + const homeDirty = stored?.homeDirty; + if (snapshotDue({ lastSnapshotMs: stored?.checkpointAtMs, homeDirty }, tdOpts, snapshotIntervalMs)) { + try { + await captureCheckpoint(scope, session); + } catch (e) { + await store.merge(scope, { checkpointError: errMessage(e) }); + reportError("sandbox_snapshot", "checkpoint_failed", errMessage(e), scope); + } + } + if (snapshotDue({ lastSnapshotMs: stored?.lastSnapshotMs, homeDirty }, tdOpts, snapshotIntervalMs)) { + try { + await snapshotHome(scope, session); + } catch (e) { + reportError("sandbox_snapshot", "teardown_snapshot_failed", errMessage(e), scope); + } + } + if (tdOpts?.keepWarm) return; + try { + await session.pause(); + await store.merge(scope, { preservationState: "paused", preservationError: undefined }); + sessionByName.delete(handle.id); + } catch (error) { + await store.merge(scope, { preservationState: "pause_failed", preservationError: errMessage(error) }); + reportError("sandbox_preservation", "pause_failed", errMessage(error), scope); + throw error; + } + } + + return sandbox; +} diff --git a/src/sandbox/managed-agents-tunnel.ts b/src/sandbox/managed-agents-tunnel.ts new file mode 100644 index 0000000000..a0557e7302 --- /dev/null +++ b/src/sandbox/managed-agents-tunnel.ts @@ -0,0 +1,151 @@ +import { createServer, type Server, type Socket } from "node:net"; +import { once } from "node:events"; +import type WebSocket from "ws"; + +export const SANDBOX_AGENT_PORT = 8443; + +const TUNNEL_CLOSE_SESSION_UNAVAILABLE = 4001; +const TUNNEL_CLOSE_GUEST_DIAL_FAILED = 4002; +const TUNNEL_CLOSE_UPSTREAM_ERROR = 4011; +const REJECTION_BODY_MAX = 300; +const PING_INTERVAL_MS = 30_000; + +export interface ManagedAgentsTunnel { + readonly localPort: number; + lastFailure(): string | null; + whenFailed(): Promise; + close(): Promise; +} + +export interface ManagedAgentsTunnelOptions { + apiBaseUrl: string; + sessionId: string; + remotePort: number; + getToken: () => Promise; + pingIntervalMs?: number; +} + +export function tunnelUrl(apiBaseUrl: string, sessionId: string, remotePort: number): string { + const url = new URL(apiBaseUrl); + if (url.protocol !== "http:" && url.protocol !== "https:") + throw new Error(`DO_AGENTS_API_BASE_URL must be http(s), got ${url.protocol}`); + url.protocol = url.protocol === "https:" ? "wss:" : "ws:"; + url.pathname = `${url.pathname.replace(/\/$/, "")}/v2/agents/sessions/${sessionId}/port-forward/${remotePort}`; + return url.toString(); +} + +type WebSocketCtor = typeof WebSocket; + +let wsModule: WebSocketCtor | null = null; +async function loadWebSocket(): Promise { + wsModule ??= (await import("ws")).default; + return wsModule; +} + +export async function openManagedAgentsTunnel(opts: ManagedAgentsTunnelOptions): Promise { + const WebSocketImpl = await loadWebSocket(); + const url = tunnelUrl(opts.apiBaseUrl, opts.sessionId, opts.remotePort); + const sockets = new Set(); + let failure: string | null = null; + let closed = false; + let announceFailure: (detail: string) => void = () => undefined; + const failed = new Promise((resolve) => { + announceFailure = resolve; + }); + const fail = (detail: string): void => { + failure = detail; + announceFailure(detail); + }; + + const bridge = (local: Socket): void => { + sockets.add(local); + local.once("close", () => sockets.delete(local)); + void opts + .getToken() + .then((token) => { + if (closed) { + local.destroy(); + return; + } + const ws = new WebSocketImpl(url, { headers: { Authorization: `Bearer ${token}` } }); + let rejected = false; + + ws.on("unexpected-response", (_req, res) => { + rejected = true; + let body = ""; + res.on("data", (chunk: Buffer) => { + body += chunk.toString("utf8"); + }); + res.on("end", () => { + fail(`${res.statusCode ?? 0} ${body.trim().slice(0, REJECTION_BODY_MAX)}`.trim()); + local.destroy(); + }); + }); + + ws.on("open", () => { + const heartbeat = setInterval(() => { + if (ws.readyState === WebSocketImpl.OPEN) ws.ping(); + }, opts.pingIntervalMs ?? PING_INTERVAL_MS); + heartbeat.unref(); + ws.once("close", () => clearInterval(heartbeat)); + local.on("data", (chunk: Buffer) => { + local.pause(); + ws.send(chunk, (err) => { + if (err) return; + local.resume(); + }); + }); + ws.on("message", (data: Buffer) => { + if (!local.write(data)) { + ws.pause(); + local.once("drain", () => ws.resume()); + } + }); + }); + + ws.on("close", (code: number, reason: Buffer) => { + if ( + code === TUNNEL_CLOSE_SESSION_UNAVAILABLE || + code === TUNNEL_CLOSE_GUEST_DIAL_FAILED || + code === TUNNEL_CLOSE_UPSTREAM_ERROR + ) + fail(`close ${code} ${reason.toString("utf8")}`.trim()); + local.destroy(); + }); + + ws.on("error", (err: Error) => { + if (!rejected) fail(err.message); + local.destroy(); + }); + + local.on("close", () => { + if (ws.readyState === WebSocketImpl.OPEN || ws.readyState === WebSocketImpl.CONNECTING) ws.close(); + }); + local.on("error", () => ws.close()); + }) + .catch((err: unknown) => { + fail(err instanceof Error ? err.message : String(err)); + local.destroy(); + }); + }; + + const server: Server = createServer(bridge); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + server.unref(); + const address = server.address(); + if (address === null || typeof address === "string") + throw new Error("Managed Agents tunnel failed to bind a local port"); + + return { + localPort: address.port, + lastFailure: () => failure, + whenFailed: () => failed, + close: async () => { + closed = true; + for (const socket of sockets) socket.destroy(); + sockets.clear(); + await new Promise((resolve) => server.close(() => resolve())); + }, + }; +} diff --git a/src/sandbox/sandbox-routing.ts b/src/sandbox/sandbox-routing.ts index 4960a78bd2..db73377801 100644 --- a/src/sandbox/sandbox-routing.ts +++ b/src/sandbox/sandbox-routing.ts @@ -17,7 +17,16 @@ import { } from "./sandbox.ts"; export type SandboxBackendName = - "sprites" | "aws" | "local" | "smolmachines" | "e2b" | "modal" | "porter" | "agent37" | "superserve"; + | "sprites" + | "aws" + | "local" + | "smolmachines" + | "e2b" + | "modal" + | "porter" + | "agent37" + | "superserve" + | "do-managed-agents"; export class NoDefaultSandboxError extends Error { constructor() { diff --git a/src/wiring.ts b/src/wiring.ts index 8fb402aba7..e3ea370e6f 100644 --- a/src/wiring.ts +++ b/src/wiring.ts @@ -230,6 +230,12 @@ import { import { createSdkSuperserveClient } from "./sandbox/superserve-client.ts"; import { createE2bSandbox, type StoredE2bSandbox } from "./sandbox/e2b-sandbox.ts"; import { createSdkE2bClient } from "./sandbox/e2b-client.ts"; +import { + createManagedAgentsSandbox, + QM_TEMPLATE_TOOLS, + type StoredManagedAgentsSandbox, +} from "./sandbox/managed-agents-sandbox.ts"; +import { createSdkManagedAgentsClient } from "./sandbox/managed-agents-client.ts"; import { createS3SnapshotStore } from "./sandbox/home-snapshot.ts"; import { createModalSandbox, type StoredModalSandbox } from "./sandbox/modal-sandbox.ts"; import { createSdkModalClient } from "./sandbox/modal-client.ts"; @@ -932,6 +938,41 @@ export function buildApp( onError: sandboxOnError, }); }; + const managedAgentsBodies = artifactMap("do_managed_agents_sandbox_bodies"); + const buildManagedAgents = (): Sandbox => { + const managedAgents = config.managedAgentsSandbox; + if (!managedAgents.apiToken) throw new Error("SANDBOX_BACKEND=do-managed-agents requires DO_AGENTS_API_TOKEN"); + return createManagedAgentsSandbox(workspace, { + client: createSdkManagedAgentsClient({ + apiToken: managedAgents.apiToken, + ...(managedAgents.apiBaseUrl ? { apiBaseUrl: managedAgents.apiBaseUrl } : {}), + ...(managedAgents.template ? { template: managedAgents.template } : {}), + ...(managedAgents.sizeSlug ? { sizeSlug: managedAgents.sizeSlug } : {}), + ...(managedAgents.idleTimeoutSec ? { idleTimeoutSec: managedAgents.idleTimeoutSec } : {}), + ...(managedAgents.egressProxyUrl ? { egressProxyUrl: managedAgents.egressProxyUrl } : {}), + }), + ...(managedAgents.namePrefix ? { namePrefix: managedAgents.namePrefix } : {}), + ...(managedAgents.defaultTimeoutSec ? { defaultTimeoutSec: managedAgents.defaultTimeoutSec } : {}), + ...(managedAgents.snapshotIntervalSec !== undefined + ? { snapshotIntervalMs: managedAgents.snapshotIntervalSec * 1000 } + : {}), + ...(managedAgents.egressProxyUrl ? { egressProxyUrl: managedAgents.egressProxyUrl } : {}), + extraTools: [...(managedAgents.template ? QM_TEMPLATE_TOOLS : []), ...deploymentLayer.advertisedTools], + credentialPaths: deploymentLayer.credentialPaths, + layerToolFiles: () => deploymentLayer.installFiles, + blobTransfer, + ...(config.signingSecret ? { signingSecret: config.signingSecret } : {}), + ...(config.capabilitySecret ? { capabilitySecret: config.capabilitySecret } : {}), + ...(config.apiBaseUrl ? { apiBaseUrl: config.apiBaseUrl } : {}), + store: managedAgentsBodies, + ...(managedAgents.snapshotS3Bucket + ? { + snapshots: createS3SnapshotStore({ bucket: managedAgents.snapshotS3Bucket, prefix: "managed-agents-home" }), + } + : {}), + onError: sandboxOnError, + }); + }; const buildModal = (): Sandbox => { const modal = config.modalSandbox; if (!modal.tokenId || !modal.tokenSecret) @@ -1081,6 +1122,7 @@ export function buildApp( porter: buildPorter, agent37: buildAgent37, superserve: buildSuperserve, + "do-managed-agents": buildManagedAgents, }; const enabledBackends = new Set(enabledSandboxBackends(config)); const sandboxBackends: Partial> = { diff --git a/test/live-slack-sandbox-providers.test.ts b/test/live-slack-sandbox-providers.test.ts index ea5634a729..9bf5081d0e 100644 --- a/test/live-slack-sandbox-providers.test.ts +++ b/test/live-slack-sandbox-providers.test.ts @@ -64,6 +64,7 @@ test("every provider has its own parallel execution scenario", () => { assert.deepEqual(sandboxProviders.toSorted(), [ "agent37", "aws", + "do-managed-agents", "e2b", "local", "modal", @@ -159,7 +160,7 @@ test("explicit provider selection retains strict coverage without treating defer "sandbox-execute-modal", ], ); - assert.equal(selectSandboxProviderScenarios("all").length, 9); + assert.equal(selectSandboxProviderScenarios("all").length, 10); assert.deepEqual(selectSandboxProviderScenarios(undefined), []); for (const value of ["", "sprites,", "typo", "sprites,sprites", "all,sprites", "constructor", "__proto__"]) assert.throws(() => selectSandboxProviderScenarios(value)); diff --git a/test/live-slack/scenarios-sandbox-providers.ts b/test/live-slack/scenarios-sandbox-providers.ts index a8675f071b..8e7183c907 100644 --- a/test/live-slack/scenarios-sandbox-providers.ts +++ b/test/live-slack/scenarios-sandbox-providers.ts @@ -14,6 +14,7 @@ const providerCoverage: Record = { porter: true, agent37: true, superserve: true, + "do-managed-agents": true, }; export const sandboxProviders = Object.keys(providerCoverage) as SandboxBackendName[]; diff --git a/test/managed-agents-client.test.ts b/test/managed-agents-client.test.ts new file mode 100644 index 0000000000..d8cfa47489 --- /dev/null +++ b/test/managed-agents-client.test.ts @@ -0,0 +1,301 @@ +import { test, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { connect } from "node:net"; +import { once } from "node:events"; +import { join } from "node:path"; +import { + createSdkManagedAgentsClient, + managedAgentsEgressAllow, + managedAgentsManifest, + managedAgentsSessionState, + ManagedAgentsSandboxGoneError, + ManagedAgentsCommandLostError, + type ManagedAgentsClient, + type ManagedAgentsSession, +} from "../src/sandbox/managed-agents-client.ts"; +import { tunnelUrl, openManagedAgentsTunnel, SANDBOX_AGENT_PORT } from "../src/sandbox/managed-agents-tunnel.ts"; +import { startFakeManagedAgentsService, type FakeManagedAgentsService } from "./support/fake-managed-agents-service.ts"; + +let service: FakeManagedAgentsService; +let client: ManagedAgentsClient; +const opened: ManagedAgentsSession[] = []; + +const session = async (name: string): Promise => { + const created = await client.create({ name }); + opened.push(created); + return created; +}; + +before(async () => { + service = await startFakeManagedAgentsService(); + client = createSdkManagedAgentsClient({ + apiToken: service.token, + apiBaseUrl: service.apiBaseUrl, + template: "base", + }); +}); + +after(async () => { + await Promise.all(opened.map((s) => s.kill().catch(() => undefined))); + await service?.close(); +}); + +test("the port-forward url targets sandbox-agent over wss", () => { + assert.equal( + tunnelUrl("https://api.digitalocean.com", "sess-1", SANDBOX_AGENT_PORT), + "wss://api.digitalocean.com/v2/agents/sessions/sess-1/port-forward/8443", + ); + assert.equal( + tunnelUrl("http://127.0.0.1:9000/edge", "sess-2", SANDBOX_AGENT_PORT), + "ws://127.0.0.1:9000/edge/v2/agents/sessions/sess-2/port-forward/8443", + ); + assert.throws(() => tunnelUrl("ftp://example.com", "s", 8443), /must be http/); +}); + +test("the manifest asks for a bare sandbox and pre-allows bash so Managed Agents never gates a command", () => { + const yaml = managedAgentsManifest({ name: "qm-personal-tester", template: "base" }); + assert.match(yaml, /^name: "qm-personal-tester"$/m); + assert.match(yaml, /^agent: "none"$/m); + assert.match(yaml, /^template: "base"$/m); + assert.match(yaml, /tool: bash/); + assert.match(yaml, /action: allow/); + assert.doesNotMatch(yaml, /persistent_workspace/, "Managed Agents deprecated the key and warns when it is sent"); +}); + +test("the manifest omits the template so Managed Agents lands a bare session on its own base", () => { + const yaml = managedAgentsManifest({ name: "n", template: "" }); + assert.match(yaml, /^agent: "none"$/m); + assert.doesNotMatch(yaml, /^template:/m); +}); + +test("the manifest carries size, idle timeout and the egress allowlist only when configured", () => { + const bare = managedAgentsManifest({ name: "n", template: "base" }); + assert.doesNotMatch(bare, /size:|idle_timeout:|egress:/); + const full = managedAgentsManifest({ + name: "n", + template: "base", + sizeSlug: "mv-2vcpu-4gb", + idleTimeoutSec: 600, + egressAllow: ["proxy.example.com"], + }); + assert.match(full, /^size: "mv-2vcpu-4gb"$/m); + assert.match(full, /^idle_timeout: "600s"$/m); + assert.match(full, /^ {2}- "proxy\.example\.com"$/m); +}); + +test("the vendored proto is the client subset and carries nothing provider-internal", () => { + const proto = readFileSync(join(process.cwd(), "src/sandbox/managed-agents-sandbox-agent.proto"), "utf8"); + assert.match(proto, /service SandboxAgentService/); + for (const rpc of ["Exec", "Upload", "Download"]) assert.match(proto, new RegExp(`rpc ${rpc}\\(`)); + for (const unused of ["Shell", "ImportFromURL", "ExportToURL", "Health", "StartChildProcess", "ProxyPort"]) + assert.doesNotMatch(proto, new RegExp(`rpc ${unused}\\(`), `${unused} is unused and should not be vendored`); + for (const internal of [/do\/doge/, /dorpc/, /sandboxsvc/, /sandbox-service/, /microvm\.v1/, /MSANDBOX/, /169\.254/]) + assert.doesNotMatch(proto, internal, `${internal.source} is provider-internal`); +}); + +test("creating a session posts a yaml manifest and waits for ready", async () => { + const s = await session("qm-create"); + assert.ok(s.sessionId); + assert.equal(service.sessions().length, 1); + assert.match(service.manifests()[0] ?? "", /template: "base"/); + assert.equal(service.sessions()[0]?.template, "base"); +}); + +test("a command runs in the guest over the port-forward tunnel", async () => { + const s = await session("qm-exec"); + const result = await s.runCommand("echo hello-from-guest; echo oops >&2; exit 3"); + assert.equal(result.exitCode, 3); + assert.match(result.stdout, /hello-from-guest/); + assert.match(result.stderr, /oops/); + assert.ok(service.tunnelCount() > 0, "exec must travel through the tunnel, not the REST exec endpoint"); +}); + +test("a port-forward the edge refuses reads as gone, carrying the rejection, not as a half-run command", async () => { + const s = await session("qm-no-tunnel"); + service.rejectPortForward("team is not entitled to port-forward"); + const err = await s.runCommand("echo never-runs").then( + () => null, + (e: unknown) => e, + ); + assert.ok(err instanceof ManagedAgentsSandboxGoneError, `expected a retryable gone error, got ${String(err)}`); + assert.match(err.message, /403/); + assert.match(err.message, /not entitled to port-forward/); + assert.doesNotMatch(err.message, /may have partially executed/); + service.rejectPortForward(null); +}); + +test("a transient lifecycle failure while waiting for readiness is polled through, not surfaced", async () => { + const created = await session("qm-transient-ready"); + service.failNextLifecycleCalls(2, 403); + const reconnected = await client.connect(created.sessionId); + opened.push(reconnected); + assert.equal(reconnected.sessionId, created.sessionId); +}); + +test("a lifecycle failure that never clears is surfaced instead of polled to the deadline", async () => { + const created = await session("qm-stuck-ready"); + service.failNextLifecycleCalls(50, 403); + const err = await client.connect(created.sessionId).then( + () => null, + (e: unknown) => e, + ); + service.failNextLifecycleCalls(0, 403); + assert.ok(err instanceof Error, "expected the persistent failure to surface"); + assert.match(err.message, /403/); + assert.doesNotMatch(err.message, /did not become ready/); +}); + +test("command env and working directory reach the guest", async () => { + const s = await session("qm-env"); + const result = await s.runCommand("echo VAR=$MY_VAR", { env: { MY_VAR: "set-by-qm" } }); + assert.match(result.stdout, /VAR=set-by-qm/); +}); + +test("files round trip through the guest upload and download streams", async () => { + const s = await session("qm-files"); + const body = Buffer.from("managed agents file payload"); + await s.writeFileBytes("/home/user/notes/a.txt", new Uint8Array(body)); + const read = await s.readFileBytes("/home/user/notes/a.txt"); + assert.ok(read); + assert.equal(Buffer.from(read).toString("utf8"), "managed agents file payload"); +}); + +test("a missing file reads as null rather than an error", async () => { + const s = await session("qm-missing"); + assert.equal(await s.readFileBytes("/home/user/nope.txt"), null); +}); + +test("resuming a paused session does not wait out a fixed poll interval", async () => { + const created = await session("qm-resume-latency"); + service.setStatus("qm-resume-latency", "SESSION_STATUS_PAUSED"); + const started = Date.now(); + const back = await client.connect(created.sessionId); + opened.push(back); + assert.equal(back.sessionId, created.sessionId); + assert.ok(Date.now() - started < 1_000, `resume took ${Date.now() - started}ms`); +}); + +test("a session paused mid-command fails fast instead of stalling until the command timeout", async () => { + const watchful = createSdkManagedAgentsClient({ + apiToken: service.token, + apiBaseUrl: service.apiBaseUrl, + stallProbeMs: 50, + }); + const s = await watchful.create({ name: "qm-paused-midcommand" }); + opened.push(s); + const started = Date.now(); + const running = s.runCommand("sleep 30", { timeoutMs: 120_000 }); + await new Promise((resolve) => setTimeout(resolve, 150)); + service.setStatus("qm-paused-midcommand", "SESSION_STATUS_PAUSED"); + const err = await running.then( + () => null, + (e: unknown) => e, + ); + assert.ok(err instanceof ManagedAgentsCommandLostError, `expected a command-lost error, got ${String(err)}`); + assert.ok(Date.now() - started < 10_000, `expected a fast failure, took ${Date.now() - started}ms`); +}); + +test("the tunnel heartbeats so an edge idle timeout cannot kill a long silent command", async () => { + const created = await session("qm-heartbeat"); + const before = service.pingCount(); + const tunnel = await openManagedAgentsTunnel({ + apiBaseUrl: service.apiBaseUrl, + sessionId: created.sessionId, + remotePort: SANDBOX_AGENT_PORT, + getToken: async () => service.token, + pingIntervalMs: 20, + }); + try { + const probe = connect(tunnel.localPort, "127.0.0.1"); + await once(probe, "connect"); + await new Promise((resolve) => setTimeout(resolve, 400)); + assert.ok(service.pingCount() > before, `expected heartbeat pings, saw ${service.pingCount() - before}`); + probe.destroy(); + } finally { + await tunnel.close(); + } +}); + +test("a large file survives the chunked upload path", async () => { + const s = await session("qm-large"); + const body = Buffer.alloc(900 * 1024, "x"); + await s.writeFileBytes("/home/user/big.bin", new Uint8Array(body)); + const read = await s.readFileBytes("/home/user/big.bin"); + assert.equal(read?.length, body.length); +}); + +test("list resolves a session by name and skips destroyed rows", async () => { + const s = await session("qm-list"); + const found = await client.list("qm-list"); + assert.equal(found.length, 1); + assert.equal(found[0]?.sessionId, s.sessionId); + await s.kill(); + assert.deepEqual(await client.list("qm-list"), []); +}); + +test("create checkpoint posts to the session checkpoint route and returns the ready id", async () => { + const s = await session("qm-checkpoint"); + const captured = await s.createCheckpoint("qm teardown"); + assert.match(captured.checkpointId, /^cp_/); + assert.equal(captured.status, "READY"); + assert.ok(captured.createdAtMs); + const stored = service.checkpoints(s.sessionId); + assert.equal(stored.length, 1); + assert.equal(stored[0]?.checkpoint_id, captured.checkpointId); + assert.equal(stored[0]?.label, "qm teardown"); + assert.equal(stored[0]?.status, "READY"); + await s.deleteCheckpoint(captured.checkpointId); + assert.equal(service.checkpoints(s.sessionId).length, 0); +}); + +test("rollback rewinds the same session onto a new sandbox without resuming it first", async () => { + const s = await session("qm-rollback"); + const captured = await s.createCheckpoint("qm teardown"); + const previousSandbox = s.sandboxId; + await s.pause(); + assert.equal((await client.info(s.sessionId)).state, "paused"); + const restored = await client.rollback(s.sessionId, captured.checkpointId); + assert.equal(restored.sessionId, s.sessionId); + assert.notEqual(restored.sandboxId, previousSandbox); + assert.equal(restored.state, "ready"); + assert.equal((await client.info(s.sessionId)).sandboxId, restored.sandboxId); + assert.equal(service.checkpoints(s.sessionId).length, 1); + await assert.rejects(() => client.rollback(s.sessionId, "missing"), /404|no such checkpoint/); +}); + +test("connect resumes a paused session before handing it back", async () => { + const s = await session("qm-paused"); + await s.pause(); + assert.equal((await client.info(s.sessionId)).state, "paused"); + const again = await client.connect(s.sessionId); + assert.equal((await client.info(again.sessionId)).state, "ready"); + const result = await again.runCommand("echo awake"); + assert.match(result.stdout, /awake/); +}); + +test("a destroyed session reports a terminal state, and reconnecting to it fails fast", async () => { + const s = await session("qm-gone"); + await client.kill(s.sessionId); + assert.equal((await client.info(s.sessionId)).state, "destroyed"); + await assert.rejects(() => client.connect(s.sessionId), ManagedAgentsSandboxGoneError); +}); + +test("a session the control plane has never heard of reads as gone", async () => { + await assert.rejects(() => client.info("11111111-2222-3333-4444-555555555555"), ManagedAgentsSandboxGoneError); +}); + +test("a bad token is refused by the control plane", async () => { + const wrong = createSdkManagedAgentsClient({ apiToken: "nope", apiBaseUrl: service.apiBaseUrl, template: "base" }); + await assert.rejects(() => wrong.create({ name: "qm-unauth" }), /401|bad token/); +}); + +test("session status wire values map onto backend states", () => { + assert.equal(managedAgentsSessionState("SESSION_STATUS_READY"), "ready"); + assert.equal(managedAgentsSessionState("SESSION_STATUS_PAUSED"), "paused"); + assert.equal(managedAgentsSessionState("nonsense"), "unspecified"); +}); + +test("the egress allowlist carries just the proxy host", () => { + assert.deepEqual(managedAgentsEgressAllow("https://egress.example.com:443/path"), ["egress.example.com"]); +}); diff --git a/test/managed-agents-sandbox.test.ts b/test/managed-agents-sandbox.test.ts new file mode 100644 index 0000000000..d651e7474a --- /dev/null +++ b/test/managed-agents-sandbox.test.ts @@ -0,0 +1,397 @@ +import { test, after, beforeEach } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pollProcess } from "../src/sandbox/process-poll.ts"; +import { + createManagedAgentsSandbox, + QM_TEMPLATE_TOOLS, + type StoredManagedAgentsSandbox, +} from "../src/sandbox/managed-agents-sandbox.ts"; +import { + managedAgentsEgressAllow, + managedAgentsSessionState, + ManagedAgentsHitlRejectedError, +} from "../src/sandbox/managed-agents-client.ts"; +import { sandboxScopeName } from "../src/sandbox/exec-sandbox-base.ts"; +import { createLocalWorkspaceStore } from "../src/workspace/workspace-store.ts"; +import type { HomeSnapshotStore } from "../src/sandbox/home-snapshot.ts"; +import { supportsProcessSessions } from "../src/sandbox/sandbox.ts"; +import { createMemoryMap, type DurableMap } from "../src/persistence/durable-map.ts"; +import { scopeId } from "../src/types.ts"; +import { installFakeManagedAgents, type FakeManagedAgents } from "./support/fake-managed-agents.ts"; +import type { Sandbox } from "../src/sandbox/sandbox.ts"; + +let fake: FakeManagedAgents; +let sandbox: Sandbox; +const scope = scopeId("personal", "tester"); +const layers = [{ scopeId: scope, mountPath: "/", mode: "rw" as const }]; +const scopeName = (): string => sandboxScopeName("qmt", scope); + +function make(extra: Record = {}): Sandbox { + return createManagedAgentsSandbox(createLocalWorkspaceStore(mkdtempSync(join(tmpdir(), "managed-agents-ws-"))), { + client: fake.client, + namePrefix: "qmt", + ...extra, + }); +} + +beforeEach(() => { + fake = installFakeManagedAgents(); + sandbox = make(); +}); + +after(() => fake?.cleanup()); + +test("provision runs commands with env and cwd", async () => { + const h = await sandbox.provision(layers, { env: { MY_VAR: "v1" } }); + assert.equal(h.coldStart, true); + const r = await sandbox.run(h, "pwd; echo VAR=$MY_VAR"); + assert.equal(r.code, 0); + assert.match(r.stdout, /workspace/); + assert.match(r.stdout, /VAR=v1/); +}); + +test("one session per scope is reused across provisions", async () => { + const first = await sandbox.provision(layers); + await sandbox.teardown(first, { keepWarm: true }); + const second = await sandbox.provision(layers); + assert.equal(second.coldStart, false); + assert.equal(fake.createdCount(scopeName()), 1); +}); + +test("the session is named after the scope so an operator can find it", async () => { + await sandbox.provision(layers); + const current = fake.current(scopeName()); + assert.ok(current, "expected a session registered under the scope name"); + assert.ok(scopeName().length <= 64, "Managed Agents caps session names at 64 characters"); + assert.doesNotMatch(scopeName(), /^[0-9a-f]{8}-[0-9a-f]{4}-/, "Managed Agents rejects UUID-shaped session names"); +}); + +test("teardown captures a provider checkpoint before pausing", async () => { + const store: DurableMap = createMemoryMap(); + sandbox = make({ store }); + const h = await sandbox.provision(layers); + await sandbox.run(h, "echo hi > kept.txt"); + await sandbox.teardown(h); + const calls = fake.checkpointCalls(); + assert.equal(calls.length, 1); + assert.equal(calls[0]?.label, "qm teardown"); + assert.equal(calls[0]?.sessionId, fake.current(scopeName())?.sessionId); + const stored = await store.get(scope); + assert.equal(stored?.checkpointId, calls[0]?.checkpointId); + const status = await sandbox.computerStatus!(scope); + assert.equal(status.recovery?.checkpointId, stored?.checkpointId); + assert.equal(status.lifecycleState, "paused"); + assert.equal(fake.pauseCalls().length, 1); +}); + +test("restart rolls a paused session back to its checkpoint without resuming the old microVM", async () => { + const store: DurableMap = createMemoryMap(); + const restored: string[] = []; + sandbox = make({ + store, + onError: (e: { code: string }) => { + if (e.code === "checkpoint_restored") restored.push(e.code); + }, + }); + const h = await sandbox.provision(layers); + await sandbox.teardown(h); + const before = await store.get(scope); + assert.ok(before?.checkpointId); + assert.equal(before.preservationState, "paused"); + const resumes = fake.resumeCalls().length; + const disconnects = fake.guestDisconnects().length; + await sandbox.restartComputer!(scope); + const after = await store.get(scope); + assert.equal(after?.sessionId, before.sessionId); + assert.notEqual(after?.sandboxId, before.sandboxId); + assert.equal(after?.preservationState, "running"); + assert.equal(after?.checkpointId, before.checkpointId); + assert.deepEqual(fake.rollbackCalls(), [{ sessionId: before.sessionId, checkpointId: before.checkpointId }]); + assert.equal(fake.resumeCalls().length, resumes); + assert.equal(fake.guestDisconnects().length, disconnects); + assert.equal(fake.createdCount(scopeName()), 1); + assert.deepEqual(restored, ["checkpoint_restored"]); + const again = await sandbox.provision(layers); + assert.equal(again.coldStart, false); + assert.equal(fake.current(scopeName())?.sandboxId, after?.sandboxId); +}); + +test("restart drops a warm guest's port-forward before rolling back", async () => { + const store: DurableMap = createMemoryMap(); + sandbox = make({ store }); + const h = await sandbox.provision(layers); + await sandbox.teardown(h, { keepWarm: true }); + const disconnects = fake.guestDisconnects().length; + await sandbox.restartComputer!(scope); + assert.equal(fake.guestDisconnects().length, disconnects + 1); + assert.equal(fake.rollbackCalls().length, 1); + assert.equal((await store.get(scope))?.preservationState, "running"); +}); + +test("restart without a checkpoint does not call rollback", async () => { + await sandbox.provision(layers); + await assert.rejects(() => sandbox.restartComputer!(scope), /no checkpoint to restore/); + assert.equal(fake.rollbackCalls().length, 0); +}); + +test("a failed rollback keeps the stored sandbox and the same session", async () => { + const store: DurableMap = createMemoryMap(); + sandbox = make({ store }); + const h = await sandbox.provision(layers); + await sandbox.teardown(h); + const before = await store.get(scope); + fake.failNextRollback(scopeName()); + await assert.rejects(() => sandbox.restartComputer!(scope), /rollback/); + assert.equal((await store.get(scope))?.sandboxId, before?.sandboxId); + assert.equal(fake.rollbackCalls().length, 0); + const again = await sandbox.provision(layers); + assert.equal(again.coldStart, false); + assert.equal(fake.createdCount(scopeName()), 1); +}); + +test("a checkpoint failure is recorded and the session still pauses", async () => { + const store: DurableMap = createMemoryMap(); + sandbox = make({ store }); + const h = await sandbox.provision(layers); + fake.failNextCheckpoint(scopeName()); + await sandbox.teardown(h); + assert.equal(fake.checkpointCalls().length, 0); + assert.equal(fake.pauseCalls().length, 1); + assert.equal(fake.current(scopeName())?.state, "paused"); + const status = await sandbox.computerStatus!(scope); + assert.match(status.recovery?.error ?? "", /create checkpoint/); +}); + +test("checkpoint capture is throttled across a pause and resume", async () => { + const h = await sandbox.provision(layers); + await sandbox.teardown(h); + const again = await sandbox.provision(layers); + await sandbox.teardown(again); + assert.equal(fake.checkpointCalls().length, 1); +}); + +test("a newer checkpoint replaces the previous one", async () => { + const store: DurableMap = createMemoryMap(); + sandbox = make({ store, snapshotIntervalMs: -1 }); + const h = await sandbox.provision(layers); + await sandbox.teardown(h); + const first = (await store.get(scope))?.checkpointId; + const again = await sandbox.provision(layers); + await sandbox.teardown(again); + const second = (await store.get(scope))?.checkpointId; + assert.ok(first); + assert.ok(second); + assert.notEqual(first, second); + assert.deepEqual(fake.deletedCheckpoints(), [first]); + assert.deepEqual((await store.get(scope))?.pendingCheckpointDeletes ?? [], []); +}); + +test("a failed checkpoint delete is remembered and retried without another capture", async () => { + const store: DurableMap = createMemoryMap(); + const fast = make({ store, snapshotIntervalMs: -1 }); + const h = await fast.provision(layers); + await fast.teardown(h); + const first = (await store.get(scope))?.checkpointId; + assert.ok(first); + fake.failNextCheckpointDelete(scopeName()); + const again = await fast.provision(layers); + await fast.teardown(again); + assert.deepEqual((await store.get(scope))?.pendingCheckpointDeletes, [first]); + assert.deepEqual(fake.deletedCheckpoints(), []); + + const slow = make({ store }); + const resumed = await slow.provision(layers); + const captures = fake.checkpointCalls().length; + await slow.teardown(resumed, { homeUnchanged: true }); + assert.equal(fake.checkpointCalls().length, captures); + assert.deepEqual(fake.deletedCheckpoints(), [first]); + assert.deepEqual((await store.get(scope))?.pendingCheckpointDeletes ?? [], []); +}); + +test("a failed home snapshot retries without recapturing the checkpoint", async () => { + const uploads: number[] = []; + const snapshots: HomeSnapshotStore = { + open: async () => null, + put: async () => undefined, + createUpload: async () => { + uploads.push(1); + throw new Error("snapshot upload failed"); + }, + }; + sandbox = make({ snapshots }); + const h = await sandbox.provision(layers); + await sandbox.teardown(h); + const again = await sandbox.provision(layers); + await sandbox.teardown(again); + assert.equal(fake.checkpointCalls().length, 1); + assert.equal(uploads.length, 2); +}); + +test("teardown pauses the session and the next provision resumes it", async () => { + const h = await sandbox.provision(layers); + await sandbox.run(h, "echo hi > kept.txt"); + await sandbox.teardown(h); + assert.deepEqual(fake.pauseCalls().length, 1); + assert.equal(fake.current(scopeName())?.state, "paused"); + + const again = await sandbox.provision(layers); + assert.equal(fake.resumeCalls().length, 1); + const r = await sandbox.run(again, "cat kept.txt"); + assert.match(r.stdout, /hi/); + assert.equal(fake.createdCount(scopeName()), 1); +}); + +test("keepWarm teardown leaves the session running for background work", async () => { + const h = await sandbox.provision(layers); + await sandbox.teardown(h, { keepWarm: true }); + assert.equal(fake.pauseCalls().length, 0); + assert.equal(fake.current(scopeName())?.state, "ready"); +}); + +test("a lost session is re-adopted by name when the durable record is gone", async () => { + const store: DurableMap = createMemoryMap(); + sandbox = make({ store }); + await sandbox.provision(layers); + const before = fake.current(scopeName()); + await store.delete(scope); + + const again = await sandbox.provision(layers); + assert.equal(again.coldStart, false); + assert.equal(fake.createdCount(scopeName()), 1); + assert.equal(fake.current(scopeName())?.sessionId, before?.sessionId); +}); + +test("a destroyed session is replaced on the next provision", async () => { + await sandbox.provision(layers); + fake.destroy(scopeName()); + const again = await sandbox.provision(layers); + assert.ok(again.id); + assert.equal(fake.createdCount(scopeName()), 2); +}); + +test("destroying a scope disconnects the live guest rather than only deleting the session", async () => { + const h = await sandbox.provision(layers); + await sandbox.run(h, "true"); + const live = fake.current(scopeName()); + assert.ok(live, "expected a live session before destroying the scope"); + await sandbox.destroyScope!(scope); + assert.deepEqual(fake.guestDisconnects(), [live.sessionId]); + assert.equal(fake.current(scopeName()), null); +}); + +test("a command lost mid-flight is surfaced rather than silently retried", async () => { + const h = await sandbox.provision(layers); + fake.loseNextCommand(scopeName()); + await assert.rejects(() => sandbox.run(h, "echo nope"), /was lost while a command was running/); +}); + +test("a Managed Agents approval rejection names the permissions fix", async () => { + const h = await sandbox.provision(layers); + fake.rejectNextCommand(scopeName()); + await assert.rejects(() => sandbox.run(h, "echo nope"), ManagedAgentsHitlRejectedError); + await assert.rejects(() => { + fake.rejectNextCommand(scopeName()); + return sandbox.run(h, "echo nope"); + }, /permissions to allow bash/); +}); + +test("files round trip through the data plane", async () => { + const h = await sandbox.provision(layers); + await sandbox.writeFile(h, "notes/a.txt", "hello managed agents"); + assert.equal(await sandbox.readFile(h, "notes/a.txt"), "hello managed agents"); + assert.equal(await sandbox.readFile(h, "notes/missing.txt"), null); +}); + +test("destroy removes the session and its durable record", async () => { + const store: DurableMap = createMemoryMap(); + sandbox = make({ store }); + const h = await sandbox.provision(layers); + await sandbox.teardown(h, { destroy: true }); + assert.equal(await store.get(scope), null); + assert.equal(fake.current(scopeName()), null); +}); + +test("computerStatus reports a paused machine without waking it", async () => { + const h = await sandbox.provision(layers); + await sandbox.teardown(h); + const status = await sandbox.computerStatus!(scope); + assert.equal(status.provisioned, true); + assert.equal(status.guestResponsive, false); + assert.equal(status.lifecycleState, "paused"); + assert.equal(fake.resumeCalls().length, 0); +}); + +test("computerStatus probes a running machine", async () => { + await sandbox.provision(layers); + const status = await sandbox.computerStatus!(scope); + assert.equal(status.provisioned, true); + assert.equal(status.guestResponsive, true); +}); + +test("computerStatus on an unprovisioned scope reports nothing provisioned", async () => { + const status = await sandbox.computerStatus!(scope); + assert.equal(status.provisioned, false); + assert.equal(status.guestResponsive, false); +}); + +test("scratch sandboxes are reference counted and never persisted", async () => { + const store: DurableMap = createMemoryMap(); + sandbox = make({ store }); + const a = await sandbox.provision(layers, { scratch: { key: "k1" } }); + const b = await sandbox.provision(layers, { scratch: { key: "k1" } }); + assert.equal(a.id, b.id); + assert.equal(a.scratch, true); + await sandbox.teardown(a); + assert.ok(fake.current(a.id), "still held by the second handle"); + await sandbox.teardown(b); + assert.equal(fake.current(a.id), null); + assert.equal(await store.get(scope), null); +}); + +test("the profile advertises provider-managed persistence and parks on teardown", () => { + assert.equal(sandbox.profile.backend, "do-managed-agents"); + assert.equal(sandbox.profile.writablePersistence, "provider_managed"); + assert.equal(sandbox.profile.parksOnTeardown, true); + assert.equal(supportsProcessSessions(sandbox), true); +}); + +test("the QM template's tools are advertised as installed, not missing", () => { + assert.ok(sandbox.profile.spec?.notInstalled?.includes("jq")); + const spec = make({ extraTools: QM_TEMPLATE_TOOLS }).profile.spec; + for (const tool of QM_TEMPLATE_TOOLS) { + assert.ok(spec?.tools?.includes(tool), tool); + assert.ok(!spec?.notInstalled?.includes(tool), tool); + } +}); + +test("egress enforcement is only claimed when a proxy is configured", () => { + assert.equal(sandbox.profile.egressEnforcement, "none"); + assert.equal(make({ egressProxyUrl: "https://egress.example.com" }).profile.egressEnforcement, "domain"); +}); + +test("the egress allowlist carries just the proxy host", () => { + assert.deepEqual(managedAgentsEgressAllow("https://egress.example.com:443/path"), ["egress.example.com"]); + assert.deepEqual(managedAgentsEgressAllow("http://10.0.0.7:8080"), ["10.0.0.7"]); +}); + +test("session status wire values map onto backend states", () => { + assert.equal(managedAgentsSessionState("SESSION_STATUS_READY"), "ready"); + assert.equal(managedAgentsSessionState("SESSION_STATUS_PAUSED"), "paused"); + assert.equal(managedAgentsSessionState("SESSION_STATUS_DESTROYED"), "destroyed"); + assert.equal(managedAgentsSessionState("nonsense"), "unspecified"); + assert.equal(managedAgentsSessionState(undefined), "unspecified"); +}); + +test("process sessions stream output through the data plane exec path", async () => { + assert.ok(supportsProcessSessions(sandbox)); + if (!supportsProcessSessions(sandbox)) return; + const h = await sandbox.provision(layers); + const { processId } = await sandbox.startProcess(h, "echo one; echo two"); + const { output, status } = await pollProcess(sandbox, h, processId, { deadlineMs: 5_000, waitMs: 100 }); + assert.equal(status.state, "exited"); + assert.match(output, /one/); + assert.match(output, /two/); +}); diff --git a/test/support/fake-managed-agents-service.ts b/test/support/fake-managed-agents-service.ts new file mode 100644 index 0000000000..71cb864cd8 --- /dev/null +++ b/test/support/fake-managed-agents-service.ts @@ -0,0 +1,348 @@ +import { createServer as createHttpServer, type Server as HttpServer, type ServerResponse } from "node:http"; +import { connect, type Socket } from "node:net"; +import { once } from "node:events"; +import { fileURLToPath } from "node:url"; +import { randomUUID } from "node:crypto"; +import { spawn } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { WebSocketServer, type WebSocket } from "ws"; +import * as loader from "@grpc/proto-loader"; +import * as grpc from "@grpc/grpc-js"; + +const PROTO_PATH = fileURLToPath(new URL("../../src/sandbox/managed-agents-sandbox-agent.proto", import.meta.url)); + +interface SessionRow { + session_id: string; + sandbox_id: string; + name: string; + status: string; + template: string; + created_at: string; +} + +export interface FakeManagedAgentsService { + apiBaseUrl: string; + token: string; + sessions(): SessionRow[]; + manifests(): string[]; + execScripts(): string[]; + home(): string; + checkpoints(sessionId: string): Array<{ checkpoint_id: string; label: string; status: string }>; + setStatus(name: string, status: string): void; + rejectPortForward(body: string | null): void; + failNextLifecycleCalls(count: number, status: number): void; + tunnelCount(): number; + pingCount(): number; + close(): Promise; +} + +function loadService(): grpc.ServiceDefinition { + const definition = loader.loadSync(PROTO_PATH, { + keepCase: true, + longs: String, + enums: String, + defaults: true, + oneofs: true, + }); + const pkg = grpc.loadPackageDefinition(definition) as unknown as { + do: { + teams: { + hosted_agents: { runtime: { sandbox_agent: { v1: { SandboxAgentService: grpc.ServiceClientConstructor } } } }; + }; + }; + }; + return pkg.do.teams.hosted_agents.runtime.sandbox_agent.v1.SandboxAgentService.service; +} + +export async function startFakeManagedAgentsService(): Promise { + const token = "dop_v1_test-token"; + const root = mkdtempSync(join(tmpdir(), "fake-managed-agents-guest-")); + const rows = new Map(); + const checkpoints = new Map>(); + const manifests: string[] = []; + const execScripts: string[] = []; + let tunnels = 0; + let pings = 0; + let lifecycleFailures = 0; + let lifecycleFailureStatus = 403; + + const guest = new grpc.Server(); + guest.addService(loadService(), { + Exec: (call: grpc.ServerDuplexStream, unknown>) => { + call.on("data", (frame: { start?: { argv?: string[]; workdir?: string; env?: Record } }) => { + if (!frame.start) return; + const argv = frame.start.argv ?? []; + const script = argv[argv.length - 1] ?? ""; + execScripts.push(script); + try { + const cwd = (frame.start.workdir || root).replace(/^\/home\/user/, root); + mkdirSync(cwd, { recursive: true }); + const child = spawn("sh", ["-c", script.replace(/\/home\/user/g, root)], { + cwd, + env: { ...process.env, ...frame.start.env, HOME: root }, + }); + child.stdout.on("data", (chunk: Buffer) => call.write({ stdout: chunk })); + child.stderr.on("data", (chunk: Buffer) => call.write({ stderr: chunk })); + child.on("close", (code: number | null) => { + call.write({ exit: { exit_code: code ?? -1, duration_ms: 1, hitl_rejected: false } }); + call.end(); + }); + child.on("error", (e: Error) => { + call.write({ stderr: Buffer.from(e.message) }); + call.write({ exit: { exit_code: 127, duration_ms: 0, hitl_rejected: false } }); + call.end(); + }); + } catch (e) { + call.write({ stderr: Buffer.from(e instanceof Error ? e.message : String(e)) }); + call.write({ exit: { exit_code: 127, duration_ms: 0, hitl_rejected: false } }); + call.end(); + } + }); + }, + Upload: (call: grpc.ServerReadableStream, unknown>, cb: grpc.sendUnaryData) => { + let path = ""; + const chunks: Buffer[] = []; + call.on("data", (frame: { header?: { path?: string }; chunk?: Buffer }) => { + if (frame.header?.path) path = frame.header.path; + if (frame.chunk?.length) chunks.push(frame.chunk); + }); + call.on("end", () => { + const hostPath = path.replace(/^\/home\/user/, root); + mkdirSync(dirname(hostPath), { recursive: true }); + const body = Buffer.concat(chunks); + writeFileSync(hostPath, body); + cb(null, { path, bytes_written: String(body.length) }); + }); + }, + Download: (call: grpc.ServerWritableStream<{ path?: string }, unknown>) => { + const hostPath = (call.request.path ?? "").replace(/^\/home\/user/, root); + let body: Buffer; + try { + body = readFileSync(hostPath); + } catch { + call.emit("error", { + code: grpc.status.INTERNAL, + details: `envd download: download status 404: {"code":404,"message":"path '${call.request.path ?? ""}' does not exist"}`, + }); + return; + } + call.write({ header: { size_bytes: String(body.length), is_archive: false } }); + call.write({ chunk: body }); + call.write({ end: { sha256: "" } }); + call.end(); + }, + }); + + const guestPort = await new Promise((resolve, reject) => { + guest.bindAsync("127.0.0.1:0", grpc.ServerCredentials.createInsecure(), (err, port) => + err ? reject(err) : resolve(port), + ); + }); + + const byName = (name: string): SessionRow | undefined => + [...rows.values()].find((r) => r.name === name && r.status !== "SESSION_STATUS_DESTROYED"); + + const json = (res: ServerResponse, code: number, body: unknown): void => { + res.writeHead(code, { "content-type": "application/json" }); + res.end(JSON.stringify(body)); + }; + + const http: HttpServer = createHttpServer((req, res) => { + if (req.headers.authorization !== `Bearer ${token}`) { + json(res, 401, { error: { code: "unauthorized", message: "bad token" } }); + return; + } + const url = new URL(req.url ?? "/", "http://localhost"); + const parts = url.pathname.split("/").filter(Boolean); + const id = parts[3] ?? ""; + const action = parts[4] ?? ""; + + if (req.method === "POST" && url.pathname === "/v2/agents/sessions") { + let body = ""; + req.on("data", (c: Buffer) => (body += c.toString("utf8"))); + req.on("end", () => { + manifests.push(body); + const name = /^name:\s*"?([^"\n]+)"?/m.exec(body)?.[1] ?? "unnamed"; + const row: SessionRow = { + session_id: randomUUID(), + sandbox_id: `sbx-${rows.size + 1}`, + name, + status: "SESSION_STATUS_READY", + template: /^template:\s*"?([^"\n]+)"?/m.exec(body)?.[1] ?? "", + created_at: new Date().toISOString(), + }; + rows.set(row.session_id, row); + json(res, 201, { session: row }); + }); + return; + } + if (req.method === "GET" && url.pathname === "/v2/agents/sessions") { + const name = url.searchParams.get("name"); + const all = [...rows.values()].filter((r) => !name || r.name === name); + json(res, 200, { sessions: all, next_page_token: "" }); + return; + } + if (lifecycleFailures > 0) { + lifecycleFailures -= 1; + json(res, lifecycleFailureStatus, { error: { code: "forbidden", message: "transient" } }); + return; + } + if (req.method === "GET" && parts.length === 4) { + const row = rows.get(id); + if (!row) { + json(res, 404, { error: { code: "not_found", message: "no such session" } }); + return; + } + json(res, 200, { session: row }); + return; + } + if (req.method === "DELETE" && parts.length === 4) { + const row = rows.get(id); + if (row) row.status = "SESSION_STATUS_DESTROYED"; + res.writeHead(204).end(); + return; + } + if (req.method === "POST" && action === "checkpoints" && parts.length === 5) { + let body = ""; + req.on("data", (c: Buffer) => (body += c.toString("utf8"))); + req.on("end", () => { + const row = rows.get(id); + if (!row || row.status === "SESSION_STATUS_DESTROYED") { + json(res, 404, { error: { code: "not_found", message: "no such session" } }); + return; + } + let label: string; + try { + label = (JSON.parse(body) as { label?: string }).label ?? ""; + } catch { + label = ""; + } + const checkpoint = { + checkpoint_id: `cp_${randomUUID().slice(0, 8)}`, + session_id: row.session_id, + status: "READY", + kind: "explicit", + label, + created_at: new Date().toISOString(), + size_bytes: 1, + }; + const list = checkpoints.get(row.session_id) ?? []; + list.unshift(checkpoint); + checkpoints.set(row.session_id, list); + json(res, 200, { checkpoint }); + }); + return; + } + if (req.method === "POST" && action === "checkpoints" && parts[6] === "rollback" && parts.length === 7) { + const row = rows.get(id); + const checkpointId = decodeURIComponent(parts[5] ?? ""); + const list = checkpoints.get(id) ?? []; + if (!row || row.status === "SESSION_STATUS_DESTROYED" || !list.some((c) => c.checkpoint_id === checkpointId)) { + json(res, 404, { error: { code: "not_found", message: "no such checkpoint" } }); + return; + } + row.sandbox_id = `sbx-restored-${randomUUID().slice(0, 8)}`; + row.status = "SESSION_STATUS_READY"; + json(res, 200, { session: row }); + return; + } + if (req.method === "DELETE" && action === "checkpoints" && parts.length === 6) { + const checkpointId = decodeURIComponent(parts[5] ?? ""); + const list = checkpoints.get(id) ?? []; + checkpoints.set( + id, + list.filter((c) => c.checkpoint_id !== checkpointId), + ); + json(res, 200, {}); + return; + } + if (req.method === "POST" && (action === "pause" || action === "resume")) { + const row = rows.get(id); + if (!row) { + json(res, 404, { error: { code: "not_found", message: "no such session" } }); + return; + } + row.status = action === "pause" ? "SESSION_STATUS_PAUSED" : "SESSION_STATUS_READY"; + res.writeHead(204).end(); + return; + } + json(res, 404, { error: { code: "not_found", message: url.pathname } }); + }); + + const wss = new WebSocketServer({ noServer: true }); + let portForwardRejection: string | null = null; + http.on("upgrade", (req, socket, head) => { + const url = new URL(req.url ?? "/", "http://localhost"); + const match = /^\/v2\/agents\/sessions\/([^/]+)\/port-forward\/(\d+)$/.exec(url.pathname); + if (req.headers.authorization !== `Bearer ${token}` || !match) { + socket.destroy(); + return; + } + if (portForwardRejection !== null) { + socket.end( + `HTTP/1.1 403 Forbidden\r\nContent-Length: ${Buffer.byteLength(portForwardRejection)}\r\nConnection: close\r\n\r\n${portForwardRejection}`, + ); + return; + } + const row = rows.get(match[1] ?? ""); + if (!row || row.status === "SESSION_STATUS_DESTROYED") { + socket.destroy(); + return; + } + wss.handleUpgrade(req, socket, head, (ws: WebSocket) => { + tunnels += 1; + ws.on("ping", () => { + pings += 1; + }); + const upstream: Socket = connect(guestPort, "127.0.0.1"); + ws.on("message", (data: Buffer) => upstream.write(data)); + upstream.on("data", (chunk: Buffer) => ws.send(chunk)); + const shutdown = (): void => { + upstream.destroy(); + if (ws.readyState === ws.OPEN) ws.close(); + }; + ws.on("close", shutdown); + ws.on("error", shutdown); + upstream.on("close", shutdown); + upstream.on("error", shutdown); + }); + }); + + http.listen(0, "127.0.0.1"); + await once(http, "listening"); + const address = http.address(); + if (address === null || typeof address === "string") throw new Error("fake managed agents service failed to bind"); + + return { + apiBaseUrl: `http://127.0.0.1:${address.port}`, + token, + sessions: () => [...rows.values()], + manifests: () => [...manifests], + execScripts: () => [...execScripts], + home: () => root, + checkpoints: (sessionId) => [...(checkpoints.get(sessionId) ?? [])], + setStatus: (name, status) => { + const row = byName(name); + if (row) row.status = status; + }, + rejectPortForward: (body) => { + portForwardRejection = body; + }, + failNextLifecycleCalls: (count, status) => { + lifecycleFailures = count; + lifecycleFailureStatus = status; + }, + tunnelCount: () => tunnels, + pingCount: () => pings, + close: async () => { + for (const ws of wss.clients) ws.terminate(); + wss.close(); + http.closeAllConnections(); + await new Promise((resolve) => http.close(() => resolve())); + guest.forceShutdown(); + rmSync(root, { recursive: true, force: true }); + }, + }; +} diff --git a/test/support/fake-managed-agents.ts b/test/support/fake-managed-agents.ts new file mode 100644 index 0000000000..39754201da --- /dev/null +++ b/test/support/fake-managed-agents.ts @@ -0,0 +1,289 @@ +import { spawnSync } from "node:child_process"; +import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { + ManagedAgentsCommandLostError, + ManagedAgentsHitlRejectedError, + ManagedAgentsSandboxGoneError, + type ManagedAgentsClient, + type ManagedAgentsCommandResult, + type ManagedAgentsCheckpoint, + type ManagedAgentsSession, + type ManagedAgentsSessionInfo, + type ManagedAgentsSessionState, + type ManagedAgentsSessionSummary, +} from "../../src/sandbox/managed-agents-client.ts"; +import { mkdtempSync } from "node:fs"; + +const GUEST_HOME_DIR = "/workspace/home"; + +interface FakeRecord { + sessionId: string; + sandboxId: string; + name: string; + state: ManagedAgentsSessionState; + home: string; + createdAt: number; + loseNextCommand: boolean; + rejectNextCommand: boolean; + failNextCheckpoint: boolean; + failNextCheckpointDelete: boolean; + failNextRollback: boolean; +} + +export interface FakeManagedAgents { + client: ManagedAgentsClient; + current(name: string): { sessionId: string; sandboxId: string; state: ManagedAgentsSessionState } | null; + createdCount(name: string): number; + homeDir(name: string): string; + pause(name: string): void; + destroy(name: string): void; + loseNextCommand(name: string): void; + rejectNextCommand(name: string): void; + execScripts(): string[]; + pauseCalls(): string[]; + resumeCalls(): string[]; + guestDisconnects(): string[]; + checkpointCalls(): Array<{ sessionId: string; checkpointId: string; label?: string }>; + deletedCheckpoints(): string[]; + rollbackCalls(): Array<{ sessionId: string; checkpointId: string }>; + failNextCheckpoint(name: string): void; + failNextCheckpointDelete(name: string): void; + failNextRollback(name: string): void; + cleanup(): void; +} + +export function installFakeManagedAgents(): FakeManagedAgents { + const root = mkdtempSync(join(tmpdir(), "fake-managed-agents-")); + const records = new Map(); + const execScripts: string[] = []; + const pauseCalls: string[] = []; + const resumeCalls: string[] = []; + const guestDisconnects: string[] = []; + const checkpointCalls: Array<{ sessionId: string; checkpointId: string; label?: string }> = []; + const deletedCheckpoints: string[] = []; + const rollbackCalls: Array<{ sessionId: string; checkpointId: string }> = []; + let nextId = 1; + let nextSandbox = 1; + let nextCheckpoint = 1; + let clock = 0; + + const gone = (state: ManagedAgentsSessionState): boolean => + state === "destroyed" || state === "destroying" || state === "failed"; + + const byName = (name: string): FakeRecord | undefined => { + const all = [...records.values()].filter((r) => r.name === name).sort((a, b) => b.createdAt - a.createdAt); + return all.find((r) => !gone(r.state)) ?? all[0]; + }; + + const need = (name: string): FakeRecord => { + const r = byName(name); + if (!r) throw new Error(`fake-managed-agents: no session named ${name}`); + return r; + }; + + const remap = (r: FakeRecord, script: string): string => { + const homeRe = r.home.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const remapPath = new RegExp(`${homeRe}/tmp/|${homeRe}(?![A-Za-z0-9._-])|/tmp/`, "g"); + return ( + `export HOME=${JSON.stringify(r.home)}; ` + + script + .replace(/\btimeout \d+ /g, "") + .replaceAll(GUEST_HOME_DIR, r.home) + .replace(remapPath, (mm) => (mm.startsWith(r.home) ? mm : `${r.home}/tmp/`)) + ); + }; + + const alive = (r: FakeRecord): void => { + if (gone(r.state)) throw new ManagedAgentsSandboxGoneError(r.sessionId, `status ${r.state}`); + if (r.state === "paused") r.state = "ready"; + }; + + const info = (r: FakeRecord): ManagedAgentsSessionInfo => ({ + sessionId: r.sessionId, + sandboxId: r.sandboxId, + name: r.name, + state: r.state, + createdAtMs: r.createdAt, + }); + + const session = (r: FakeRecord): ManagedAgentsSession => ({ + sessionId: r.sessionId, + sandboxId: r.sandboxId, + async runCommand(command): Promise { + alive(r); + if (r.loseNextCommand) { + r.loseNextCommand = false; + r.state = "destroyed"; + throw new ManagedAgentsCommandLostError(r.sessionId, "sandbox is not running anymore"); + } + if (r.rejectNextCommand) { + r.rejectNextCommand = false; + throw new ManagedAgentsHitlRejectedError(r.sessionId); + } + execScripts.push(command); + mkdirSync(join(r.home, "tmp"), { recursive: true }); + const spawned = spawnSync("sh", ["-c", remap(r, command)], { + encoding: "buffer", + maxBuffer: 128 * 1024 * 1024, + env: { ...process.env, COPYFILE_DISABLE: "1" }, + }); + return { + stdout: (spawned.stdout ?? Buffer.alloc(0)).toString("utf8"), + stderr: (spawned.stderr ?? Buffer.alloc(0)).toString("utf8"), + exitCode: spawned.status ?? (spawned.signal ? 137 : -1), + }; + }, + async readFileBytes(absPath): Promise { + alive(r); + const hostPath = absPath.startsWith(GUEST_HOME_DIR) ? r.home + absPath.slice(GUEST_HOME_DIR.length) : absPath; + if (!existsSync(hostPath)) return null; + return new Uint8Array(readFileSync(hostPath)); + }, + async writeFileBytes(absPath, data): Promise { + alive(r); + const hostPath = absPath.startsWith(GUEST_HOME_DIR) ? r.home + absPath.slice(GUEST_HOME_DIR.length) : absPath; + mkdirSync(dirname(hostPath), { recursive: true }); + writeFileSync(hostPath, Buffer.from(data)); + }, + async createCheckpoint(label?: string): Promise { + if (gone(r.state)) throw new ManagedAgentsSandboxGoneError(r.sessionId, `status ${r.state}`); + if (r.failNextCheckpoint) { + r.failNextCheckpoint = false; + throw new Error("do-managed-agents create checkpoint: http 500 capture failed"); + } + const checkpointId = `cp-${nextCheckpoint++}`; + checkpointCalls.push({ sessionId: r.sessionId, checkpointId, ...(label ? { label } : {}) }); + return { checkpointId, status: "READY", createdAtMs: Date.now() }; + }, + async deleteCheckpoint(checkpointId: string): Promise { + if (gone(r.state)) throw new ManagedAgentsSandboxGoneError(r.sessionId, `status ${r.state}`); + if (r.failNextCheckpointDelete) { + r.failNextCheckpointDelete = false; + throw new Error("do-managed-agents delete checkpoint: http 500 delete failed"); + } + deletedCheckpoints.push(checkpointId); + }, + async close(): Promise { + guestDisconnects.push(r.sessionId); + }, + async pause(): Promise { + if (gone(r.state)) throw new ManagedAgentsSandboxGoneError(r.sessionId, `status ${r.state}`); + guestDisconnects.push(r.sessionId); + pauseCalls.push(r.sessionId); + r.state = "paused"; + }, + async resume(): Promise { + if (gone(r.state)) throw new ManagedAgentsSandboxGoneError(r.sessionId, `status ${r.state}`); + resumeCalls.push(r.sessionId); + r.state = "ready"; + }, + async kill(): Promise { + guestDisconnects.push(r.sessionId); + r.state = "destroyed"; + rmSync(r.home, { recursive: true, force: true }); + }, + }); + + const client: ManagedAgentsClient = { + nativePause: true, + async info(sessionId): Promise { + const r = records.get(sessionId); + if (!r || gone(r.state)) throw new ManagedAgentsSandboxGoneError(sessionId, "session was not found"); + return info(r); + }, + async create(opts): Promise { + const id = `sess-${nextId++}`; + const r: FakeRecord = { + sessionId: id, + sandboxId: `sbx-${id}`, + name: opts.name, + state: "ready", + home: join(root, id), + createdAt: ++clock, + loseNextCommand: false, + rejectNextCommand: false, + failNextCheckpoint: false, + failNextCheckpointDelete: false, + failNextRollback: false, + }; + mkdirSync(r.home, { recursive: true }); + records.set(id, r); + return session(r); + }, + async connect(sessionId): Promise { + const r = records.get(sessionId); + if (!r || gone(r.state)) throw new ManagedAgentsSandboxGoneError(sessionId, "session was not found"); + if (r.state === "paused") { + resumeCalls.push(r.sessionId); + r.state = "ready"; + } + return session(r); + }, + async list(name): Promise { + return [...records.values()] + .filter((r) => !gone(r.state) && r.name === name) + .map((r) => ({ sessionId: r.sessionId, sandboxId: r.sandboxId, name: r.name, state: r.state })); + }, + async rollback(sessionId, checkpointId): Promise { + const r = records.get(sessionId); + if (!r || gone(r.state)) throw new ManagedAgentsSandboxGoneError(sessionId, "session was not found"); + if (r.failNextRollback) { + r.failNextRollback = false; + throw new Error("do-managed-agents rollback: http 409 checkpoint is not ready"); + } + rollbackCalls.push({ sessionId, checkpointId }); + r.sandboxId = `sbx-restored-${nextSandbox++}`; + r.state = "ready"; + return info(r); + }, + async kill(sessionId): Promise { + const r = records.get(sessionId); + if (!r) return; + r.state = "destroyed"; + rmSync(r.home, { recursive: true, force: true }); + }, + }; + + return { + client, + current: (name) => { + const r = byName(name); + return r && !gone(r.state) ? { sessionId: r.sessionId, sandboxId: r.sandboxId, state: r.state } : null; + }, + createdCount: (name) => [...records.values()].filter((r) => r.name === name).length, + homeDir: (name) => need(name).home, + pause: (name) => { + need(name).state = "paused"; + }, + destroy: (name) => { + const r = need(name); + r.state = "destroyed"; + rmSync(r.home, { recursive: true, force: true }); + }, + loseNextCommand: (name) => { + need(name).loseNextCommand = true; + }, + rejectNextCommand: (name) => { + need(name).rejectNextCommand = true; + }, + checkpointCalls: () => [...checkpointCalls], + deletedCheckpoints: () => [...deletedCheckpoints], + failNextCheckpoint: (name) => { + need(name).failNextCheckpoint = true; + }, + failNextCheckpointDelete: (name) => { + need(name).failNextCheckpointDelete = true; + }, + rollbackCalls: () => [...rollbackCalls], + failNextRollback: (name) => { + need(name).failNextRollback = true; + }, + execScripts: () => [...execScripts], + pauseCalls: () => [...pauseCalls], + resumeCalls: () => [...resumeCalls], + guestDisconnects: () => [...guestDisconnects], + cleanup: () => rmSync(root, { recursive: true, force: true }), + }; +}