diff --git a/.changeset/add-sandbox-extensions.md b/.changeset/add-sandbox-extensions.md index 70ab1c68d..960068532 100644 --- a/.changeset/add-sandbox-extensions.md +++ b/.changeset/add-sandbox-extensions.md @@ -2,4 +2,22 @@ '@cloudflare/sandbox': minor --- -Add the experimental `@cloudflare/sandbox/extensions` framework for attaching opt-in SDK extensions and lazily started container sidecars to a Sandbox subclass. Sidecars are distributed as npm-style `.tgz` packages: the SDK ships the bytes, the container provisions by content hash, derives identity from the embedded `package.json`, and `bun add`s the package. Host ↔ sidecar IPC runs over capnweb on a unix socket, so sidecar methods are a typed remote stub via `await this.sidecar()` — streaming is just a typed callback parameter. Sidecar authors get a `@cloudflare/sandbox/sidecar` helper (`SandboxSidecar` + `serveSandboxSidecar`). npm distribution of third-party extensions is not yet wired up; the wire shape is the one a future authoring story will use. +Add the experimental `@cloudflare/sandbox/extensions` helpers for optional higher-level APIs on a Sandbox subclass. An extension can ship its own helper program as a `.tgz` sidecar that starts on first use, so features like the code interpreter stay out of the core SDK. + +Attach a shipped extension and call it directly: + +```ts +import { Sandbox as BaseSandbox } from '@cloudflare/sandbox'; +import { withInterpreter } from '@cloudflare/sandbox/interpreter'; + +export class Sandbox extends BaseSandbox { + interpreter = withInterpreter(this); +} + +const context = await sandbox.interpreter.createCodeContext({ + language: 'python' +}); +const result = await sandbox.interpreter.runCode('print("hello")', { context }); +``` + +To write your own, extend `SandboxExtension` and export a `withYourExtension(sandbox)` helper. Sidecar-backed extensions call their methods through `this.withSidecar(...)`. This is experimental; publishing third-party extensions on npm is not set up yet. diff --git a/.changeset/establish-runtime-incarnations.md b/.changeset/establish-runtime-incarnations.md new file mode 100644 index 000000000..81e9ec499 --- /dev/null +++ b/.changeset/establish-runtime-incarnations.md @@ -0,0 +1,5 @@ +--- +'@cloudflare/sandbox': minor +--- + +Keep each sandbox operation on the container that started it. If that container is replaced after sleep, eviction, or a crash, in-flight work and old process or terminal handles fail instead of continuing against the new container. Create or look up handles again after the new container is running. Extension authors should run runtime work inside `withRuntime()` and `withSidecar()`. diff --git a/.changeset/fix-execution-defaults.md b/.changeset/fix-execution-defaults.md new file mode 100644 index 000000000..63bf8709c --- /dev/null +++ b/.changeset/fix-execution-defaults.md @@ -0,0 +1,5 @@ +--- +'@cloudflare/sandbox': patch +--- + +Apply sandbox environment variables and the default `/workspace` directory on every process launch again. Stopping a warm-pool sandbox now finishes container teardown before the slot is reused. diff --git a/.changeset/stabilize-sandbox-proxies.md b/.changeset/stabilize-sandbox-proxies.md new file mode 100644 index 000000000..d3fdf3cd8 --- /dev/null +++ b/.changeset/stabilize-sandbox-proxies.md @@ -0,0 +1,5 @@ +--- +'@cloudflare/sandbox': patch +--- + +Keep filesystem watch and terminal output streams open across Worker to Durable Object calls. HTTPS preview and proxy requests now reach the container over HTTP as expected. Sandbox setup finishes before the first forwarded request runs, repeated restores reapply the chosen backup, and idle expiry stops the container cleanly. diff --git a/.github/scripts/wait-for-container-rollout.sh b/.github/scripts/wait-for-container-rollout.sh index 06f029edf..76c4e0b98 100755 --- a/.github/scripts/wait-for-container-rollout.sh +++ b/.github/scripts/wait-for-container-rollout.sh @@ -23,6 +23,7 @@ expected_image() { local worker=$1 image_tag=$2 app_name=$3 image=sandbox case "$app_name" in "$worker") ;; + "$worker-browser") ;; "$worker-python") image=sandbox-python ;; "$worker-opencode") image=sandbox-opencode ;; "$worker-standalone") image=sandbox-standalone ;; @@ -66,6 +67,11 @@ if [[ ${1:-} == --evaluate ]]; then exit 0 fi +if [[ ${1:-} == --expected-image ]]; then + expected_image "$2" "$3" "$4" + exit 0 +fi + worker=${1:?worker name required} image_tag=${2:?image tag required} timeout_seconds=${ROLLOUT_TIMEOUT_SECONDS:-600} @@ -73,11 +79,21 @@ drain_grace_seconds=${ROLLOUT_DRAIN_GRACE_SECONDS:-180} poll_seconds=${ROLLOUT_POLL_SECONDS:-10} deadline=$((SECONDS + timeout_seconds)) drain_deadline=$((SECONDS + drain_grace_seconds)) -app_names=("$worker" "$worker-python" "$worker-opencode" "$worker-standalone" "$worker-musl") +app_names=("$worker" "$worker-browser" "$worker-python" "$worker-opencode" "$worker-standalone" "$worker-musl") + +if command -v wrangler >/dev/null 2>&1; then + wrangler_command=(wrangler) +elif command -v npx >/dev/null 2>&1; then + echo 'Wrangler is not installed globally; running it through npx' + wrangler_command=(npx --yes wrangler@latest) +else + echo '::error::Neither wrangler nor npx is available' >&2 + exit 1 +fi echo "Waiting for container applications to serve image tag $image_tag" while ((SECONDS < deadline)); do - apps=$(wrangler containers list --json) + apps=$("${wrangler_command[@]}" containers list --json) all_ready=true for app_name in "${app_names[@]}"; do @@ -89,9 +105,9 @@ while ((SECONDS < deadline)); do fi app_id=$(jq -r '.id' <<<"$app") - app=$(wrangler containers info "$app_id") + app=$("${wrangler_command[@]}" containers info "$app_id") expected=$(expected_image "$worker" "$image_tag" "$app_name") - instances=$(wrangler containers instances "$app_id" --json) + instances=$("${wrangler_command[@]}" containers instances "$app_id" --json) reasons=$(readiness_reasons "$app" "$instances" "$expected") if [[ -z $reasons ]]; then diff --git a/.github/scripts/wait-for-container-rollout.test.sh b/.github/scripts/wait-for-container-rollout.test.sh index d3f9af89e..b68310a77 100755 --- a/.github/scripts/wait-for-container-rollout.test.sh +++ b/.github/scripts/wait-for-container-rollout.test.sh @@ -18,6 +18,70 @@ assert_output() { echo "PASS $name" } +assert_expected_image() { + local name=$1 worker=$2 image_tag=$3 app_name=$4 expected_image=$5 actual + actual=$(CLOUDFLARE_ACCOUNT_ID=account \ + "$script" --expected-image "$worker" "$image_tag" "$app_name") + if [[ $actual != "$expected_image" ]]; then + printf 'FAIL %s\nexpected: %q\nactual: %q\n' \ + "$name" "$expected_image" "$actual" >&2 + exit 1 + fi + echo "PASS $name" +} + +assert_expected_image browser-image worker ci-expected worker-browser \ + 'registry.cloudflare.com/account/sandbox:ci-expected' + +mkdir -p "$tmp/bin" +cat >"$tmp/bin/npx" <<'SH' +#!/usr/bin/env bash +printf '%s\n' "$*" >>"$WRANGLER_INVOCATIONS" +shift 2 +case "$1 $2" in + 'containers list') + cat <<'JSON' +[ + {"id":"default","name":"worker"}, + {"id":"browser","name":"worker-browser"}, + {"id":"python","name":"worker-python"}, + {"id":"opencode","name":"worker-opencode"}, + {"id":"standalone","name":"worker-standalone"}, + {"id":"musl","name":"worker-musl"} +] +JSON + ;; + 'containers info') + case "$3" in + default|browser) image=sandbox ;; + *) image="sandbox-$3" ;; + esac + printf '{"version":1,"configuration":{"image":"registry.cloudflare.com/account/%s:ci-expected"},"health":{"errors":[],"instances":{}}}\n' "$image" + ;; + 'containers instances') printf '[]\n' ;; + *) exit 1 ;; +esac +SH +chmod +x "$tmp/bin/npx" + +invocations="$tmp/wrangler-invocations" +fallback_output=$(PATH="$tmp/bin:/usr/bin:/bin" \ + WRANGLER_INVOCATIONS="$invocations" \ + CLOUDFLARE_ACCOUNT_ID=account \ + CLOUDFLARE_API_TOKEN=token \ + ROLLOUT_TIMEOUT_SECONDS=5 \ + "$script" worker ci-expected) +if [[ $fallback_output != *'All container applications are ready'* ]]; then + printf 'FAIL npx-fallback\n%s\n' "$fallback_output" >&2 + exit 1 +fi +if ! grep -qx -- '--yes wrangler@latest containers list --json' "$invocations"; then + printf 'FAIL npx-fallback invocation\n' >&2 + cat "$invocations" >&2 + exit 1 +fi +echo 'PASS npx-fallback' + cat >"$tmp/ready.json" <<'JSON' {"version":2,"configuration":{"image":"registry.cloudflare.com/account/sandbox:ci-expected"},"health":{"errors":[],"instances":{"starting":0,"scheduling":0,"failed":0}}} JSON diff --git a/.github/workflows/reusable-e2e.yml b/.github/workflows/reusable-e2e.yml index 19265d06c..062284df0 100644 --- a/.github/workflows/reusable-e2e.yml +++ b/.github/workflows/reusable-e2e.yml @@ -93,7 +93,7 @@ jobs: if: ${{ steps.deploy-check.outputs.skip != 'true' }} run: | SAFE_BUDGET=84 - EXPECTED_PR_APPS=5 + EXPECTED_PR_APPS=6 CONTAINERS=$(wrangler containers list --json 2>/dev/null) if [ $? -ne 0 ] || [ -z "$CONTAINERS" ]; then @@ -326,7 +326,7 @@ jobs: } pids=() - for type in default python opencode standalone musl; do + for type in default browser python opencode standalone musl; do smoke_type "$type" & pids+=("$!") done diff --git a/bridge/worker/package.json b/bridge/worker/package.json index 1bacf756f..b9ed4ee38 100644 --- a/bridge/worker/package.json +++ b/bridge/worker/package.json @@ -17,8 +17,8 @@ "devDependencies": { "@biomejs/biome": "2.3.7", "@cloudflare/sandbox": "*", - "@cloudflare/vitest-pool-workers": "^0.16.20", - "@cloudflare/workers-types": "^4.20251126.0", + "@cloudflare/vitest-pool-workers": "^0.18.7", + "@cloudflare/workers-types": "^5.20260721.1", "@types/node": "^24.10.1", "hono": "^4.12.26", "typescript": "^5.9.3", diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index d5bfdf11a..a02419b85 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -28,6 +28,10 @@ See [PROCESS_EXECUTION.md](./PROCESS_EXECUTION.md). `createTerminal()` is the single PTY primitive for interactive shells. A terminal has its own ID, cursor-retained output, input, resize, interrupt, terminate, and reconnect path via `getTerminal(id)`. These terminal controls are intentionally separate: use terminals for interactive PTY state and `exec()` for supervised argv processes and numeric signals. +## Runtime authority + +The Durable Object owns sandbox-lifetime state, while each live container runtime owns runtime-local truth. Before runtime RPCs, the SDK establishes or observes the control process, validates its runtime incarnation, activates one scoped control session, and admits the semantic operation through a runtime lease. Waking operations choose to start at the operation boundary; non-waking discovery and cleanup observe only an existing exact runtime and never create a replacement. Runtime authority, control domains, extension sidecars, streams, and WebSockets are callback- or transport-scoped and are not replayed after interruption. + ## Active resources The current runtime owns active process and terminal leases. Active resources pin the live Sandbox independent of the Worker request that launched them. Durable Object storage records durable sandbox configuration such as preview ports and mounts, not process or terminal truth. diff --git a/docs/ERROR_HANDLING.md b/docs/ERROR_HANDLING.md index 4129293e2..2b943653a 100644 --- a/docs/ERROR_HANDLING.md +++ b/docs/ERROR_HANDLING.md @@ -1,20 +1,31 @@ -# Error Handling & Retry Behavior +# Error Handling & Runtime Admission ## HTTP Status Code Semantics -The SDK uses proper HTTP status codes for container startup errors: +The SDK uses HTTP status codes to distinguish startup/admission failures from +already-admitted runtime work: -| Status | Meaning | SDK Behavior | -| ------- | ------------------------------------------------ | ------------------------------ | -| **503** | Container unavailable before operation admission | Retry with exponential backoff | -| **500** | Permanent (config error, missing image) | Fail immediately | -| **400** | Client error (capacity limits, validation) | Fail immediately | +| Status | Meaning | SDK behavior | +| ------- | ------------------------------------------------ | -------------------------------- | +| **503** | Container unavailable before operation admission | Caller may retry a new operation | +| **500** | Permanent startup/configuration error | Fail immediately | +| **400** | Client error (capacity limits, validation) | Fail immediately | -## Retry Logic +## Runtime Admission, Not Operation Replay -- **Total budget**: 2 minutes (configurable per Sandbox via `containerTimeouts`). -- **Backoff**: 3s → 6s → 12s → 24s → 30s (capped at 30s). -- **Only retries**: 503 Service Unavailable. +Before runtime RPCs, the SDK establishes or observes the current container +runtime, probes its control-process metadata, validates the runtime +incarnation, and activates an exact control session. A semantic operation then +runs inside one admitted runtime lease. Waking operations choose to start the +runtime at that boundary; non-waking discovery and cleanup only use an already +active exact runtime and never create a replacement. + +The transport is single-attempt for each activated session. If a runtime changes +or the control WebSocket is lost after admission, the SDK surfaces +`OperationInterruptedError`, `RPCTransportError`, or the domain-specific +runtime error. It does not replay ambiguous side-effecting work. Applications +that want recovery should start a new semantic operation after observing the +failure. ## Container Unavailable @@ -32,115 +43,86 @@ an operation was admitted. For example, if a command starts and the WebSocket later closes, the SDK surfaces an execution error or `RPCTransportError` because the operation may have already produced side effects. -### Retry surface - -The SDK has one startup retry surface: `ContainerControlConnection.fetchUpgradeWithRetry()` retries 503 responses on the `/rpc` WebSocket upgrade fetch during cold start. The retry budget comes from `Sandbox.computeRetryTimeoutMs()` and is pushed into the active `ContainerControlClient` when container timeouts change. - -`containerFetch()` cannot be used for the WebSocket upgrade itself. The control connection calls `stub.fetch()` directly and uses 503 responses as the readiness signal. `Sandbox.fetch()` delegates WebSocket upgrade requests to `super.fetch()`, while non-upgrade container startup and port readiness still flow through `containerFetch()` and `startAndWaitForPorts()`. - -Production-only instance allocation failures surface as 503 while workerd allocates a container VM. The `/rpc` upgrade retry loop continues until the retry budget is exhausted. - ## Container Boot Lifecycle -When a request triggers a cold start, the container goes through five distinct phases. -Errors are surfaced as 503 (retried) for any phase that can self-heal, and 500 (immediate -fail) for phases where retrying makes no difference. +When a request triggers a cold start, the container goes through four relevant +phases. Errors are surfaced as 503 for phases that can self-heal, and 500 for +phases where retrying the same deployment makes no difference. ```text -[1] Instance allocation ──┐ -[2] Container start ──┤ startAndWaitForPorts() owns 1–3 -[3] Port readiness ──┘ driven by containerFetch() / Sandbox.fetch() -[4] onStart hook ── runs inside blockConcurrencyWhile -[5] Request proxying ── tcpPort.fetch() forwards the original request +[1] Instance allocation ──┐ +[2] Control port start ──┤ runtime lifecycle establishment owns 1–3 +[3] Control session activate ──┘ exact runtime incarnation is validated +[4] Admitted forwarding/RPC ── admitted TCP-port fetch or activated RPC ``` ### [1] Instance allocation **Budget:** `containerTimeouts.instanceGetTimeoutMS` (default 30 s). -**What happens:** workerd's container scheduler tries to assign a VM to the DO. On a -cold start or right after eviction there may not be one ready yet. +**What happens:** workerd's container scheduler tries to assign a VM to the DO. +On a cold start or right after eviction there may not be one ready yet. -| Failure | Surfaced as | SDK behavior | -| ----------------------- | ----------- | -------------------------------------- | -| `no container instance` | 503 | **Retry** (cold-start race) | -| `SURPASSED_*_LIMITS` | 400 | Fail immediately (account-level limit) | +| Failure | Surfaced as | SDK behavior | +| ----------------------- | ----------- | -------------------------------- | +| `no container instance` | 503 | Caller may retry a new operation | +| `SURPASSED_*_LIMITS` | 400 | Fail immediately (account limit) | -Production-only: `wrangler dev` always has an instance ready, so phase 1 never returns -503 locally. +Production-only: `wrangler dev` always has an instance ready, so phase 1 never +returns 503 locally. -### [2] Container start +### [2] Control port start -**What happens:** `containerStart` boots the configured Docker image with the requested -env, entrypoint, and outbound config. +**What happens:** the runtime lifecycle starts the configured container and +waits for the control port to be reachable. Image and configuration errors are +not retryable by the SDK because they will keep failing until the deployment is +fixed. | Failure | Surfaced as | SDK behavior | | -------------------------------- | ----------- | -------------------------------------------------------- | | `No such image available` | 500 | Fail — misconfigured `wrangler.jsonc` or registry mirror | | `Container already exists` | 500 | Fail — name collision in DO state | -| `Container exited before health` | 500 | Fail — image entrypoint crashed before phase 3 | - -Image / config errors are not retryable; they will keep failing until the deployment is -fixed. - -### [3] Port readiness - -**Budget:** `containerTimeouts.portReadyTimeoutMS` (default 90 s). -**What happens:** `waitForPort()` polls TCP on the requested port (default 3000) every -500 ms until it accepts a connection. - -| Failure | Surfaced as | SDK behavior | -| ---------------------------------- | ----------- | ----------------------------------------------------------------- | -| `the container is not listening` | 503 | **Retry** — app still starting up | -| `failed to verify port` | 503 | **Retry** — health check timeout | -| `container port not found` | 503 | **Retry** — workerd hasn't picked up the Docker port mapping yet | -| `Monitor failed to find container` | 503 | **Retry** — monitor restarted between provisioning and port check | -| Total wait > `portReadyTimeoutMS` | 503 | **Retry** until the SDK's overall budget is exhausted | - -Slow-booting images (large dependencies, JIT warm-up, restoring snapshots) most often -trip up here. Increase `portReadyTimeoutMS` rather than `instanceGetTimeoutMS` for slow -containers — they govern different phases. - -### [4] onStart hook - -**What happens:** Once the port is up, `@cloudflare/containers` calls -`this.state.setHealthy()` and then `await this.onStart()` inside -`blockConcurrencyWhile`. `Sandbox.onStart()` rehydrates exposed-port tokens, restores -syncs, and initializes runtime-local control resources. - -| Failure | Surfaced as | SDK behavior | -| ------------------ | ----------- | ------------------------------------------------------------------------------- | -| `onStart()` throws | bubble | The DO gate stays held; later requests see the same exception until it succeeds | - -Anything `onStart` does that re-enters the DO via `stub.fetch()` will deadlock against -its own `blockConcurrencyWhile`. The SDK avoids this by talking to the container over -`stub.fetch()` directly (which routes via `containerFetch()` for non-WS calls and -`super.fetch()` for the WS upgrade), never via the DO's own `fetch()` handler. - -### [5] Request proxying - -**What happens:** `tcpPort.fetch(containerUrl, request)` forwards the original request -to the container. For WebSocket upgrades, the response carries a `webSocket` property -that gets `accept()`ed and handed to capnweb. - -| Failure | Surfaced as | SDK behavior | -| ----------------------------- | ----------- | -------------------------------------------------------------------------------------------------- | -| `network connection lost` | 503 | **Retry** — socket dropped mid-request | -| WebSocket close 1000 + reason | bubble | Surfaced as `RPCTransportError(peer_closed)`; not retried (call may have already had side effects) | -| Container-side handler error | passthrough | The container returns a typed error, the SDK maps it to a `SandboxError` subclass | - -### Where the retries live, by phase - -| Phase | Retried by | -| ------------------ | --------------------------------------------------------------------------- | -| 1 Instance alloc | `/rpc` upgrade retry loop when opening the control channel | -| 2 Container start | None — 500 is permanent | -| 3 Port readiness | `/rpc` upgrade retry loop when opening the control channel | -| 4 onStart | None — the exception bubbles | -| 5 Request proxying | The control connection is reopened on the next call after a transport error | +| `Container exited before health` | 500 | Fail — image entrypoint crashed before readiness | +| Control port not ready | 503 | Caller may retry a new operation | + +### [3] Control session activation + +**What happens:** the SDK probes runtime metadata, validates the runtime +incarnation, and activates the capnweb control session for that exact +incarnation. Domain RPCs are rejected before activation. A replacement runtime +must perform a new probe and activation; old sessions are interrupted rather +than reused. + +| Failure | Surfaced as | SDK behavior | +| ------------------------------- | ---------------------------- | --------------------------------- | +| Missing/invalid metadata | Runtime control protocol err | Fail current operation | +| Incarnation mismatch | Operation interrupted | Do not replay ambiguous operation | +| WebSocket upgrade/transport err | Transport/runtime error | Surface failure, no hidden retry | + +### [4] Admitted forwarding/RPC + +**What happens:** once an operation is admitted, runtime control RPCs use the +activated session. Direct HTTP/WebSocket forwarding first waits for the target +port using the admitted runtime lease, then calls +`this.ctx.container.getTcpPort(port).fetch(request)` inside that same lease. +HTTP response bodies and WebSockets retain runtime authority until EOF, +cancellation, close, or interruption. + +| Failure | Surfaced as | +| ----------------------------- | ------------------------------------------------------ | +| Runtime replacement | `OperationInterruptedError`; no replay | +| WebSocket close/transport | `RPCTransportError` or domain-specific interruption | +| Container-side handler error | Passthrough response or mapped `SandboxError` subclass | +| Caller abort before admission | Caller abort reason | + +Direct forwarding no longer routes through inherited `Container.fetch()` or +`Container.containerFetch()` after admission. Preview, terminal, and direct +container forwarding all use exact runtime ownership and non-starting paths +where their public API requires non-waking behavior. ## Capacity Limit Errors (Production Only) -When hitting account limits, the Containers API returns 400 with these error codes: +When hitting account limits, the Containers API returns 400 with these error +codes: | Error Code | Meaning | | -------------------------------- | --------------------------------- | @@ -158,42 +140,32 @@ These cannot be reproduced locally - they only occur in production. | Concurrent vCPU | 100 | | Concurrent Disk | 2 TB | -See [Containers limits](https://developers.cloudflare.com/containers/platform-details/limits/) for current values. +## Testing Strategy -## Best Practices +### Local Unit Tests -- Call `destroy()` when done to free resources -- Use `keepAlive: false` (default) for auto-timeout -- Monitor concurrent container usage in production +Mock each startup phase separately: -## Error Sources & Test Coverage +1. Instance allocation errors via the container start path. +2. Control port readiness errors via lifecycle establishment. +3. Activation errors via runtime metadata/session mocks. +4. Post-admission interruption via runtime replacement or transport close. -The SDK handles errors from two layers: +### E2E Tests -### workerd (container-client.c++) +Production-only paths require deployed Workers tests: -| Error Message | Condition | SDK Response | -| ---------------------------------- | --------------------------------- | ------------ | -| `container port not found` | Port not in Docker mappings | 503 | -| `Monitor failed to find container` | Container not found after retries | 503 | -| `No such image available` | Docker image missing | 500 | -| `Container already exists` | Name collision | 500 | +- Cold start under load. +- Image pull failures. +- Account capacity limits. +- Runtime replacement while streams/WebSockets are retained. -### @cloudflare/containers (container.ts) +## Monitoring -| Error Message | Condition | SDK Response | -| -------------------------------- | ------------------------ | ------------ | -| `the container is not listening` | App not ready on port | 503 | -| `failed to verify port` | Port health check failed | 503 | -| `container did not start` | Startup timeout | 503 | -| `network connection lost` | Connection dropped | 503 | -| `no container instance` | VM still provisioning | 503 | - -### Test Coverage - -| Test File | What It Tests | -| -------------------------------- | --------------------------------- | -| `sandbox-error-handling.test.ts` | Error classification (503 vs 500) | -| `base-client.test.ts` | Retry logic based on status codes | +Track these error patterns in production: -All error patterns are verified against the actual error messages from workerd and @cloudflare/containers source code. +- High 503 rate with `container_starting`: cold start or readiness tuning. +- 500s with image/config messages: deployment misconfiguration. +- `OperationInterruptedError`: expected during runtime replacement; unexpected + spikes indicate churn. +- `RPCTransportError`: control transport closed after admission. diff --git a/examples/codex-app-server/package.json b/examples/codex-app-server/package.json index 85cede50e..abc1ed89f 100644 --- a/examples/codex-app-server/package.json +++ b/examples/codex-app-server/package.json @@ -23,6 +23,6 @@ "author": "", "license": "MIT", "dependencies": { - "@cloudflare/containers": "^0.3.5" + "@cloudflare/containers": "^0.3.7" } } diff --git a/examples/codex-app-server/src/index.test.ts b/examples/codex-app-server/src/index.test.ts index 057ca5bc5..ca5dee8e4 100644 --- a/examples/codex-app-server/src/index.test.ts +++ b/examples/codex-app-server/src/index.test.ts @@ -384,20 +384,17 @@ describe('Codex App-Server Setup & Admission', () => { expect(mockExecCalls).toHaveLength(1); }); - it('waitForPortReady succeeds on ready event and rejects on error/done', async () => { + it('waitForPortReady delegates to admitted public port readiness', async () => { const sandbox = new Sandbox(mockCtx, mockEnv); + const startAndWaitForPorts = vi.fn(async (_port: number) => undefined); + Object.assign(sandbox, { startAndWaitForPorts }); - setupMockPorts(sandbox, [{ type: 'ready' }]); await expect(sandbox.waitForPortReady(4500)).resolves.toBeUndefined(); + expect(startAndWaitForPorts).toHaveBeenCalledWith(4500); - setupMockPorts(sandbox, [{ type: 'error' }]); - await expect(sandbox.waitForPortReady(4500)).rejects.toThrow( - 'Port 4500 watch reported error' - ); - - setupMockPorts(sandbox, []); + startAndWaitForPorts.mockRejectedValueOnce(new Error('Port unavailable')); await expect(sandbox.waitForPortReady(4500)).rejects.toThrow( - 'Port 4500 watch closed before ready' + 'Port unavailable' ); }); diff --git a/examples/codex-app-server/src/index.ts b/examples/codex-app-server/src/index.ts index cad169e67..b66c9d9b3 100644 --- a/examples/codex-app-server/src/index.ts +++ b/examples/codex-app-server/src/index.ts @@ -82,28 +82,7 @@ export class Sandbox extends BaseSandbox { } async waitForPortReady(port: number): Promise { - const watch = await this.client.ports.openWatch(port); - const stream = await watch.stream(); - const reader = stream.getReader(); - try { - while (true) { - const { value, done } = await reader.read(); - if (done) { - throw new Error(`Port ${port} watch closed before ready`); - } - if (value) { - if (value.type === 'ready') { - return; - } - if (value.type === 'error') { - throw new Error(`Port ${port} watch reported error`); - } - } - } - } finally { - await reader.cancel(); - watch[Symbol.dispose]?.(); - } + await this.startAndWaitForPorts(port); } } diff --git a/examples/collaborative-terminal/package.json b/examples/collaborative-terminal/package.json index 4642b4f1c..9eec37186 100644 --- a/examples/collaborative-terminal/package.json +++ b/examples/collaborative-terminal/package.json @@ -24,7 +24,7 @@ }, "devDependencies": { "@cloudflare/sandbox": "*", - "@cloudflare/vite-plugin": "^1.42.0", + "@cloudflare/vite-plugin": "^1.46.0", "@react-router/dev": "^7.15.0", "@tailwindcss/vite": "^4.1.17", "@types/node": "^24.10.1", diff --git a/examples/openai-agents/package.json b/examples/openai-agents/package.json index e848b606a..02493edce 100644 --- a/examples/openai-agents/package.json +++ b/examples/openai-agents/package.json @@ -21,7 +21,7 @@ "react-dom": "^19.2.0" }, "devDependencies": { - "@cloudflare/vite-plugin": "^1.42.0", + "@cloudflare/vite-plugin": "^1.46.0", "@types/react": "^19.2.7", "@types/react-dom": "^19.2.3", "@vitejs/plugin-react": "^5.1.1", diff --git a/examples/time-machine/tsconfig.json b/examples/time-machine/tsconfig.json index 1715e3d6d..214e54f4e 100644 --- a/examples/time-machine/tsconfig.json +++ b/examples/time-machine/tsconfig.json @@ -5,7 +5,7 @@ "moduleResolution": "bundler", "strict": true, "skipLibCheck": true, - "types": ["@cloudflare/workers-types/2023-07-01", "node"], + "types": ["@cloudflare/workers-types", "node"], "noEmit": true }, "include": ["src/**/*.ts", "worker-configuration.d.ts"] diff --git a/examples/typescript-validator/package.json b/examples/typescript-validator/package.json index 6c5c0f9a1..0d7cd9e20 100644 --- a/examples/typescript-validator/package.json +++ b/examples/typescript-validator/package.json @@ -20,7 +20,7 @@ "react-dom": "^19.2.0" }, "devDependencies": { - "@cloudflare/vite-plugin": "^1.42.0", + "@cloudflare/vite-plugin": "^1.46.0", "@tailwindcss/vite": "^4.1.17", "@types/node": "^24.10.1", "@vitejs/plugin-react": "^5.1.1", diff --git a/examples/vite-sandbox/package.json b/examples/vite-sandbox/package.json index 5619ef182..76c7d862b 100644 --- a/examples/vite-sandbox/package.json +++ b/examples/vite-sandbox/package.json @@ -22,6 +22,6 @@ "author": "", "license": "MIT", "dependencies": { - "@cloudflare/vite-plugin": "^1.42.0" + "@cloudflare/vite-plugin": "^1.46.0" } } diff --git a/extensions/interpreter/src/index.ts b/extensions/interpreter/src/index.ts index 4fc556222..755f04a58 100644 --- a/extensions/interpreter/src/index.ts +++ b/extensions/interpreter/src/index.ts @@ -106,11 +106,15 @@ export class Interpreter extends SandboxExtension { options: CreateContextOptions = {} ): Promise { validateLanguage(options.language); - const api = await this.sidecar(); - const raw = await api.createContext({ - language: options.language, - cwd: options.cwd - }); + const raw = await this.withSidecar< + InterpreterSidecarAPI, + Awaited> + >('interpreter.createContext', (api) => + api.createContext({ + language: options.language, + cwd: options.cwd + }) + ); const context = toCodeContext(raw); this.#contexts.set(context.id, context); return context; @@ -127,10 +131,13 @@ export class Interpreter extends SandboxExtension { const execution = new Execution(code, context); - const api = await this.sidecar(); - await api.runCode(context.id, code, options.language, async (event) => { - await this.#applyEvent(execution, event, options); - }); + await this.withSidecar( + 'interpreter.runCode', + (api) => + api.runCode(context.id, code, options.language, async (event) => { + await this.#applyEvent(execution, event, options); + }) + ); return execution.toJSON(); } @@ -146,27 +153,55 @@ export class Interpreter extends SandboxExtension { const encoder = new TextEncoder(); const self = this; + let closed = false; + let interruptContext: (() => Promise) | undefined; + let interruptSettled: Promise = Promise.resolve(); return new ReadableStream({ async start(controller) { try { - const api = await self.sidecar(); - await api.runCode(context.id, code, options.language, (event) => { - controller.enqueue( - encoder.encode(`data: ${JSON.stringify(event)}\n\n`) - ); - }); - controller.close(); + await self.withSidecar( + 'interpreter.runCodeStream', + async (api) => { + interruptContext = () => api.interruptContext(context.id); + if (closed) return; + await api.runCode(context.id, code, options.language, (event) => { + if (closed) return; + controller.enqueue( + encoder.encode(`data: ${JSON.stringify(event)}\n\n`) + ); + }); + } + ); + if (!closed) { + closed = true; + controller.close(); + } } catch (error) { - controller.error(error); + if (!closed) { + closed = true; + controller.error(error); + } + } finally { + await interruptSettled; + interruptContext = undefined; } + }, + cancel() { + closed = true; + const interrupt = interruptContext; + if (!interrupt) return; + interruptSettled = interrupt().catch(() => {}); + return interruptSettled; } }); } /** List all code contexts (refreshes the local cache). */ async listCodeContexts(): Promise { - const api = await this.sidecar(); - const raw = await api.listContexts(); + const raw = await this.withSidecar< + InterpreterSidecarAPI, + Awaited> + >('interpreter.listContexts', (api) => api.listContexts()); const contexts = raw.map(toCodeContext); for (const context of contexts) { this.#contexts.set(context.id, context); @@ -176,8 +211,10 @@ export class Interpreter extends SandboxExtension { /** Delete a code context. */ async deleteCodeContext(contextId: string): Promise { - const api = await this.sidecar(); - await api.deleteContext(contextId); + await this.withSidecar( + 'interpreter.deleteContext', + (api) => api.deleteContext(contextId) + ); this.#contexts.delete(contextId); } diff --git a/extensions/interpreter/src/sidecar-api.ts b/extensions/interpreter/src/sidecar-api.ts index 1a3bfc8a1..05054d57f 100644 --- a/extensions/interpreter/src/sidecar-api.ts +++ b/extensions/interpreter/src/sidecar-api.ts @@ -24,6 +24,7 @@ export interface InterpreterSidecarAPI { }): Promise; listContexts(): Promise; deleteContext(contextId: string): Promise; + interruptContext(contextId: string): Promise; runCode( contextId: string, code: string, diff --git a/extensions/interpreter/src/sidecar/lifecycle.ts b/extensions/interpreter/src/sidecar/lifecycle.ts index 8461beb5d..25f31e040 100644 --- a/extensions/interpreter/src/sidecar/lifecycle.ts +++ b/extensions/interpreter/src/sidecar/lifecycle.ts @@ -176,6 +176,33 @@ export class SidecarProcessLifecycle { const cleanup = () => { if (timer) clearTimeout(timer); process.process.stdout?.removeListener('data', responseHandler); + process.process.stdout?.removeListener('close', closeHandler); + process.process.removeListener('error', errorHandler); + process.process.removeListener('exit', exitHandler); + }; + + const fail = (error: Error) => { + cleanup(); + reject(error); + }; + + const errorHandler = (error: Error) => { + fail(error); + }; + + const exitHandler = ( + code: number | null, + signal: NodeJS.Signals | null + ) => { + fail( + new Error( + `Interpreter executor exited during execution (${signal ?? code ?? 'unknown'})` + ) + ); + }; + + const closeHandler = () => { + fail(new Error('Interpreter executor output closed during execution')); }; if (timeout !== undefined) { @@ -206,7 +233,12 @@ export class SidecarProcessLifecycle { }; process.process.stdout?.on('data', responseHandler); - process.process.stdin?.write(`${request}\n`); + process.process.stdout?.once('close', closeHandler); + process.process.once('error', errorHandler); + process.process.once('exit', exitHandler); + process.process.stdin?.write(`${request}\n`, (error) => { + if (error) fail(error); + }); }); } diff --git a/extensions/interpreter/src/sidecar/pool.ts b/extensions/interpreter/src/sidecar/pool.ts index b6261544c..288f6534e 100644 --- a/extensions/interpreter/src/sidecar/pool.ts +++ b/extensions/interpreter/src/sidecar/pool.ts @@ -315,7 +315,38 @@ export class ProcessPoolManager { }); return; } - this.logger.debug('Releasing executor for context', { + this.terminateContextExecutor(contextId, language, executor, 'Releasing'); + await this.ensureMinimumPool(language); + } + async interruptContext( + contextId: string, + language: InterpreterLanguage + ): Promise { + const executor = this.contextExecutors.get(contextId); + if (!executor) { + this.logger.debug( + 'Context interrupt ignored because no executor exists', + { + contextId + } + ); + return; + } + this.terminateContextExecutor( + contextId, + language, + executor, + 'Interrupting' + ); + await this.reserveExecutorForContext(contextId, language); + } + private terminateContextExecutor( + contextId: string, + language: InterpreterLanguage, + executor: InterpreterProcess, + action: 'Releasing' | 'Interrupting' + ): void { + this.logger.debug(`${action} executor for context`, { contextId, language, executorId: executor.id @@ -326,7 +357,6 @@ export class ProcessPoolManager { executor.process.kill(); this.removeExecutorFromState(executor); this.releaseProcessSlot(executor.id); - await this.ensureMinimumPool(language); } isContextExecutorHealthy(contextId: string): boolean { const executor = this.contextExecutors.get(contextId); diff --git a/extensions/interpreter/src/sidecar/server.ts b/extensions/interpreter/src/sidecar/server.ts index 0c796859a..6b67c089c 100644 --- a/extensions/interpreter/src/sidecar/server.ts +++ b/extensions/interpreter/src/sidecar/server.ts @@ -69,6 +69,15 @@ class InterpreterSidecar } } + async interruptContext(contextId: string): Promise { + const context = this.#contexts.get(contextId); + if (!context) return; + await this.#pool.interruptContext( + contextId, + context.language as InterpreterLanguage + ); + } + async runCode( contextId: string, code: string, diff --git a/extensions/interpreter/tests/interpreter.test.ts b/extensions/interpreter/tests/interpreter.test.ts index 4430a1489..c21991950 100644 --- a/extensions/interpreter/tests/interpreter.test.ts +++ b/extensions/interpreter/tests/interpreter.test.ts @@ -5,7 +5,12 @@ import type { import { EXTENSION_TARBALL_REQUIRED } from '@repo/shared'; import type { Mock } from 'vitest'; import { afterEach, describe, expect, it, vi } from 'vitest'; -import type { SandboxLike } from '../../../packages/sandbox/src/extensions'; +import { + type ExtensionRuntimeCall, + type ExtensionRuntimeControl, + type SandboxLike, + sandboxRuntimeCall +} from '../../../packages/sandbox/src/extensions'; import { Interpreter, withInterpreter } from '../src/index'; import type { InterpreterSidecarAPI } from '../src/sidecar-api'; @@ -31,10 +36,22 @@ function makeSandbox(): { sandbox: SandboxLike; api: ExtensionsApiMock } { })), stop: vi.fn(async () => {}) }; + const unusedDomain = {}; + const runtimeCall = (async (_operation, call) => + await call({ + files: unusedDomain, + ports: unusedDomain, + backup: unusedDomain, + watch: unusedDomain, + tunnels: unusedDomain, + terminals: unusedDomain, + extensions: api, + utils: unusedDomain + } as unknown as ExtensionRuntimeControl)) as ExtensionRuntimeCall; return { sandbox: { - client: { extensions: api as unknown as SandboxExtensionsAPI } - } as unknown as SandboxLike, + [sandboxRuntimeCall]: runtimeCall + }, api }; } @@ -70,6 +87,7 @@ describe('withInterpreter', () => { createContext: vi.fn(async () => RAW_CONTEXT), listContexts: vi.fn(async () => []), deleteContext: vi.fn(async () => {}), + interruptContext: vi.fn(async () => {}), runCode: vi.fn(async () => {}) }; api.connect @@ -112,6 +130,7 @@ describe('withInterpreter', () => { createContext: vi.fn(async () => RAW_CONTEXT), listContexts: vi.fn(async () => []), deleteContext: vi.fn(async () => {}), + interruptContext: vi.fn(async () => {}), runCode: vi.fn(async (_contextId, _code, _language, onEvent) => { await onEvent({ type: 'stdout', text: 'hello\n' }); await onEvent({ type: 'result', text: '42', metadata: {} }); @@ -149,6 +168,7 @@ describe('withInterpreter', () => { createContext: vi.fn(async () => RAW_CONTEXT), listContexts: vi.fn(async () => []), deleteContext: vi.fn(async () => {}), + interruptContext: vi.fn(async () => {}), runCode: vi.fn(async (_contextId, _code, _language, onEvent) => { await onEvent({ type: 'result', text: '42', metadata: {} }); }) @@ -187,6 +207,7 @@ describe('withInterpreter', () => { createContext: vi.fn(async () => RAW_CONTEXT), listContexts: vi.fn(async () => []), deleteContext: vi.fn(async () => {}), + interruptContext: vi.fn(async () => {}), runCode: vi.fn(async (_contextId, _code, _language, onEvent) => { await onEvent({ type: 'error', @@ -222,6 +243,7 @@ describe('withInterpreter', () => { createContext: vi.fn(async () => RAW_CONTEXT), listContexts: vi.fn(async () => [RAW_CONTEXT]), deleteContext: vi.fn(async () => {}), + interruptContext: vi.fn(async () => {}), runCode: vi.fn(async () => {}) }; api.connect.mockResolvedValue(stub); @@ -236,6 +258,83 @@ describe('withInterpreter', () => { expect(stub.deleteContext).toHaveBeenCalledWith('ctx-1'); }); + it('interrupts the active context when stream consumption is canceled', async () => { + const { sandbox, api } = makeSandbox(); + let finishRunCode: (() => void) | undefined; + const stub: InterpreterSidecarAPI = { + createContext: vi.fn(async () => RAW_CONTEXT), + listContexts: vi.fn(async () => []), + deleteContext: vi.fn(async () => {}), + interruptContext: vi.fn(async () => { + finishRunCode?.(); + }), + runCode: vi.fn( + async (_contextId, _code, _language, onEvent) => + await new Promise((resolve) => { + finishRunCode = resolve; + void onEvent({ type: 'stdout', text: 'hello\n' }); + }) + ) + }; + api.connect.mockResolvedValue(stub); + + const ext = withInterpreter(sandbox); + const stream = await ext.runCodeStream('print("hello")', { + context: { + id: 'ctx-1', + language: 'python', + cwd: '/workspace', + createdAt: new Date(), + lastUsed: new Date() + } + }); + + const reader = stream.getReader(); + const first = await reader.read(); + expect(first.done).toBe(false); + await reader.cancel(); + + expect(stub.interruptContext).toHaveBeenCalledWith('ctx-1'); + expect(stub.runCode).toHaveBeenCalledTimes(1); + }); + + it('does not start code after cancellation during sidecar connection', async () => { + const { sandbox, api } = makeSandbox(); + let finishConnect: ((stub: InterpreterSidecarAPI) => void) | undefined; + const stub: InterpreterSidecarAPI = { + createContext: vi.fn(async () => RAW_CONTEXT), + listContexts: vi.fn(async () => []), + deleteContext: vi.fn(async () => {}), + interruptContext: vi.fn(async () => {}), + runCode: vi.fn(async () => {}) + }; + api.connect.mockImplementation( + async () => + await new Promise((resolve) => { + finishConnect = resolve; + }) + ); + + const ext = withInterpreter(sandbox); + const stream = await ext.runCodeStream('while True: pass', { + context: { + id: 'ctx-1', + language: 'python', + cwd: '/workspace', + createdAt: new Date(), + lastUsed: new Date() + } + }); + + await vi.waitFor(() => expect(finishConnect).toBeDefined()); + await stream.cancel(); + finishConnect?.(stub); + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(stub.runCode).not.toHaveBeenCalled(); + expect(stub.interruptContext).not.toHaveBeenCalled(); + }); + it('returns an Interpreter instance from the factory', () => { const { sandbox } = makeSandbox(); expect(withInterpreter(sandbox)).toBeInstanceOf(Interpreter); diff --git a/package-lock.json b/package-lock.json index cce92a1e8..45e5d7ad5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,8 +19,8 @@ "@biomejs/biome": "2.3.7", "@changesets/changelog-github": "^0.5.1", "@changesets/cli": "^2.29.7", - "@cloudflare/vitest-pool-workers": "^0.16.20", - "@cloudflare/workers-types": "^4.20251126.0", + "@cloudflare/vitest-pool-workers": "^0.18.7", + "@cloudflare/workers-types": "^5.20260721.1", "@playwright/test": "^1.50.0", "@types/bun": "^1.3.14", "@types/node": "^24.10.1", @@ -54,8 +54,8 @@ "devDependencies": { "@biomejs/biome": "2.3.7", "@cloudflare/sandbox": "*", - "@cloudflare/vitest-pool-workers": "^0.16.20", - "@cloudflare/workers-types": "^4.20251126.0", + "@cloudflare/vitest-pool-workers": "^0.18.7", + "@cloudflare/workers-types": "^5.20260721.1", "@types/node": "^24.10.1", "hono": "^4.12.26", "typescript": "^5.9.3", @@ -96,6 +96,7 @@ "@cloudflare/sandbox": "*", "@types/node": "^24.10.1", "typescript": "^5.9.3", + "vitest": "^4.1.8", "wrangler": "^4.102.0" } }, @@ -130,6 +131,7 @@ "@cloudflare/sandbox": "*", "@types/node": "^24.10.1", "typescript": "^5.9.3", + "vitest": "^4.1.8", "wrangler": "^4.102.0" } }, @@ -138,12 +140,13 @@ "version": "1.0.0", "license": "MIT", "dependencies": { - "@cloudflare/containers": "^0.3.5" + "@cloudflare/containers": "^0.3.7" }, "devDependencies": { "@cloudflare/sandbox": "*", "@types/node": "^24.10.1", "typescript": "^5.9.3", + "vitest": "^4.1.8", "wrangler": "^4.102.0" } }, @@ -161,7 +164,7 @@ }, "devDependencies": { "@cloudflare/sandbox": "*", - "@cloudflare/vite-plugin": "^1.42.0", + "@cloudflare/vite-plugin": "^1.46.0", "@react-router/dev": "^7.15.0", "@tailwindcss/vite": "^4.1.17", "@types/node": "^24.10.1", @@ -171,6 +174,7 @@ "typescript": "^5.9.3", "vite": "^7.3.5", "vite-tsconfig-paths": "^5.1.4", + "vitest": "^4.1.8", "wrangler": "^4.102.0" } }, @@ -233,7 +237,7 @@ "react-dom": "^19.2.0" }, "devDependencies": { - "@cloudflare/vite-plugin": "^1.42.0", + "@cloudflare/vite-plugin": "^1.46.0", "@types/react": "^19.2.7", "@types/react-dom": "^19.2.3", "@vitejs/plugin-react": "^5.1.1", @@ -281,6 +285,7 @@ "aws4fetch": "^1.0.20", "hono": "^4.12.26", "typescript": "^5.9.3", + "vitest": "^4.1.8", "wrangler": "^4.102.0" } }, @@ -306,7 +311,7 @@ "react-dom": "^19.2.0" }, "devDependencies": { - "@cloudflare/vite-plugin": "^1.42.0", + "@cloudflare/vite-plugin": "^1.46.0", "@tailwindcss/vite": "^4.1.17", "@types/node": "^24.10.1", "@vitejs/plugin-react": "^5.1.1", @@ -320,7 +325,7 @@ "version": "1.0.0", "license": "MIT", "dependencies": { - "@cloudflare/vite-plugin": "^1.42.0" + "@cloudflare/vite-plugin": "^1.46.0" }, "devDependencies": { "@cloudflare/sandbox": "*", @@ -328,6 +333,7 @@ "react": "^19.2.0", "react-dom": "^19.2.0", "vite": "^7.3.5", + "vitest": "^4.1.8", "wrangler": "^4.102.0" } }, @@ -1719,9 +1725,9 @@ } }, "node_modules/@cloudflare/containers": { - "version": "0.3.5", - "resolved": "https://registry.npmjs.org/@cloudflare/containers/-/containers-0.3.5.tgz", - "integrity": "sha512-P6jYEDkw1Q9qWRr9iFBxe1fozI5HfGMY6XrNg/jROPGZykcYrrzOluUqXv+q4N8gIoRXPCqJJ1FGALbTqnYTkg==", + "version": "0.3.7", + "resolved": "https://registry.npmjs.org/@cloudflare/containers/-/containers-0.3.7.tgz", + "integrity": "sha512-DM9dm3FnIBSyiSJ1FLavKwl/lk3oAmTaynCzZQ9pZR0ncRPquSxkxd8Nu2MFILxmDDsPkxKsSNEh9mHHMty4Fw==", "license": "MIT OR Apache-2.0" }, "node_modules/@cloudflare/kv-asset-handler": { @@ -1825,15 +1831,16 @@ } }, "node_modules/@cloudflare/vite-plugin": { - "version": "1.42.0", - "resolved": "https://registry.npmjs.org/@cloudflare/vite-plugin/-/vite-plugin-1.42.0.tgz", - "integrity": "sha512-U8Bpcn9l10NNCyYo6kMI2RPZhKRWU0i3udrS/+LHHBDSa61Ra6r7OaDY5LnZw86tOC5vZIKRUm5E51MKjOvfwg==", + "version": "1.46.0", + "resolved": "https://registry.npmjs.org/@cloudflare/vite-plugin/-/vite-plugin-1.46.0.tgz", + "integrity": "sha512-+pnxcFWo+kMozeCah9CxYI6VywMQmBBfEiGJZgYkIji+vnNWkWEC0WkL5MQWHCBIiEPIbcoZDQiKr6yruLFI2Q==", "license": "MIT", "dependencies": { "@cloudflare/unenv-preset": "2.16.1", - "miniflare": "4.20260617.0", + "miniflare": "4.20260721.0", "unenv": "2.0.0-rc.24", - "wrangler": "4.102.0", + "workerd": "1.20260721.1", + "wrangler": "4.113.0", "ws": "8.21.0" }, "bin": { @@ -1841,20 +1848,20 @@ }, "peerDependencies": { "vite": "^6.1.0 || ^7.0.0 || ^8.0.0", - "wrangler": "^4.102.0" + "wrangler": "^4.113.0" } }, "node_modules/@cloudflare/vitest-pool-workers": { - "version": "0.16.20", - "resolved": "https://registry.npmjs.org/@cloudflare/vitest-pool-workers/-/vitest-pool-workers-0.16.20.tgz", - "integrity": "sha512-buw0YgsAMT7s60wcmyxbtciEJjMJzKcWzayDMPhWaqMqfQzW+0WPLV67Lobn4C80nkNQhYocEJPnrEhLWnOf+A==", + "version": "0.18.7", + "resolved": "https://registry.npmjs.org/@cloudflare/vitest-pool-workers/-/vitest-pool-workers-0.18.7.tgz", + "integrity": "sha512-PGYToiFoGpuRV2Uh33S4fI7JcmSkDxk6LQeneYtgfsITEkVi5iGxc3Vms6yW9Jr6uSzVK4Be9XZfdyZDr5GWYw==", "dev": true, "license": "MIT", "dependencies": { "cjs-module-lexer": "1.2.3", "esbuild": "0.28.1", - "miniflare": "4.20260625.0", - "wrangler": "4.105.0", + "miniflare": "4.20260721.0", + "wrangler": "4.113.0", "zod": "3.25.76" }, "peerDependencies": { @@ -1863,91 +1870,6 @@ "vitest": "^4.1.0" } }, - "node_modules/@cloudflare/vitest-pool-workers/node_modules/@cloudflare/workerd-darwin-64": { - "version": "1.20260625.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20260625.1.tgz", - "integrity": "sha512-naCfBv0WnnTQIQPTniqMoUlklOIFjrAcSn1X+IAOhY8aFLF/xGYtFjs1eEE8sFib3ZuChGGpU23FFORVczqr0A==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=16" - } - }, - "node_modules/@cloudflare/vitest-pool-workers/node_modules/@cloudflare/workerd-darwin-arm64": { - "version": "1.20260625.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20260625.1.tgz", - "integrity": "sha512-jmH6zjp6Wrux46+qtFwDwrj+vd7s5bdwEqeGvdnwE0a4IEeAhKs0L42HQOyID+g5lkrHq9m55+AbhtmRAm63Pw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=16" - } - }, - "node_modules/@cloudflare/vitest-pool-workers/node_modules/@cloudflare/workerd-linux-64": { - "version": "1.20260625.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20260625.1.tgz", - "integrity": "sha512-MiQkpA/dX8d83Zp64pzHUKfd6ca4cvwxnNobSP6CnXvfESvnNI9pfa+nfwnParla36sPmnYntNkjR7NjRuDeKQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=16" - } - }, - "node_modules/@cloudflare/vitest-pool-workers/node_modules/@cloudflare/workerd-linux-arm64": { - "version": "1.20260625.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20260625.1.tgz", - "integrity": "sha512-LxxW7Qv60Xvv37+w6gUSDpYZziyqMy+cZWd9IvSA5ehVgKAxmzEaYPMiSZlxk32nbIWL9u/tfjXYCOKJ4Lo+XQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=16" - } - }, - "node_modules/@cloudflare/vitest-pool-workers/node_modules/@cloudflare/workerd-windows-64": { - "version": "1.20260625.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20260625.1.tgz", - "integrity": "sha512-LH6iIX1HHaTwVKV5VokDxxUErXJzQoNZFRwVm7Vx/3fB/ApcTcRCUaMqcxI4as94jEUqg+pmX5czOndiveohow==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=16" - } - }, "node_modules/@cloudflare/vitest-pool-workers/node_modules/@esbuild/aix-ppc64": { "version": "0.28.1", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz", @@ -2432,103 +2354,10 @@ "@esbuild/win32-x64": "0.28.1" } }, - "node_modules/@cloudflare/vitest-pool-workers/node_modules/fsevents": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", - "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, - "node_modules/@cloudflare/vitest-pool-workers/node_modules/miniflare": { - "version": "4.20260625.0", - "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-4.20260625.0.tgz", - "integrity": "sha512-3kKXwRUObJsnBYPBgR0NiNZYKF/yv8GFyha1cx2EeAEraxNODgRVcyeRo+F1ok1tg5Mg7iUpOWSkknQTHuFhwA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@cspotcode/source-map-support": "0.8.1", - "sharp": "0.34.5", - "undici": "7.28.0", - "workerd": "1.20260625.1", - "ws": "8.21.0", - "youch": "4.1.0-beta.10" - }, - "bin": { - "miniflare": "bootstrap.js" - }, - "engines": { - "node": ">=22.0.0" - } - }, - "node_modules/@cloudflare/vitest-pool-workers/node_modules/workerd": { - "version": "1.20260625.1", - "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20260625.1.tgz", - "integrity": "sha512-GApQvFX52SDM6L4u0+RRnUDB1wJOnEwoXjinkmOPtIyofWBxrlZckdegJSYc1leg++lLZ3+DQ4zMVmBqYVtzfA==", - "dev": true, - "hasInstallScript": true, - "license": "Apache-2.0", - "bin": { - "workerd": "bin/workerd" - }, - "engines": { - "node": ">=16" - }, - "optionalDependencies": { - "@cloudflare/workerd-darwin-64": "1.20260625.1", - "@cloudflare/workerd-darwin-arm64": "1.20260625.1", - "@cloudflare/workerd-linux-64": "1.20260625.1", - "@cloudflare/workerd-linux-arm64": "1.20260625.1", - "@cloudflare/workerd-windows-64": "1.20260625.1" - } - }, - "node_modules/@cloudflare/vitest-pool-workers/node_modules/wrangler": { - "version": "4.105.0", - "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-4.105.0.tgz", - "integrity": "sha512-7dXFH6OLj1Fv0y6ZeRPUxFTkp+duWD7/xxVi/1c0vfOeEYwIFKWB7cdqnY05DvY1Ta3BnqAwRkXfLs8PDj538g==", - "dev": true, - "license": "MIT OR Apache-2.0", - "dependencies": { - "@cloudflare/kv-asset-handler": "0.5.0", - "@cloudflare/unenv-preset": "2.16.1", - "blake3-wasm": "2.1.5", - "esbuild": "0.28.1", - "miniflare": "4.20260625.0", - "path-to-regexp": "6.3.0", - "unenv": "2.0.0-rc.24", - "workerd": "1.20260625.1" - }, - "bin": { - "cf-wrangler": "bin/cf-wrangler.js", - "wrangler": "bin/wrangler.js", - "wrangler2": "bin/wrangler.js" - }, - "engines": { - "node": ">=22.0.0" - }, - "optionalDependencies": { - "fsevents": "2.3.3" - }, - "peerDependencies": { - "@cloudflare/workers-types": "^4.20260625.1" - }, - "peerDependenciesMeta": { - "@cloudflare/workers-types": { - "optional": true - } - } - }, "node_modules/@cloudflare/workerd-darwin-64": { - "version": "1.20260617.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20260617.1.tgz", - "integrity": "sha512-jWwmgEVVWbsHNrLSNXzwjJaH90VzRxq1cWkQFUidxyeUPnMxemeNE8I9qFAfrpzGgE11e9sKDcE3ettJW08swQ==", + "version": "1.20260721.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20260721.1.tgz", + "integrity": "sha512-VivNMhiEdZIB4JBWxf1RMJGROErv53qmQ+dvhjA1evrCouvqRYW718VqDideU3PSV7Ythl5Df48NqZYWoaEHpQ==", "cpu": [ "x64" ], @@ -2542,9 +2371,9 @@ } }, "node_modules/@cloudflare/workerd-darwin-arm64": { - "version": "1.20260617.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20260617.1.tgz", - "integrity": "sha512-LHH7b565g9znfCUOkwbec6FG2rmRbsgCy6aJiU9KN662mNheWl5sw/iKleiFSiljPKQQP3HkjnC/NSkdgi/aSA==", + "version": "1.20260721.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20260721.1.tgz", + "integrity": "sha512-k7oye1ZiuwnnBBA2eTMduconr/ud5ZxFtRNTsYwMdmJeeeislw2+M72otrHxxvybCP7JWPPlJ38uhfajpcyhOA==", "cpu": [ "arm64" ], @@ -2558,9 +2387,9 @@ } }, "node_modules/@cloudflare/workerd-linux-64": { - "version": "1.20260617.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20260617.1.tgz", - "integrity": "sha512-FMnaAKXe4Cfd8TQurCVd9fs2XQVBFRCsP+Id/SRdUv89MlwYu9zXfoyx6BxM+brPTIUK38SHbo8iaxiwzLi9JQ==", + "version": "1.20260721.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20260721.1.tgz", + "integrity": "sha512-hon0lW4ZQ4boAVgaw+0ZFTNS8v5MWPWvK0HZnt4tDpKYnDUviLZawtUW3KqvFmCQTipVHl1S34j3J8Eqb93hGQ==", "cpu": [ "x64" ], @@ -2574,9 +2403,9 @@ } }, "node_modules/@cloudflare/workerd-linux-arm64": { - "version": "1.20260617.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20260617.1.tgz", - "integrity": "sha512-MRoifFYcqbxxIIQy7PqO5tFY/qPFSnjXzakWl0sO93l+HLyG35jRAgOi6jfqa4kBxc7gKKtH861DcewjxUfkjA==", + "version": "1.20260721.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20260721.1.tgz", + "integrity": "sha512-nAl+HRQqpX5b7xVwWcvLPZmCk8NQ2yjI0yvJTWcHiRswbMEg1ZZckVmjJUAn0PHzZARbCSyIV7v3UjM+SPRmIQ==", "cpu": [ "arm64" ], @@ -2590,9 +2419,9 @@ } }, "node_modules/@cloudflare/workerd-windows-64": { - "version": "1.20260617.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20260617.1.tgz", - "integrity": "sha512-rgBV9wQrv0OSKgCTTbhFUFY3sLGNANZ88aqaLvtmEn2gmbFVb1J4PDGochVUdB7NSEp4D/ghHva6/8SZmbONpw==", + "version": "1.20260721.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20260721.1.tgz", + "integrity": "sha512-9paFG5cMTKz/CRixnEEnZbe5uvFPBFSDthxJHANfCWhUtBj49GSL1FPIokIg+Q+H8DGJEExU0lL92LtxD0lTxQ==", "cpu": [ "x64" ], @@ -2606,9 +2435,9 @@ } }, "node_modules/@cloudflare/workers-types": { - "version": "4.20260626.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workers-types/-/workers-types-4.20260626.1.tgz", - "integrity": "sha512-fBnpQyFRS3Ce1l2IUd3k+aUxgy/7VMlVXF4F672/eSrpXFeezCy3Ha6Z2uTyGgqu9sGvQPOj8nqKBv2yeI+ciw==", + "version": "5.20260722.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workers-types/-/workers-types-5.20260722.1.tgz", + "integrity": "sha512-8+kivCgFGzwrAfNOWgSpzy/VDvmT/i5KWBgQhnygv3d1kajNn6mCYTbLKpouG0aY8mXjhv+IQm1a8r2K/H4pqQ==", "devOptional": true, "license": "MIT OR Apache-2.0" }, @@ -10134,15 +9963,15 @@ } }, "node_modules/miniflare": { - "version": "4.20260617.0", - "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-4.20260617.0.tgz", - "integrity": "sha512-A+H5gcOCQZsKFg7/daZUtx8WHn4gGxwUfH1jnNDAisyAWSvvSZHe+GCeQWs16uthnUDcm72UQIQ1NXDJtnuo9Q==", + "version": "4.20260721.0", + "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-4.20260721.0.tgz", + "integrity": "sha512-fBLaCxZ2i/nPH8iyLzvza0C8/sSF4sjD1ma1Skf+pkZVK0TlaW5ujHJlUHwcwR66v2JZt+Q28d4DCX/oaLG0cA==", "license": "MIT", "dependencies": { "@cspotcode/source-map-support": "0.8.1", "sharp": "0.34.5", "undici": "7.28.0", - "workerd": "1.20260617.1", + "workerd": "1.20260721.1", "ws": "8.21.0", "youch": "4.1.0-beta.10" }, @@ -13814,9 +13643,9 @@ } }, "node_modules/workerd": { - "version": "1.20260617.1", - "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20260617.1.tgz", - "integrity": "sha512-Re5pl6pdowt3ZmWUzGlOuB7jbRIIPetgKalmo4cYmucQnVhpo7/3e4MfpekbhLi2EhZZz5EY9NWRu8zFzuEZew==", + "version": "1.20260721.1", + "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20260721.1.tgz", + "integrity": "sha512-b/DWhpV0jTudzQpLhDovcOgBz233386q+3Hbari7CLCNT9UXxjQziSTZ9yCoKdT2K3TSx5jrwlOisq8hlLWXYg==", "hasInstallScript": true, "license": "Apache-2.0", "bin": { @@ -13826,11 +13655,11 @@ "node": ">=16" }, "optionalDependencies": { - "@cloudflare/workerd-darwin-64": "1.20260617.1", - "@cloudflare/workerd-darwin-arm64": "1.20260617.1", - "@cloudflare/workerd-linux-64": "1.20260617.1", - "@cloudflare/workerd-linux-arm64": "1.20260617.1", - "@cloudflare/workerd-windows-64": "1.20260617.1" + "@cloudflare/workerd-darwin-64": "1.20260721.1", + "@cloudflare/workerd-darwin-arm64": "1.20260721.1", + "@cloudflare/workerd-linux-64": "1.20260721.1", + "@cloudflare/workerd-linux-arm64": "1.20260721.1", + "@cloudflare/workerd-windows-64": "1.20260721.1" } }, "node_modules/workers-ai-provider": { @@ -13844,19 +13673,19 @@ } }, "node_modules/wrangler": { - "version": "4.102.0", - "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-4.102.0.tgz", - "integrity": "sha512-GPljlQs9a+/Ai2h0TdEUYaaWv9upK4fUteSWTPlruas7tdixiIwr74CZSWHcEPuRgZYkyYKHIBbT1w+BYPkPrw==", + "version": "4.113.0", + "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-4.113.0.tgz", + "integrity": "sha512-ROGzSloJv0y21It6Oc9LaruNcu1tdiQ/XzL3Jc3YkFjzXEMXzTqVhA8vQaGMTdZHTjFP0PVcwAHNgaw3gXu4wA==", "license": "MIT OR Apache-2.0", "dependencies": { "@cloudflare/kv-asset-handler": "0.5.0", "@cloudflare/unenv-preset": "2.16.1", "blake3-wasm": "2.1.5", "esbuild": "0.28.1", - "miniflare": "4.20260617.0", + "miniflare": "4.20260721.0", "path-to-regexp": "6.3.0", "unenv": "2.0.0-rc.24", - "workerd": "1.20260617.1" + "workerd": "1.20260721.1" }, "bin": { "cf-wrangler": "bin/cf-wrangler.js", @@ -13870,7 +13699,7 @@ "fsevents": "2.3.3" }, "peerDependencies": { - "@cloudflare/workers-types": "^4.20260617.1" + "@cloudflare/workers-types": "^5.20260721.1" }, "peerDependenciesMeta": { "@cloudflare/workers-types": { @@ -14569,7 +14398,7 @@ "version": "0.12.1", "license": "Apache-2.0", "dependencies": { - "@cloudflare/containers": "^0.3.5", + "@cloudflare/containers": "^0.3.7", "aws4fetch": "^1.0.20", "capnweb": "^0.8.0", "hono": "^4.12.26" @@ -14645,7 +14474,7 @@ "dependencies": { "@astrojs/check": "^0.9.5", "@astrojs/react": "^5.0.4", - "@cloudflare/vite-plugin": "^1.42.0", + "@cloudflare/vite-plugin": "^1.46.0", "@tailwindcss/vite": "^4.1.17", "astro": "^6.4.6", "clsx": "^2.1.1", diff --git a/package.json b/package.json index 8baf4a34b..550b664c9 100644 --- a/package.json +++ b/package.json @@ -44,8 +44,8 @@ "@biomejs/biome": "2.3.7", "@changesets/changelog-github": "^0.5.1", "@changesets/cli": "^2.29.7", - "@cloudflare/vitest-pool-workers": "^0.16.20", - "@cloudflare/workers-types": "^4.20251126.0", + "@cloudflare/vitest-pool-workers": "^0.18.7", + "@cloudflare/workers-types": "^5.20260721.1", "@playwright/test": "^1.50.0", "@types/bun": "^1.3.14", "@types/node": "^24.10.1", diff --git a/packages/sandbox-container/src/control-plane/api.ts b/packages/sandbox-container/src/control-plane/api.ts index 8e1e91ac0..01b1a46ee 100644 --- a/packages/sandbox-container/src/control-plane/api.ts +++ b/packages/sandbox-container/src/control-plane/api.ts @@ -17,6 +17,7 @@ import type { ReadFileBinaryOptions, ReadFileOptions, ReadFileStreamOptions, + RuntimeMetadata, SandboxAPI, SandboxPortsAPI, StopTunnelRunRequest, @@ -46,6 +47,7 @@ import type { WatchService } from '../services/watch-service'; import { WorkspaceArchiveService } from '../services/workspace-archive-service'; import { MountsRPCAPI } from './mounts-rpc'; import { ProcessesRPCAPI } from './processes-rpc'; +import type { ControlSession } from './session'; import { StreamSubscriptionRPC } from './subscription-rpc'; import { TerminalsRPCAPI } from './terminals-rpc'; import { WorkspaceRPCAPI } from './workspace-rpc'; @@ -94,45 +96,58 @@ function extractData( */ export class SandboxControlAPI extends RpcTarget implements SandboxAPI { #deps: SandboxAPIDeps; - constructor(deps: SandboxAPIDeps) { + #session: ControlSession; + + constructor(deps: SandboxAPIDeps, session: ControlSession) { super(); this.#deps = deps; + this.#session = session; } get files() { + this.#session.assertActive(); return new FilesRPCAPI(this.#deps.fileService); } get ports() { + this.#session.assertActive(); return new PortsRPCAPI(this.#deps.portService); } get processes() { + this.#session.assertActive(); return new ProcessesRPCAPI(this.#deps.processService); } get mounts() { + this.#session.assertActive(); return new MountsRPCAPI(new MountService(this.#deps.commandContextService)); } get workspace() { + this.#session.assertActive(); const service = new WorkspaceArchiveService( this.#deps.commandContextService ); return new WorkspaceRPCAPI(service); } get utils() { - return new UtilsRPCAPI(); + return new UtilsRPCAPI(this.#session); } get backup() { + this.#session.assertActive(); return new BackupRPCAPI(this.#deps.backupService); } get watch() { + this.#session.assertActive(); return new WatchRPCAPI(this.#deps.watchService); } get tunnels() { + this.#session.assertActive(); return new TunnelsRPCAPI(this.#deps.tunnelService); } get terminals() { + this.#session.assertActive(); return new TerminalsRPCAPI(this.#deps.terminalManager); } get extensions() { + this.#session.assertActive(); return new ExtensionsRPCAPI(this.#deps.extensionHost); } } @@ -361,16 +376,25 @@ class PortsRPCAPI extends RpcTarget implements SandboxPortsAPI { // =========================================================================== class UtilsRPCAPI extends RpcTarget { + #session: ControlSession; + + constructor(session: ControlSession) { + super(); + this.#session = session; + } + async ping(): Promise { return 'healthy'; } - async getVersion(): Promise { - try { - return process.env.SANDBOX_VERSION || 'unknown'; - } catch { - return 'unknown'; - } + async getRuntimeMetadata(): Promise { + return this.#session.metadata; + } + + async activateControlSession( + expectedRuntimeIncarnationID: string + ): Promise { + return this.#session.activate(expectedRuntimeIncarnationID); } } diff --git a/packages/sandbox-container/src/control-plane/session.ts b/packages/sandbox-container/src/control-plane/session.ts new file mode 100644 index 000000000..5f2f9dcda --- /dev/null +++ b/packages/sandbox-container/src/control-plane/session.ts @@ -0,0 +1,90 @@ +import { + ErrorCode, + type RuntimeMetadata, + type SandboxControlCallback +} from '@repo/shared'; + +export const CONTROL_PROTOCOL_VERSION = 1; + +export interface ControlSessionOptions { + metadata: RuntimeMetadata; + connectionID: string; + peerCallback: SandboxControlCallback | undefined; + registerControlCallback: ( + connectionID: string, + callback: SandboxControlCallback + ) => void; + clearControlCallback: (connectionID: string) => void; +} + +export function controlProtocolIncompatible(message: string): Error { + return Object.assign(new Error(message), { + code: ErrorCode.CONTROL_PROTOCOL_INCOMPATIBLE + }); +} + +export class ControlSession { + #metadata: RuntimeMetadata; + #connectionID: string; + #peerCallback: SandboxControlCallback | undefined; + #registerControlCallback: ( + connectionID: string, + callback: SandboxControlCallback + ) => void; + #clearControlCallback: (connectionID: string) => void; + #active = false; + #registered = false; + #closed = false; + + constructor(options: ControlSessionOptions) { + this.#metadata = options.metadata; + this.#connectionID = options.connectionID; + this.#peerCallback = options.peerCallback; + this.#registerControlCallback = options.registerControlCallback; + this.#clearControlCallback = options.clearControlCallback; + } + + get metadata(): RuntimeMetadata { + return this.#metadata; + } + + setPeerCallback(callback: SandboxControlCallback): void { + this.#peerCallback = callback; + } + + assertActive(): void { + if (!this.#active) { + throw controlProtocolIncompatible('Control session is not activated'); + } + } + + async activate( + expectedRuntimeIncarnationID: string + ): Promise { + if (expectedRuntimeIncarnationID !== this.#metadata.runtimeIncarnationID) { + throw controlProtocolIncompatible('Runtime incarnation does not match'); + } + + if (this.#closed) { + if (this.#active) return this.#metadata; + throw controlProtocolIncompatible('Control session is closed'); + } + + if (!this.#active) { + this.#active = true; + if (this.#peerCallback) { + this.#registerControlCallback(this.#connectionID, this.#peerCallback); + this.#registered = true; + } + } + + return this.#metadata; + } + + close(): void { + this.#closed = true; + if (this.#registered) { + this.#clearControlCallback(this.#connectionID); + } + } +} diff --git a/packages/sandbox-container/src/core/container.ts b/packages/sandbox-container/src/core/container.ts index 45745fe5a..1bb353cc3 100644 --- a/packages/sandbox-container/src/core/container.ts +++ b/packages/sandbox-container/src/core/container.ts @@ -43,7 +43,10 @@ export class Container { // Updated on every session open so tunnel exits and other future // container→DO events route to the current peer. Cleared by the WS // close handler. `null` between connections. - private controlCallback: SandboxControlCallback | null = null; + private controlCallback: { + connectionID: string; + callback: SandboxControlCallback; + } | null = null; get(key: T): Dependencies[T] { if (!this.initialized) { @@ -69,17 +72,26 @@ export class Container { } /** - * Set / clear the DO-side control callback exposed via the current - * capnweb session's remote main. Called from `server.ts` on each - * `capnweb` WS open (with the new peer) and on close (with `null`). + * Store the DO-side control callback after a capnweb session activates. + * Close handling supplies the connection ID so stale sessions cannot clear + * a callback registered by a newer activated session. */ - setControlCallback(cb: SandboxControlCallback | null): void { - this.controlCallback = cb; + setControlCallback( + connectionID: string, + callback: SandboxControlCallback + ): void { + this.controlCallback = { connectionID, callback }; + } + + clearControlCallback(connectionID: string): void { + if (this.controlCallback?.connectionID === connectionID) { + this.controlCallback = null; + } } /** Returns the current peer's control callback or `null`. */ getControlCallback(): SandboxControlCallback | null { - return this.controlCallback; + return this.controlCallback?.callback ?? null; } async initialize(): Promise { diff --git a/packages/sandbox-container/src/handlers/terminal-ws-handler.ts b/packages/sandbox-container/src/handlers/terminal-ws-handler.ts index f0a798ae9..72e21f3ed 100644 --- a/packages/sandbox-container/src/handlers/terminal-ws-handler.ts +++ b/packages/sandbox-container/src/handlers/terminal-ws-handler.ts @@ -17,6 +17,7 @@ export interface TerminalWSData { cursor?: string; cols?: number; rows?: number; + runtimeIncarnationID: string; } interface TerminalOutputReader { diff --git a/packages/sandbox-container/src/server.ts b/packages/sandbox-container/src/server.ts index 7951a0fa1..8e1244b39 100644 --- a/packages/sandbox-container/src/server.ts +++ b/packages/sandbox-container/src/server.ts @@ -1,10 +1,18 @@ -import { createLogger, type SandboxControlCallback } from '@repo/shared'; +import { + createLogger, + type RuntimeMetadata, + type SandboxControlCallback +} from '@repo/shared'; import type { ServerWebSocket } from 'bun'; import { serve } from 'bun'; import { type BunWebSocketTransport, newBunWebSocketRpcSession } from 'capnweb'; import { trustRuntimeCert } from './cert'; import { CONFIG } from './config'; -import { SandboxControlAPI } from './control-plane'; +import { type SandboxAPIDeps, SandboxControlAPI } from './control-plane'; +import { + CONTROL_PROTOCOL_VERSION, + ControlSession +} from './control-plane/session'; import { Container } from './core/container'; import type { TerminalWSData } from './handlers/terminal-ws-handler'; @@ -12,6 +20,7 @@ export type CapnwebWSData = { type: 'capnweb'; connectionId: string; transport?: BunWebSocketTransport; + controlSession?: ControlSession; }; export type WSData = TerminalWSData | CapnwebWSData; @@ -27,6 +36,43 @@ export function webSocketUpgradeFailedResponse(): Response { return new Response('WebSocket upgrade failed', { status: 503 }); } +function terminalWebSocketUpgradeResponse( + req: Request, + server: { upgrade(req: Request, options: { data: TerminalWSData }): boolean }, + runtimeIncarnationID: string +): Response { + const url = new URL(req.url); + const terminalId = url.searchParams.get('terminalId'); + if (!terminalId) { + return new Response('terminalId query parameter required', { status: 400 }); + } + + const expectedRuntimeIncarnationID = url.searchParams.get( + 'runtimeIncarnationID' + ); + if (expectedRuntimeIncarnationID !== runtimeIncarnationID) { + return new Response('Runtime incarnation mismatch', { status: 409 }); + } + + const colsParam = url.searchParams.get('cols'); + const rowsParam = url.searchParams.get('rows'); + const cursor = url.searchParams.get('cursor') ?? undefined; + + const upgraded = server.upgrade(req, { + data: { + type: 'terminal' as const, + terminalId, + connectionId: generateConnectionId(), + cursor, + runtimeIncarnationID: expectedRuntimeIncarnationID, + cols: colsParam ? Number.parseInt(colsParam, 10) : undefined, + rows: rowsParam ? Number.parseInt(rowsParam, 10) : undefined + } + }); + if (upgraded) return undefined as unknown as Response; + return webSocketUpgradeFailedResponse(); +} + // Global error handlers to prevent fragmented stack traces in logs // Bun's default handler writes stack traces line-by-line to stderr, // which Cloudflare captures as separate log entries @@ -54,13 +100,19 @@ async function createApplication(): Promise<{ server: ReturnType> ) => Promise; container: Container; - controlPlaneAPI: SandboxControlAPI; + controlPlaneMetadata: RuntimeMetadata; + controlPlaneDeps: SandboxAPIDeps; }> { const container = new Container(); await container.initialize(); - // Create the control-plane API that calls services directly. - const controlPlaneAPI = new SandboxControlAPI({ + const controlPlaneMetadata: RuntimeMetadata = { + runtimeIncarnationID: crypto.randomUUID(), + sandboxVersion: process.env.SANDBOX_VERSION || 'unknown', + controlProtocolVersion: CONTROL_PROTOCOL_VERSION + }; + + const controlPlaneDeps: SandboxAPIDeps = { fileService: container.get('fileService'), portService: container.get('portService'), processService: container.get('processService'), @@ -71,7 +123,7 @@ async function createApplication(): Promise<{ extensionHost: container.get('extensionHost'), commandContextService: container.get('commandContextService'), logger - }); + }; return { fetch: async ( @@ -83,35 +135,11 @@ async function createApplication(): Promise<{ const url = new URL(req.url); if (url.pathname === '/ws/terminal') { - const terminalId = url.searchParams.get('terminalId'); - if (!terminalId) { - return new Response('terminalId query parameter required', { - status: 400 - }); - } - - const colsParam = url.searchParams.get('cols'); - const rowsParam = url.searchParams.get('rows'); - const cursor = url.searchParams.get('cursor') ?? undefined; - - const upgraded = server.upgrade(req, { - data: { - type: 'terminal' as const, - terminalId, - connectionId: generateConnectionId(), - cursor, - cols: colsParam ? Number.parseInt(colsParam, 10) : undefined, - rows: rowsParam ? Number.parseInt(rowsParam, 10) : undefined - } - }); - if (upgraded) { - // Bun's server.upgrade() handles the response internally — at runtime the - // fetch handler returns `undefined` to signal a successful upgrade. The Bun - // type signature requires `MaybePromise` (no `undefined`), so we - // cast through `unknown`. See: https://bun.sh/docs/api/websockets#upgrade - return undefined as unknown as Response; - } - return webSocketUpgradeFailedResponse(); + return terminalWebSocketUpgradeResponse( + req, + server, + controlPlaneMetadata.runtimeIncarnationID + ); } if (url.pathname === '/rpc') { @@ -132,7 +160,8 @@ async function createApplication(): Promise<{ return new Response('Not Found', { status: 404 }); }, container, - controlPlaneAPI + controlPlaneMetadata, + controlPlaneDeps }; } @@ -172,15 +201,25 @@ export async function startServer(): Promise { } catch {} }); } else if (ws.data.type === 'capnweb') { + const session = new ControlSession({ + metadata: app.controlPlaneMetadata, + connectionID: ws.data.connectionId, + peerCallback: undefined, + registerControlCallback: (connectionID, callback) => { + app.container.setControlCallback(connectionID, callback); + }, + clearControlCallback: (connectionID) => { + app.container.clearControlCallback(connectionID); + } + }); + const api = new SandboxControlAPI(app.controlPlaneDeps, session); const { stub, transport } = newBunWebSocketRpcSession< SandboxControlCallback, WSData - >(ws, app.controlPlaneAPI); + >(ws, api); ws.data.transport = transport; - // Capture the peer's remote main (the DO's - // SandboxControlCallback) so the container can push - // events back — e.g. tunnel-exit notifications. - app.container.setControlCallback(stub); + session.setPeerCallback(stub); + ws.data.controlSession = session; logger.debug('RPC session initialized', { connectionId: ws.data.connectionId }); @@ -200,10 +239,7 @@ export async function startServer(): Promise { .onClose(ws as ServerWebSocket, code, reason); } else if (ws.data.type === 'capnweb') { ws.data.transport?.dispatchClose(code, reason); - // Forget the peer's control callback. Subsequent tunnel - // exits resolve `null` from the accessor and become no-ops - // until a new session opens. - app.container.setControlCallback(null); + ws.data.controlSession?.close(); } } catch (error) { logger.error( diff --git a/packages/sandbox-container/src/services/process-service.ts b/packages/sandbox-container/src/services/process-service.ts index 498541bce..c4578beec 100644 --- a/packages/sandbox-container/src/services/process-service.ts +++ b/packages/sandbox-container/src/services/process-service.ts @@ -16,6 +16,7 @@ import { type ProcessStatus, type SandboxCommand } from '@repo/shared'; +import { CONFIG } from '../config'; export const MAX_RETAINED_TERMINAL_PROCESSES = 64; @@ -72,7 +73,7 @@ export class ProcessService { const process = await this.#supervisor.start({ runId: id, command, - cwd: options.cwd, + cwd: options.cwd ?? CONFIG.DEFAULT_CWD, env: options.env, timeoutMs: options.timeout, onTerminal: (status) => this.onTerminal(id, status) diff --git a/packages/sandbox-container/tests/handlers/terminal-ws-handler.test.ts b/packages/sandbox-container/tests/handlers/terminal-ws-handler.test.ts index 4e9a1423a..de02bf220 100644 --- a/packages/sandbox-container/tests/handlers/terminal-ws-handler.test.ts +++ b/packages/sandbox-container/tests/handlers/terminal-ws-handler.test.ts @@ -18,8 +18,11 @@ type MockWebSocket = Pick< type MockPty = RuntimeTerminalProcess; -const createMockWS = (data: TerminalWSData): MockWebSocket => ({ - data, +const createMockWS = ( + data: Omit & + Partial> +): MockWebSocket => ({ + data: { runtimeIncarnationID: 'runtime-incarnation-test', ...data }, send: mock(() => 1), sendBinary: mock(() => 1), close: mock(() => {}) diff --git a/packages/sandbox-container/tests/rpc/sandbox-api-backup.test.ts b/packages/sandbox-container/tests/rpc/sandbox-api-backup.test.ts index e473a8fd8..f09168e24 100644 --- a/packages/sandbox-container/tests/rpc/sandbox-api-backup.test.ts +++ b/packages/sandbox-container/tests/rpc/sandbox-api-backup.test.ts @@ -1,10 +1,8 @@ import { beforeEach, describe, expect, it, vi } from 'bun:test'; import type { Logger } from '@repo/shared'; -import { - type SandboxAPIDeps, - SandboxControlAPI -} from '@sandbox-container/control-plane'; +import type { SandboxAPIDeps } from '@sandbox-container/control-plane'; import type { BackupService } from '@sandbox-container/services/backup-service'; +import { createActivatedSandboxControlAPI } from './session-helper'; const mockLogger = { info: vi.fn(), @@ -15,8 +13,8 @@ const mockLogger = { } as Logger; mockLogger.child = vi.fn(() => mockLogger); -function buildApi(backupService: BackupService): SandboxControlAPI { - return new SandboxControlAPI({ +async function buildApi(backupService: BackupService) { + return createActivatedSandboxControlAPI({ backupService, logger: mockLogger } as unknown as SandboxAPIDeps); @@ -61,7 +59,7 @@ describe('SandboxControlAPI backup', () => { }); it('routes backup calls to stateless service operations', async () => { - const api = buildApi(mockBackupService); + const api = await buildApi(mockBackupService); await api.backup.createArchive('/workspace/app', '/var/backups/app.sqsh', { gitignore: true, diff --git a/packages/sandbox-container/tests/rpc/sandbox-api-files.test.ts b/packages/sandbox-container/tests/rpc/sandbox-api-files.test.ts index 3c1670551..8f37be86d 100644 --- a/packages/sandbox-container/tests/rpc/sandbox-api-files.test.ts +++ b/packages/sandbox-container/tests/rpc/sandbox-api-files.test.ts @@ -1,10 +1,8 @@ import { describe, expect, it, vi } from 'bun:test'; import type { Logger } from '@repo/shared'; -import { - type SandboxAPIDeps, - SandboxControlAPI -} from '@sandbox-container/control-plane'; +import type { SandboxAPIDeps } from '@sandbox-container/control-plane'; import type { FileService } from '@sandbox-container/services/file-service'; +import { createActivatedSandboxControlAPI } from './session-helper'; const logger = { info: vi.fn(), @@ -31,7 +29,7 @@ describe('SandboxControlAPI files', () => { writeFile: vi.fn().mockResolvedValue({ success: true }), exists: vi.fn().mockResolvedValue({ success: true, data: true }) } as unknown as FileService; - const api = new SandboxControlAPI({ + const api = await createActivatedSandboxControlAPI({ fileService, logger } as unknown as SandboxAPIDeps); diff --git a/packages/sandbox-container/tests/rpc/sandbox-api-ports.test.ts b/packages/sandbox-container/tests/rpc/sandbox-api-ports.test.ts index d7958b85a..85930bd18 100644 --- a/packages/sandbox-container/tests/rpc/sandbox-api-ports.test.ts +++ b/packages/sandbox-container/tests/rpc/sandbox-api-ports.test.ts @@ -1,11 +1,9 @@ import { describe, expect, it, vi } from 'bun:test'; import type { Logger, PortWatchEvent } from '@repo/shared'; -import { - type SandboxAPIDeps, - SandboxControlAPI -} from '@sandbox-container/control-plane'; +import type { SandboxAPIDeps } from '@sandbox-container/control-plane'; import type { PortService } from '@sandbox-container/services/port-service'; import { StreamSubscriptionRPC } from '../../src/control-plane/subscription-rpc'; +import { createActivatedSandboxControlAPI } from './session-helper'; const logger = { info: vi.fn(), @@ -27,7 +25,7 @@ describe('SandboxControlAPI ports', () => { const portService = { openWatch: vi.fn(() => stream) } as unknown as PortService; - const api = new SandboxControlAPI({ + const api = await createActivatedSandboxControlAPI({ portService, logger } as unknown as SandboxAPIDeps); diff --git a/packages/sandbox-container/tests/rpc/sandbox-api-session.test.ts b/packages/sandbox-container/tests/rpc/sandbox-api-session.test.ts new file mode 100644 index 000000000..d45a4d0a3 --- /dev/null +++ b/packages/sandbox-container/tests/rpc/sandbox-api-session.test.ts @@ -0,0 +1,179 @@ +import { describe, expect, it, vi } from 'bun:test'; +import { + ErrorCode, + type RuntimeMetadata, + type SandboxControlCallback +} from '@repo/shared'; +import { + type SandboxAPIDeps, + SandboxControlAPI +} from '@sandbox-container/control-plane'; +import { ControlSession } from '@sandbox-container/control-plane/session'; + +const metadata: RuntimeMetadata = { + runtimeIncarnationID: 'runtime-1', + sandboxVersion: '1.2.3', + controlProtocolVersion: 1 +}; + +function createAPI(connectionID = 'conn-1', callback?: SandboxControlCallback) { + const fileService = { + exists: vi.fn().mockResolvedValue({ success: true, data: true }) + }; + const registerControlCallback = vi.fn(); + const clearControlCallback = vi.fn(); + const session = new ControlSession({ + metadata, + connectionID, + peerCallback: callback, + registerControlCallback, + clearControlCallback + }); + const api = new SandboxControlAPI( + { + fileService + } as unknown as SandboxAPIDeps, + session + ); + return { + api, + fileService, + registerControlCallback, + clearControlCallback, + session + }; +} + +async function expectIncompatible(promise: Promise) { + await expect(promise).rejects.toMatchObject({ + code: ErrorCode.CONTROL_PROTOCOL_INCOMPATIBLE + }); +} + +describe('SandboxControlAPI session activation', () => { + it('returns stable process metadata before activation', async () => { + const { api } = createAPI(); + + await expect(api.utils.ping()).resolves.toBe('healthy'); + await expect(api.utils.getRuntimeMetadata()).resolves.toEqual(metadata); + await expect(api.utils.getRuntimeMetadata()).resolves.toBe( + await api.utils.getRuntimeMetadata() + ); + }); + + it('rejects all non-utils domains before activation', async () => { + const { api, fileService } = createAPI(); + const gatedDomains = [ + 'files', + 'ports', + 'processes', + 'terminals', + 'backup', + 'mounts', + 'tunnels', + 'watch', + 'workspace', + 'extensions' + ] as const; + + for (const domain of gatedDomains) { + expect(() => api[domain]).toThrow( + expect.objectContaining({ + code: ErrorCode.CONTROL_PROTOCOL_INCOMPATIBLE + }) + ); + } + expect(fileService.exists).not.toHaveBeenCalled(); + }); + + it('rejects activation for the wrong incarnation', async () => { + const { api, registerControlCallback } = createAPI(); + + await expectIncompatible(api.utils.activateControlSession('runtime-2')); + expect(registerControlCallback).not.toHaveBeenCalled(); + }); + + it('activates idempotently only for the matching incarnation', async () => { + const callback = { onTunnelRunExit: vi.fn() }; + const { api, registerControlCallback } = createAPI('conn-1', callback); + + await expect( + api.utils.activateControlSession('runtime-1') + ).resolves.toEqual(metadata); + await expect( + api.files.exists('/workspace/file.txt') + ).resolves.toMatchObject({ + success: true, + exists: true + }); + await expect( + api.utils.activateControlSession('runtime-1') + ).resolves.toEqual(metadata); + await expectIncompatible(api.utils.activateControlSession('runtime-2')); + expect(registerControlCallback).toHaveBeenCalledTimes(1); + expect(registerControlCallback).toHaveBeenCalledWith('conn-1', callback); + }); + + it('rejects first activation after the session closes', async () => { + const { api, session, registerControlCallback } = createAPI(); + session.close(); + + await expectIncompatible(api.utils.activateControlSession('runtime-1')); + expect(registerControlCallback).not.toHaveBeenCalled(); + }); + + it('registers callbacks only after successful activation', async () => { + const callback = { onTunnelRunExit: vi.fn() }; + const { api, registerControlCallback } = createAPI('conn-1', callback); + + expect(registerControlCallback).not.toHaveBeenCalled(); + await api.utils.activateControlSession('runtime-1'); + expect(registerControlCallback).toHaveBeenCalledWith('conn-1', callback); + }); + + it('clears callback ownership only for the matching connection', async () => { + const firstCallback = { onTunnelRunExit: vi.fn() }; + const secondCallback = { onTunnelRunExit: vi.fn() }; + let current: { + connectionID: string; + callback: SandboxControlCallback; + } | null = null; + const registerControlCallback = vi.fn( + (connectionID: string, callback: SandboxControlCallback) => { + current = { connectionID, callback }; + } + ); + const clearControlCallback = vi.fn((connectionID: string) => { + if (current?.connectionID === connectionID) current = null; + }); + const session1 = new ControlSession({ + metadata, + connectionID: 'conn-1', + peerCallback: firstCallback, + registerControlCallback, + clearControlCallback + }); + const session2 = new ControlSession({ + metadata, + connectionID: 'conn-2', + peerCallback: secondCallback, + registerControlCallback, + clearControlCallback + }); + + await session1.activate('runtime-1'); + await session2.activate('runtime-1'); + session1.close(); + expect(current as unknown).toEqual({ + connectionID: 'conn-2', + callback: secondCallback + }); + await session1.activate('runtime-1'); + expect(current as unknown).toEqual({ + connectionID: 'conn-2', + callback: secondCallback + }); + session2.close(); + expect(current).toBeNull(); + }); +}); diff --git a/packages/sandbox-container/tests/rpc/sandbox-api-terminals.test.ts b/packages/sandbox-container/tests/rpc/sandbox-api-terminals.test.ts index 70690decf..6164388d2 100644 --- a/packages/sandbox-container/tests/rpc/sandbox-api-terminals.test.ts +++ b/packages/sandbox-container/tests/rpc/sandbox-api-terminals.test.ts @@ -5,14 +5,14 @@ import type { TerminalOutputEvent, TerminalSnapshot } from '@repo/shared'; -import { - type SandboxAPIDeps, - SandboxControlAPI -} from '@sandbox-container/control-plane'; +import type { SandboxAPIDeps } from '@sandbox-container/control-plane'; import type { TerminalManager } from '@sandbox-container/services/terminal-manager'; +import { createActivatedSandboxControlAPI } from './session-helper'; -function buildApi(terminalManager: TerminalManager): SandboxControlAPI { - return new SandboxControlAPI({ terminalManager } as SandboxAPIDeps); +async function buildApi(terminalManager: TerminalManager) { + return createActivatedSandboxControlAPI({ + terminalManager + } as SandboxAPIDeps); } describe('SandboxControlAPI terminals', () => { @@ -20,7 +20,7 @@ describe('SandboxControlAPI terminals', () => { let manager: TerminalManager; let terminals: SandboxTerminalsAPI; - beforeEach(() => { + beforeEach(async () => { snapshot = { id: 'generated-terminal-id', pid: 123, @@ -39,7 +39,7 @@ describe('SandboxControlAPI terminals', () => { terminate: mock(async () => undefined), hasActive: mock(async () => true) } as unknown as TerminalManager; - terminals = buildApi(manager).terminals; + terminals = (await buildApi(manager)).terminals; }); it('creates terminals through the focused terminals RPC API', async () => { diff --git a/packages/sandbox-container/tests/rpc/sandbox-api-watch.test.ts b/packages/sandbox-container/tests/rpc/sandbox-api-watch.test.ts index 2530d0ca9..5fd267d83 100644 --- a/packages/sandbox-container/tests/rpc/sandbox-api-watch.test.ts +++ b/packages/sandbox-container/tests/rpc/sandbox-api-watch.test.ts @@ -1,23 +1,21 @@ import { beforeEach, describe, expect, it, mock } from 'bun:test'; import type { SandboxWatchAPI, WatchRequest } from '@repo/shared'; -import { - type SandboxAPIDeps, - SandboxControlAPI -} from '@sandbox-container/control-plane'; +import type { SandboxAPIDeps } from '@sandbox-container/control-plane'; import type { WatchService } from '@sandbox-container/services/watch-service'; +import { createActivatedSandboxControlAPI } from './session-helper'; -function buildApi(watchService: WatchService): SandboxControlAPI { - return new SandboxControlAPI({ watchService } as SandboxAPIDeps); +async function buildApi(watchService: WatchService) { + return createActivatedSandboxControlAPI({ watchService } as SandboxAPIDeps); } describe('SandboxControlAPI watch', () => { let watchDirectory: ReturnType; let watch: SandboxWatchAPI; - beforeEach(() => { + beforeEach(async () => { watchDirectory = mock(); const service = { watchDirectory } as unknown as WatchService; - watch = buildApi(service).watch; + watch = (await buildApi(service)).watch; }); it('owns watch streams through a disposable subscription', async () => { diff --git a/packages/sandbox-container/tests/rpc/session-helper.ts b/packages/sandbox-container/tests/rpc/session-helper.ts new file mode 100644 index 000000000..e75898355 --- /dev/null +++ b/packages/sandbox-container/tests/rpc/session-helper.ts @@ -0,0 +1,24 @@ +import type { RuntimeMetadata } from '@repo/shared'; +import type { SandboxAPIDeps } from '@sandbox-container/control-plane'; +import { SandboxControlAPI } from '@sandbox-container/control-plane'; +import { ControlSession } from '@sandbox-container/control-plane/session'; + +const metadata: RuntimeMetadata = { + runtimeIncarnationID: 'test-runtime', + sandboxVersion: 'test-version', + controlProtocolVersion: 1 +}; + +export async function createActivatedSandboxControlAPI( + deps: SandboxAPIDeps +): Promise { + const session = new ControlSession({ + metadata, + connectionID: 'test-connection', + peerCallback: undefined, + registerControlCallback: () => {}, + clearControlCallback: () => {} + }); + await session.activate(metadata.runtimeIncarnationID); + return new SandboxControlAPI(deps, session); +} diff --git a/packages/sandbox-container/tests/server.test.ts b/packages/sandbox-container/tests/server.test.ts index 3265b8d00..f8a55d78a 100644 --- a/packages/sandbox-container/tests/server.test.ts +++ b/packages/sandbox-container/tests/server.test.ts @@ -1,6 +1,7 @@ -import { afterEach, describe, expect, it } from 'bun:test'; +import { afterAll, beforeAll, describe, expect, it } from 'bun:test'; import { registerShutdownHandlers, + startServer, webSocketUpgradeFailedResponse } from '../src/server'; @@ -13,10 +14,58 @@ describe('server WebSocket upgrade failures', () => { }); }); +describe('terminal WebSocket runtime fencing', () => { + const runtimeIncarnationID = '00000000-0000-4000-8000-000000000001'; + const originalRandomUUID = crypto.randomUUID; + let cleanup: (() => Promise) | undefined; + + beforeAll(async () => { + crypto.randomUUID = (() => + runtimeIncarnationID) as typeof crypto.randomUUID; + const server = await startServer(); + cleanup = server.cleanup; + crypto.randomUUID = originalRandomUUID; + }); + + afterAll(async () => { + crypto.randomUUID = originalRandomUUID; + await cleanup?.(); + }); + + it('rejects missing runtime incarnation before upgrade', async () => { + const response = await fetch( + 'http://localhost:3000/ws/terminal?terminalId=terminal-1', + { + headers: { Upgrade: 'websocket' } + } + ); + + expect(response.status).toBe(409); + }); + + it('rejects mismatched runtime incarnation before upgrade', async () => { + const response = await fetch( + 'http://localhost:3000/ws/terminal?terminalId=terminal-1&runtimeIncarnationID=other', + { headers: { Upgrade: 'websocket' } } + ); + + expect(response.status).toBe(409); + }); + + it('accepts matching runtime incarnation through pre-upgrade validation', async () => { + const response = await fetch( + `http://localhost:3000/ws/terminal?terminalId=terminal-1&runtimeIncarnationID=${runtimeIncarnationID}`, + { headers: { Upgrade: 'websocket' } } + ); + + expect(response.status).not.toBe(409); + }); +}); + describe('registerShutdownHandlers', () => { const originalExit = process.exit; - afterEach(() => { + afterAll(() => { process.removeAllListeners('SIGTERM'); process.removeAllListeners('SIGINT'); process.exit = originalExit; diff --git a/packages/sandbox-container/tests/services/process-service.test.ts b/packages/sandbox-container/tests/services/process-service.test.ts index dd2a412f7..82f14c61d 100644 --- a/packages/sandbox-container/tests/services/process-service.test.ts +++ b/packages/sandbox-container/tests/services/process-service.test.ts @@ -166,6 +166,17 @@ describe('ProcessService', () => { } }); + it('defaults process cwd to the workspace', async () => { + const stub = supervisor(); + const service = new ProcessService({ supervisor: stub, logger }); + + await service.start(['pwd']); + + expect(stub.start).toHaveBeenCalledWith( + expect.objectContaining({ cwd: '/workspace' }) + ); + }); + it('validates only argv[0], cwd, environment and timeout before starting', async () => { const service = new ProcessService({ supervisor: supervisor(), logger }); const temp = await mkdtemp(join(tmpdir(), 'process-service-')); diff --git a/packages/sandbox/package.json b/packages/sandbox/package.json index 38936d528..78735144a 100644 --- a/packages/sandbox/package.json +++ b/packages/sandbox/package.json @@ -8,7 +8,7 @@ "description": "A sandboxed environment for running commands", "type": "module", "dependencies": { - "@cloudflare/containers": "^0.3.5", + "@cloudflare/containers": "^0.3.7", "aws4fetch": "^1.0.20", "capnweb": "^0.8.0", "hono": "^4.12.26" @@ -57,6 +57,7 @@ "typecheck": "tsc --noEmit && tsc --noEmit -p tsconfig.sidecar.json", "docker:local": "scripts/docker-local.sh", "test": "vitest run --config vitest.config.ts \"$@\"", + "test:harness": "vitest run --config vitest.harness.config.ts \"$@\"", "test:sidecar": "bun test ../../extensions/interpreter/src/sidecar", "test:e2e": "npm run test:e2e:vitest && npm run test:e2e:browser", "test:e2e:vitest": "cd ../../tests/e2e/test-worker && ./generate-config.sh && cd ../../.. && vitest run --config vitest.e2e.config.ts \"$@\"", diff --git a/packages/sandbox/src/backup/backup-service.ts b/packages/sandbox/src/backup/backup-service.ts index 037ee861e..2673becce 100644 --- a/packages/sandbox/src/backup/backup-service.ts +++ b/packages/sandbox/src/backup/backup-service.ts @@ -5,7 +5,6 @@ import type { } from '@repo/shared'; import { type createLogger, logCanonicalEvent } from '@repo/shared'; import type { ContainerControlClient } from '../container-control'; -import type { CurrentRuntimeIdentity } from '../current-runtime-identity'; import { BackupCreateError, BackupExpiredError, @@ -14,6 +13,7 @@ import { ErrorCode, InvalidBackupConfigError } from '../errors'; +import type { RuntimeIdentity, RuntimeIdentityReader } from '../runtime'; import type { CurrentSandboxLifetime } from '../sandbox-lifetime'; import { isR2Bucket } from '../storage-mount'; import { @@ -37,12 +37,23 @@ import { validateBackupDir } from './validation'; export type { BackupRestoreTestFault } from './restore-fault-injection'; +export type BackupAttemptLease = { + runtime: RuntimeIdentity; + control: ContainerControlClient; + retain(onInterrupt?: () => void): { release(): void }; +}; + +type BackupAttemptCall = ( + operation: string, + call: (lease: BackupAttemptLease) => Promise +) => Promise; + type BackupServiceDeps = { ctx: DurableObjectState<{}>; getEnv: () => unknown; logger: ReturnType; - getClient: () => ContainerControlClient; - currentRuntime: CurrentRuntimeIdentity; + runBackupAttempt: BackupAttemptCall; + runtimeReader: RuntimeIdentityReader; currentLifetime: CurrentSandboxLifetime; }; @@ -64,18 +75,17 @@ export class BackupService { ); this.restoreLifecycle = new RestoreLifecycleRunner({ storage: this.ctx.storage, - currentRuntime: deps.currentRuntime, + runtimeReader: deps.runtimeReader, currentLifetime: deps.currentLifetime, faultInjector: this.restoreFaults }); this.transfer = new BackupTransfer({ getEnv: deps.getEnv, - getClient: deps.getClient, logger: deps.logger }); this.creator = new BackupCreator({ getEnv: deps.getEnv, - getClient: deps.getClient, + runBackupAttempt: deps.runBackupAttempt, logger: deps.logger, transfer: this.transfer }); @@ -85,10 +95,6 @@ export class BackupService { return this.deps.getEnv(); } - private get client(): ContainerControlClient { - return this.deps.getClient(); - } - private static readonly UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; @@ -265,30 +271,36 @@ export class BackupService { }); } - await lifecycle.runtimeReady(archiveHead.size); const archivePath = `${BACKUP_CONTAINER_DIR}/${id}.sqsh`; - - const prepareResult = await this.client.backup.prepareRestore({ - dir, - backupId: id, - archivePath - }); - - if (prepareResult.existingSize !== archiveHead.size) { - await this.transfer.downloadBackupParallel( - archivePath, - r2Key, - archiveHead.size, - id, - dir - ); - } - - await lifecycle.archiveReady(archiveHead.size); - - const restoreResult = await this.client.backup.restoreArchive( - dir, - archivePath + const restoreResult = await this.deps.runBackupAttempt( + 'backup.restore', + async ({ runtime, control }) => { + await lifecycle.runtimeReady(runtime, archiveHead.size); + try { + const prepareResult = await control.backup.prepareRestore({ + dir, + backupId: id, + archivePath + }); + + if (prepareResult.existingSize !== archiveHead.size) { + await this.transfer.downloadBackupParallel( + archivePath, + r2Key, + archiveHead.size, + id, + dir, + control + ); + } + + await lifecycle.archiveReady(archiveHead.size); + return await control.backup.restoreArchive(dir, archivePath); + } catch (error) { + await control.backup.cleanupArchive(archivePath).catch(() => {}); + throw error; + } + } ); if (!restoreResult.success) { @@ -313,10 +325,6 @@ export class BackupService { return result; } catch (error) { caughtError = error instanceof Error ? error : new Error(String(error)); - if (id) { - const cleanupPath = `${BACKUP_CONTAINER_DIR}/${id}.sqsh`; - await this.client.backup.cleanupArchive(cleanupPath).catch(() => {}); - } throw error; } finally { logCanonicalEvent(this.logger, { @@ -427,18 +435,6 @@ export class BackupService { }); } - await lifecycle.runtimeReady(metadata.sizeBytes); - const archivePath = `${BACKUP_CONTAINER_DIR}/${id}.sqsh`; - - await this.client.backup.prepareRestore({ - dir, - backupId: id, - archivePath - }); - - // Stream the archive into the container to avoid base64-encoding the - // whole archive in Worker memory and hitting workerd's 32 MiB RPC - // payload cap. const body = archiveObject.body; if (!body) { throw new BackupRestoreError({ @@ -449,14 +445,35 @@ export class BackupService { timestamp: new Date().toISOString() }); } - await this.client.files.writeFileStream(archivePath, body); - - await lifecycle.archiveReady(metadata.sizeBytes); - - await this.client.backup.extractArchive(dir, archivePath); + const archivePath = `${BACKUP_CONTAINER_DIR}/${id}.sqsh`; - // Clean up archive after extraction (no FUSE mount holds it open) - await this.client.backup.cleanupArchive(archivePath).catch(() => {}); + await this.deps.runBackupAttempt( + 'backup.restore', + async ({ runtime, control }) => { + await lifecycle.runtimeReady(runtime, metadata.sizeBytes); + try { + await control.backup.prepareRestore({ + dir, + backupId: id, + archivePath + }); + + // Stream the archive into the container to avoid base64-encoding the + // whole archive in Worker memory and hitting workerd's 32 MiB RPC + // payload cap. + await control.files.writeFileStream(archivePath, body); + + await lifecycle.archiveReady(metadata.sizeBytes); + await control.backup.extractArchive(dir, archivePath); + + // Clean up archive after extraction (no FUSE mount holds it open) + await control.backup.cleanupArchive(archivePath).catch(() => {}); + } catch (error) { + await control.backup.cleanupArchive(archivePath).catch(() => {}); + throw error; + } + } + ); const result = { success: true as const, @@ -470,10 +487,6 @@ export class BackupService { return result; } catch (error) { caughtError = error instanceof Error ? error : new Error(String(error)); - if (id) { - const archivePath = `${BACKUP_CONTAINER_DIR}/${id}.sqsh`; - await this.client.backup.cleanupArchive(archivePath).catch(() => {}); - } throw error; } finally { logCanonicalEvent(this.logger, { diff --git a/packages/sandbox/src/backup/create.ts b/packages/sandbox/src/backup/create.ts index d94ec8f7e..830fff4e5 100644 --- a/packages/sandbox/src/backup/create.ts +++ b/packages/sandbox/src/backup/create.ts @@ -1,6 +1,5 @@ import type { BackupOptions, DirectoryBackup } from '@repo/shared'; import { type createLogger, logCanonicalEvent } from '@repo/shared'; -import type { ContainerControlClient } from '../container-control'; import { BackupCreateError, ErrorCode, @@ -8,6 +7,7 @@ import { } from '../errors'; import { streamFile } from '../file-stream'; import { isR2Bucket } from '../storage-mount'; +import type { BackupAttemptLease } from './backup-service'; import { BACKUP_ARCHIVE_OBJECT_NAME, BACKUP_CONTAINER_DIR, @@ -26,7 +26,10 @@ import { type BackupCreatorDeps = { getEnv: () => unknown; - getClient: () => ContainerControlClient; + runBackupAttempt( + operation: string, + call: (lease: BackupAttemptLease) => Promise + ): Promise; logger: ReturnType; transfer: BackupTransfer; }; @@ -38,10 +41,6 @@ export class BackupCreator { return this.deps.getEnv(); } - private get client(): ContainerControlClient { - return this.deps.getClient(); - } - private get logger(): ReturnType { return this.deps.logger; } @@ -80,131 +79,89 @@ export class BackupCreator { let caughtError: Error | undefined; try { - validateBackupDir(dir, 'BackupOptions.dir'); - if (name !== undefined) { - if (typeof name !== 'string' || name.length > BACKUP_MAX_NAME_LENGTH) { - throw new InvalidBackupConfigError({ - message: `BackupOptions.name must be a string of at most ${BACKUP_MAX_NAME_LENGTH} characters`, - code: ErrorCode.INVALID_BACKUP_CONFIG, - httpStatus: 400, - context: { - reason: `name must be a string of at most ${BACKUP_MAX_NAME_LENGTH} characters` - }, - timestamp: new Date().toISOString() - }); - } - // biome-ignore lint/suspicious/noControlCharactersInRegex: intentionally matching control chars - if (/[\u0000-\u001f\u007f]/.test(name)) { - throw new InvalidBackupConfigError({ - message: 'BackupOptions.name must not contain control characters', - code: ErrorCode.INVALID_BACKUP_CONFIG, - httpStatus: 400, - context: { reason: 'name must not contain control characters' }, - timestamp: new Date().toISOString() - }); - } - } - if (ttl <= 0) { - throw new InvalidBackupConfigError({ - message: 'BackupOptions.ttl must be a positive number of seconds', - code: ErrorCode.INVALID_BACKUP_CONFIG, - httpStatus: 400, - context: { reason: 'ttl must be a positive number of seconds' }, - timestamp: new Date().toISOString() - }); - } - if (typeof gitignore !== 'boolean') { - throw new InvalidBackupConfigError({ - message: 'BackupOptions.gitignore must be a boolean', - code: ErrorCode.INVALID_BACKUP_CONFIG, - httpStatus: 400, - context: { reason: 'gitignore must be a boolean' }, - timestamp: new Date().toISOString() - }); - } - if ( - !Array.isArray(excludes) || - !excludes.every((e: unknown) => typeof e === 'string') - ) { - throw new InvalidBackupConfigError({ - message: 'BackupOptions.excludes must be an array of strings', - code: ErrorCode.INVALID_BACKUP_CONFIG, - httpStatus: 400, - context: { reason: 'excludes must be an array of strings' }, - timestamp: new Date().toISOString() - }); - } + this.validateOptions({ dir, name, ttl, gitignore, excludes }); const resolvedCompression = resolveBackupCompression(compression); const normalizedExcludes = normalizeBackupExcludes(excludes, this.logger); - backupId = crypto.randomUUID(); - const archivePath = `${BACKUP_CONTAINER_DIR}/${backupId}.sqsh`; + const currentBackupId = crypto.randomUUID(); + backupId = currentBackupId; + const archivePath = `${BACKUP_CONTAINER_DIR}/${currentBackupId}.sqsh`; + const r2Key = `${BACKUP_STORAGE_PREFIX}/${currentBackupId}/${BACKUP_ARCHIVE_OBJECT_NAME}`; + const metaKey = `${BACKUP_STORAGE_PREFIX}/${currentBackupId}/${BACKUP_METADATA_OBJECT_NAME}`; + + const result = await this.deps.runBackupAttempt( + 'backup.create', + async ({ control }) => { + try { + const createResult = await control.backup.createArchive( + dir, + archivePath, + { + gitignore, + excludes: normalizedExcludes, + compression: resolvedCompression + } + ); + + if (!createResult.success) { + throw new BackupCreateError({ + message: 'Container failed to create backup archive', + code: ErrorCode.BACKUP_CREATE_FAILED, + httpStatus: 500, + context: { dir, backupId }, + timestamp: new Date().toISOString() + }); + } - const createResult = await this.client.backup.createArchive( - dir, - archivePath, - { - gitignore, - excludes: normalizedExcludes, - compression: resolvedCompression + sizeBytes = createResult.sizeBytes; + if ( + multipart && + createResult.sizeBytes >= BACKUP_MULTIPART_MIN_SIZE + ) { + await this.transfer.uploadBackupMultipart( + archivePath, + r2Key, + createResult.sizeBytes, + currentBackupId, + dir, + control + ); + } else { + await this.transfer.uploadBackupPresigned( + archivePath, + r2Key, + createResult.sizeBytes, + currentBackupId, + dir, + control + ); + } + + const metadata = { + id: currentBackupId, + dir, + name: name || null, + sizeBytes: createResult.sizeBytes, + ttl, + createdAt: new Date().toISOString() + }; + await bucket.put(metaKey, JSON.stringify(metadata)); + + outcome = 'success'; + await control.backup.cleanupArchive(archivePath).catch(() => {}); + return { id: currentBackupId, dir }; + } catch (error) { + await control.backup.cleanupArchive(archivePath).catch(() => {}); + await bucket.delete(r2Key).catch(() => {}); + await bucket.delete(metaKey).catch(() => {}); + throw error; + } } ); - if (!createResult.success) { - throw new BackupCreateError({ - message: 'Container failed to create backup archive', - code: ErrorCode.BACKUP_CREATE_FAILED, - httpStatus: 500, - context: { dir, backupId }, - timestamp: new Date().toISOString() - }); - } - - sizeBytes = createResult.sizeBytes; - const r2Key = `${BACKUP_STORAGE_PREFIX}/${backupId}/${BACKUP_ARCHIVE_OBJECT_NAME}`; - const metaKey = `${BACKUP_STORAGE_PREFIX}/${backupId}/${BACKUP_METADATA_OBJECT_NAME}`; - - if (multipart && createResult.sizeBytes >= BACKUP_MULTIPART_MIN_SIZE) { - await this.transfer.uploadBackupMultipart( - archivePath, - r2Key, - createResult.sizeBytes, - backupId, - dir - ); - } else { - await this.transfer.uploadBackupPresigned( - archivePath, - r2Key, - createResult.sizeBytes, - backupId, - dir - ); - } - - const metadata = { - id: backupId, - dir, - name: name || null, - sizeBytes: createResult.sizeBytes, - ttl, - createdAt: new Date().toISOString() - }; - await bucket.put(metaKey, JSON.stringify(metadata)); - - outcome = 'success'; - await this.client.backup.cleanupArchive(archivePath).catch(() => {}); - return { id: backupId, dir }; + return result; } catch (error) { caughtError = error instanceof Error ? error : new Error(String(error)); - if (backupId) { - const archivePath = `${BACKUP_CONTAINER_DIR}/${backupId}.sqsh`; - const r2Key = `${BACKUP_STORAGE_PREFIX}/${backupId}/${BACKUP_ARCHIVE_OBJECT_NAME}`; - const metaKey = `${BACKUP_STORAGE_PREFIX}/${backupId}/${BACKUP_METADATA_OBJECT_NAME}`; - await this.client.backup.cleanupArchive(archivePath).catch(() => {}); - await bucket.delete(r2Key).catch(() => {}); - await bucket.delete(metaKey).catch(() => {}); - } throw error; } finally { logCanonicalEvent(this.logger, { @@ -220,6 +177,72 @@ export class BackupCreator { } } + private validateOptions(options: { + dir: string; + name: string | undefined; + ttl: number; + gitignore: boolean; + excludes: string[]; + }): void { + validateBackupDir(options.dir, 'BackupOptions.dir'); + if (options.name !== undefined) { + if ( + typeof options.name !== 'string' || + options.name.length > BACKUP_MAX_NAME_LENGTH + ) { + throw new InvalidBackupConfigError({ + message: `BackupOptions.name must be a string of at most ${BACKUP_MAX_NAME_LENGTH} characters`, + code: ErrorCode.INVALID_BACKUP_CONFIG, + httpStatus: 400, + context: { + reason: `name must be a string of at most ${BACKUP_MAX_NAME_LENGTH} characters` + }, + timestamp: new Date().toISOString() + }); + } + // biome-ignore lint/suspicious/noControlCharactersInRegex: intentionally matching control chars + if (/[\u0000-\u001f\u007f]/.test(options.name)) { + throw new InvalidBackupConfigError({ + message: 'BackupOptions.name must not contain control characters', + code: ErrorCode.INVALID_BACKUP_CONFIG, + httpStatus: 400, + context: { reason: 'name must not contain control characters' }, + timestamp: new Date().toISOString() + }); + } + } + if (options.ttl <= 0) { + throw new InvalidBackupConfigError({ + message: 'BackupOptions.ttl must be a positive number of seconds', + code: ErrorCode.INVALID_BACKUP_CONFIG, + httpStatus: 400, + context: { reason: 'ttl must be a positive number of seconds' }, + timestamp: new Date().toISOString() + }); + } + if (typeof options.gitignore !== 'boolean') { + throw new InvalidBackupConfigError({ + message: 'BackupOptions.gitignore must be a boolean', + code: ErrorCode.INVALID_BACKUP_CONFIG, + httpStatus: 400, + context: { reason: 'gitignore must be a boolean' }, + timestamp: new Date().toISOString() + }); + } + if ( + !Array.isArray(options.excludes) || + !options.excludes.every((e: unknown) => typeof e === 'string') + ) { + throw new InvalidBackupConfigError({ + message: 'BackupOptions.excludes must be an array of strings', + code: ErrorCode.INVALID_BACKUP_CONFIG, + httpStatus: 400, + context: { reason: 'excludes must be an array of strings' }, + timestamp: new Date().toISOString() + }); + } + } + /** * Local-dev implementation of createBackup. * Uses the R2 binding directly instead of presigned URLs. @@ -243,7 +266,6 @@ export class BackupCreator { let outcome: 'success' | 'error' = 'error'; let caughtError: Error | undefined; - // Resolve backup bucket from env as an R2 binding const envObj = this.env as Record; const bucket = envObj.BACKUP_BUCKET; if (!bucket || !isR2Bucket(bucket)) { @@ -259,159 +281,111 @@ export class BackupCreator { } try { - validateBackupDir(dir, 'BackupOptions.dir'); - if (name !== undefined) { - if (typeof name !== 'string' || name.length > BACKUP_MAX_NAME_LENGTH) { - throw new InvalidBackupConfigError({ - message: `BackupOptions.name must be a string of at most ${BACKUP_MAX_NAME_LENGTH} characters`, - code: ErrorCode.INVALID_BACKUP_CONFIG, - httpStatus: 400, - context: { - reason: `name must be a string of at most ${BACKUP_MAX_NAME_LENGTH} characters` - }, - timestamp: new Date().toISOString() - }); - } - // biome-ignore lint/suspicious/noControlCharactersInRegex: intentionally matching control chars - if (/[\u0000-\u001f\u007f]/.test(name)) { - throw new InvalidBackupConfigError({ - message: 'BackupOptions.name must not contain control characters', - code: ErrorCode.INVALID_BACKUP_CONFIG, - httpStatus: 400, - context: { reason: 'name must not contain control characters' }, - timestamp: new Date().toISOString() - }); - } - } - if (ttl <= 0) { - throw new InvalidBackupConfigError({ - message: 'BackupOptions.ttl must be a positive number of seconds', - code: ErrorCode.INVALID_BACKUP_CONFIG, - httpStatus: 400, - context: { reason: 'ttl must be a positive number of seconds' }, - timestamp: new Date().toISOString() - }); - } - if (typeof gitignore !== 'boolean') { - throw new InvalidBackupConfigError({ - message: 'BackupOptions.gitignore must be a boolean', - code: ErrorCode.INVALID_BACKUP_CONFIG, - httpStatus: 400, - context: { reason: 'gitignore must be a boolean' }, - timestamp: new Date().toISOString() - }); - } - if ( - !Array.isArray(excludes) || - !excludes.every((e: unknown) => typeof e === 'string') - ) { - throw new InvalidBackupConfigError({ - message: 'BackupOptions.excludes must be an array of strings', - code: ErrorCode.INVALID_BACKUP_CONFIG, - httpStatus: 400, - context: { reason: 'excludes must be an array of strings' }, - timestamp: new Date().toISOString() - }); - } - + this.validateOptions({ dir, name, ttl, gitignore, excludes }); const resolvedCompression = resolveBackupCompression(compression); - const normalizedExcludes = normalizeBackupExcludes(excludes, this.logger); - backupId = crypto.randomUUID(); - const archivePath = `${BACKUP_CONTAINER_DIR}/${backupId}.sqsh`; - - // Step 1: Create squashfs archive in the container (same as production) - const createResult = await this.client.backup.createArchive( - dir, - archivePath, - { - gitignore, - excludes: normalizedExcludes, - compression: resolvedCompression - } - ); - - if (!createResult.success) { - throw new BackupCreateError({ - message: 'Container failed to create backup archive', - code: ErrorCode.BACKUP_CREATE_FAILED, - httpStatus: 500, - context: { dir, backupId }, - timestamp: new Date().toISOString() - }); - } - - sizeBytes = createResult.sizeBytes; - const r2Key = `${BACKUP_STORAGE_PREFIX}/${backupId}/${BACKUP_ARCHIVE_OBJECT_NAME}`; - const metaKey = `${BACKUP_STORAGE_PREFIX}/${backupId}/${BACKUP_METADATA_OBJECT_NAME}`; + const currentBackupId = crypto.randomUUID(); + backupId = currentBackupId; + const archivePath = `${BACKUP_CONTAINER_DIR}/${currentBackupId}.sqsh`; + const r2Key = `${BACKUP_STORAGE_PREFIX}/${currentBackupId}/${BACKUP_ARCHIVE_OBJECT_NAME}`; + const metaKey = `${BACKUP_STORAGE_PREFIX}/${currentBackupId}/${BACKUP_METADATA_OBJECT_NAME}`; - // Step 2: Read archive from container and stream it into R2 via binding. - // readFileStream returns SSE-framed base64 chunks, so we pipe it through - // streamFile (which decodes SSE frames + base64 on the fly) into a - // FixedLengthStream backed by the known archive size. This avoids - // buffering the whole archive in Worker memory. - const archiveStream = await this.client.files.readFileStream( - archivePath, - {} - ); - const sseDecoded = new ReadableStream({ - async start(controller) { + const result = await this.deps.runBackupAttempt( + 'backup.create', + async ({ control }) => { try { - for await (const chunk of streamFile(archiveStream)) { - if (chunk instanceof Uint8Array) { - controller.enqueue(chunk); + const createResult = await control.backup.createArchive( + dir, + archivePath, + { + gitignore, + excludes: normalizedExcludes, + compression: resolvedCompression } + ); + + if (!createResult.success) { + throw new BackupCreateError({ + message: 'Container failed to create backup archive', + code: ErrorCode.BACKUP_CREATE_FAILED, + httpStatus: 500, + context: { dir, backupId }, + timestamp: new Date().toISOString() + }); } - controller.close(); - } catch (err) { - controller.error(err); - } - } - }); - const fixedStream = new FixedLengthStream(createResult.sizeBytes); - sseDecoded.pipeTo(fixedStream.writable).catch(() => {}); - await bucket.put(r2Key, fixedStream.readable); - - // Verify upload — size comes from createArchive result, not the stream. - const head = await bucket.head(r2Key); - if (!head || head.size !== createResult.sizeBytes) { - throw new BackupCreateError({ - message: `Upload verification failed: expected ${createResult.sizeBytes} bytes, got ${head?.size ?? 0}`, - code: ErrorCode.BACKUP_CREATE_FAILED, - httpStatus: 500, - context: { dir, backupId }, - timestamp: new Date().toISOString() - }); - } - // Step 3: Write metadata - const metadata = { - id: backupId, - dir, - name: name || null, - sizeBytes: createResult.sizeBytes, - ttl, - createdAt: new Date().toISOString() - }; - await bucket.put(metaKey, JSON.stringify(metadata)); - - outcome = 'success'; + sizeBytes = createResult.sizeBytes; + const archiveStream = await control.files.readFileStream( + archivePath, + {} + ); + const fixedStream = new FixedLengthStream(createResult.sizeBytes); + const writer = fixedStream.writable.getWriter(); + const uploadArchive = bucket + .put(r2Key, fixedStream.readable) + .catch(async (error) => { + await writer.abort(error).catch(() => {}); + throw error; + }); + const decodeAndWrite = (async () => { + try { + for await (const chunk of streamFile(archiveStream)) { + if (chunk instanceof Uint8Array) { + await writer.write(chunk); + } + } + await writer.close(); + } catch (error) { + await writer.abort(error).catch(() => {}); + throw error; + } + })(); + const results = await Promise.allSettled([ + decodeAndWrite, + uploadArchive + ]); + const rejected = [...results] + .reverse() + .find((result) => result.status === 'rejected'); + if (rejected) throw rejected.reason; + + const head = await bucket.head(r2Key); + if (!head || head.size !== createResult.sizeBytes) { + throw new BackupCreateError({ + message: `Upload verification failed: expected ${createResult.sizeBytes} bytes, got ${head?.size ?? 0}`, + code: ErrorCode.BACKUP_CREATE_FAILED, + httpStatus: 500, + context: { dir, backupId }, + timestamp: new Date().toISOString() + }); + } - // Clean up local archive - await this.client.backup.cleanupArchive(archivePath).catch(() => {}); + const metadata = { + id: currentBackupId, + dir, + name: name || null, + sizeBytes: createResult.sizeBytes, + ttl, + createdAt: new Date().toISOString() + }; + await bucket.put(metaKey, JSON.stringify(metadata)); + + outcome = 'success'; + await control.backup.cleanupArchive(archivePath).catch(() => {}); + return { id: currentBackupId, dir, localBucket: true }; + } catch (error) { + await control.backup.cleanupArchive(archivePath).catch(() => {}); + await bucket.delete(r2Key).catch(() => {}); + await bucket.delete(metaKey).catch(() => {}); + throw error; + } + } + ); - return { id: backupId, dir, localBucket: true }; + return result; } catch (error) { caughtError = error instanceof Error ? error : new Error(String(error)); - if (backupId) { - const archivePath = `${BACKUP_CONTAINER_DIR}/${backupId}.sqsh`; - const r2Key = `${BACKUP_STORAGE_PREFIX}/${backupId}/${BACKUP_ARCHIVE_OBJECT_NAME}`; - const metaKey = `${BACKUP_STORAGE_PREFIX}/${backupId}/${BACKUP_METADATA_OBJECT_NAME}`; - await this.client.backup.cleanupArchive(archivePath).catch(() => {}); - await bucket.delete(r2Key).catch(() => {}); - await bucket.delete(metaKey).catch(() => {}); - } throw error; } finally { logCanonicalEvent(this.logger, { diff --git a/packages/sandbox/src/backup/restore-lifecycle.ts b/packages/sandbox/src/backup/restore-lifecycle.ts index f8491c18c..6d68d76ce 100644 --- a/packages/sandbox/src/backup/restore-lifecycle.ts +++ b/packages/sandbox/src/backup/restore-lifecycle.ts @@ -1,19 +1,15 @@ -import type { - CurrentRuntimeIdentity, - RuntimeIdentity -} from '../current-runtime-identity'; -import { RuntimeIdentityInactiveError } from '../current-runtime-identity'; import { ErrorCode, OperationInterruptedError, RPCTransportError } from '../errors'; +import type { RuntimeIdentity, RuntimeIdentityReader } from '../runtime'; +import { RuntimeIdentityInactiveError } from '../runtime/types'; import type { CurrentSandboxLifetime, SandboxLifetime } from '../sandbox-lifetime'; import { SandboxLifetimeChangedError } from '../sandbox-lifetime'; -import { BACKUP_RESTORE_MAX_RECOVERY_ATTEMPTS } from './constants'; import type { BackupRestoreFaultInjector } from './restore-fault-injection'; import { type BackupRestoreOperationPhase, @@ -27,14 +23,17 @@ import { type RestoreLifecycleDeps = { storage: DurableObjectStorage; - currentRuntime: CurrentRuntimeIdentity; + runtimeReader: RuntimeIdentityReader; currentLifetime: CurrentSandboxLifetime; faultInjector: BackupRestoreFaultInjector; }; export type RestoreLifecycleContext = { lifetime: SandboxLifetime; - runtimeReady: (archiveSize?: number) => Promise<{ + runtimeReady: ( + runtime: RuntimeIdentity, + archiveSize?: number + ) => Promise<{ runtime: RuntimeIdentity; operation: BackupRestoreOperationRecord; }>; @@ -69,11 +68,6 @@ export class RestoreLifecycleRunner { lifetime.id ); - // Short-circuit: return the stored result without restoring again. - if (existing?.status === 'committed' && existing.result) { - return existing.result; - } - const now = new Date().toISOString(); let operation: BackupRestoreOperationRecord; @@ -96,7 +90,7 @@ export class RestoreLifecycleRunner { } await this.operationRecords.put(operation); - return await this.runWithRecovery(operation, lifetime, params.attempt); + return await this.runAttempt(operation, lifetime, params.attempt); } private async runAttempt( @@ -111,10 +105,10 @@ export class RestoreLifecycleRunner { const context: RestoreLifecycleContext = { lifetime, - runtimeReady: async (archiveSize) => { + runtimeReady: async (admittedRuntime, archiveSize) => { try { - runtime = await this.captureRuntime(); - await this.deps.currentLifetime.assertCurrent(lifetime); + runtime = admittedRuntime; + await this.assertFences(runtime, lifetime); } catch (error) { const interrupted = await this.translateFenceError( error, @@ -203,63 +197,18 @@ export class RestoreLifecycleRunner { try { return await attempt(context); } catch (error) { + if (error instanceof OperationInterruptedError) { + throw await this.translateOperationInterrupted(error, currentOperation); + } const translated = await this.translateRPCError(error, currentOperation); if (translated) { throw translated; } - if (!(error instanceof OperationInterruptedError)) { - await this.markFailed(currentOperation, error); - } + await this.markFailed(currentOperation, error); throw error; } } - private async runWithRecovery( - initialOperation: BackupRestoreOperationRecord, - lifetime: SandboxLifetime, - attempt: ( - context: RestoreLifecycleContext - ) => Promise - ): Promise { - let recoveryAttempts = 0; - let currentOperation = initialOperation; - - while (true) { - try { - return await this.runAttempt(currentOperation, lifetime, attempt); - } catch (error) { - if (!(error instanceof OperationInterruptedError)) { - throw error; - } - - if (!error.context.retryable) { - throw error; - } - - if (recoveryAttempts >= BACKUP_RESTORE_MAX_RECOVERY_ATTEMPTS) { - throw this.createRecoveryExhaustedError(error, recoveryAttempts); - } - - recoveryAttempts++; - // Re-read the interrupted record written by markInterrupted() and - // advance to the next attempt while preserving the operationId. - const interrupted = - (await this.operationRecords.get(currentOperation.operationKey)) ?? - currentOperation; - const now = new Date().toISOString(); - currentOperation = nextBackupRestoreAttempt(interrupted, now); - await this.operationRecords.put(currentOperation); - } - } - } - - async captureRuntime(): Promise { - let runtime = await this.deps.currentRuntime.get(); - runtime = runtime ?? (await this.deps.currentRuntime.markStarted()); - await this.deps.currentRuntime.assertActive(runtime); - return runtime; - } - async markRuntimeReady( operation: BackupRestoreOperationRecord, runtime: RuntimeIdentity, @@ -269,6 +218,7 @@ export class RestoreLifecycleRunner { ...operation, phase: 'runtime_ready' as const, runtimeIdentityID: runtime.id, + runtimeIncarnationID: runtime.runtimeIncarnationID, payload: { ...operation.payload, ...(archiveSize !== undefined && { archiveSize }) @@ -288,6 +238,7 @@ export class RestoreLifecycleRunner { ...operation, phase: 'archive_ready' as const, runtimeIdentityID: runtime.id, + runtimeIncarnationID: runtime.runtimeIncarnationID, payload: { ...operation.payload, ...(archiveSize !== undefined && { archiveSize }) @@ -302,7 +253,7 @@ export class RestoreLifecycleRunner { runtime: RuntimeIdentity, lifetime: SandboxLifetime ): Promise { - await this.deps.currentRuntime.assertActive(runtime); + await this.deps.runtimeReader.assertActive(runtime); await this.deps.currentLifetime.assertCurrent(lifetime); } @@ -318,6 +269,7 @@ export class RestoreLifecycleRunner { phase: 'verified' as const, status: 'committed' as const, runtimeIdentityID: runtime.id, + runtimeIncarnationID: runtime.runtimeIncarnationID, payload: { ...operation.payload, ...(archiveSize !== undefined && { archiveSize }) @@ -365,6 +317,32 @@ export class RestoreLifecycleRunner { }); } + private async translateOperationInterrupted( + error: OperationInterruptedError, + operation: BackupRestoreOperationRecord + ): Promise { + if (error.context.operationId === operation.operationId) { + return error; + } + + const phase = operation.phase; + const message = + 'Backup restore was interrupted before completion could be verified'; + const retryable = this.isRestoreRetryable(error.context.reason); + const interrupted = await this.markInterrupted( + operation, + message, + retryable + ); + return this.createInterruptedError({ + operation: interrupted, + reason: error.context.reason, + phase, + admitted: error.context.admitted, + message + }); + } + async translateRPCError( error: unknown, operation: BackupRestoreOperationRecord @@ -436,17 +414,24 @@ export class RestoreLifecycleRunner { return next; } + private isRestoreRetryable( + reason: OperationInterruptedError['context']['reason'] + ): boolean { + return ![ + 'sandbox_destroyed', + 'sandbox_lifetime_changed', + 'recovery_exhausted' + ].includes(reason); + } + private createInterruptedError(params: { operation: BackupRestoreOperationRecord; - reason: - | 'runtime_replaced' - | 'sandbox_lifetime_changed' - | 'transport_disposed'; + reason: OperationInterruptedError['context']['reason']; phase: BackupRestoreOperationPhase; admitted: true | 'unknown'; message: string; }): OperationInterruptedError { - const retryable = params.reason !== 'sandbox_lifetime_changed'; + const retryable = this.isRestoreRetryable(params.reason); const suggestion = retryable ? 'Retry restoreBackup() with the same backup handle so the SDK can reconcile the restore operation.' : 'Start a new restoreBackup() call only if restoring this backup is still desired for the current sandbox.'; @@ -471,33 +456,4 @@ export class RestoreLifecycleRunner { suggestion }); } - - private createRecoveryExhaustedError( - error: OperationInterruptedError, - recoveryAttempts: number - ): OperationInterruptedError { - const context = error.context; - return new OperationInterruptedError({ - message: 'Backup restore recovery attempts were exhausted', - code: ErrorCode.OPERATION_INTERRUPTED, - httpStatus: 409, - context: { - reason: 'recovery_exhausted', - operation: context.operation, - operationId: context.operationId, - operationKey: context.operationKey, - idempotencyKey: context.idempotencyKey, - backupId: context.backupId, - dir: context.dir, - phase: 'interrupted', - admitted: context.admitted, - retryable: true, - recoveryAttempts, - maxRecoveryAttempts: BACKUP_RESTORE_MAX_RECOVERY_ATTEMPTS - }, - timestamp: new Date().toISOString(), - suggestion: - 'Retry restoreBackup() with the same backup handle so the SDK can reconcile the restore operation.' - }); - } } diff --git a/packages/sandbox/src/backup/restore-operation-store.ts b/packages/sandbox/src/backup/restore-operation-store.ts index 6a1dbe8ee..225d66aec 100644 --- a/packages/sandbox/src/backup/restore-operation-store.ts +++ b/packages/sandbox/src/backup/restore-operation-store.ts @@ -1,4 +1,5 @@ -import type { RuntimeIdentityID } from '../current-runtime-identity'; +import type { RuntimeIncarnationID } from '../runtime'; +import type { RuntimeIdentityID } from '../runtime/types'; import type { SandboxLifetimeID } from '../sandbox-lifetime'; export type BackupRestoreOperationStatus = @@ -41,6 +42,7 @@ export type BackupRestoreOperationRecord = { phase: BackupRestoreOperationPhase; status: BackupRestoreOperationStatus; runtimeIdentityID?: RuntimeIdentityID; + runtimeIncarnationID?: RuntimeIncarnationID; payload: BackupRestoreOperationPayload; result?: BackupRestoreOperationResult; error?: BackupRestoreOperationError; @@ -100,8 +102,12 @@ export function nextBackupRestoreAttempt( ...record, phase: 'validating', status: 'running', + runtimeIdentityID: undefined, + runtimeIncarnationID: undefined, error: undefined, + result: undefined, completedAt: undefined, + lastInterruptedAt: undefined, updatedAt: now, attempt: (record.attempt ?? 0) + 1 }; diff --git a/packages/sandbox/src/backup/transfer.ts b/packages/sandbox/src/backup/transfer.ts index 67e9f4008..557172078 100644 --- a/packages/sandbox/src/backup/transfer.ts +++ b/packages/sandbox/src/backup/transfer.ts @@ -1,4 +1,8 @@ -import { type createLogger, getEnvString } from '@repo/shared'; +import { + type createLogger, + getEnvString, + type UploadPartsResponse +} from '@repo/shared'; import { AwsClient } from 'aws4fetch'; import type { ContainerControlClient } from '../container-control'; import { @@ -6,6 +10,8 @@ import { BackupRestoreError, ErrorCode, InvalidBackupConfigError, + OperationInterruptedError, + RPCTransportError, SandboxError } from '../errors'; import { isR2Bucket } from '../storage-mount'; @@ -21,7 +27,6 @@ import { type BackupTransferDeps = { getEnv: () => unknown; - getClient: () => ContainerControlClient; logger: ReturnType; }; @@ -36,10 +41,6 @@ export class BackupTransfer { return this.deps.getEnv(); } - private get client(): ContainerControlClient { - return this.deps.getClient(); - } - private parseBackupBucketEndpoint( envObj: Record ): string | null { @@ -234,11 +235,12 @@ export class BackupTransfer { r2Key: string, archiveSize: number, backupId: string, - dir: string + dir: string, + control: ContainerControlClient ): Promise { const presignedURL = await this.generatePresignedPutURL(r2Key); - await this.client.backup.uploadArchive({ + await control.backup.uploadArchive({ archivePath, url: presignedURL, timeoutMs: 1_810_000 @@ -303,7 +305,8 @@ export class BackupTransfer { r2Key: string, sizeBytes: number, backupId: string, - dir: string + dir: string, + control: ContainerControlClient ): Promise { const targetParts = calculatePartCount( sizeBytes, @@ -321,7 +324,8 @@ export class BackupTransfer { r2Key, sizeBytes, backupId, - dir + dir, + control ); } @@ -384,11 +388,9 @@ export class BackupTransfer { })) ); - let uploadResult: Awaited< - ReturnType - >; + let uploadResult: UploadPartsResponse; try { - uploadResult = await this.client.backup.uploadParts({ + uploadResult = await control.backup.uploadParts({ archivePath, parts }); @@ -403,7 +405,8 @@ export class BackupTransfer { r2Key, sizeBytes, backupId, - dir + dir, + control ); } throw err; @@ -476,7 +479,8 @@ export class BackupTransfer { r2Key: string, expectedSize: number, backupId: string, - dir: string + dir: string, + control: ContainerControlClient ): Promise { const presignedURL = await this.generatePresignedGetURL(r2Key); const parts = @@ -502,13 +506,19 @@ export class BackupTransfer { })(); try { - await this.client.backup.downloadArchive({ + await control.backup.downloadArchive({ archivePath, expectedSize, parts, timeoutMs: 1_810_000 }); } catch (error) { + if ( + error instanceof OperationInterruptedError || + error instanceof RPCTransportError + ) { + throw error; + } throw new BackupRestoreError({ message: `Presigned URL download failed: ${error instanceof Error ? error.message : String(error)}`, code: ErrorCode.BACKUP_RESTORE_FAILED, diff --git a/packages/sandbox/src/bridge/openapi/lifecycle.ts b/packages/sandbox/src/bridge/openapi/lifecycle.ts index b1686193a..2fa7cc3b5 100644 --- a/packages/sandbox/src/bridge/openapi/lifecycle.ts +++ b/packages/sandbox/src/bridge/openapi/lifecycle.ts @@ -156,7 +156,7 @@ export const LIFECYCLEPaths = { operationId: 'shutdownPrewarmed', summary: 'Shutdown pre-warmed containers', description: - 'Stops all idle (unassigned) warm containers. Does not affect containers assigned to sandbox instances.', + 'Destroys all idle (unassigned) warm containers. Does not affect containers assigned to sandbox instances.', 'x-codeSamples': [ { lang: 'curl', @@ -168,7 +168,7 @@ export const LIFECYCLEPaths = { ], responses: { '200': { - description: 'All pre-warmed containers stopped.', + description: 'All pre-warmed containers destroyed.', content: { 'application/json': { schema: { $ref: '#/components/schemas/OkResponse' } diff --git a/packages/sandbox/src/bridge/warm-pool.ts b/packages/sandbox/src/bridge/warm-pool.ts index 0dc1bdc70..b395e57e0 100644 --- a/packages/sandbox/src/bridge/warm-pool.ts +++ b/packages/sandbox/src/bridge/warm-pool.ts @@ -41,7 +41,7 @@ export interface PoolStats { interface ContainerRpc { startAndWaitForPorts(): Promise; - stop(signal?: string): Promise; + destroy(): Promise; renewActivityTimeout(): void; } @@ -202,11 +202,11 @@ export class WarmPool extends DurableObject { for (const containerUUID of [...this.warmContainers]) { try { const stub = this.getSandboxStub(containerUUID); - await (stub as unknown as ContainerRpc).stop(); + await (stub as unknown as ContainerRpc).destroy(); this.warmContainers.delete(containerUUID); } catch (error) { console.error({ - message: 'Failed to stop container', + message: 'Failed to destroy container', component: 'warm-pool', containerUUID, error @@ -443,20 +443,20 @@ export class WarmPool extends DurableObject { console.info({ message: 'Scaling down pool', component: 'warm-pool', - stopping: excess + destroying: excess }); - const toStop = [...this.warmContainers].slice(0, excess); - const stopped: string[] = []; + const toDestroy = [...this.warmContainers].slice(0, excess); + const destroyed: string[] = []; - for (const uuid of toStop) { + for (const uuid of toDestroy) { try { const stub = this.getSandboxStub(uuid); - await (stub as unknown as ContainerRpc).stop(); - stopped.push(uuid); + await (stub as unknown as ContainerRpc).destroy(); + destroyed.push(uuid); } catch (error) { console.error({ - message: 'Failed to stop container', + message: 'Failed to destroy container', component: 'warm-pool', containerUUID: uuid, error @@ -464,7 +464,7 @@ export class WarmPool extends DurableObject { } } - for (const uuid of stopped) { + for (const uuid of destroyed) { this.warmContainers.delete(uuid); } await this.persist(); diff --git a/packages/sandbox/src/container-control/client.ts b/packages/sandbox/src/container-control/client.ts index 5dc603239..0bccf61b0 100644 --- a/packages/sandbox/src/container-control/client.ts +++ b/packages/sandbox/src/container-control/client.ts @@ -15,7 +15,6 @@ import type { SandboxWorkspaceAPI } from '@repo/shared'; import { createNoOpLogger } from '@repo/shared'; -import type { ResourceActivityOperation } from '../resource-activity-gate'; import { ContainerControlConnection, type ContainerControlConnectionOptions @@ -75,13 +74,14 @@ export interface ContainerControlClientOptions extends ContainerControlConnectio * Mirrors what `containerFetch()` does at the top of each HTTP request. */ onActivity?: () => void; - onOperationStarted?: () => ResourceActivityOperation; /** Verifies that the owning runtime lease still permits RPC dispatch. */ onDispatch?: () => void; /** Fires when the underlying connection closes or fails. */ onConnectionClose?: () => void; /** Maps transport loss to operation interruption for the waking facade. */ translateTransportErrorsAsInterruptions?: boolean; + connection?: ContainerControlConnection; + externallyOwnedConnection?: boolean; /** * Fires once when the capnweb session transitions from idle to busy * (an RPC call was started or a stream return is now in flight). The @@ -112,14 +112,14 @@ export interface ContainerControlClientOptions extends ContainerControlConnectio * the file-level comment for the full rationale). */ export class ContainerControlClient { - private readonly connOptions: ContainerControlConnectionOptions; + private readonly connOptions: ContainerControlConnectionOptions & { + connection?: ContainerControlConnection; + externallyOwnedConnection?: boolean; + }; private readonly idleDisconnectMs: number; private readonly busyPollIntervalMs: number; private readonly logger: Logger; private readonly onActivity: (() => void) | undefined; - private readonly onOperationStarted: - | (() => ResourceActivityOperation) - | undefined; private readonly onDispatch: (() => void) | undefined; private readonly onConnectionClose: (() => void) | undefined; private readonly translateTransportErrorsAsInterruptions: boolean; @@ -140,7 +140,8 @@ export class ContainerControlClient { port: options.port, localMain: options.localMain, logger: options.logger, - retryTimeoutMs: options.retryTimeoutMs, + connection: options.connection, + externallyOwnedConnection: options.externallyOwnedConnection, // Event-driven failure recovery: when the live WebSocket closes // or errors, tear the connection down inside the same turn of // the event loop so the next RPC call builds a fresh one. The @@ -159,7 +160,6 @@ export class ContainerControlClient { options.busyPollIntervalMs ?? BUSY_POLL_INTERVAL_MS; this.logger = options.logger ?? createNoOpLogger(); this.onActivity = options.onActivity; - this.onOperationStarted = options.onOperationStarted; this.onDispatch = options.onDispatch; this.onConnectionClose = options.onConnectionClose; this.translateTransportErrorsAsInterruptions = @@ -179,8 +179,10 @@ export class ContainerControlClient { private getConnection(): ContainerControlConnection { this.onDispatch?.(); if (!this.conn) { - this.conn = new ContainerControlConnection(this.connOptions); - this.startBusyPoll(); + this.conn = + this.connOptions.connection ?? + new ContainerControlConnection(this.connOptions); + if (!this.connOptions.externallyOwnedConnection) this.startBusyPoll(); } return this.conn; } @@ -189,20 +191,6 @@ export class ContainerControlClient { // Activity & busy/idle tracking // ------------------------------------------------------------------------- - /** - * Called synchronously at the start of each RPC method invocation. - * Renews the DO activity timeout so the sleepAfter alarm is pushed - * forward before the container processes the call. - */ - private beginOperation = (): ResourceActivityOperation => { - return ( - this.onOperationStarted?.() ?? { - beforeCall: Promise.resolve(), - finish: () => undefined - } - ); - }; - /** * Sample `getStats()` and update busy/idle state. While busy, renews the * activity timeout each tick so an in-flight stream keeps pushing the @@ -311,7 +299,9 @@ export class ContainerControlClient { this.onSessionIdle?.(); } if (this.conn) { - this.conn.disconnect(); + if (!this.connOptions.externallyOwnedConnection) { + this.conn.disconnect(); + } this.conn = null; } } @@ -327,13 +317,11 @@ export class ContainerControlClient { private createDomainProxy( getStub: () => T, - domain: string, - onCallStarted = this.beginOperation + domain: string ): T { return createControlDomainProxy( getStub, domain, - onCallStarted, this.translateTransportErrorsAsInterruptions ); } @@ -358,16 +346,6 @@ export class ContainerControlClient { 'processes' ) as unknown as SandboxProcessesAPI; } - processesWithoutActivity(): SandboxProcessesAPI { - return this.createDomainProxy( - () => this.getConnection().rpc().processes, - 'processes', - () => ({ - beforeCall: Promise.resolve(), - finish: () => undefined - }) - ) as unknown as SandboxProcessesAPI; - } get mounts(): SandboxMountsAPI { return this.createDomainProxy( () => this.getConnection().rpc().mounts, @@ -410,16 +388,6 @@ export class ContainerControlClient { 'terminals' ) as unknown as SandboxTerminalsAPI; } - terminalsWithoutActivity(): SandboxTerminalsAPI { - return this.createDomainProxy( - () => this.getConnection().rpc().terminals, - 'terminals', - () => ({ - beforeCall: Promise.resolve(), - finish: () => undefined - }) - ) as unknown as SandboxTerminalsAPI; - } get extensions(): SandboxExtensionsAPI { return this.createDomainProxy( () => this.getConnection().rpc().extensions, @@ -427,31 +395,15 @@ export class ContainerControlClient { ); } - /** - * Update the upgrade retry budget. Applies to the current connection - * (if any) and is remembered for any future connections created after the - * client is torn down and reconnected. - */ - setRetryTimeoutMs(ms: number): void { - this.connOptions.retryTimeoutMs = ms; - this.conn?.setRetryTimeoutMs(ms); - } - isWebSocketConnected(): boolean { return this.conn?.isConnected() ?? false; } async connect(): Promise { - const activity = this.beginOperation(); - try { - await activity.beforeCall; - await this.getConnection().connect(); - } finally { - activity.finish(); - } + await this.getConnection().connect(); } - retainConnection(): () => void { + retainRuntimeHold(): () => void { this.getConnection(); const generation = this.connectionGeneration; this.connectionRetainers += 1; diff --git a/packages/sandbox/src/container-control/connection.ts b/packages/sandbox/src/container-control/connection.ts index 7bd710125..9e6e9e85d 100644 --- a/packages/sandbox/src/container-control/connection.ts +++ b/packages/sandbox/src/container-control/connection.ts @@ -8,6 +8,7 @@ import type { Logger, + RuntimeMetadata, SandboxAPI, SandboxBackupAPI, SandboxControlCallback, @@ -30,18 +31,12 @@ import { type RpcTransport } from 'capnweb'; import { createErrorFromResponse } from '../errors/adapter'; -import { - fetchWithResponseRetry, - isRetryableWebSocketUpgradeResponse -} from '../response-retry'; // --------------------------------------------------------------------------- // Connection manager // --------------------------------------------------------------------------- const DEFAULT_CONNECT_TIMEOUT_MS = 30_000; -const DEFAULT_RETRY_TIMEOUT_MS = 120_000; // 2 minute total budget for upgrade retries -const MIN_TIME_FOR_RETRY_MS = 15_000; // Need at least 15s remaining to attempt a retry /** Stub that can issue a WebSocket-upgrade fetch through the DO's Container base class. */ export interface ContainerFetchStub { @@ -98,12 +93,6 @@ export interface ContainerControlConnectionOptions { stub: ContainerFetchStub; port?: number; logger?: Logger; - /** - * Total retry budget (ms) for retryable upgrade responses while the - * container is unavailable. Defaults to 120 000 (2 minutes). Set to 0 to - * disable retries. - */ - retryTimeoutMs?: number; /** * Optional `localMain` exposed to the container side of the capnweb * session. The container reaches it via @@ -139,13 +128,18 @@ export class ContainerControlConnection { private readonly session: RpcSession; private readonly transport: DeferredTransport; private ws: WebSocket | null = null; - private connected = false; + private state: + | 'disconnected' + | 'connecting' + | 'connected' + | 'activating' + | 'active' + | 'disposed' = 'disconnected'; private connectPromise: Promise | null = null; private activeUpgradeAbortController: AbortController | null = null; private readonly containerStub: ContainerFetchStub; private readonly port: number; private readonly logger: Logger; - private retryTimeoutMs: number; private readonly onClose: (() => void) | undefined; private readonly disposalError = new Error( 'Container control connection was disconnected' @@ -156,7 +150,6 @@ export class ContainerControlConnection { this.containerStub = options.stub; this.port = options.port ?? 3000; this.logger = options.logger ?? createNoOpLogger(); - this.retryTimeoutMs = options.retryTimeoutMs ?? DEFAULT_RETRY_TIMEOUT_MS; this.onClose = options.onClose; this.transport = new DeferredTransport(); @@ -175,12 +168,40 @@ export class ContainerControlConnection { * the WebSocket is established. */ rpc(): RpcStub { - if (!this.connected && !this.connectPromise) { - this.connect().catch(() => {}); + if (this.state !== 'active') { + throw new Error('Container control connection is not activated'); } return this.stub; } + async getRuntimeMetadata(): Promise { + await this.connect(); + return this.stub.utils.getRuntimeMetadata(); + } + + async activateControlSession( + expectedRuntimeIncarnationID: string + ): Promise { + await this.connect(); + if (this.state === 'active') return this.stub.utils.getRuntimeMetadata(); + if (this.state !== 'connected') { + throw new Error( + 'Container control connection cannot activate from current state' + ); + } + this.state = 'activating'; + try { + const metadata = await this.stub.utils.activateControlSession( + expectedRuntimeIncarnationID + ); + this.state = 'active'; + return metadata; + } catch (error) { + if (this.state === 'activating') this.state = 'connected'; + throw error; + } + } + /** * Return capnweb session statistics. The `imports` and `exports` counts * reflect all in-flight RPC calls, streams, and peer-held references. @@ -191,12 +212,16 @@ export class ContainerControlConnection { } isConnected(): boolean { - return this.connected; + return ( + this.state === 'connected' || + this.state === 'activating' || + this.state === 'active' + ); } async connect(): Promise { - if (this.disposed) throw this.disposalError; - if (this.connected) return; + if (this.state === 'disposed') throw this.disposalError; + if (this.isConnected()) return; if (this.connectPromise) { return this.connectPromise; @@ -211,7 +236,8 @@ export class ContainerControlConnection { } disconnect(): void { - if (this.disposed) return; + if (this.state === 'disposed') return; + this.state = 'disposed'; this.disposed = true; this.activeUpgradeAbortController?.abort(); this.activeUpgradeAbortController = null; @@ -228,16 +254,6 @@ export class ContainerControlConnection { // Stub may already be disposed } this.ws = null; - this.connected = false; - } - - /** - * Update the upgrade retry budget without recreating the connection. Takes - * effect on the next `connect()`; an in-flight connect uses the value - * captured at start. - */ - setRetryTimeoutMs(ms: number): void { - this.retryTimeoutMs = ms; } // ----------------------------------------------------------------------- @@ -267,8 +283,8 @@ export class ContainerControlConnection { * fail to unbind. */ private onWebSocketClose = (): void => { - const wasConnected = this.connected; - this.connected = false; + const wasConnected = this.isConnected(); + if (this.state !== 'disposed') this.state = 'disconnected'; this.ws = null; this.logger.debug('ContainerControlConnection WebSocket closed'); if (wasConnected) this.fireOnClose(); @@ -279,15 +295,16 @@ export class ContainerControlConnection { * {@link onWebSocketClose}. */ private onWebSocketError = (): void => { - const wasConnected = this.connected; - this.connected = false; + const wasConnected = this.isConnected(); + if (this.state !== 'disposed') this.state = 'disconnected'; this.ws = null; if (wasConnected) this.fireOnClose(); }; private async doConnect(): Promise { + this.state = 'connecting'; try { - const response = await this.fetchUpgradeWithRetry(); + const response = await this.fetchUpgradeAttempt(); if (this.disposed) { this.closeUpgradeWebSocket(response); @@ -305,21 +322,6 @@ export class ContainerControlConnection { if (structuredError) { throw createErrorFromResponse(structuredError); } - if (isRetryableWebSocketUpgradeResponse(response)) { - const context = { - reason: 'rpc_upgrade_failed' as const, - retryable: true as const - }; - throw createErrorFromResponse({ - code: ErrorCode.CONTAINER_UNAVAILABLE, - message: - 'Container was unavailable after exhausting upgrade retry budget.', - context, - httpStatus: getHttpStatus(ErrorCode.CONTAINER_UNAVAILABLE), - timestamp: new Date().toISOString(), - suggestion: getSuggestion(ErrorCode.CONTAINER_UNAVAILABLE, context) - }); - } throw new Error( `WebSocket upgrade failed: ${response.status} ${response.statusText}` ); @@ -340,13 +342,13 @@ export class ContainerControlConnection { this.ws = ws; this.transport.activate(ws); - this.connected = true; + this.state = 'connected'; this.logger.debug('ContainerControlConnection established', { port: this.port }); } catch (error) { - this.connected = false; + if (!this.disposed) this.state = 'disconnected'; this.transport.abort(error); if (this.disposed) throw this.disposalError; this.logger.error( @@ -425,25 +427,8 @@ export class ContainerControlConnection { } /** - * Issue WebSocket upgrade fetches, retrying transient control-plane - * unavailability responses until either the upgrade succeeds, a - * non-retryable status is returned, or the retry budget runs out. - */ - private async fetchUpgradeWithRetry(): Promise { - return fetchWithResponseRetry(() => this.fetchUpgradeAttempt(), { - retryTimeoutMs: this.retryTimeoutMs, - minTimeForRetryMs: MIN_TIME_FOR_RETRY_MS, - logger: this.logger, - retryLogMessage: - 'ContainerControlConnection upgrade returned retryable status, retrying', - shouldRetry: isRetryableWebSocketUpgradeResponse - }); - } - - /** - * Single WebSocket-upgrade fetch attempt. Owns its own AbortController so - * each retry gets a fresh per-attempt connect timeout independent of the - * total retry budget. + * Single WebSocket-upgrade fetch attempt. Owns its own AbortController for + * the connect timeout. */ private async fetchUpgradeAttempt(): Promise { const controller = new AbortController(); diff --git a/packages/sandbox/src/container-control/index.ts b/packages/sandbox/src/container-control/index.ts index 5c04fd7ef..4d5c963f2 100644 --- a/packages/sandbox/src/container-control/index.ts +++ b/packages/sandbox/src/container-control/index.ts @@ -1,2 +1,2 @@ export { ContainerControlClient } from './client'; -export { RuntimeControlClient } from './runtime-client'; +export { ContainerControlConnection } from './connection'; diff --git a/packages/sandbox/src/container-control/rpc-proxy.ts b/packages/sandbox/src/container-control/rpc-proxy.ts index b432abaed..ddbcd3264 100644 --- a/packages/sandbox/src/container-control/rpc-proxy.ts +++ b/packages/sandbox/src/container-control/rpc-proxy.ts @@ -1,58 +1,42 @@ -import type { ResourceActivityOperation } from '../resource-activity-gate'; import { translateRPCError } from './rpc-error'; export function createControlDomainProxy( getStub: () => T, domain: string, - onCallStarted: () => ResourceActivityOperation, translateTransportErrorsAsInterruptions = true ): T { return new Proxy(Object.create(null) as T, { get(_target, prop) { return (...args: unknown[]) => { - const activity = onCallStarted(); const operation = typeof prop === 'string' ? `${domain}.${prop}` : domain; - const invoke = () => { - try { - const target = getStub(); - const value = Reflect.get(target, prop, target); - if (typeof value !== 'function') { - activity.finish(); - return value; - } - const result = Reflect.apply( - value as (...a: unknown[]) => unknown, - target, - args + try { + const target = getStub(); + const value = Reflect.get(target, prop, target); + if (typeof value !== 'function') return value; + const result = Reflect.apply( + value as (...a: unknown[]) => unknown, + target, + args + ); + if ( + result != null && + typeof (result as { then?: unknown }).then === 'function' + ) { + return (result as Promise).catch((err: unknown) => + translateRPCError(err, { + operation, + translateTransportErrorsAsInterruptions + }) ); - if ( - result != null && - typeof (result as { then?: unknown }).then === 'function' - ) { - return (result as Promise) - .catch((err: unknown) => - translateRPCError(err, { - operation, - translateTransportErrorsAsInterruptions - }) - ) - .finally(activity.finish); - } - activity.finish(); - return result; - } catch (err) { - activity.finish(); - translateRPCError(err, { - operation, - translateTransportErrorsAsInterruptions - }); } - }; - return activity.beforeCall.then(invoke, (err: unknown) => { - activity.finish(); - throw err; - }); + return result; + } catch (err) { + translateRPCError(err, { + operation, + translateTransportErrorsAsInterruptions + }); + } }; } }); diff --git a/packages/sandbox/src/container-control/runtime-client.ts b/packages/sandbox/src/container-control/runtime-client.ts deleted file mode 100644 index 80269b780..000000000 --- a/packages/sandbox/src/container-control/runtime-client.ts +++ /dev/null @@ -1,84 +0,0 @@ -import type { Logger, SandboxControlCallback } from '@repo/shared'; -import type { RpcTarget } from 'capnweb'; -import type { RuntimeIdentityID } from '../current-runtime-identity'; -import type { ResourceActivityOperation } from '../resource-activity-gate'; -import { - ContainerControlClient, - type ContainerControlClientOptions -} from './client'; -import type { ContainerFetchStub } from './connection'; -import { translateRPCError } from './rpc-error'; - -type RuntimeControlClientOptions = { - getTcpPort: (port: number) => ContainerFetchStub; - beginNonWakingOperation: () => ResourceActivityOperation; - logger?: Logger; - localMain?: SandboxControlCallback & RpcTarget; -}; - -type RuntimeClientLease = { - active: boolean; -}; - -type CachedRuntimeClient = { - runtimeIdentityID: RuntimeIdentityID; - client: ContainerControlClient; - lease: RuntimeClientLease; -}; - -/** Owns at most one direct, non-starting control connection for one runtime. */ -export class RuntimeControlClient { - private cached: CachedRuntimeClient | null = null; - - constructor(private readonly options: RuntimeControlClientOptions) {} - - get(runtimeIdentityID: RuntimeIdentityID): ContainerControlClient { - if (this.cached?.runtimeIdentityID === runtimeIdentityID) { - return this.cached.client; - } - - this.dispose(); - const lease: RuntimeClientLease = { active: true }; - let stub: ContainerFetchStub; - try { - stub = this.options.getTcpPort(3000); - } catch (error) { - translateRPCError(error); - } - const clientOptions: ContainerControlClientOptions = { - stub, - retryTimeoutMs: 0, - logger: this.options.logger, - localMain: this.options.localMain, - onOperationStarted: this.options.beginNonWakingOperation, - translateTransportErrorsAsInterruptions: false, - onDispatch: () => { - if (!lease.active) { - throw new Error( - 'RPC session was shut down by disposing the main stub' - ); - } - } - }; - const client = new ContainerControlClient({ - ...clientOptions, - onConnectionClose: () => { - lease.active = false; - if (this.cached?.client === client) { - this.cached = null; - } - } - }); - this.cached = { runtimeIdentityID, client, lease }; - return client; - } - - dispose(): void { - const cached = this.cached; - this.cached = null; - if (cached) { - cached.lease.active = false; - cached.client.disconnect(); - } - } -} diff --git a/packages/sandbox/src/current-runtime-identity.ts b/packages/sandbox/src/current-runtime-identity.ts deleted file mode 100644 index 793ef6a51..000000000 --- a/packages/sandbox/src/current-runtime-identity.ts +++ /dev/null @@ -1,225 +0,0 @@ -export type RuntimeIdentityID = string & { - readonly __runtimeIdentityID: unique symbol; -}; - -export type RuntimeIdentityRecord = { - id: RuntimeIdentityID; -}; - -export type RuntimeScoped = T & { - readonly runtimeIdentityID: RuntimeIdentityID; -}; - -export type CurrentRuntimeStatus = - | { status: 'active'; runtime: RuntimeIdentity; containerStatus: string } - | { - status: 'inactive'; - reason: - | 'runtime-not-healthy' - | 'runtime-not-running' - | 'missing-runtime-id'; - containerStatus?: string; - }; - -type RuntimeIdentityStorage = Pick< - DurableObjectStorage | DurableObjectTransaction, - 'get' ->; - -const CURRENT_RUNTIME_IDENTITY_STORAGE_KEY = 'currentRuntimeIdentity'; - -export class RuntimeIdentityInactiveError extends Error { - constructor() { - super('Runtime identity is no longer active'); - this.name = 'RuntimeIdentityInactiveError'; - } -} - -export class RuntimeIdentity { - readonly id: RuntimeIdentityID; - - constructor(record: RuntimeIdentityRecord) { - this.id = record.id; - } - - owns(record: { readonly runtimeIdentityID: RuntimeIdentityID }): boolean { - return record.runtimeIdentityID === this.id; - } - - scope(value: T): RuntimeScoped { - return { - ...value, - runtimeIdentityID: this.id - }; - } -} - -export class CurrentRuntimeIdentity { - private readonly changeListeners = new Set<() => void>(); - private transitionEpoch = 0; - private pendingTransitions = 0; - private mutationRunning = false; - private readonly mutationQueue: Array<() => void> = []; - - /** - * Runtime identity is stored in Durable Object storage so a reconstructed DO - * can still recognize the live container runtime it owns. In-memory state is - * only a cache and cannot define runtime-scoped correctness. - */ - constructor( - private readonly storage: DurableObjectState['storage'], - private readonly getContainerState: () => Promise<{ status: string }>, - private readonly isContainerRunning: () => boolean - ) {} - - async get(): Promise { - const status = await this.getStatus(); - return status.status === 'active' ? status.runtime : null; - } - - async getStatus(): Promise { - const epoch = this.transitionEpoch; - if (!this.isCurrentEpoch(epoch)) { - return { status: 'inactive', reason: 'missing-runtime-id' }; - } - - const state = await this.getContainerState(); - if (!this.isCurrentEpoch(epoch)) { - return { status: 'inactive', reason: 'missing-runtime-id' }; - } - if (state.status !== 'healthy') { - return { - status: 'inactive', - reason: 'runtime-not-healthy', - containerStatus: state.status - }; - } - - if (!this.isContainerRunning()) { - return { - status: 'inactive', - reason: 'runtime-not-running', - containerStatus: state.status - }; - } - - const runtime = await this.getStored(); - if (!this.isCurrentEpoch(epoch)) { - return { status: 'inactive', reason: 'missing-runtime-id' }; - } - if (!runtime) { - return { - status: 'inactive', - reason: 'missing-runtime-id', - containerStatus: state.status - }; - } - - return { - status: 'active', - runtime, - containerStatus: state.status - }; - } - - async getStored( - storage: RuntimeIdentityStorage = this.storage - ): Promise { - const epoch = this.transitionEpoch; - if (!this.isCurrentEpoch(epoch)) return null; - - const record = - (await storage.get( - CURRENT_RUNTIME_IDENTITY_STORAGE_KEY - )) ?? null; - if (!this.isCurrentEpoch(epoch)) return null; - return record ? new RuntimeIdentity(record) : null; - } - - markStarted(): Promise { - const record: RuntimeIdentityRecord = { - id: crypto.randomUUID() as RuntimeIdentityID - }; - return this.enqueueMutation(async () => { - await this.storage.put(CURRENT_RUNTIME_IDENTITY_STORAGE_KEY, record); - return new RuntimeIdentity(record); - }); - } - - clear(): Promise { - return this.enqueueMutation(async () => { - await this.storage.delete(CURRENT_RUNTIME_IDENTITY_STORAGE_KEY); - }); - } - - onChange(listener: () => void): () => void { - this.changeListeners.add(listener); - return () => this.changeListeners.delete(listener); - } - - private notifyChanged(): void { - for (const listener of this.changeListeners) listener(); - } - - private isCurrentEpoch(epoch: number): boolean { - return this.pendingTransitions === 0 && this.transitionEpoch === epoch; - } - - private enqueueMutation(operation: () => Promise): Promise { - this.transitionEpoch++; - this.pendingTransitions++; - try { - this.notifyChanged(); - } catch (error) { - this.pendingTransitions--; - return Promise.reject(error); - } - - return new Promise((resolve, reject) => { - const run = (): void => { - this.mutationRunning = true; - let result: Promise; - try { - result = operation(); - } catch (error) { - this.completeMutation(); - reject(error); - return; - } - result.then( - (value) => { - this.completeMutation(); - resolve(value); - }, - (error: unknown) => { - this.completeMutation(); - reject(error); - } - ); - }; - - if (this.mutationRunning) { - this.mutationQueue.push(run); - } else { - run(); - } - }); - } - - private completeMutation(): void { - this.pendingTransitions--; - this.mutationRunning = false; - this.mutationQueue.shift()?.(); - } - - async isActive(runtime: RuntimeIdentity): Promise { - const current = await this.get(); - return current?.id === runtime.id; - } - - async assertActive(runtime: RuntimeIdentity): Promise { - if (!(await this.isActive(runtime))) { - throw new RuntimeIdentityInactiveError(); - } - } -} diff --git a/packages/sandbox/src/errors/adapter.ts b/packages/sandbox/src/errors/adapter.ts index 3be3ce0f5..d25595700 100644 --- a/packages/sandbox/src/errors/adapter.ts +++ b/packages/sandbox/src/errors/adapter.ts @@ -44,6 +44,7 @@ import type { ProcessWaitTimeoutContext, RPCTransportContext, StaleProcessHandleContext, + StaleTerminalHandleContext, TerminalControlErrorContext, TerminalNotFoundContext, ValidationFailedContext @@ -96,6 +97,7 @@ import { SandboxError, ServiceNotRespondingError, StaleProcessHandleError, + StaleTerminalHandleError, TerminalControlError, TerminalNotFoundError, ValidationFailedError @@ -105,6 +107,10 @@ type StaleProcessHandleResponse = ErrorResponse & { code: typeof ErrorCode.STALE_PROCESS_HANDLE; }; +type StaleTerminalHandleResponse = ErrorResponse & { + code: typeof ErrorCode.STALE_TERMINAL_HANDLE; +}; + type ProcessWaitTimeoutResponse = ErrorResponse & { code: typeof ErrorCode.PROCESS_WAIT_TIMEOUT; }; @@ -113,14 +119,15 @@ type ProcessAbortedResponse = ErrorResponse & { code: typeof ErrorCode.PROCESS_ABORTED; }; -type ProcessLifecycleResponse = +type ProcessErrorResponse = | ErrorResponse | StaleProcessHandleResponse + | StaleTerminalHandleResponse | ProcessWaitTimeoutResponse | ProcessAbortedResponse; function isStaleProcessHandleResponse( - errorResponse: ProcessLifecycleResponse + errorResponse: ProcessErrorResponse ): errorResponse is StaleProcessHandleResponse { const { context } = errorResponse; return ( @@ -134,8 +141,21 @@ function isStaleProcessHandleResponse( ); } +function isStaleTerminalHandleResponse( + errorResponse: ProcessErrorResponse +): errorResponse is StaleTerminalHandleResponse { + const { context } = errorResponse; + return ( + errorResponse.code === ErrorCode.STALE_TERMINAL_HANDLE && + 'terminalId' in context && + typeof context.terminalId === 'string' && + 'operation' in context && + typeof context.operation === 'string' + ); +} + function isProcessWaitTimeoutResponse( - errorResponse: ProcessLifecycleResponse + errorResponse: ProcessErrorResponse ): errorResponse is ProcessWaitTimeoutResponse { const { context } = errorResponse; return ( @@ -152,7 +172,7 @@ function isProcessWaitTimeoutResponse( } function isProcessAbortedResponse( - errorResponse: ProcessLifecycleResponse + errorResponse: ProcessErrorResponse ): errorResponse is ProcessAbortedResponse { const { context } = errorResponse; return ( @@ -251,7 +271,7 @@ export function createErrorFromResponse( case ErrorCode.STALE_PROCESS_HANDLE: if ( isStaleProcessHandleResponse( - errorResponse as unknown as ProcessLifecycleResponse + errorResponse as unknown as ProcessErrorResponse ) ) { return new StaleProcessHandleError( @@ -260,10 +280,22 @@ export function createErrorFromResponse( } return new SandboxError(errorResponse); + case ErrorCode.STALE_TERMINAL_HANDLE: + if ( + isStaleTerminalHandleResponse( + errorResponse as unknown as ProcessErrorResponse + ) + ) { + return new StaleTerminalHandleError( + errorResponse as unknown as StaleTerminalHandleResponse + ); + } + return new SandboxError(errorResponse); + case ErrorCode.PROCESS_WAIT_TIMEOUT: if ( isProcessWaitTimeoutResponse( - errorResponse as unknown as ProcessLifecycleResponse + errorResponse as unknown as ProcessErrorResponse ) ) { return new ProcessWaitTimeoutError( @@ -275,7 +307,7 @@ export function createErrorFromResponse( case ErrorCode.PROCESS_ABORTED: if ( isProcessAbortedResponse( - errorResponse as unknown as ProcessLifecycleResponse + errorResponse as unknown as ProcessErrorResponse ) ) { return new ProcessAbortedError( diff --git a/packages/sandbox/src/errors/classes.ts b/packages/sandbox/src/errors/classes.ts index 70276f601..adc21d9f4 100644 --- a/packages/sandbox/src/errors/classes.ts +++ b/packages/sandbox/src/errors/classes.ts @@ -4,3 +4,5 @@ export * from './classes/filesystem'; export * from './classes/git-code'; export * from './classes/port'; export * from './classes/process'; +export * from './classes/runtime-control-protocol'; +export * from './classes/runtime-inactive'; diff --git a/packages/sandbox/src/errors/classes/backup-terminal-lifecycle.ts b/packages/sandbox/src/errors/classes/backup-terminal-lifecycle.ts index 50dc2b8bc..62abc9dc1 100644 --- a/packages/sandbox/src/errors/classes/backup-terminal-lifecycle.ts +++ b/packages/sandbox/src/errors/classes/backup-terminal-lifecycle.ts @@ -32,6 +32,7 @@ import type { ProcessReadyTimeoutContext, RPCTransportContext, RPCTransportErrorKind, + StaleTerminalHandleContext, TerminalControlErrorContext, TerminalNotFoundContext, ValidationFailedContext @@ -175,6 +176,21 @@ export class TerminalControlError extends SandboxError { + constructor(errorResponse: ErrorResponse) { + super(errorResponse); + this.name = 'StaleTerminalHandleError'; + } + + get terminalId() { + return this.context.terminalId; + } + + get operation() { + return this.context.operation; + } +} + // ============================================================================ // Container Availability Errors // ============================================================================ diff --git a/packages/sandbox/src/errors/classes/runtime-control-protocol.ts b/packages/sandbox/src/errors/classes/runtime-control-protocol.ts new file mode 100644 index 000000000..0449e1572 --- /dev/null +++ b/packages/sandbox/src/errors/classes/runtime-control-protocol.ts @@ -0,0 +1,34 @@ +import { ErrorCode, getHttpStatus } from '@repo/shared/errors'; +import { SandboxError } from './base'; + +export type RuntimeControlProtocolErrorReason = + | 'missing-metadata' + | 'malformed-metadata' + | 'unsupported-protocol-version' + | 'activation-mismatch'; + +export class RuntimeControlProtocolError extends SandboxError<{ + reason: RuntimeControlProtocolErrorReason; + operation?: string; +}> { + constructor( + message: string, + context: { + reason: RuntimeControlProtocolErrorReason; + operation?: string; + }, + options?: { cause?: unknown } + ) { + super( + { + code: ErrorCode.INTERNAL_ERROR, + message, + context, + httpStatus: getHttpStatus(ErrorCode.INTERNAL_ERROR), + timestamp: new Date().toISOString() + }, + options + ); + this.name = 'RuntimeControlProtocolError'; + } +} diff --git a/packages/sandbox/src/errors/classes/runtime-inactive.ts b/packages/sandbox/src/errors/classes/runtime-inactive.ts new file mode 100644 index 000000000..30fb7869a --- /dev/null +++ b/packages/sandbox/src/errors/classes/runtime-inactive.ts @@ -0,0 +1,6 @@ +export class RuntimeIdentityInactiveError extends Error { + constructor() { + super('Runtime identity is no longer active'); + this.name = 'RuntimeIdentityInactiveError'; + } +} diff --git a/packages/sandbox/src/errors/index.ts b/packages/sandbox/src/errors/index.ts index 58282972a..e04c2291b 100644 --- a/packages/sandbox/src/errors/index.ts +++ b/packages/sandbox/src/errors/index.ts @@ -84,6 +84,7 @@ export type { RPCTransportContext, RPCTransportErrorKind, StaleProcessHandleContext, + StaleTerminalHandleContext, TerminalControlErrorContext, TerminalNotFoundContext, ValidationFailedContext @@ -148,9 +149,12 @@ export { ProcessWaitTimeoutError, // RPC Transport Errors (SDK-side, raised on WebSocket failures) RPCTransportError, + RuntimeControlProtocolError, + RuntimeIdentityInactiveError, SandboxError, ServiceNotRespondingError, StaleProcessHandleError, + StaleTerminalHandleError, TerminalControlError, // Terminal Errors TerminalNotFoundError, diff --git a/packages/sandbox/src/extensions/index.ts b/packages/sandbox/src/extensions/index.ts index 7bf180575..e02a63d6e 100644 --- a/packages/sandbox/src/extensions/index.ts +++ b/packages/sandbox/src/extensions/index.ts @@ -6,9 +6,9 @@ * {@link SandboxExtension}, captured lazily via a `withX(this)` factory. * * - No sidecar? Extend {@link SandboxExtension} and use `this.exec()` or - * `this.client.` (`files`, `ports`, \u2026). Don't pass a package. + * `this.withRuntime()` for scoped control APIs. Don't pass a package. * - Need a container sidecar? Pass an {@link ExtensionPackage} to `super()`. - * Then call {@link SandboxExtension.sidecar} to obtain the sidecar's typed + * Then use {@link SandboxExtension.withSidecar} with the sidecar's typed * capnweb remote main. Calls on that stub stream through capnweb \u2014 callback * parameters round-trip across both the DO\u2192container and container\u2192sidecar * hops. @@ -43,11 +43,7 @@ import type { ProcessRPCDescriptor } from '../processes/rpc-types'; // subpath without reaching into `@repo/shared` directly. export type { ExtensionHealth, ExtensionPackage } from '@repo/shared'; -/** - * The slice of the Sandbox an extension captures: just its control `client`. - * Narrow on purpose \u2014 an extension never holds the whole instance. - */ -export type ExtensionControlClient = { +export type ExtensionRuntimeControl = { readonly files: SandboxFilesAPI; readonly ports: SandboxPortsAPI; readonly backup: SandboxBackupAPI; @@ -58,6 +54,37 @@ export type ExtensionControlClient = { readonly utils: SandboxUtilsAPI; }; +type ExtensionRuntimeDomain = + ExtensionRuntimeControl[keyof ExtensionRuntimeControl]; + +export type ExtensionRuntimeCallback = ( + control: ExtensionRuntimeControl +) => Promise; + +type ExtensionRuntimeResult = T extends + | ExtensionRuntimeControl + | ExtensionRuntimeDomain + ? never + : T; + +export type ExtensionRuntimeCallbackResult< + Call extends ExtensionRuntimeCallback +> = Awaited>; + +type RejectEscapingRuntimeCallback = + ExtensionRuntimeCallbackResult extends + | ExtensionRuntimeControl + | ExtensionRuntimeDomain + ? never + : unknown; + +export type ExtensionRuntimeCall = ( + operation: string, + call: Call & RejectEscapingRuntimeCallback +) => Promise>; + +export const sandboxRuntimeCall: unique symbol = Symbol('sandboxRuntimeCall'); + export interface HTTPAuthHostConfig { token: string; username?: string; @@ -69,7 +96,7 @@ export interface HTTPAuthInterceptorParams { } export type SandboxLike = { - readonly client: ExtensionControlClient; + readonly [sandboxRuntimeCall]: ExtensionRuntimeCall; readonly exec?: ( command: SandboxCommand, options?: ExecOptions @@ -140,9 +167,9 @@ export function createExtensionProcessSandbox( * * - SDK-only: just drives existing sub-APIs. * - Sidecar: pass an {@link ExtensionPackage} to `super(sandbox, pkg)` and - * call `this.sidecar()` to get a typed capnweb stub of the sidecar. + * call `this.withSidecar(operation, callback)` to use the typed capnweb stub inside one runtime callback. * - * The sidecar accessor throws a clear error if no package was supplied, so an + * The sidecar helper throws a clear error if no package was supplied, so an * extension only "becomes" a sidecar extension when it opts in. * * ```ts @@ -162,16 +189,16 @@ export function createExtensionProcessSandbox( * class MyExt extends SandboxExtension { * constructor(s: SandboxLike) { super(s, { tarball: new Uint8Array(sidecarTarballBytes) }); } * async run(input: string): Promise { - * const api = await this.sidecar(); - * return api.run(input); + * return this.withSidecar('my-ext.run', (api) => + * api.run(input) + * ); * } * } * ``` * - * RPC-safety: the sandbox lives in `#sandbox` and is reached only through the - * `protected` `client` getter (a prototype accessor, not an own property), - * so it is never serialised across RPC. Only the public methods you add form - * the extension's RPC surface. + * RPC-safety: the sandbox lives in `#sandbox` and runtime control is reached + * only through a symbol-keyed capability, so it is not exposed as a named RPC + * method. Only the public methods you add form the extension's RPC surface. */ export abstract class SandboxExtension extends RpcTarget { readonly #sandbox: SandboxLike; @@ -184,9 +211,24 @@ export abstract class SandboxExtension extends RpcTarget { this.#pkg = pkg; } - /** The container control client. Use inside your own methods, lazily. */ - protected get client(): ExtensionControlClient { - return this.#sandbox.client; + protected withRuntime( + operation: string, + call: Call & RejectEscapingRuntimeCallback + ): Promise>> { + const guardedCall = async (runtimeControl: ExtensionRuntimeControl) => { + const { control, revoke } = scopedRuntimeControl(runtimeControl); + try { + const result = await call(control); + assertRuntimeControlDidNotEscape(result, control); + return result; + } finally { + revoke(); + } + }; + return this.#sandbox[sandboxRuntimeCall]( + operation, + guardedCall as ExtensionRuntimeCallback + ) as Promise>>; } /** Launch an argv process through the owning Sandbox. */ @@ -212,52 +254,71 @@ export abstract class SandboxExtension extends RpcTarget { } /** - * Return the sidecar's capnweb remote main, provisioning + spawning on - * demand. `T` is the typed interface the sidecar exposes (its - * `SandboxSidecar` subclass shape). Each call reconnects through the host - * so a crashed sidecar can be restarted transparently on the next use. - * - * Streaming is just a method that takes a callback parameter: capnweb - * stubs the callback and routes invocations back through the SDK\u2192container - * \u2192sidecar hops. + * Provision/connect to the sidecar and use its typed capnweb remote inside + * one runtime callback. The remote is invalid after `call` resolves or + * rejects, so extension code cannot silently reconnect or keep using an old + * runtime's sidecar. */ - protected sidecar(): Promise { - // Wrap the synchronous `#requirePackage` check in an async closure so a - // missing-package error surfaces as a rejected promise, not a sync throw - // -- callers always treat `sidecar()` as awaitable. - return (async () => this.#connect(this.#requirePackage()))() as Promise; + protected async withSidecar( + operation: string, + call: (api: T) => Promise + ): Promise { + const pkg = this.#requirePackage(); + const packageHash = await this.#hashOnce(); + const scopedCall = async (control: ExtensionRuntimeControl) => { + const connected = await this.#connect( + control.extensions, + pkg, + packageHash + ); + const { proxy, revoke } = scopedSidecarRemote(connected); + try { + const result = await call(proxy); + assertSidecarRemoteDidNotEscape(result); + return detachPlainData(result) as Result; + } finally { + revoke(); + } + }; + return (await this.withRuntime( + operation, + scopedCall as ExtensionRuntimeCallback + )) as Result; } /** Health snapshot for this extension's sidecar. */ protected async sidecarHealth(): Promise { const hash = await this.#hashOnce(); - return this.#sandbox.client.extensions.health(hash); + return await this.withRuntime('extension.health', (control) => + control.extensions.health(hash) + ); } /** - * Stop this extension's sidecar. The next `sidecar()` call will respawn on - * demand. + * Stop this extension's sidecar. The next `withSidecar()` call will respawn + * it on demand. */ protected async stopSidecar(): Promise { const hash = await this.#hashOnce(); - await this.#sandbox.client.extensions.stop(hash); + await this.withRuntime('extension.stop', (control) => + control.extensions.stop(hash) + ); } // --- internals ----------------------------------------------------------- - async #connect(pkg: ExtensionPackage): Promise { - const packageHash = await this.#hashOnce(); - const api = this.#sandbox.client.extensions; - - // Hash-first: ask the host whether this process already has the package. - // If it doesn't, retry once with the tarball bytes attached. + async #connect( + api: SandboxExtensionsAPI, + pkg: ExtensionPackage, + packageHash: string + ): Promise { try { return (await api.connect({ packageHash, bin: pkg.bin, readinessTimeoutMs: pkg.readinessTimeoutMs, allowInstallScripts: pkg.allowInstallScripts - })) as object; + })) as T; } catch (error) { if (!isTarballRequiredError(error)) throw error; try { @@ -267,7 +328,7 @@ export abstract class SandboxExtension extends RpcTarget { bin: pkg.bin, readinessTimeoutMs: pkg.readinessTimeoutMs, allowInstallScripts: pkg.allowInstallScripts - })) as object; + })) as T; } catch (retryError) { throw createSidecarProvisioningError(packageHash, retryError); } @@ -300,6 +361,202 @@ export abstract class SandboxExtension extends RpcTarget { * `name` but preserving the message text. We fall back to matching the * message so the tarball retry still fires in that case. */ +function assertRuntimeControlDidNotEscape( + result: unknown, + control: ExtensionRuntimeControl +): void { + const forbiddenHandles: unknown[] = [ + control, + control.files, + control.ports, + control.backup, + control.watch, + control.tunnels, + control.terminals, + control.extensions, + control.utils + ].filter((handle) => handle !== undefined); + + if (!forbiddenHandles.includes(result)) return; + + throw new Error( + 'Sandbox extension runtime callbacks must not return runtime control handles' + ); +} + +function scopedRuntimeControl(target: ExtensionRuntimeControl): { + control: ExtensionRuntimeControl; + revoke(): void; +} { + let active = true; + const inactiveError = () => + new Error( + 'Sandbox extension runtime control is no longer valid outside its runtime callback' + ); + const wrappedDomains = new WeakMap(); + const scopeDomain = (domain: Domain): Domain => { + const existing = wrappedDomains.get(domain); + if (existing) return existing as Domain; + const proxy = new Proxy(domain, { + get(domainTarget, property, receiver) { + if (!active) return () => Promise.reject(inactiveError()); + const value = Reflect.get(domainTarget, property, receiver) as unknown; + if (typeof value === 'function') { + return (...args: unknown[]) => { + if (!active) return Promise.reject(inactiveError()); + const result = Reflect.apply(value, domainTarget, args) as unknown; + return scopeRuntimeResult(result); + }; + } + if (typeof value === 'object' && value !== null) { + if (objectHasCallableMember(value)) return scopeDomain(value); + return detachPlainData(value); + } + return value; + } + }); + wrappedDomains.set(domain, proxy); + return proxy as Domain; + }; + const scopeRuntimeResult = (result: unknown): unknown => { + if (result instanceof Promise) { + return result.then(scopeRuntimeResult); + } + if (typeof result === 'object' && result !== null) { + if (objectHasCallableMember(result)) return scopeDomain(result); + return detachPlainData(result); + } + return result; + }; + + const control: ExtensionRuntimeControl = { + files: scopeDomain(target.files), + ports: scopeDomain(target.ports), + backup: scopeDomain(target.backup), + watch: scopeDomain(target.watch), + tunnels: scopeDomain(target.tunnels), + terminals: scopeDomain(target.terminals), + extensions: scopeDomain(target.extensions), + utils: scopeDomain(target.utils) + }; + return { + control, + revoke: () => { + active = false; + } + }; +} + +const sidecarRemoteProxies = new WeakSet(); + +function scopedSidecarRemote( + target: T +): { + proxy: T; + revoke(): void; +} { + let active = true; + const wrappedTargets = new WeakMap(); + const createProxy = (currentTarget: Value): Value => { + const existing = wrappedTargets.get(currentTarget); + if (existing) return existing as Value; + const proxy = new Proxy(currentTarget, { + get(targetObject, property, receiver) { + if (!active) { + return () => + Promise.reject( + new Error( + 'Sandbox extension sidecar remote is no longer valid outside its runtime callback' + ) + ); + } + const value = Reflect.get(targetObject, property, receiver) as unknown; + if (typeof value === 'function') { + return (...args: unknown[]) => { + if (!active) { + return Promise.reject( + new Error( + 'Sandbox extension sidecar remote is no longer valid outside its runtime callback' + ) + ); + } + const result = Reflect.apply(value, targetObject, args) as unknown; + return wrapSidecarResult(result, createProxy); + }; + } + if (typeof value === 'object' && value !== null) { + if (objectHasCallableMember(value)) return createProxy(value); + return detachPlainData(value); + } + return value; + } + }); + wrappedTargets.set(currentTarget, proxy); + sidecarRemoteProxies.add(proxy); + return proxy as Value; + }; + + return { + proxy: createProxy(target), + revoke: () => { + active = false; + } + }; +} + +function wrapSidecarResult( + result: unknown, + wrapObject: (value: Value) => Value +): unknown { + if (result instanceof Promise) { + return result.then((value) => wrapSidecarResult(value, wrapObject)); + } + if (typeof result === 'object' && result !== null) { + if (objectHasCallableMember(result)) return wrapObject(result); + return detachPlainData(result); + } + return result; +} + +function objectHasCallableMember(value: object): boolean { + if (Array.isArray(value)) + return value.some((entry) => typeof entry === 'function'); + let current: object | null = value; + while (current && current !== Object.prototype) { + for (const property of Reflect.ownKeys(current)) { + if (property === 'constructor') continue; + const descriptor = Reflect.getOwnPropertyDescriptor(current, property); + if (!descriptor) continue; + if (typeof descriptor.value === 'function') return true; + if (typeof descriptor.get === 'function') return true; + } + current = Reflect.getPrototypeOf(current); + } + return false; +} + +function assertSidecarRemoteDidNotEscape(result: unknown): void { + const seen = new WeakSet(); + const visit = (value: unknown): boolean => { + if (typeof value !== 'object' || value === null) return false; + if (seen.has(value)) return false; + seen.add(value); + if (sidecarRemoteProxies.has(value)) return true; + if (Array.isArray(value)) return value.some(visit); + return Object.values(value as Record).some(visit); + }; + + if (!visit(result)) return; + throw new Error( + 'Sandbox extension sidecar callbacks must not return sidecar remotes' + ); +} + +function detachPlainData(value: T): T { + if (typeof value !== 'object' || value === null) return value; + return structuredClone(value) as T; +} + function isTarballRequiredError(error: unknown): boolean { if (typeof error !== 'object' || error === null) return false; const candidate = error as { name?: unknown; message?: unknown }; diff --git a/packages/sandbox/src/index.ts b/packages/sandbox/src/index.ts index baea64acf..5b05e9ef4 100644 --- a/packages/sandbox/src/index.ts +++ b/packages/sandbox/src/index.ts @@ -73,7 +73,10 @@ export { ProcessWaitTimeoutError, // RPC transport error (raised on capnweb WebSocket session failures) RPCTransportError, + RuntimeControlProtocolError, + RuntimeIdentityInactiveError, StaleProcessHandleError, + StaleTerminalHandleError, TerminalControlError, TerminalNotFoundError } from './errors'; diff --git a/packages/sandbox/src/local-mount-sync.ts b/packages/sandbox/src/local-mount-sync.ts index b44a8485d..492101e14 100644 --- a/packages/sandbox/src/local-mount-sync.ts +++ b/packages/sandbox/src/local-mount-sync.ts @@ -4,6 +4,10 @@ import type { ContainerControlClient } from './container-control'; import { openRemoteSubscription } from './processes/remote-subscription'; import { parseSSEStream } from './sse-parser'; import { validatePrefix } from './storage-mount'; +import type { + MountRuntimeCall, + MountRuntimeHold +} from './storage-mount/runtime-call'; const DEFAULT_POLL_INTERVAL_MS = 1000; const DEFAULT_ECHO_SUPPRESS_TTL_MS = 2000; @@ -20,7 +24,8 @@ interface LocalMountSyncOptions { mountPath: string; prefix: string | undefined; readOnly: boolean; - client: ContainerControlClient; + runRuntimeCall: MountRuntimeCall; + runtimeHold?: MountRuntimeHold; logger: Logger; pollIntervalMs?: number; echoSuppressTtlMs?: number; @@ -37,7 +42,8 @@ export class LocalMountSyncManager { private readonly mountPath: string; private readonly prefix: string | undefined; private readonly readOnly: boolean; - private readonly client: ContainerControlClient; + private readonly runRuntimeCall: MountRuntimeCall; + private readonly runtimeHold: MountRuntimeHold; private readonly logger: Logger; private readonly pollIntervalMs: number; @@ -49,8 +55,12 @@ export class LocalMountSyncManager { private watchReconnectTimer: ReturnType | null = null; private watchAbortController: AbortController | null = null; private running = false; + private generation = 0; + private activePollCycle: Promise | null = null; + private activeWatchLoop: Promise | null = null; private consecutivePollFailures = 0; private consecutiveWatchFailures = 0; + private runtimeHoldReleased = false; constructor(options: LocalMountSyncOptions) { this.bucket = options.bucket; @@ -62,7 +72,8 @@ export class LocalMountSyncManager { // value into bare R2 key format for list() and put(). this.prefix = options.prefix?.replace(/^\//, '') || undefined; this.readOnly = options.readOnly; - this.client = options.client; + this.runRuntimeCall = options.runRuntimeCall; + this.runtimeHold = options.runtimeHold ?? { release: () => {} }; this.logger = options.logger.child({ operation: 'local-mount-sync' }); this.pollIntervalMs = options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS; this.echoSuppressTtlMs = @@ -75,12 +86,21 @@ export class LocalMountSyncManager { */ async start(): Promise { this.running = true; + this.generation += 1; + const generation = this.generation; + + await this.runRuntimeCallIfCurrent( + generation, + 'mount.local.mkdir', + (control) => + control.files.mkdir(this.mountPath, { + recursive: true + }) + ); - await this.client.files.mkdir(this.mountPath, { - recursive: true - }); - - await this.fullSyncR2ToContainer(); + if (!this.isCurrentGeneration(generation)) return; + await this.fullSyncR2ToContainer(generation); + if (!this.isCurrentGeneration(generation)) return; this.schedulePoll(); if (!this.readOnly) { @@ -99,6 +119,21 @@ export class LocalMountSyncManager { * Stop all sync activity and clean up resources. */ async stop(): Promise { + this.interrupt(); + + const pollCycle = this.activePollCycle; + const watchLoop = this.activeWatchLoop; + await Promise.allSettled([pollCycle, watchLoop].filter(isPromise)); + + this.snapshot.clear(); + this.echoSuppressSet.clear(); + + this.logger.info('Local mount sync stopped', { + mountPath: this.mountPath + }); + } + + interrupt(): void { this.running = false; if (this.pollTimer) { @@ -116,32 +151,38 @@ export class LocalMountSyncManager { this.watchAbortController = null; } - this.snapshot.clear(); - this.echoSuppressSet.clear(); - - this.logger.info('Local mount sync stopped', { - mountPath: this.mountPath - }); + if (!this.runtimeHoldReleased) { + this.runtimeHoldReleased = true; + this.runtimeHold.release(); + } } - private async fullSyncR2ToContainer(): Promise { + private async fullSyncR2ToContainer(generation: number): Promise { const objects = await this.listAllR2Objects(); + if (!this.isCurrentGeneration(generation)) return; const newSnapshot = new Map(); // No echo suppression needed: this runs before startContainerWatch() in start(). // Process in batches to limit concurrent HTTP requests for (let i = 0; i < objects.length; i += SYNC_CONCURRENCY) { + if (!this.isCurrentGeneration(generation)) return; const batch = objects.slice(i, i + SYNC_CONCURRENCY); await Promise.all( batch.map(async (obj) => { + if (!this.isCurrentGeneration(generation)) return; const containerPath = this.r2KeyToContainerPath(obj.key); newSnapshot.set(obj.key, { etag: obj.etag, size: obj.size }); - await this.ensureParentDir(containerPath); - await this.transferR2ObjectToContainer(obj.key, containerPath); + await this.ensureParentDir(containerPath, generation); + await this.transferR2ObjectToContainer( + obj.key, + containerPath, + generation + ); }) ); } + if (!this.isCurrentGeneration(generation)) return; this.snapshot = newSnapshot; this.logger.debug('Initial R2 -> Container sync complete', { objectCount: objects.length @@ -159,23 +200,39 @@ export class LocalMountSyncManager { ) : this.pollIntervalMs; - this.pollTimer = setTimeout(async () => { - try { - await this.pollR2ForChanges(); - this.consecutivePollFailures = 0; - } catch (error) { - this.consecutivePollFailures++; - this.logger.error( - 'R2 poll cycle failed', - error instanceof Error ? error : new Error(String(error)) - ); - } - this.schedulePoll(); + const generation = this.generation; + this.pollTimer = setTimeout(() => { + this.pollTimer = null; + if (!this.isCurrentGeneration(generation)) return; + const cycle = this.pollR2ForChanges(generation) + .then(() => { + if (this.isCurrentGeneration(generation)) { + this.consecutivePollFailures = 0; + } + }) + .catch((error) => { + if (!this.isCurrentGeneration(generation)) return; + this.consecutivePollFailures++; + this.logger.error( + 'R2 poll cycle failed', + error instanceof Error ? error : new Error(String(error)) + ); + }) + .finally(() => { + if (this.activePollCycle === cycle) { + this.activePollCycle = null; + } + if (this.isCurrentGeneration(generation)) { + this.schedulePoll(); + } + }); + this.activePollCycle = cycle; }, backoffMs); } - private async pollR2ForChanges(): Promise { + private async pollR2ForChanges(generation: number): Promise { const objects = await this.listAllR2Objects(); + if (!this.isCurrentGeneration(generation)) return; const newSnapshot = new Map(); // Collect changed objects first, then transfer in batches @@ -196,10 +253,16 @@ export class LocalMountSyncManager { await Promise.all( batch.map(async ({ key, action }) => { try { + if (!this.isCurrentGeneration(generation)) return; const containerPath = this.r2KeyToContainerPath(key); - await this.ensureParentDir(containerPath); + await this.ensureParentDir(containerPath, generation); + if (!this.isCurrentGeneration(generation)) return; this.suppressEcho(containerPath); - await this.transferR2ObjectToContainer(key, containerPath); + await this.transferR2ObjectToContainer( + key, + containerPath, + generation + ); this.logger.debug('R2 -> Container: synced object', { key, action @@ -220,7 +283,12 @@ export class LocalMountSyncManager { this.suppressEcho(containerPath); try { - await this.client.files.deleteFile(containerPath); + if (!this.isCurrentGeneration(generation)) return; + await this.runRuntimeCallIfCurrent( + generation, + 'mount.local.deleteFile', + (control) => control.files.deleteFile(containerPath) + ); this.logger.debug('R2 -> Container: deleted file', { key }); } catch (error) { this.logger.error( @@ -231,6 +299,7 @@ export class LocalMountSyncManager { } } + if (!this.isCurrentGeneration(generation)) return; this.snapshot = newSnapshot; } @@ -258,7 +327,8 @@ export class LocalMountSyncManager { private async transferR2ObjectToContainer( key: string, - containerPath: string + containerPath: string, + generation?: number ): Promise { const obj = await this.bucket.get(key); if (!obj) return; @@ -266,20 +336,33 @@ export class LocalMountSyncManager { const arrayBuffer = await obj.arrayBuffer(); const base64 = uint8ArrayToBase64(new Uint8Array(arrayBuffer)); - await this.client.files.writeFile(containerPath, base64, { - encoding: 'base64' - }); + await this.runRuntimeCallIfCurrent( + generation, + 'mount.local.writeFile', + (control) => + control.files.writeFile(containerPath, base64, { + encoding: 'base64' + }) + ); } - private async ensureParentDir(containerPath: string): Promise { + private async ensureParentDir( + containerPath: string, + generation?: number + ): Promise { const parentDir = containerPath.substring( 0, containerPath.lastIndexOf('/') ); if (parentDir && parentDir !== this.mountPath) { - await this.client.files.mkdir(parentDir, { - recursive: true - }); + await this.runRuntimeCallIfCurrent( + generation, + 'mount.local.mkdir', + (control) => + control.files.mkdir(parentDir, { + recursive: true + }) + ); } } @@ -291,21 +374,29 @@ export class LocalMountSyncManager { private runWatchWithRetry(): void { if (!this.running) return; - this.runContainerWatchLoop() + const generation = this.generation; + const loop = this.runContainerWatchLoop(generation) .then(() => { + if (!this.isCurrentGeneration(generation)) return; // Stream ended cleanly (e.g. server closed it). Reconnect unless stopped. this.consecutiveWatchFailures = 0; this.scheduleWatchReconnect(); }) .catch((error) => { - if (!this.running) return; + if (!this.isCurrentGeneration(generation)) return; this.consecutiveWatchFailures++; this.logger.error( 'Container watch loop failed', error instanceof Error ? error : new Error(String(error)) ); this.scheduleWatchReconnect(); + }) + .finally(() => { + if (this.activeWatchLoop === loop) { + this.activeWatchLoop = null; + } }); + this.activeWatchLoop = loop; } private scheduleWatchReconnect(): void { @@ -332,70 +423,79 @@ export class LocalMountSyncManager { }, backoffMs); } - private async runContainerWatchLoop(): Promise { - const stream = await openRemoteSubscription( - this.client.watch.watch({ - path: this.mountPath, - recursive: true - }), - { - signal: this.watchAbortController?.signal, - operation: 'open local mount filesystem watch' - } - ); - - for await (const event of parseSSEStream( - stream, - this.watchAbortController?.signal - )) { - if (!this.running) break; + private async runContainerWatchLoop(generation: number): Promise { + await this.runRuntimeCallIfCurrent( + generation, + 'mount.local.watch', + async (control) => { + const stream = await openRemoteSubscription( + control.watch.watch({ + path: this.mountPath, + recursive: true + }), + { + signal: this.watchAbortController?.signal, + operation: 'open local mount filesystem watch', + protocol: 'stream' + } + ); - // Successful event received — reset failure counter - this.consecutiveWatchFailures = 0; + for await (const event of parseSSEStream( + stream, + this.watchAbortController?.signal + )) { + if (!this.isCurrentGeneration(generation)) break; - if (event.type !== 'event') continue; - if (event.isDirectory) continue; + // Successful event received — reset failure counter + this.consecutiveWatchFailures = 0; - const containerPath = event.path; + if (event.type !== 'event') continue; + if (event.isDirectory) continue; - // Skip echo from our own R2 -> Container writes - if (this.echoSuppressSet.has(containerPath)) continue; + const containerPath = event.path; - const r2Key = this.containerPathToR2Key(containerPath); - if (!r2Key) continue; + // Skip echo from our own R2 -> Container writes + if (this.echoSuppressSet.has(containerPath)) continue; - try { - switch (event.eventType) { - case 'create': - case 'modify': - case 'move_to': { - await this.uploadFileToR2(containerPath, r2Key); - this.logger.debug('Container -> R2: synced file', { - path: containerPath, - key: r2Key, - action: event.eventType - }); - break; - } + const r2Key = this.containerPathToR2Key(containerPath); + if (!r2Key) continue; - case 'delete': - case 'move_from': { - await this.bucket.delete(r2Key); - this.snapshot.delete(r2Key); - this.logger.debug('Container -> R2: deleted object', { - path: containerPath, - key: r2Key - }); - break; + try { + switch (event.eventType) { + case 'create': + case 'modify': + case 'move_to': { + await this.uploadFileToR2(containerPath, r2Key, generation); + this.logger.debug('Container -> R2: synced file', { + path: containerPath, + key: r2Key, + action: event.eventType + }); + break; + } + + case 'delete': + case 'move_from': { + if (!this.isCurrentGeneration(generation)) break; + await this.bucket.delete(r2Key); + if (!this.isCurrentGeneration(generation)) break; + this.snapshot.delete(r2Key); + this.logger.debug('Container -> R2: deleted object', { + path: containerPath, + key: r2Key + }); + break; + } + } + } catch (error) { + this.logger.error( + `Container -> R2 sync failed for ${containerPath}`, + error instanceof Error ? error : new Error(String(error)) + ); } } - } catch (error) { - this.logger.error( - `Container -> R2 sync failed for ${containerPath}`, - error instanceof Error ? error : new Error(String(error)) - ); } - } + ); } /** @@ -404,13 +504,21 @@ export class LocalMountSyncManager { */ private async uploadFileToR2( containerPath: string, - r2Key: string + r2Key: string, + generation?: number ): Promise { - const result = await this.client.files.readFile(containerPath, { - encoding: 'base64' - }); + const result = await this.runRuntimeCallIfCurrent( + generation, + 'mount.local.readFile', + (control) => + control.files.readFile(containerPath, { + encoding: 'base64' + }) + ); + if (!this.isCurrentOrUnscoped(generation)) return; const bytes = base64ToUint8Array(result.content); await this.bucket.put(r2Key, bytes); + if (!this.isCurrentOrUnscoped(generation)) return; const head = await this.bucket.head(r2Key); if (head) { @@ -418,6 +526,30 @@ export class LocalMountSyncManager { } } + private async runRuntimeCallIfCurrent( + generation: number | undefined, + operation: string, + call: (control: ContainerControlClient) => Promise + ): Promise { + if (generation !== undefined && !this.isCurrentGeneration(generation)) { + throw new Error('local mount sync stopped'); + } + return await this.runRuntimeCall(operation, async (control) => { + if (generation !== undefined && !this.isCurrentGeneration(generation)) { + throw new Error('local mount sync stopped'); + } + return await call(control); + }); + } + + private isCurrentOrUnscoped(generation: number | undefined): boolean { + return generation === undefined || this.isCurrentGeneration(generation); + } + + private isCurrentGeneration(generation: number): boolean { + return this.running && this.generation === generation; + } + private suppressEcho(containerPath: string): void { this.echoSuppressSet.add(containerPath); setTimeout(() => { @@ -448,6 +580,10 @@ export class LocalMountSyncManager { } } +function isPromise(value: Promise | null): value is Promise { + return value !== null; +} + function uint8ArrayToBase64(bytes: Uint8Array): string { return Buffer.from(bytes).toString('base64'); } diff --git a/packages/sandbox/src/preview/forwarding.ts b/packages/sandbox/src/preview/forwarding.ts index 94bf57b53..ed1dbd481 100644 --- a/packages/sandbox/src/preview/forwarding.ts +++ b/packages/sandbox/src/preview/forwarding.ts @@ -1,3 +1,6 @@ +import { ErrorCode, OperationInterruptedError } from '../errors'; +import type { RuntimeLease } from '../runtime'; + export type PreviewTCPPort = { fetch( input: Request | string, @@ -5,10 +8,7 @@ export type PreviewTCPPort = { ): Promise; }; -export type PreviewForwardingLifecycle = { - beginForward(): () => void; - renewActivity(): void; -}; +export type PreviewForwardingLease = Pick; export type PreviewForwardingResult = | { status: 'response'; response: Response } @@ -17,34 +17,55 @@ export type PreviewForwardingResult = export async function forwardPreviewRequest( tcpPort: PreviewTCPPort, request: Request, - lifecycle: PreviewForwardingLifecycle + lease: PreviewForwardingLease ): Promise { const containerURL = request.url.replace('https:', 'http:'); - const settleForward = lifecycle.beginForward(); + let interruptedError: OperationInterruptedError | undefined; + let closeAssignedResponse: + | ((error: OperationInterruptedError) => void) + | undefined; + const hold = lease.retain(() => { + interruptedError = previewForwardInterrupted(); + closeAssignedResponse?.(interruptedError); + }); + const release = once(() => hold.release()); + if (interruptedError) { + release(); + throw interruptedError; + } try { const response = await tcpPort.fetch(containerURL, request); + if (interruptedError) { + closeLateResponse(response, interruptedError); + throw interruptedError; + } if (response.webSocket !== null) { return { status: 'response', - response: bridgePreviewWebSocket(response, lifecycle, settleForward) + response: bridgePreviewWebSocket(response, release, (close) => { + closeAssignedResponse = close; + if (interruptedError) close(interruptedError); + }) }; } if (response.body !== null) { - const { readable, writable } = new TransformStream(); - response.body - .pipeTo(writable) - .finally(settleForward) - .catch(() => {}); - return { status: 'response', response: new Response(readable, response) }; + const retained = retainPreviewBody(response.body, release, (close) => { + closeAssignedResponse = close; + if (interruptedError) close(interruptedError); + }); + return { + status: 'response', + response: new Response(retained, response) + }; } - settleForward(); + release(); return { status: 'response', response }; } catch (error) { - settleForward(); + release(); if ( error instanceof Error && error.message.includes('Network connection lost.') @@ -55,31 +76,89 @@ export async function forwardPreviewRequest( } } +function retainPreviewBody( + body: ReadableStream, + release: () => void, + bindInterrupt: (close: (error: OperationInterruptedError) => void) => void +): ReadableStream { + const reader = body.getReader(); + let controller: ReadableStreamDefaultController | undefined; + let interruptedError: OperationInterruptedError | undefined; + let sourceCancelled = false; + const cancelSource = (reason?: unknown) => { + if (sourceCancelled) return; + sourceCancelled = true; + release(); + void reader.cancel(reason).catch(() => undefined); + }; + const fail = once((error: OperationInterruptedError) => { + interruptedError = error; + controller?.error(error); + cancelSource(error); + }); + bindInterrupt(fail); + + return new ReadableStream({ + start(streamController) { + controller = streamController; + if (interruptedError) streamController.error(interruptedError); + }, + async pull(streamController) { + try { + if (interruptedError) throw interruptedError; + const result = await reader.read(); + if (interruptedError) throw interruptedError; + if (result.done) { + release(); + streamController.close(); + return; + } + streamController.enqueue(result.value); + } catch (error) { + release(); + if (error !== interruptedError) { + streamController.error(error); + } + } + }, + cancel(reason) { + cancelSource(reason); + } + }); +} + function bridgePreviewWebSocket( response: Response, - lifecycle: PreviewForwardingLifecycle, - settleForward: () => void + release: () => void, + bindInterrupt: (close: (error: OperationInterruptedError) => void) => void ): Response { const containerWebSocket = response.webSocket; if (containerWebSocket === null) { - settleForward(); + release(); return response; } const [client, server] = Object.values(new WebSocketPair()); let settled = false; - const settle = () => { - if (!settled) { - settled = true; - settleForward(); + const settle = once(() => { + settled = true; + release(); + }); + const closeInterrupted = once(() => { + try { + containerWebSocket.close(1012, 'Runtime replaced'); + server.close(1012, 'Runtime replaced'); + } finally { + settle(); } - }; + }); + bindInterrupt(closeInterrupted); containerWebSocket.accept(); server.accept(); server.addEventListener('message', async (event) => { - lifecycle.renewActivity(); + if (settled) return; try { const data = event.data instanceof Blob @@ -92,7 +171,7 @@ function bridgePreviewWebSocket( }); containerWebSocket.addEventListener('message', async (event) => { - lifecycle.renewActivity(); + if (settled) return; try { const data = event.data instanceof Blob @@ -132,3 +211,44 @@ function bridgePreviewWebSocket( headers: response.headers }); } + +function closeLateResponse( + response: Response, + error: OperationInterruptedError +): void { + if (response.webSocket !== null) { + try { + response.webSocket.accept(); + } catch { + // The socket may already be accepted by the platform. + } + response.webSocket.close(1012, 'Runtime replaced'); + return; + } + void response.body?.cancel(error).catch(() => undefined); +} + +function previewForwardInterrupted(): OperationInterruptedError { + return new OperationInterruptedError({ + code: ErrorCode.OPERATION_INTERRUPTED, + message: + 'Sandbox operation preview.forward was interrupted because the runtime changed', + context: { + reason: 'runtime_replaced', + operation: 'preview.forward', + admitted: true, + retryable: false + }, + httpStatus: 409, + timestamp: new Date().toISOString() + }); +} + +function once void>(fn: T): T { + let called = false; + return ((...args: Parameters) => { + if (called) return; + called = true; + fn(...(args as never[])); + }) as T; +} diff --git a/packages/sandbox/src/preview/proxy-request.ts b/packages/sandbox/src/preview/proxy-request.ts index ae386159e..2c923f924 100644 --- a/packages/sandbox/src/preview/proxy-request.ts +++ b/packages/sandbox/src/preview/proxy-request.ts @@ -42,5 +42,7 @@ function stripPreviewProxyHeaders(source: Headers): Headers { for (const header of PREVIEW_PROXY_HEADERS) { headers.delete(header); } + headers.delete('cf-container-target-port'); + headers.delete('x-sandbox-port-route-token'); return headers; } diff --git a/packages/sandbox/src/preview/service.ts b/packages/sandbox/src/preview/service.ts index 348920be7..bc82cbae8 100644 --- a/packages/sandbox/src/preview/service.ts +++ b/packages/sandbox/src/preview/service.ts @@ -1,12 +1,20 @@ import { type Logger, logCanonicalEvent } from '@repo/shared'; -import type { - CurrentRuntimeIdentity, - RuntimeIdentity -} from '../current-runtime-identity'; import type { ErrorResponse } from '../errors'; -import { CustomDomainRequiredError, ErrorCode } from '../errors'; +import { + CustomDomainRequiredError, + ErrorCode, + OperationInterruptedError, + RuntimeControlProtocolError +} from '../errors'; +import type { RuntimeLease } from '../runtime'; +import type { RuntimeIdentity } from '../runtime/types'; +import { RuntimeIdentityInactiveError } from '../runtime/types'; import { SandboxSecurityError, validatePort } from '../security'; -import { forwardPreviewRequest, type PreviewTCPPort } from './forwarding'; +import { + forwardPreviewRequest, + type PreviewForwardingLease, + type PreviewTCPPort +} from './forwarding'; import { readPreviewProxyMetadata } from './protocol'; import { buildPreviewProxyRequest } from './proxy-request'; import { constructPreviewURL } from './route'; @@ -15,6 +23,7 @@ import { clearActivePreviewPorts, PORT_TOKENS_STORAGE_KEY, type PortTokenEntry, + type PreviewPortActivation, type PreviewPortActivations, readActivePreviewPorts, readPortTokens, @@ -47,7 +56,7 @@ type StalePreviewRuntime = { type PreviewURLRuntimeValidation = | { status: 'invalid' } | StalePreviewRuntime - | { status: 'active'; runtime: RuntimeIdentity }; + | { status: 'active'; activation: PreviewPortActivation }; type PreviewRuntimeAvailability = | StalePreviewRuntime @@ -61,17 +70,37 @@ type PreviewRuntimeSnapshot = { runtime: RuntimeIdentity | null; }; +type PreviewExposureLease = Pick; + +type PreviewExposureCommit = { + portKey: string; + previousEntry: PortTokenEntry | undefined; + previousActivation: PreviewPortActivation | undefined; + entry: PortTokenEntry; + activation: PreviewPortActivation; +}; + export interface PreviewServiceDeps { storage: DurableObjectStorage; logger: Logger; - currentRuntime: CurrentRuntimeIdentity; + getStoredRuntime( + storage: DurableObjectTransaction + ): Promise; + assertRuntimeActive(runtime: RuntimeIdentity): Promise; getContainerState(): Promise<{ status: string }>; getForwardingContainer(): PreviewForwardingContainer | undefined; - ensureRuntimeActiveForPreview(): Promise; + runWaking( + operation: string, + call: (lease: PreviewExposureLease) => Promise + ): Promise; + runExisting( + operation: string, + call: ( + lease: PreviewForwardingLease & { runtime: RuntimeIdentity } + ) => Promise + ): Promise; getSandboxName(): string | null; getNormalizeID(): boolean; - beginForward(): () => void; - renewActivity(): void; } export class PreviewService { @@ -95,6 +124,7 @@ export class PreviewService { const exposeStartTime = Date.now(); let outcome: 'success' | 'error' = 'error'; let caughtError: Error | undefined; + let committed: PreviewExposureCommit | undefined; try { if (!validatePort(port)) { throw new SandboxSecurityError( @@ -124,55 +154,107 @@ export class PreviewService { assertValidCustomPreviewToken(options.token); } - const runtime = await this.deps.ensureRuntimeActiveForPreview(); - await this.deps.currentRuntime.assertActive(runtime); - - const token = await this.deps.storage.transaction(async (txn) => { - const tokens = await readPortTokens(txn); - const existingEntry = tokens[port.toString()]; - const nextToken = - options.token ?? existingEntry?.token ?? generatePreviewToken(); - - const existingPort = Object.entries(tokens).find( - ([p, entry]) => entry.token === nextToken && p !== port.toString() - ); - if (existingPort) { - throw new SandboxSecurityError( - `Token '${nextToken}' is already in use by port ${existingPort[0]}. Please use a different token.` - ); + await this.preflightTokenCollision(port, options.token); + + const result = await this.deps.runWaking( + 'preview.expose', + async (lease) => { + let interrupted = false; + const hold = lease.retain(() => { + interrupted = true; + }); + const assertLeaseActive = () => { + if (interrupted) throw new RuntimeIdentityInactiveError(); + }; + const runtime = lease.runtime; + try { + assertLeaseActive(); + await this.deps.assertRuntimeActive(runtime); + + const exposure = await this.deps.storage.transaction( + async (txn) => { + const portKey = port.toString(); + const tokens = await readPortTokens(txn); + const previousEntry = tokens[portKey]; + const nextToken = + options.token ?? + previousEntry?.token ?? + generatePreviewToken(); + + const existingPort = Object.entries(tokens).find( + ([p, entry]) => entry.token === nextToken && p !== portKey + ); + if (existingPort) { + throw new SandboxSecurityError( + `Token '${nextToken}' is already in use by port ${existingPort[0]}. Please use a different token.` + ); + } + + const activations = await readActivePreviewPorts(txn); + const previousActivation = activations[portKey]; + const storedRuntime = await this.deps.getStoredRuntime(txn); + assertLeaseActive(); + if (!storedRuntime || !sameRuntime(storedRuntime, runtime)) { + throw new RuntimeIdentityInactiveError(); + } + + const entry = { token: nextToken, name: options.name }; + const activation = { + runtimeIdentityID: runtime.id, + runtimeIncarnationID: runtime.runtimeIncarnationID, + token: nextToken + }; + tokens[portKey] = entry; + activations[portKey] = activation; + await Promise.all([ + txn.put(PORT_TOKENS_STORAGE_KEY, tokens), + writeActivePreviewPorts(activations, txn) + ]); + + return { + token: nextToken, + commit: { + portKey, + previousEntry, + previousActivation, + entry, + activation + } + }; + } + ); + committed = exposure.commit; + + assertLeaseActive(); + await this.deps.assertRuntimeActive(runtime); + assertLeaseActive(); + const token = exposure.token; + + const url = constructPreviewURL({ + port, + sandboxId: sandboxName, + effectiveId: sandboxName, + hostname: options.hostname, + token, + normalizeId: this.deps.getNormalizeID() + }); + + return { + url, + port, + name: options.name + }; + } finally { + hold.release(); + } } - - const activations = await readActivePreviewPorts(txn); - - tokens[port.toString()] = { token: nextToken, name: options.name }; - activations[port.toString()] = runtime.scope({ token: nextToken }); - await Promise.all([ - txn.put(PORT_TOKENS_STORAGE_KEY, tokens), - writeActivePreviewPorts(activations, txn) - ]); - - return nextToken; - }); - - await this.deps.currentRuntime.assertActive(runtime); - - const url = constructPreviewURL({ - port, - sandboxId: sandboxName, - effectiveId: sandboxName, - hostname: options.hostname, - token, - normalizeId: this.deps.getNormalizeID() - }); + ); outcome = 'success'; - return { - url, - port, - name: options.name - }; + return result; } catch (error) { + if (committed) await this.rollbackExposureIfUnchanged(committed); caughtError = error instanceof Error ? error : new Error(String(error)); throw error; } finally { @@ -188,6 +270,53 @@ export class PreviewService { } } + private async rollbackExposureIfUnchanged( + commit: PreviewExposureCommit + ): Promise { + await this.deps.storage.transaction(async (txn) => { + const [tokens, activations] = await Promise.all([ + readPortTokens(txn), + readActivePreviewPorts(txn) + ]); + if ( + !samePortTokenEntry(tokens[commit.portKey], commit.entry) || + !isSameActivation(activations[commit.portKey], commit.activation) + ) { + return; + } + + if (commit.previousEntry) tokens[commit.portKey] = commit.previousEntry; + else delete tokens[commit.portKey]; + if (commit.previousActivation) { + activations[commit.portKey] = commit.previousActivation; + } else { + delete activations[commit.portKey]; + } + await Promise.all([ + txn.put(PORT_TOKENS_STORAGE_KEY, tokens), + writeActivePreviewPorts(activations, txn) + ]); + }); + } + + private async preflightTokenCollision( + port: number, + requestedToken: string | undefined + ): Promise { + const tokens = await readPortTokens(this.deps.storage); + const existingEntry = tokens[port.toString()]; + const candidate = requestedToken ?? existingEntry?.token; + if (!candidate) return; + const existingPort = Object.entries(tokens).find( + ([p, entry]) => entry.token === candidate && p !== port.toString() + ); + if (existingPort) { + throw new SandboxSecurityError( + `Token '${candidate}' is already in use by port ${existingPort[0]}. Please use a different token.` + ); + } + } + async unexposePort(port: number): Promise { const unexposeStartTime = Date.now(); let outcome: 'success' | 'error' = 'error'; @@ -301,11 +430,31 @@ export class PreviewService { return this.stalePreviewURLResponse(); } - return await this.fetchPreviewIfRunning( - proxyRequest, - port, - validation.runtime - ); + try { + const response = await this.deps.runExisting( + 'preview.forward', + async (lease) => + await this.fetchPreviewIfRunning( + proxyRequest, + port, + validation.activation, + lease + ) + ); + + if (response) return response; + await this.clearActivationIfUnchanged(port, validation.activation); + return this.stalePreviewURLResponse(); + } catch (error) { + if ( + error instanceof OperationInterruptedError || + isActivationMismatch(error) + ) { + await this.clearActivationIfUnchanged(port, validation.activation); + return this.stalePreviewURLResponse(); + } + throw error; + } } private invalidPreviewTokenResponse(): Response { @@ -341,34 +490,25 @@ export class PreviewService { private async fetchPreviewIfRunning( request: Request, port: number, - runtime: RuntimeIdentity + activation: PreviewPortActivation, + lease: PreviewForwardingLease & { runtime: RuntimeIdentity } ): Promise { - const container = this.deps.getForwardingContainer(); - const state = await this.deps.getContainerState(); - - if (!container?.running || state.status !== 'healthy') { + if (!samePreviewRuntime(lease.runtime, activation)) { + await this.clearActivationIfUnchanged(port, activation); return this.stalePreviewURLResponse(); } - if (!(await this.deps.currentRuntime.isActive(runtime))) { + const container = this.deps.getForwardingContainer(); + if (!container?.running) { return this.stalePreviewURLResponse(); } const tcpPort = container.getTcpPort(port); - const result = await forwardPreviewRequest(tcpPort, request, { - beginForward: () => this.deps.beginForward(), - renewActivity: () => this.deps.renewActivity() - }); + const result = await forwardPreviewRequest(tcpPort, request, lease); if (result.status === 'network-lost') { - if (!(await this.deps.currentRuntime.isActive(runtime))) { - return this.stalePreviewURLResponse(); - } - - return new Response('Container suddenly disconnected, try again', { - status: 500 - }); + return this.stalePreviewURLResponse(); } return result.response; @@ -378,8 +518,10 @@ export class PreviewService { port: number, token: string ): Promise { - const snapshot = await this.readRuntimeSnapshot(); - const entry = snapshot.tokens[port.toString()]; + const { tokens, activations } = await this.deps.storage.transaction( + async (txn) => readPreviewState(txn) + ); + const entry = tokens[port.toString()]; if (!entry) { return { status: 'invalid' }; } @@ -389,25 +531,12 @@ export class PreviewService { return { status: 'invalid' }; } - const availability = this.getRuntimeAvailability(snapshot); - if (availability.status === 'stale') { - return availability; - } - - const activation = snapshot.activations[port.toString()]; - if (!activation) { + const activation = activations[port.toString()]; + if (!isPreviewActivation(activation)) { + if (activation) await this.clearActivationIfUnchanged(port, activation); return { status: 'stale', - reason: 'missing-activation', - containerStatus: snapshot.containerStatus - }; - } - - if (!availability.runtime.owns(activation)) { - return { - status: 'stale', - reason: 'runtime-mismatch', - containerStatus: snapshot.containerStatus + reason: 'missing-activation' }; } @@ -415,16 +544,29 @@ export class PreviewService { if (!activationTokenMatches) { this.deps.logger.warn('Preview URL activation token mismatch', { port, - runtimeIdentityID: availability.runtime.id + runtimeIdentityID: activation.runtimeIdentityID }); + await this.clearActivationIfUnchanged(port, activation); return { status: 'stale', - reason: 'token-mismatch', - containerStatus: snapshot.containerStatus + reason: 'token-mismatch' }; } - return { status: 'active', runtime: availability.runtime }; + return { status: 'active', activation }; + } + + private async clearActivationIfUnchanged( + port: number, + expected: PreviewPortActivation + ): Promise { + await this.deps.storage.transaction(async (txn) => { + const activations = await readActivePreviewPorts(txn); + const current = activations[port.toString()]; + if (!isSameActivation(current, expected)) return; + delete activations[port.toString()]; + await writeActivePreviewPorts(activations, txn); + }); } private async getCurrentPreviewPorts(): Promise { @@ -443,7 +585,10 @@ export class PreviewService { continue; } - if (!availability.runtime.owns(activation)) { + if ( + !isPreviewActivation(activation) || + !samePreviewRuntime(availability.runtime, activation) + ) { continue; } @@ -465,7 +610,7 @@ export class PreviewService { await this.deps.storage.transaction(async (txn) => { const [previewState, runtime] = await Promise.all([ readPreviewState(txn), - this.deps.currentRuntime.getStored(txn) + this.deps.getStoredRuntime(txn) ]); return { ...previewState, runtime }; }); @@ -509,3 +654,62 @@ export class PreviewService { return { status: 'active', runtime: snapshot.runtime }; } } + +function sameRuntime(left: RuntimeIdentity, right: RuntimeIdentity): boolean { + return ( + left.id === right.id && + left.runtimeIncarnationID === right.runtimeIncarnationID + ); +} + +function samePreviewRuntime( + runtime: RuntimeIdentity, + activation: PreviewPortActivation +): boolean { + return ( + runtime.id === activation.runtimeIdentityID && + runtime.runtimeIncarnationID === activation.runtimeIncarnationID + ); +} + +function samePortTokenEntry( + left: PortTokenEntry | undefined, + right: PortTokenEntry +): boolean { + return Boolean( + left && left.token === right.token && left.name === right.name + ); +} + +function isPreviewActivation(value: unknown): value is PreviewPortActivation { + if (!value || typeof value !== 'object') return false; + const activation = value as Partial< + Record + >; + return ( + typeof activation.runtimeIdentityID === 'string' && + activation.runtimeIdentityID.length > 0 && + typeof activation.runtimeIncarnationID === 'string' && + activation.runtimeIncarnationID.length > 0 && + typeof activation.token === 'string' + ); +} + +function isSameActivation( + left: PreviewPortActivation | undefined, + right: PreviewPortActivation +): boolean { + return Boolean( + left && + left.runtimeIdentityID === right.runtimeIdentityID && + left.runtimeIncarnationID === right.runtimeIncarnationID && + left.token === right.token + ); +} + +function isActivationMismatch(error: unknown): boolean { + return ( + error instanceof RuntimeControlProtocolError && + error.context.reason === 'activation-mismatch' + ); +} diff --git a/packages/sandbox/src/preview/state.ts b/packages/sandbox/src/preview/state.ts index 0a96fa522..3e6ebfa84 100644 --- a/packages/sandbox/src/preview/state.ts +++ b/packages/sandbox/src/preview/state.ts @@ -1,4 +1,4 @@ -import type { RuntimeScoped } from '../current-runtime-identity'; +import type { RuntimeIdentityID, RuntimeIncarnationID } from '../runtime/types'; /** * Persisted record for a single exposed port. `token` authorizes preview @@ -10,9 +10,11 @@ export type PortTokenEntry = { name?: string; }; -export type PreviewPortActivation = RuntimeScoped<{ +export type PreviewPortActivation = { + runtimeIdentityID: RuntimeIdentityID; + runtimeIncarnationID: RuntimeIncarnationID; token: string; -}>; +}; export type PreviewPortActivations = Record; diff --git a/packages/sandbox/src/processes/process-capability.ts b/packages/sandbox/src/processes/process-capability.ts index 4178d4194..f6e9ddf16 100644 --- a/packages/sandbox/src/processes/process-capability.ts +++ b/packages/sandbox/src/processes/process-capability.ts @@ -20,10 +20,11 @@ import type { export interface ProcessCapabilityRuntime { readonly id: string; + readonly runtimeIncarnationID: string; } export interface ProcessCapabilityControl { - retainConnection(): () => void; + retainRuntimeHold(): () => void; getProcess(id: string): Promise; openLogs( id: string, @@ -86,7 +87,7 @@ export class ProcessCapabilityTarget extends RpcTarget { this.#runtime, 'process.logs.open', async (control) => { - const releaseConnection = control.retainConnection(); + const releaseConnection = control.retainRuntimeHold(); try { this.#verifiedStatus(await control.getProcess(this.#id)); return { @@ -118,7 +119,7 @@ export class ProcessCapabilityTarget extends RpcTarget { this.#runtime, 'process.port.open', async (control) => { - const releaseConnection = control.retainConnection(); + const releaseConnection = control.retainRuntimeHold(); try { this.#verifiedStatus(await control.getProcess(this.#id)); return { diff --git a/packages/sandbox/src/processes/process-lifecycle.ts b/packages/sandbox/src/processes/process-lifecycle.ts deleted file mode 100644 index b01870074..000000000 --- a/packages/sandbox/src/processes/process-lifecycle.ts +++ /dev/null @@ -1,140 +0,0 @@ -import type { ContainerControlClient } from '../container-control/client'; -import type { - CurrentRuntimeIdentity, - RuntimeIdentity -} from '../current-runtime-identity'; -import { - ErrorCode, - type OperationInterruptedContext, - OperationInterruptedError, - StaleProcessHandleError -} from '../errors'; -import type { ResourceActivityOperation } from '../resource-activity-gate'; - -export type ProcessLifecycleRuntimeClient = { - get(runtimeIdentityID: RuntimeIdentity['id']): ContainerControlClient; - dispose(): void; -}; - -type ProcessLifecycleOptions = { - currentRuntime: Pick; - runtimeClient: ProcessLifecycleRuntimeClient; - beginNonWakingOperation: () => ResourceActivityOperation; - process?: { id: string; pid: number }; -}; - -type OperationEffect = 'none' | 'unknown'; - -/** Applies non-starting admission and runtime fences around process RPCs. */ -export class ProcessLifecycle { - constructor(private readonly options: ProcessLifecycleOptions) {} - - captureCurrent(): Promise { - return this.options.currentRuntime.get(); - } - - runRead( - runtime: RuntimeIdentity, - operation: string, - fn: (client: ContainerControlClient) => Promise - ): Promise { - return this.run(runtime, operation, 'none', fn); - } - - runControl( - runtime: RuntimeIdentity, - operation: string, - fn: (client: ContainerControlClient) => Promise - ): Promise { - return this.run(runtime, operation, 'unknown', fn); - } - - private async run( - runtime: RuntimeIdentity, - operation: string, - effect: OperationEffect, - fn: (client: ContainerControlClient) => Promise - ): Promise { - const admission = this.options.beginNonWakingOperation(); - try { - await admission.beforeCall; - await this.assertPreFence(runtime, operation, effect); - try { - const client = this.options.runtimeClient.get(runtime.id); - const result = await fn(client); - await this.assertPostFence(runtime, operation, effect); - return result; - } catch (error) { - try { - await this.options.currentRuntime.assertActive(runtime); - } catch { - this.options.runtimeClient.dispose(); - throw interrupted(operation, effect); - } - throw error; - } - } finally { - admission.finish(); - } - } - - private async assertPreFence( - runtime: RuntimeIdentity, - operation: string, - effect: OperationEffect - ): Promise { - try { - await this.options.currentRuntime.assertActive(runtime); - } catch { - this.options.runtimeClient.dispose(); - const process = this.options.process; - if (process) { - throw new StaleProcessHandleError({ - code: ErrorCode.STALE_PROCESS_HANDLE, - message: `Process handle ${process.id} belongs to an inactive runtime`, - context: { - processId: process.id, - pid: process.pid, - operation - }, - httpStatus: 409, - timestamp: new Date().toISOString() - }); - } - throw interrupted(operation, effect); - } - } - - private async assertPostFence( - runtime: RuntimeIdentity, - operation: string, - effect: OperationEffect - ): Promise { - try { - await this.options.currentRuntime.assertActive(runtime); - } catch { - this.options.runtimeClient.dispose(); - throw interrupted(operation, effect); - } - } -} - -function interrupted( - operation: string, - effect: OperationEffect -): OperationInterruptedError { - const context: OperationInterruptedContext = { - reason: 'runtime_replaced', - operation, - admitted: true, - retryable: false, - effect - }; - return new OperationInterruptedError({ - code: ErrorCode.OPERATION_INTERRUPTED, - message: `Sandbox operation ${operation} was interrupted because the runtime changed`, - context, - httpStatus: 409, - timestamp: new Date().toISOString() - }); -} diff --git a/packages/sandbox/src/processes/remote-subscription.ts b/packages/sandbox/src/processes/remote-subscription.ts index 415c520f2..a1f90de75 100644 --- a/packages/sandbox/src/processes/remote-subscription.ts +++ b/packages/sandbox/src/processes/remote-subscription.ts @@ -1,3 +1,4 @@ +import { RpcTarget } from 'cloudflare:workers'; import { translateRPCError } from '../container-control/rpc-error'; import type { ProcessPullSubscriptionRPC, @@ -5,17 +6,67 @@ import type { } from './rpc-types'; interface RemoteSubscriptionOptions { + protocol: 'stream' | 'pull'; signal?: AbortSignal; operation?: string; abortError?: () => Error; } +/** Keeps a stream local to its Durable Object and exposes pull RPC methods. */ +export class PullSubscriptionTarget + extends RpcTarget + implements ProcessPullSubscriptionRPC +{ + readonly #reader: ReadableStreamDefaultReader; + readonly #onRelease: (() => void) | undefined; + #released = false; + + constructor(stream: ReadableStream, onRelease?: () => void) { + super(); + this.#reader = stream.getReader(); + this.#onRelease = onRelease; + } + + async next(): Promise> { + if (this.#released) return { done: true, value: undefined }; + try { + const result = await this.#reader.read(); + if (result.done) this.#release(); + return result; + } catch (error) { + this.#release(); + throw error; + } + } + + async cancel(): Promise { + if (this.#released) return; + try { + await this.#reader.cancel(); + } finally { + this.#release(); + } + } + + [Symbol.dispose](): void { + if (this.#released) return; + void this.#reader.cancel().catch(() => undefined); + this.#release(); + } + + #release(): void { + if (this.#released) return; + this.#released = true; + this.#onRelease?.(); + } +} + /** Exposes a remote Workers RPC subscription as a caller-owned local stream. */ export async function openRemoteSubscription( subscriptionPromise: Promise< ProcessSubscriptionRPC | ProcessPullSubscriptionRPC >, - options: RemoteSubscriptionOptions = {} + options: RemoteSubscriptionOptions ): Promise> { let subscription: | ProcessSubscriptionRPC @@ -86,14 +137,15 @@ export async function openRemoteSubscription( let read: () => Promise>; let cancelReader: () => void; - if ('next' in subscription) { - const pullSubscription = subscription; + if (options.protocol === 'pull') { + const pullSubscription = subscription as ProcessPullSubscriptionRPC; read = () => pullSubscription.next(); cancelReader = () => undefined; } else { + const streamSubscription = subscription as ProcessSubscriptionRPC; let source: ReadableStream; try { - source = await raceSetup(subscription.stream()); + source = await raceSetup(streamSubscription.stream()); } catch (error) { release(); if (signal?.aborted) throw error; diff --git a/packages/sandbox/src/processes/rpc-types.ts b/packages/sandbox/src/processes/rpc-types.ts index 428778167..3f6ee8e87 100644 --- a/packages/sandbox/src/processes/rpc-types.ts +++ b/packages/sandbox/src/processes/rpc-types.ts @@ -26,15 +26,11 @@ export interface ProcessCapabilityRPC { status(): Promise; openLogs( options?: ProcessLogsRPCOptions - ): Promise< - ProcessLogSubscriptionRPC | ProcessPullSubscriptionRPC - >; + ): Promise>; openPortWatch( port: number, options?: PortWatchRPCOptions - ): Promise< - ProcessPortSubscriptionRPC | ProcessPullSubscriptionRPC - >; + ): Promise>; kill(signal: number): Promise; } diff --git a/packages/sandbox/src/processes/sandbox-process.ts b/packages/sandbox/src/processes/sandbox-process.ts index 53841659c..69ae9c2ed 100644 --- a/packages/sandbox/src/processes/sandbox-process.ts +++ b/packages/sandbox/src/processes/sandbox-process.ts @@ -49,6 +49,7 @@ export class SandboxProcessImpl implements SandboxProcess { ): Promise> { const { signal, ...rpcOptions } = options; return openRemoteSubscription(this.capability.openLogs(rpcOptions), { + protocol: 'pull', signal, operation: 'read process logs', abortError: @@ -156,7 +157,11 @@ export class SandboxProcessImpl implements SandboxProcess { ): Promise { const portStream = await openRemoteSubscription( this.capability.openPortWatch(port, options), - { operation: `watch port ${port}`, signal: settlementSignal } + { + operation: `watch port ${port}`, + protocol: 'pull', + signal: settlementSignal + } ); let logStream: ReadableStream | undefined; let portReader: ReadableStreamDefaultReader | undefined; @@ -165,7 +170,11 @@ export class SandboxProcessImpl implements SandboxProcess { portReader = portStream.getReader(); logStream = await openRemoteSubscription( this.capability.openLogs({ replay: true, follow: true }), - { operation: 'read process logs', signal: settlementSignal } + { + operation: 'read process logs', + protocol: 'pull', + signal: settlementSignal + } ); logReader = logStream.getReader(); await waitForReadiness(portReader, logReader, { @@ -191,7 +200,7 @@ export class SandboxProcessImpl implements SandboxProcess { ): Promise { const stream = await openRemoteSubscription( this.capability.openLogs({ replay: true, follow: true }), - { signal, operation: 'read process logs' } + { signal, operation: 'read process logs', protocol: 'pull' } ); return this.consume(stream, consume); } diff --git a/packages/sandbox/src/pty/index.ts b/packages/sandbox/src/pty/index.ts index 326e63884..c33926f12 100644 --- a/packages/sandbox/src/pty/index.ts +++ b/packages/sandbox/src/pty/index.ts @@ -3,5 +3,6 @@ export { getTerminalHandle, listTerminalHandles, proxyTerminal, - terminalHandle + terminalHandle, + terminalHandleFromRPCDescriptor } from './proxy'; diff --git a/packages/sandbox/src/pty/proxy.ts b/packages/sandbox/src/pty/proxy.ts index 0854c2a34..18e101c6d 100644 --- a/packages/sandbox/src/pty/proxy.ts +++ b/packages/sandbox/src/pty/proxy.ts @@ -3,11 +3,11 @@ import type { CreateTerminalOptions, ErrorResponse, ProcessExit, - SandboxTerminalsAPI, Terminal, TerminalOutputCursor, TerminalOutputEvent, TerminalOutputOptions, + TerminalOutputSubscriptionAPI, TerminalSnapshot, WaitForExitOptions } from '@repo/shared'; @@ -17,11 +17,30 @@ import { } from '@repo/shared/errors'; import { TerminalControlError } from '../errors'; import { openRemoteSubscription } from '../processes/remote-subscription'; +import type { ProcessPullSubscriptionRPC } from '../processes/rpc-types'; +import type { TerminalRPCDescriptor } from './rpc-types'; -interface SandboxTerminalStub extends SandboxTerminalsAPI { +interface TerminalHandleStub { + get(id: string): Promise; + output( + id: string, + options?: Omit + ): Promise< + | TerminalOutputSubscriptionAPI + | ProcessPullSubscriptionRPC + >; + write(id: string, data: Uint8Array): Promise; + resize(id: string, cols: number, rows: number): Promise; + interrupt(id: string): Promise; + terminate(id: string): Promise; fetch(request: Request): Promise; } +interface SandboxTerminalStub extends TerminalHandleStub { + create(options: CreateTerminalOptions): Promise; + list(): Promise; +} + export async function createTerminalHandle( stub: SandboxTerminalStub, options: CreateTerminalOptions @@ -46,8 +65,10 @@ export async function listTerminalHandles( } export function terminalHandle( - stub: SandboxTerminalStub, - snapshot: TerminalSnapshot + stub: TerminalHandleStub, + snapshot: TerminalSnapshot, + runtimeIncarnationID?: string, + outputProtocol: 'stream' | 'pull' = 'stream' ): Terminal { const id = snapshot.id; return { @@ -59,23 +80,56 @@ export function terminalHandle( }, write: (data) => stub.write(id, data), resize: (cols, rows) => stub.resize(id, cols, rows), - output: (options) => terminalOutput(stub, id, options), - waitForExit: (options) => waitForTerminalExit(stub, id, options), + output: (options) => terminalOutput(stub, id, options, outputProtocol), + waitForExit: (options) => + waitForTerminalExit(stub, id, options, outputProtocol), interrupt: () => stub.interrupt(id), terminate: () => stub.terminate(id), - connect: (request, options) => proxyTerminal(stub, id, request, options) + connect: (request, options) => { + if (!runtimeIncarnationID) { + throw new Error('terminal.connect() requires a runtime incarnation ID'); + } + return proxyTerminal(stub, id, request, { + ...options, + runtimeIncarnationID + }); + } }; } +export function terminalHandleFromRPCDescriptor( + descriptor: TerminalRPCDescriptor, + fetch: (request: Request) => Promise +): Terminal { + const capability = descriptor.capability; + const stub: TerminalHandleStub = { + get: () => capability.getSnapshot(), + output: (_id, options) => capability.openOutput(options), + write: (_id, data) => capability.write(data), + resize: (_id, cols, rows) => capability.resize(cols, rows), + interrupt: () => capability.interrupt(), + terminate: () => capability.terminate(), + fetch + }; + return terminalHandle( + stub, + descriptor.snapshot, + descriptor.runtimeIncarnationID, + 'pull' + ); +} + async function terminalOutput( - stub: SandboxTerminalStub, + stub: TerminalHandleStub, id: string, - options?: TerminalOutputOptions + options: TerminalOutputOptions | undefined, + protocol: 'stream' | 'pull' ) { const { signal, ...rpcOptions } = options ?? {}; - const stream = await openRemoteSubscription(stub.output(id, rpcOptions), { - operation: 'open terminal output' - }); + const stream = await openRemoteSubscription( + stub.output(id, rpcOptions), + { operation: 'open terminal output', protocol, signal } + ); if (!signal) return stream; let reader: ReadableStreamDefaultReader | undefined; return new ReadableStream({ @@ -124,9 +178,10 @@ async function terminalOutput( } async function waitForTerminalExit( - stub: SandboxTerminalStub, + stub: TerminalHandleStub, id: string, - options: WaitForExitOptions = {} + options: WaitForExitOptions = {}, + protocol: 'stream' | 'pull' ): Promise { const abortController = new AbortController(); let timeout: ReturnType | undefined; @@ -146,19 +201,24 @@ async function waitForTerminalExit( } if (options.timeout !== undefined) timeout = setTimeout(timeoutAbort, options.timeout); - const stream = await openRemoteSubscription( - stub.output(id, { replay: true, follow: true }), - { operation: 'wait for terminal exit' } - ); - const reader = stream.getReader(); - abortController.signal.addEventListener( - 'abort', - () => { - void reader.cancel().catch(() => {}); - }, - { once: true } - ); + let reader: ReadableStreamDefaultReader | undefined; try { + const stream = await openRemoteSubscription( + stub.output(id, { replay: true, follow: true }), + { + operation: 'wait for terminal exit', + protocol, + signal: abortController.signal + } + ); + reader = stream.getReader(); + abortController.signal.addEventListener( + 'abort', + () => { + void reader?.cancel().catch(() => {}); + }, + { once: true } + ); while (!abortController.signal.aborted) { const result = await reader.read(); if (result.done) break; @@ -176,7 +236,7 @@ async function waitForTerminalExit( } finally { if (timeout) clearTimeout(timeout); if (options.signal) options.signal.removeEventListener('abort', abort); - await reader.cancel().catch(() => {}); + await reader?.cancel().catch(() => {}); } } @@ -207,18 +267,27 @@ function abortReason(reason: unknown): Error { } export async function proxyTerminal( - stub: Pick, + stub: Pick, terminalId: string, request: Request, - options?: { cursor?: TerminalOutputCursor; cols?: number; rows?: number } + options: { + cursor?: TerminalOutputCursor; + cols?: number; + rows?: number; + runtimeIncarnationID: string; + } ): Promise { const upgradeHeader = request.headers.get('Upgrade'); if (upgradeHeader?.toLowerCase() !== 'websocket') throw new Error('terminal.connect() requires a WebSocket upgrade request'); const params = new URLSearchParams({ terminalId }); - if (options?.cursor) params.set('cursor', options.cursor); - if (options?.cols) params.set('cols', String(options.cols)); - if (options?.rows) params.set('rows', String(options.rows)); + if (!options.runtimeIncarnationID) { + throw new Error('terminal.connect() requires a runtime incarnation ID'); + } + if (options.cursor) params.set('cursor', options.cursor); + if (options.cols) params.set('cols', String(options.cols)); + if (options.rows) params.set('rows', String(options.rows)); + params.set('runtimeIncarnationID', options.runtimeIncarnationID); return stub.fetch( switchPort( new Request(`http://localhost/ws/terminal?${params}`, request), diff --git a/packages/sandbox/src/pty/rpc-types.ts b/packages/sandbox/src/pty/rpc-types.ts new file mode 100644 index 000000000..7c77f7088 --- /dev/null +++ b/packages/sandbox/src/pty/rpc-types.ts @@ -0,0 +1,24 @@ +import type { + TerminalOutputEvent, + TerminalOutputOptions, + TerminalSnapshot +} from '@repo/shared'; +import type { ProcessPullSubscriptionRPC } from '../processes/rpc-types'; + +export interface TerminalCapabilityRPC { + getSnapshot(): Promise; + openOutput( + options?: Omit + ): Promise>; + write(data: Uint8Array): Promise; + resize(cols: number, rows: number): Promise; + interrupt(): Promise; + terminate(): Promise; + authorizeConnection(): Promise; +} + +export interface TerminalRPCDescriptor { + snapshot: TerminalSnapshot; + runtimeIncarnationID: string; + capability: TerminalCapabilityRPC; +} diff --git a/packages/sandbox/src/pty/terminal-capability.ts b/packages/sandbox/src/pty/terminal-capability.ts new file mode 100644 index 000000000..ec870f58c --- /dev/null +++ b/packages/sandbox/src/pty/terminal-capability.ts @@ -0,0 +1,65 @@ +import { RpcTarget } from 'cloudflare:workers'; +import type { + TerminalOutputEvent, + TerminalOutputOptions, + TerminalSnapshot +} from '@repo/shared'; +import type { ProcessPullSubscriptionRPC } from '../processes/rpc-types'; +import type { TerminalCapabilityRPC } from './rpc-types'; + +export interface TerminalCapabilityControl { + get(id: string): Promise; + openOutput( + id: string, + options?: Omit + ): Promise>; + write(id: string, data: Uint8Array): Promise; + resize(id: string, cols: number, rows: number): Promise; + interrupt(id: string): Promise; + terminate(id: string): Promise; + authorizeConnection(): Promise; +} + +export class TerminalCapabilityTarget + extends RpcTarget + implements TerminalCapabilityRPC +{ + readonly #id: string; + readonly #control: TerminalCapabilityControl; + + constructor(id: string, control: TerminalCapabilityControl) { + super(); + this.#id = id; + this.#control = control; + } + + getSnapshot(): Promise { + return this.#control.get(this.#id); + } + + openOutput( + options?: Omit + ): Promise> { + return this.#control.openOutput(this.#id, options); + } + + write(data: Uint8Array): Promise { + return this.#control.write(this.#id, data); + } + + resize(cols: number, rows: number): Promise { + return this.#control.resize(this.#id, cols, rows); + } + + interrupt(): Promise { + return this.#control.interrupt(this.#id); + } + + terminate(): Promise { + return this.#control.terminate(this.#id); + } + + authorizeConnection(): Promise { + return this.#control.authorizeConnection(); + } +} diff --git a/packages/sandbox/src/resource-activity-gate.ts b/packages/sandbox/src/resource-activity-gate.ts index d081c5c2f..7104d14bd 100644 --- a/packages/sandbox/src/resource-activity-gate.ts +++ b/packages/sandbox/src/resource-activity-gate.ts @@ -27,20 +27,36 @@ export class ResourceActivityGate { this.renewActivity(); } - beginOperation(): ResourceActivityOperation { - return this.beginTrackedOperation(true); + beginActivity(): ResourceActivityOperation { + return this.beginTrackedOperation('activity'); } /** - * Admits observation of an already-live runtime without renewing activity. - * A committed inactivity stop remains authoritative: observers await it and - * then inspect the resulting inactive state without restarting the runtime. + * Admits already-live work without renewing activity. The hold blocks an + * expiry decision while the operation is in flight. */ - beginNonWakingOperation(): ResourceActivityOperation { - return this.beginTrackedOperation(false); + beginExistingHold(): ResourceActivityOperation { + return this.beginTrackedOperation('hold'); } - private beginTrackedOperation(renew: boolean): ResourceActivityOperation { + /** + * Admits observation of an already-live runtime without renewing activity or + * blocking the expiry decision that the observation may inform. + */ + beginProbe(): ResourceActivityOperation { + const stopToAwait = this.committedStop; + return { + beforeCall: stopToAwait + ? this.awaitCommittedTeardowns(stopToAwait) + : Promise.resolve(), + finish: () => {} + }; + } + + private beginTrackedOperation( + mode: 'activity' | 'hold' + ): ResourceActivityOperation { + const renew = mode === 'activity'; if (renew) { this.recordActivity(); this.activityInFlight += 1; @@ -52,7 +68,7 @@ export class ResourceActivityGate { return { beforeCall: stopToAwait - ? stopToAwait.then(() => { + ? this.awaitCommittedTeardowns(stopToAwait).then(() => { if (renew) this.recordActivity(); }) : Promise.resolve(), @@ -69,6 +85,41 @@ export class ResourceActivityGate { }; } + private async awaitCommittedTeardowns(initial: Promise): Promise { + let current = initial; + while (true) { + let failed = false; + let failure: unknown; + try { + await current; + } catch (error) { + failed = true; + failure = error; + } + const latest = this.committedStop; + if (latest && latest !== current) { + current = latest; + continue; + } + if (failed) throw failure; + return; + } + } + + async runStopTeardown(teardown: () => Promise): Promise { + if (!this.committedStop) { + this.committedStop = this.wrapCommittedStop(teardown()); + } + await this.committedStop; + } + + runDestroyTeardown(teardown: () => Promise): Promise { + const previous = this.committedStop; + const work = previous ? previous.then(teardown, teardown) : teardown(); + this.committedStop = this.wrapCommittedStop(work); + return this.committedStop; + } + async runExpiry( probe: ResourceActivityProbe, keepAlive: boolean @@ -163,11 +214,18 @@ export class ResourceActivityGate { private async commitStop(): Promise { if (!this.committedStop) { - this.committedStop = this.stopInactive().finally(() => { - this.committedStop = null; - this.recordActivity(); - }); + this.committedStop = this.wrapCommittedStop(this.stopInactive()); } await this.committedStop; } + + private wrapCommittedStop(work: Promise): Promise { + const committed = work.finally(() => { + if (this.committedStop === committed) { + this.committedStop = null; + } + this.recordActivity(); + }); + return committed; + } } diff --git a/packages/sandbox/src/response-retry.ts b/packages/sandbox/src/response-retry.ts deleted file mode 100644 index 157d43ebf..000000000 --- a/packages/sandbox/src/response-retry.ts +++ /dev/null @@ -1,74 +0,0 @@ -import type { LogContext, Logger } from '@repo/shared'; - -const DEFAULT_INITIAL_RETRY_DELAY_MS = 3_000; -const DEFAULT_MAX_RETRY_DELAY_MS = 30_000; -const RETRYABLE_WEBSOCKET_UPGRADE_STATUSES = new Set([500, 502, 503, 504]); - -export function isRetryableWebSocketUpgradeResponse( - response: Response -): boolean { - return RETRYABLE_WEBSOCKET_UPGRADE_STATUSES.has(response.status); -} - -export interface ResponseRetryOptions { - retryTimeoutMs: number; - minTimeForRetryMs: number; - logger: Logger; - retryLogMessage: string; - shouldRetry(response: Response): boolean; - getRetryLogContext?: (response: Response) => Partial; - onRetryExhausted?: (params: { - attempts: number; - elapsedMs: number; - response: Response; - }) => void; -} - -/** - * Retry Response-returning operations while their response remains retryable. - * The retry budget covers the whole operation; each attempt owns any - * per-request timeout inside the caller-provided `fetchResponse` function. - */ -export async function fetchWithResponseRetry( - fetchResponse: () => Promise, - options: ResponseRetryOptions -): Promise { - const startTime = Date.now(); - let attempt = 0; - - while (true) { - const response = await fetchResponse(); - - if (!options.shouldRetry(response)) { - return response; - } - - const elapsed = Date.now() - startTime; - const remaining = options.retryTimeoutMs - elapsed; - - if (remaining <= options.minTimeForRetryMs) { - options.onRetryExhausted?.({ - attempts: attempt + 1, - elapsedMs: elapsed, - response - }); - return response; - } - - const delay = Math.min( - DEFAULT_INITIAL_RETRY_DELAY_MS * 2 ** attempt, - DEFAULT_MAX_RETRY_DELAY_MS - ); - - options.logger.info(options.retryLogMessage, { - status: response.status, - attempt: attempt + 1, - delayMs: delay, - remainingSec: Math.floor(remaining / 1000), - ...options.getRetryLogContext?.(response) - }); - - await new Promise((resolve) => setTimeout(resolve, delay)); - attempt++; - } -} diff --git a/packages/sandbox/src/runtime/bootstrap-probe.ts b/packages/sandbox/src/runtime/bootstrap-probe.ts new file mode 100644 index 000000000..b694df69e --- /dev/null +++ b/packages/sandbox/src/runtime/bootstrap-probe.ts @@ -0,0 +1,72 @@ +import type { RuntimeMetadata } from '@repo/shared'; +import { + ContainerControlConnection, + type ContainerFetchStub +} from '../container-control/connection'; +import { RuntimeControlProtocolError } from '../errors'; +import type { RuntimeBootstrapProbe as RuntimeBootstrapProbeContract } from './types'; + +const CONTROL_PROTOCOL_VERSION = 1; + +export type RuntimeBootstrapProbeOptions = { + getTcpPort: (port: number) => ContainerFetchStub; +}; + +export function validateRuntimeMetadata( + value: unknown, + operation: string +): RuntimeMetadata { + if (value == null) { + throw new RuntimeControlProtocolError('Runtime metadata is missing', { + reason: 'missing-metadata', + operation + }); + } + if (typeof value !== 'object') { + throw new RuntimeControlProtocolError('Runtime metadata is malformed', { + reason: 'malformed-metadata', + operation + }); + } + const metadata = value as Partial; + if ( + typeof metadata.runtimeIncarnationID !== 'string' || + metadata.runtimeIncarnationID.length === 0 || + typeof metadata.sandboxVersion !== 'string' || + metadata.sandboxVersion.length === 0 + ) { + throw new RuntimeControlProtocolError('Runtime metadata is malformed', { + reason: 'malformed-metadata', + operation + }); + } + if (metadata.controlProtocolVersion !== CONTROL_PROTOCOL_VERSION) { + throw new RuntimeControlProtocolError( + 'Runtime control protocol version is unsupported', + { + reason: 'unsupported-protocol-version', + operation + } + ); + } + return metadata as RuntimeMetadata; +} + +export class RuntimeBootstrapProbe implements RuntimeBootstrapProbeContract { + constructor(private readonly options: RuntimeBootstrapProbeOptions) {} + + async probe(): Promise { + const connection = new ContainerControlConnection({ + stub: this.options.getTcpPort(3000) + }); + try { + await connection.connect(); + return validateRuntimeMetadata( + await connection.getRuntimeMetadata(), + 'utils.getRuntimeMetadata' + ); + } finally { + connection.disconnect(); + } + } +} diff --git a/packages/sandbox/src/runtime/index.ts b/packages/sandbox/src/runtime/index.ts new file mode 100644 index 000000000..a5f762077 --- /dev/null +++ b/packages/sandbox/src/runtime/index.ts @@ -0,0 +1,26 @@ +export type { RuntimeBootstrapProbeOptions } from './bootstrap-probe'; +export { + RuntimeBootstrapProbe, + validateRuntimeMetadata +} from './bootstrap-probe'; +export type { SandboxRuntimeLifecycleOptions } from './lifecycle'; +export { SandboxRuntimeLifecycle } from './lifecycle'; +export type { + RuntimeAbsent, + RuntimeLease, + RuntimeOperationRunnerOptions, + RuntimeOperationTarget +} from './operation-runner'; +export { RUNTIME_ABSENT, RuntimeOperationRunner } from './operation-runner'; +export type { RuntimeSessionManagerOptions } from './session-manager'; +export { RuntimeSessionManager } from './session-manager'; +export { + type RuntimeBootstrapProbe as RuntimeBootstrapProbeContract, + type RuntimeConnectionHold, + RuntimeIdentity, + type RuntimeIdentityReader, + type RuntimeIncarnationID, + type RuntimeRecord, + type RuntimeRecordStorage, + type RuntimeSessionManager as RuntimeSessionManagerContract +} from './types'; diff --git a/packages/sandbox/src/runtime/lifecycle.ts b/packages/sandbox/src/runtime/lifecycle.ts new file mode 100644 index 000000000..11c981203 --- /dev/null +++ b/packages/sandbox/src/runtime/lifecycle.ts @@ -0,0 +1,306 @@ +import type { ContainerStartConfigOptions } from '@cloudflare/containers'; +import type { ContainerControlClient } from '../container-control/client'; +import { + type RuntimeIdentityID, + RuntimeIdentityInactiveError +} from '../runtime/types'; +import { validateRuntimeMetadata } from './bootstrap-probe'; +import type { RuntimeBootstrapProbe, RuntimeSessionManager } from './types'; +import { + RuntimeIdentity, + type RuntimeIdentityReader, + type RuntimeIncarnationID, + type RuntimeRecord, + type RuntimeRecordStorage +} from './types'; + +const RUNTIME_RECORD_KEY = 'currentRuntimeIdentity'; + +type LifecycleStorage = RuntimeRecordStorage & + Pick; + +type ReplacementStartTransition = { + phase: 'reconciling-previous-stop' | 'replacement-started'; +}; + +type RuntimeStopDisposition = 'reconciled-previous-stop' | 'hard-invalidation'; + +export type RuntimeEstablishOptions = { + signal?: AbortSignal; + startOptions?: ContainerStartConfigOptions; +}; + +export type SandboxRuntimeLifecycleOptions = { + readonly storage: LifecycleStorage; + readonly isRuntimeRunning: () => boolean; + readonly startControlPort: ( + port: 3000, + options?: RuntimeEstablishOptions + ) => Promise; + readonly waitForControlPort: ( + port: 3000, + options?: { signal?: AbortSignal } + ) => Promise; + readonly stopControlPort?: (port: 3000) => Promise; + readonly probe: RuntimeBootstrapProbe; + readonly sessions: RuntimeSessionManager; + readonly observeVersionCompatibility: ( + client: ContainerControlClient, + runtime: RuntimeIdentity + ) => Promise; + readonly reconcileReplacement: (runtime: RuntimeIdentity) => Promise; +}; + +export class SandboxRuntimeLifecycle implements RuntimeIdentityReader { + private readonly changeListeners = new Set<() => void>(); + private generation = 0; + private establishing: Promise | null = null; + private mutationGate: Promise = Promise.resolve(); + private replacementStart: ReplacementStartTransition | null = null; + + constructor(private readonly options: SandboxRuntimeLifecycleOptions) {} + + get sessions(): RuntimeSessionManager { + return this.options.sessions; + } + + establish(options?: RuntimeEstablishOptions): Promise { + if (!this.establishing) { + this.establishing = this.doEstablish(options).finally(() => { + this.establishing = null; + }); + } + return this.establishing; + } + + observeStoredActive(): Promise { + return this.getStored(); + } + + async get(): Promise { + if (!this.options.isRuntimeRunning()) return null; + return this.getStored(); + } + + async getStored( + storage: RuntimeRecordStorage = this.options.storage + ): Promise { + const record = (await storage.get(RUNTIME_RECORD_KEY)) ?? null; + if (!isRuntimeRecord(record)) return null; + return new RuntimeIdentity(record); + } + + async isActive(runtime: RuntimeIdentity): Promise { + const active = await this.get(); + return ( + active?.id === runtime.id && + active.runtimeIncarnationID === runtime.runtimeIncarnationID + ); + } + + async assertActive(runtime: RuntimeIdentity): Promise { + if (!(await this.isActive(runtime))) + throw new RuntimeIdentityInactiveError(); + } + + async invalidate(expected?: RuntimeIdentity): Promise { + if (!expected) { + await this.invalidateAndObserveStoredActive(); + return; + } + + await this.withMutationGate(async () => { + const stored = await this.getStored(); + if (!stored || !sameIdentity(stored, expected)) return; + + this.generation++; + this.notifyChanged(); + this.options.sessions.closeActive(); + await this.options.storage.delete(RUNTIME_RECORD_KEY); + await this.options.stopControlPort?.(3000); + }); + } + + invalidateAndObserveStoredActive(): Promise { + return this.invalidateCurrent(true); + } + + markRuntimeStarted(): boolean { + if (this.replacementStart?.phase === 'reconciling-previous-stop') { + this.replacementStart.phase = 'replacement-started'; + return true; + } + return false; + } + + async reconcileObservedStop(): Promise { + const invalidateEstablishment = + this.replacementStart?.phase !== 'reconciling-previous-stop'; + await this.invalidateCurrent(invalidateEstablishment); + return invalidateEstablishment + ? 'hard-invalidation' + : 'reconciled-previous-stop'; + } + + onChange(listener: () => void): () => void { + this.changeListeners.add(listener); + return () => this.changeListeners.delete(listener); + } + + private async doEstablish( + options?: RuntimeEstablishOptions + ): Promise { + const needsPhysicalStart = !this.options.isRuntimeRunning(); + const replacementStart = needsPhysicalStart + ? this.beginReplacementStart() + : null; + const generation = this.generation; + + try { + const stored = await this.getStored(); + + if (needsPhysicalStart) { + await this.options.startControlPort(3000, options); + if (replacementStart?.phase !== 'replacement-started') { + throw new RuntimeIdentityInactiveError(); + } + this.assertGeneration(generation); + await this.options.waitForControlPort(3000, options); + this.assertGeneration(generation); + } + + const metadata = validateRuntimeMetadata( + await this.options.probe.probe(), + 'runtime.lifecycle.probe' + ); + this.assertGeneration(generation); + + const runtimeIncarnationID = + metadata.runtimeIncarnationID as RuntimeIncarnationID; + const runtime = + stored?.runtimeIncarnationID === runtimeIncarnationID + ? stored + : new RuntimeIdentity({ + id: crypto.randomUUID() as RuntimeIdentityID, + runtimeIncarnationID + }); + const client = await this.options.sessions.acquire(runtime); + this.assertGeneration(generation); + await this.options.observeVersionCompatibility(client, runtime); + this.assertGeneration(generation); + + if (!stored || !sameIdentity(stored, runtime)) { + await this.options.reconcileReplacement(runtime); + this.assertGeneration(generation); + } + + if (!this.options.isRuntimeRunning()) + throw new RuntimeIdentityInactiveError(); + return await this.withMutationGate(async () => { + this.assertGeneration(generation); + await this.options.storage.put(RUNTIME_RECORD_KEY, toRecord(runtime)); + if (this.generation !== generation) { + await this.deleteIfStored(runtime); + throw new RuntimeIdentityInactiveError(); + } + this.notifyChanged(); + return runtime; + }); + } finally { + this.endReplacementStart(replacementStart); + } + } + + private beginReplacementStart(): ReplacementStartTransition { + const transition: ReplacementStartTransition = { + phase: 'reconciling-previous-stop' + }; + this.replacementStart = transition; + return transition; + } + + private endReplacementStart( + transition: ReplacementStartTransition | null + ): void { + if (transition && this.replacementStart === transition) { + this.replacementStart = null; + } + } + + private invalidateCurrent( + invalidateEstablishment: boolean + ): Promise { + if (invalidateEstablishment) this.generation++; + this.notifyChanged(); + this.options.sessions.closeActive(); + return this.withMutationGate(async () => { + const stored = await this.getStored(); + if (stored) await this.options.storage.delete(RUNTIME_RECORD_KEY); + await this.options.stopControlPort?.(3000); + return stored; + }); + } + + private assertGeneration(generation: number): void { + if (this.generation !== generation) + throw new RuntimeIdentityInactiveError(); + } + + private async withMutationGate(operation: () => Promise): Promise { + const previous = this.mutationGate; + let release!: () => void; + this.mutationGate = new Promise((resolve) => { + release = resolve; + }); + await previous; + try { + return await operation(); + } finally { + release(); + } + } + + private async deleteIfStored(runtime: RuntimeIdentity): Promise { + const stored = await this.getStored(); + if (stored && sameIdentity(stored, runtime)) { + await this.options.storage.delete(RUNTIME_RECORD_KEY); + } + } + + private notifyChanged(): void { + for (const listener of this.changeListeners) { + try { + listener(); + } catch { + // Listener failures must not interrupt lifecycle state transitions. + } + } + } +} + +function toRecord(runtime: RuntimeIdentity): RuntimeRecord { + return { + schemaVersion: 1, + id: runtime.id, + runtimeIncarnationID: runtime.runtimeIncarnationID + }; +} + +function sameIdentity(left: RuntimeIdentity, right: RuntimeIdentity): boolean { + return ( + left.id === right.id && + left.runtimeIncarnationID === right.runtimeIncarnationID + ); +} + +function isRuntimeRecord(value: unknown): value is RuntimeRecord { + if (!value || typeof value !== 'object') return false; + const record = value as Partial; + return ( + record.schemaVersion === 1 && + typeof record.id === 'string' && + record.id.length > 0 && + typeof record.runtimeIncarnationID === 'string' && + record.runtimeIncarnationID.length > 0 + ); +} diff --git a/packages/sandbox/src/runtime/operation-runner.ts b/packages/sandbox/src/runtime/operation-runner.ts new file mode 100644 index 000000000..0b6488bb3 --- /dev/null +++ b/packages/sandbox/src/runtime/operation-runner.ts @@ -0,0 +1,275 @@ +import { ErrorCode } from '@repo/shared'; +import { getHttpStatus } from '@repo/shared/errors'; +import type { ContainerControlClient } from '../container-control/client'; +import { OperationInterruptedError } from '../errors'; +import type { ResourceActivityGate } from '../resource-activity-gate'; +import { RuntimeIdentityInactiveError } from '../runtime/types'; +import type { + RuntimeEstablishOptions, + SandboxRuntimeLifecycle +} from './lifecycle'; +import type { + RuntimeConnectionHold, + RuntimeIdentity, + RuntimeSession +} from './types'; + +export type RuntimeAbsent = { status: 'absent' }; +export const RUNTIME_ABSENT: RuntimeAbsent = { status: 'absent' }; + +export type RuntimeLease = { + runtime: RuntimeIdentity; + control: ContainerControlClient; + retain(onInterrupt?: () => void): RuntimeConnectionHold; +}; + +export type RuntimeOperationTarget = + | { kind: 'current' } + | { kind: 'runtime'; runtime: RuntimeIdentity }; + +export type RuntimeOperationRunnerOptions = { + lifecycle: SandboxRuntimeLifecycle; + activityGate: ResourceActivityGate; +}; + +type ActivityAdmission = ReturnType; + +export class RuntimeOperationRunner { + constructor(private readonly options: RuntimeOperationRunnerOptions) {} + + async runWaking( + operation: string, + call: (lease: RuntimeLease) => Promise, + establishOptions?: RuntimeEstablishOptions + ): Promise { + const owner = new OperationActivityOwner( + this.options.activityGate.beginActivity() + ); + try { + await owner.beforeCall; + let runtime: RuntimeIdentity; + try { + runtime = await this.options.lifecycle.establish(establishOptions); + } catch (error) { + if ( + error instanceof RuntimeIdentityInactiveError || + error instanceof OperationInterruptedError + ) { + throw interrupted(operation); + } + throw error; + } + return await this.runWithLease(runtime, operation, owner, call); + } finally { + owner.releaseBase(); + } + } + + async runExisting( + target: RuntimeOperationTarget, + operation: string, + call: (lease: RuntimeLease) => Promise + ): Promise { + const owner = new OperationActivityOwner( + this.options.activityGate.beginExistingHold() + ); + try { + await owner.beforeCall; + const runtime = await this.resolveExisting(target); + if (!runtime) return RUNTIME_ABSENT; + return await this.runWithLease(runtime, operation, owner, call); + } finally { + owner.releaseBase(); + } + } + + async probeExisting( + target: RuntimeOperationTarget, + operation: string, + call: (lease: RuntimeLease) => Promise + ): Promise { + const owner = new OperationActivityOwner( + this.options.activityGate.beginProbe() + ); + try { + await owner.beforeCall; + const runtime = await this.resolveExisting(target); + if (!runtime) return RUNTIME_ABSENT; + return await this.runWithLease(runtime, operation, owner, call); + } finally { + owner.releaseBase(); + } + } + + private async resolveExisting( + target: RuntimeOperationTarget + ): Promise { + const current = await this.options.lifecycle.get(); + if (!current) return null; + if (target.kind === 'current') return current; + if (sameIdentity(current, target.runtime)) return target.runtime; + return null; + } + + private async runWithLease( + runtime: RuntimeIdentity, + operation: string, + owner: OperationActivityOwner, + call: (lease: RuntimeLease) => Promise + ): Promise { + const session = + await this.options.lifecycle.sessions.acquireSession(runtime); + const sessionHold = session.retain(); + const releaseChangeListener = this.options.lifecycle.onChange(() => { + owner.interruptRetainedHolds(); + }); + try { + await this.assertActive(runtime, operation); + if (session.isInterrupted()) throw interrupted(operation); + const lease: RuntimeLease = { + runtime, + control: session.client, + retain: (onInterrupt) => { + if (!owner.canRetain() || session.isInterrupted()) { + onInterrupt?.(); + return { release: () => {} }; + } + const activityHold = owner.retain(); + const retainedSessionHold = session.retain(() => { + activityHold.release(); + onInterrupt?.(); + }); + return { + release: once(() => { + retainedSessionHold.release(); + activityHold.release(); + }) + }; + } + }; + const result = await this.raceSession(session, operation, call(lease)); + await this.assertActive(runtime, operation); + if (session.isInterrupted()) throw interrupted(operation); + return result; + } catch (error) { + if (await this.shouldTranslateInterruption(runtime, session, error)) { + throw interrupted(operation); + } + throw error; + } finally { + releaseChangeListener(); + sessionHold.release(); + } + } + + private async raceSession( + session: RuntimeSession, + operation: string, + promise: Promise + ): Promise { + try { + return await Promise.race([promise, session.interrupted]); + } catch (error) { + if (session.isInterrupted()) throw interrupted(operation); + throw error; + } + } + + private async shouldTranslateInterruption( + runtime: RuntimeIdentity, + session: RuntimeSession, + error: unknown + ): Promise { + if (error instanceof RuntimeIdentityInactiveError) return true; + if (session.isInterrupted()) return true; + return !(await this.options.lifecycle.isActive(runtime)); + } + + private async assertActive( + runtime: RuntimeIdentity, + operation: string + ): Promise { + try { + await this.options.lifecycle.assertActive(runtime); + } catch { + throw interrupted(operation); + } + } +} + +class OperationActivityOwner { + private references = 1; + private finished = false; + private readonly retained = new Set<{ release(): void }>(); + + constructor(private readonly activity: ActivityAdmission) {} + + get beforeCall(): Promise { + return this.activity.beforeCall; + } + + canRetain(): boolean { + return !this.finished; + } + + retain(): RuntimeConnectionHold { + if (!this.canRetain()) return { release: () => {} }; + this.references += 1; + const hold = { release: once(() => this.releaseRetained(hold)) }; + this.retained.add(hold); + return hold; + } + + releaseBase(): void { + this.releaseReference(); + } + + interruptRetainedHolds(): void { + for (const hold of [...this.retained]) hold.release(); + } + + private releaseRetained(hold: { release(): void }): void { + this.retained.delete(hold); + this.releaseReference(); + } + + private releaseReference(): void { + if (this.finished) return; + this.references -= 1; + if (this.references === 0) { + this.finished = true; + this.activity.finish(); + } + } +} + +export function interrupted(operation: string): OperationInterruptedError { + return new OperationInterruptedError({ + code: ErrorCode.OPERATION_INTERRUPTED, + message: `Sandbox operation ${operation} was interrupted because the runtime changed`, + context: { + reason: 'runtime_replaced', + operation, + admitted: true, + retryable: false + }, + httpStatus: getHttpStatus(ErrorCode.OPERATION_INTERRUPTED), + timestamp: new Date().toISOString() + }); +} + +function sameIdentity(left: RuntimeIdentity, right: RuntimeIdentity): boolean { + return ( + left.id === right.id && + left.runtimeIncarnationID === right.runtimeIncarnationID + ); +} + +function once(fn: () => void): () => void { + let called = false; + return () => { + if (called) return; + called = true; + fn(); + }; +} diff --git a/packages/sandbox/src/runtime/port-readiness.ts b/packages/sandbox/src/runtime/port-readiness.ts new file mode 100644 index 000000000..e89bea2d4 --- /dev/null +++ b/packages/sandbox/src/runtime/port-readiness.ts @@ -0,0 +1,101 @@ +import type { + PortWatchEvent, + PortWatchSubscriptionAPI, + WaitForPortOptions +} from '@repo/shared'; +import type { RuntimeLease } from './operation-runner'; + +export async function waitForRuntimePort( + lease: RuntimeLease, + port: number, + options: WaitForPortOptions = {} +): Promise { + const hold = lease.retain(); + const abort = createCombinedAbort(options.signal, options.timeout); + let subscription: PortWatchSubscriptionAPI | undefined; + let reader: ReadableStreamDefaultReader | undefined; + + try { + subscription = await abort.race( + lease.control.ports.openWatch(port, { + mode: options.mode, + path: options.path, + status: options.status, + interval: options.interval + }) + ); + const stream = await abort.race(subscription.stream()); + reader = stream.getReader(); + while (true) { + const { done, value } = await abort.race(reader.read()); + if (done) break; + if (value.type === 'ready') return; + if (value.type === 'error') throw new Error(value.error); + } + throw new Error(`Port ${port} readiness watch ended before ready`); + } finally { + abort.cleanup(); + observe(reader?.cancel()); + observe(subscription?.cancel()); + subscription?.[Symbol.dispose](); + hold.release(); + } +} + +function observe(promise: Promise | undefined): void { + promise?.catch(() => undefined); +} + +type CombinedAbort = { + race(promise: Promise): Promise; + cleanup(): void; +}; + +function createCombinedAbort( + externalSignal: AbortSignal | undefined, + timeoutMs: number | undefined +): CombinedAbort { + const controller = new AbortController(); + const listeners: Array<() => void> = []; + let timer: ReturnType | undefined; + + const abort = (reason: unknown) => { + if (!controller.signal.aborted) controller.abort(reason); + }; + + if (externalSignal?.aborted) { + abort(externalSignal.reason); + } else if (externalSignal) { + const listener = () => abort(externalSignal.reason); + externalSignal.addEventListener('abort', listener, { once: true }); + listeners.push(() => externalSignal.removeEventListener('abort', listener)); + } + + if (timeoutMs !== undefined) { + timer = setTimeout(() => { + abort(new Error('Timed out waiting for runtime port')); + }, timeoutMs); + } + + return { + race: (promise: Promise) => { + if (controller.signal.aborted) + return Promise.reject(controller.signal.reason); + return Promise.race([ + promise, + new Promise((_, reject) => { + const listener = () => reject(controller.signal.reason); + controller.signal.addEventListener('abort', listener, { once: true }); + promise.then( + () => controller.signal.removeEventListener('abort', listener), + () => controller.signal.removeEventListener('abort', listener) + ); + }) + ]); + }, + cleanup: () => { + if (timer !== undefined) clearTimeout(timer); + for (const remove of listeners) remove(); + } + }; +} diff --git a/packages/sandbox/src/runtime/session-manager.ts b/packages/sandbox/src/runtime/session-manager.ts new file mode 100644 index 000000000..6c49fac71 --- /dev/null +++ b/packages/sandbox/src/runtime/session-manager.ts @@ -0,0 +1,305 @@ +import type { Logger, SandboxControlCallback } from '@repo/shared'; +import type { RpcTarget } from 'capnweb'; +import { ContainerControlClient } from '../container-control/client'; +import { + ContainerControlConnection, + type ContainerFetchStub +} from '../container-control/connection'; +import { translateRPCError } from '../container-control/rpc-error'; +import { RuntimeControlProtocolError } from '../errors'; +import { RuntimeIdentityInactiveError } from '../runtime/types'; +import { validateRuntimeMetadata } from './bootstrap-probe'; +import { interrupted } from './operation-runner'; +import type { + RuntimeConnectionHold, + RuntimeIdentity, + RuntimeSession, + RuntimeSessionManager as RuntimeSessionManagerContract +} from './types'; + +export type RuntimeControlCallbackBinder = ( + runtime: RuntimeIdentity, + isSessionCurrent: () => boolean +) => (SandboxControlCallback & RpcTarget) | undefined; + +export type RuntimeSessionManagerOptions = { + getTcpPort: (port: number) => ContainerFetchStub; + logger?: Logger; + callbackBinder?: RuntimeControlCallbackBinder; + onConnectionClose?: () => void; +}; + +type CachedSession = RuntimeSession & { + generation: number; + key: string; + connection: ContainerControlConnection; + client: ContainerControlClient; + holds: Set; + interrupt(operation: string): void; +}; + +type OpeningSession = { + key: string; + generation: number; + connection: ContainerControlConnection; + promise: Promise; +}; + +export class RuntimeSessionManager implements RuntimeSessionManagerContract { + private cached: CachedSession | null = null; + private opening: OpeningSession | null = null; + private generation = 0; + private disposed = false; + + constructor(private readonly options: RuntimeSessionManagerOptions) {} + + async acquire(runtime: RuntimeIdentity): Promise { + return (await this.acquireSession(runtime)).client; + } + + async acquireSession(runtime: RuntimeIdentity): Promise { + const key = this.cacheKey(runtime); + if (this.disposed) throw new Error('Runtime session manager is disposed'); + if (this.cached?.key === key) return this.cached; + if (this.opening?.key === key) return await this.opening.promise; + + this.supersedeCurrentSession(); + const generation = ++this.generation; + const connection = this.createConnection(runtime, generation); + const promise = this.open(runtime, key, generation, connection); + this.opening = { key, generation, connection, promise }; + + try { + return await promise; + } finally { + if (this.opening?.promise === promise) this.opening = null; + } + } + + closeActive(): void { + this.generation += 1; + this.opening?.connection.disconnect(); + this.opening = null; + const cached = this.cached; + this.cached = null; + this.interruptSession(cached, 'runtime.session.closeActive'); + cached?.connection.disconnect(); + } + + dispose(): void { + this.disposed = true; + this.closeActive(); + } + + private supersedeCurrentSession(): void { + this.opening?.connection.disconnect(); + this.opening = null; + const previous = this.cached; + this.cached = null; + this.interruptSession(previous, 'runtime.session.superseded'); + previous?.connection.disconnect(); + } + + private retainSession( + session: CachedSession, + onInterrupt?: () => void + ): RuntimeConnectionHold { + if (session.isInterrupted()) { + onInterrupt?.(); + return { release: () => {} }; + } + const hold = new ManagedRuntimeConnectionHold(onInterrupt, () => { + session.holds.delete(hold); + }); + session.holds.add(hold); + if (session.isInterrupted()) hold.forceRelease(); + return hold; + } + + private interruptSession( + session: CachedSession | null, + operation: string + ): void { + if (!session) return; + session.interrupt(operation); + for (const hold of [...session.holds]) hold.forceRelease(); + } + + private createConnection( + runtime: RuntimeIdentity, + generation: number + ): ContainerControlConnection { + let connection: ContainerControlConnection; + connection = new ContainerControlConnection({ + stub: this.options.getTcpPort(3000), + logger: this.options.logger, + localMain: this.bindLocalMain(runtime, generation), + onClose: () => { + const cached = + this.cached?.connection === connection ? this.cached : null; + if (cached) this.cached = null; + this.interruptSession(cached, 'runtime.session.transportClosed'); + this.options.onConnectionClose?.(); + } + }); + return connection; + } + + private async open( + runtime: RuntimeIdentity, + key: string, + generation: number, + connection: ContainerControlConnection + ): Promise { + try { + const metadata = validateRuntimeMetadata( + await this.activate(connection, runtime), + 'utils.activateControlSession' + ); + if (metadata.runtimeIncarnationID !== runtime.runtimeIncarnationID) { + throw this.activationMismatch(); + } + this.assertCurrentOpening(generation, connection); + + const client = new ContainerControlClient({ + stub: this.options.getTcpPort(3000), + logger: this.options.logger, + translateTransportErrorsAsInterruptions: false, + connection, + externallyOwnedConnection: true, + onConnectionClose: () => { + const cached = + this.cached?.connection === connection ? this.cached : null; + if (cached) this.cached = null; + this.interruptSession(cached, 'runtime.session.transportClosed'); + this.options.onConnectionClose?.(); + } + }); + let rejectInterrupted!: (error: Error) => void; + let poisoned = false; + const session: CachedSession = { + generation, + key, + connection, + client, + holds: new Set(), + interrupted: new Promise((_, reject) => { + rejectInterrupted = reject; + }), + isInterrupted: () => poisoned, + interrupt: (operation) => { + if (poisoned) return; + poisoned = true; + rejectInterrupted(interrupted(operation)); + }, + retain: (onInterrupt) => this.retainSession(session, onInterrupt) + }; + session.interrupted.catch(() => undefined); + this.assertCurrentOpening(generation, connection); + this.cached = session; + return session; + } catch (error) { + connection.disconnect(); + throw error; + } + } + + private assertCurrentOpening( + generation: number, + connection: ContainerControlConnection + ): void { + if ( + this.disposed || + generation !== this.generation || + this.opening?.connection !== connection + ) { + connection.disconnect(); + throw new RuntimeIdentityInactiveError(); + } + } + + private async activate( + connection: ContainerControlConnection, + runtime: RuntimeIdentity + ) { + try { + return await connection.activateControlSession( + runtime.runtimeIncarnationID + ); + } catch (error) { + if (this.isActivationMismatchError(error)) + throw this.activationMismatch(error); + translateRPCError(error, { + operation: 'utils.activateControlSession', + translateTransportErrorsAsInterruptions: false + }); + } + } + + private isActivationMismatchError(error: unknown): boolean { + if (!(error instanceof Error)) return false; + const code = (error as { code?: unknown }).code; + return ( + code === 'CONTROL_PROTOCOL_INCOMPATIBLE' || + /runtime incarnation does not match|control protocol incompatible/i.test( + error.message + ) + ); + } + + private activationMismatch(cause?: unknown): RuntimeControlProtocolError { + return new RuntimeControlProtocolError( + 'Activated runtime incarnation does not match expected runtime', + { + reason: 'activation-mismatch', + operation: 'utils.activateControlSession' + }, + { cause } + ); + } + + private bindLocalMain( + runtime: RuntimeIdentity, + generation: number + ): (SandboxControlCallback & RpcTarget) | undefined { + return this.options.callbackBinder?.(runtime, () => + this.isSessionGenerationCurrent(generation) + ); + } + + private isSessionGenerationCurrent(generation: number): boolean { + if (this.disposed || this.generation !== generation) return false; + return ( + this.cached?.generation === generation || + this.opening?.generation === generation + ); + } + + private cacheKey(runtime: RuntimeIdentity): string { + return `${runtime.id}\0${runtime.runtimeIncarnationID}`; + } +} + +class ManagedRuntimeConnectionHold implements RuntimeConnectionHold { + private active = true; + + constructor( + private readonly onInterrupt: (() => void) | undefined, + private readonly onRelease: () => void + ) {} + + release(): void { + this.releaseInternal(false); + } + + forceRelease(): void { + this.releaseInternal(true); + } + + private releaseInternal(interrupted: boolean): void { + if (!this.active) return; + this.active = false; + this.onRelease(); + if (interrupted) this.onInterrupt?.(); + } +} diff --git a/packages/sandbox/src/runtime/types.ts b/packages/sandbox/src/runtime/types.ts new file mode 100644 index 000000000..9bed0afe1 --- /dev/null +++ b/packages/sandbox/src/runtime/types.ts @@ -0,0 +1,80 @@ +import type { RuntimeMetadata } from '@repo/shared'; +import type { ContainerControlClient } from '../container-control/client'; +import { RuntimeIdentityInactiveError } from '../errors/classes/runtime-inactive'; + +export { RuntimeIdentityInactiveError }; + +export type RuntimeIdentityID = string & { + readonly __runtimeIdentityID: unique symbol; +}; + +export type RuntimeScoped = T & { + readonly runtimeIdentityID: RuntimeIdentityID; +}; + +export type RuntimeIncarnationID = string & { + readonly __runtimeIncarnationID: unique symbol; +}; + +export class RuntimeIdentity { + readonly id: RuntimeIdentityID; + readonly runtimeIncarnationID: RuntimeIncarnationID; + + constructor(record: { + readonly id: RuntimeIdentityID; + readonly runtimeIncarnationID: RuntimeIncarnationID; + }) { + this.id = record.id; + this.runtimeIncarnationID = record.runtimeIncarnationID; + } + + owns(record: { readonly runtimeIdentityID: RuntimeIdentityID }): boolean { + return record.runtimeIdentityID === this.id; + } + + scope(value: T): RuntimeScoped { + return { + ...value, + runtimeIdentityID: this.id + }; + } +} + +export type RuntimeRecord = { + readonly schemaVersion: 1; + readonly id: RuntimeIdentityID; + readonly runtimeIncarnationID: RuntimeIncarnationID; +}; + +export type RuntimeRecordStorage = { + get(key: string): Promise; +}; + +export interface RuntimeIdentityReader { + get(): Promise; + getStored(storage?: RuntimeRecordStorage): Promise; + isActive(runtime: RuntimeIdentity): Promise; + assertActive(runtime: RuntimeIdentity): Promise; +} + +export interface RuntimeBootstrapProbe { + probe(): Promise; +} + +export type RuntimeConnectionHold = { + release(): void; +}; + +export type RuntimeSession = { + readonly client: ContainerControlClient; + readonly interrupted: Promise; + isInterrupted(): boolean; + retain(onInterrupt?: () => void): RuntimeConnectionHold; +}; + +export interface RuntimeSessionManager { + acquire(runtime: RuntimeIdentity): Promise; + acquireSession(runtime: RuntimeIdentity): Promise; + closeActive(): void; + dispose(): void; +} diff --git a/packages/sandbox/src/sandbox.ts b/packages/sandbox/src/sandbox.ts index b3ce6f862..952d3dff7 100644 --- a/packages/sandbox/src/sandbox.ts +++ b/packages/sandbox/src/sandbox.ts @@ -1,3 +1,4 @@ +import type { ContainerStartConfigOptions } from '@cloudflare/containers'; import { Container, getContainer, switchPort } from '@cloudflare/containers'; import type { BackupOptions, @@ -17,7 +18,11 @@ import type { RestoreBackupResult, SandboxCommand, SandboxOptions, - Terminal, + SandboxTerminalsAPI, + SandboxTunnelsAPI, + TerminalOutputEvent, + TerminalOutputSubscriptionAPI, + TerminalSnapshot, WaitForPortOptions, WatchOptions } from '@repo/shared'; @@ -37,12 +42,7 @@ import { type BackupRestoreTestFault, BackupService } from './backup/backup-service'; -import { ContainerControlClient } from './container-control'; -import { RuntimeControlClient } from './container-control/runtime-client'; -import { - CurrentRuntimeIdentity, - type RuntimeIdentity -} from './current-runtime-identity'; +import type { ContainerControlClient } from './container-control'; import type { ErrorResponse } from './errors'; import { ContainerUnavailableError, @@ -51,10 +51,16 @@ import { ProcessExitedBeforeReadyError, ProcessNotFoundError, ProcessReadyTimeoutError, - SandboxError + SandboxError, + StaleProcessHandleError, + StaleTerminalHandleError } from './errors'; -import type { HTTPAuthInterceptorParams as GitAuthInterceptorParams } from './extensions'; -import { SandboxExtension } from './extensions'; +import type { + ExtensionRuntimeCall, + ExtensionRuntimeControl, + HTTPAuthInterceptorParams as GitAuthInterceptorParams +} from './extensions'; +import { SandboxExtension, sandboxRuntimeCall } from './extensions'; import { collectFile, streamFile } from './file-stream'; import { isPlatformTransientError } from './platform-errors'; import { isPreviewProxyRequest } from './preview/protocol'; @@ -67,14 +73,31 @@ import { type ProcessCapabilityControl, ProcessCapabilityTarget } from './processes/process-capability'; -import { ProcessLifecycle } from './processes/process-lifecycle'; -import { openRemoteSubscription } from './processes/remote-subscription'; -import type { ProcessRPCDescriptor } from './processes/rpc-types'; -import { terminalHandle as terminalHandleFromSnapshot } from './pty'; import { - ResourceActivityGate, - type ResourceActivityOperation -} from './resource-activity-gate'; + openRemoteSubscription, + PullSubscriptionTarget +} from './processes/remote-subscription'; +import type { + ProcessPullSubscriptionRPC, + ProcessRPCDescriptor +} from './processes/rpc-types'; +import { terminalHandleFromRPCDescriptor } from './pty'; +import type { TerminalRPCDescriptor } from './pty/rpc-types'; +import { + type TerminalCapabilityControl, + TerminalCapabilityTarget +} from './pty/terminal-capability'; +import { ResourceActivityGate } from './resource-activity-gate'; +import { + RUNTIME_ABSENT, + RuntimeBootstrapProbe, + type RuntimeIdentity, + type RuntimeLease, + RuntimeOperationRunner, + RuntimeSessionManager, + SandboxRuntimeLifecycle +} from './runtime'; +import { waitForRuntimePort } from './runtime/port-readiness'; import { CurrentSandboxLifetime } from './sandbox-lifetime'; import { SandboxSecurityError, @@ -92,15 +115,17 @@ import { import { NamedTunnelConfigResolver } from './tunnels/named-tunnel-config'; import { createTunnelsHandle, + pruneTunnelsForRestart, type TunnelExitHandler, type TunnelsHandle, type TunnelsHandler } from './tunnels/rpc-target'; import { SandboxControlCallbackImpl } from './tunnels/sandbox-control-callback'; -import { SDK_VERSION } from './version'; export { ContainerProxy }; +const DESTROY_RUNTIME_CLEANUP_TIMEOUT_MS = 5_000; + function validateExecArgv(command: SandboxCommand): SandboxCommand { if (!Array.isArray(command)) { throw invalidCommand('exec() requires argv as an array of strings.'); @@ -133,16 +158,103 @@ function invalidCommand( }); } -function runtimeInterrupted( - operation: string, - effect: 'none' | 'unknown' -): OperationInterruptedError { +function toContainerHTTPRequest(request: Request): Request { + const url = new URL(request.url); + if (url.protocol === 'https:') url.protocol = 'http:'; + const headers = new Headers(request.headers); + headers.delete('cf-container-target-port'); + headers.delete('x-sandbox-port-route-token'); + return new Request(new Request(url, request), { headers }); +} + +function retainedStream( + stream: ReadableStream, + lease: RuntimeLease, + operation = 'stream.read' +): { + stream: ReadableStream; + cancel(reason?: unknown): void; + release(): void; +} { + const reader = stream.getReader(); + let released = false; + let interruptedError: Error | undefined; + let hold: ReturnType | undefined; + let interrupt!: (error: Error) => void; + const interrupted = new Promise((_, reject) => { + interrupt = reject; + }); + interrupted.catch(() => undefined); + const release = () => { + if (released) return; + released = true; + hold?.release(); + }; + const cancelSource = (reason?: unknown) => { + if (released) return; + release(); + void reader.cancel(reason).catch(() => undefined); + }; + hold = lease.retain(() => { + if (released) return; + interruptedError = runtimeInterrupted(operation); + interrupt(interruptedError); + cancelSource(interruptedError); + }); + return { + stream: new ReadableStream({ + async pull(streamController) { + try { + const result = await Promise.race([reader.read(), interrupted]); + if (interruptedError) throw interruptedError; + if (result.done) { + release(); + streamController.close(); + return; + } + streamController.enqueue(result.value); + } catch (error) { + release(); + streamController.error(error); + } + }, + cancel(reason) { + cancelSource(reason); + } + }), + cancel: cancelSource, + release + }; +} + +function retainStream( + stream: ReadableStream, + lease: RuntimeLease +): ReadableStream { + return retainedStream(stream, lease).stream; +} + +function extensionRuntimeControl( + control: ContainerControlClient +): ExtensionRuntimeControl { + return { + files: control.files, + ports: control.ports, + backup: control.backup, + watch: control.watch, + tunnels: control.tunnels, + terminals: control.terminals, + extensions: control.extensions, + utils: control.utils + }; +} + +function runtimeInterrupted(operation: string): OperationInterruptedError { const context: OperationInterruptedContext = { reason: 'runtime_replaced', operation, admitted: true, - retryable: false, - effect + retryable: false }; return new OperationInterruptedError({ code: ErrorCode.OPERATION_INTERRUPTED, @@ -153,12 +265,174 @@ function runtimeInterrupted( }); } +function isRuntimeAbsent(value: unknown): value is typeof RUNTIME_ABSENT { + return value === RUNTIME_ABSENT; +} + +function staleTerminal( + terminalId: string, + operation: string +): StaleTerminalHandleError { + return new StaleTerminalHandleError({ + code: ErrorCode.STALE_TERMINAL_HANDLE, + message: 'Terminal handle refers to a previous runtime incarnation', + context: { terminalId, operation }, + httpStatus: 409, + timestamp: new Date().toISOString() + }); +} + +type InterruptibleTerminalSubscription = TerminalOutputSubscriptionAPI & { + interrupt(): void; +}; + +function retainedTerminalSubscription( + terminalId: string, + operation: string, + subscription: TerminalOutputSubscriptionAPI, + releaseConnection: () => void +): InterruptibleTerminalSubscription { + let released = false; + let reader: ReadableStreamDefaultReader | undefined; + let controller: + | ReadableStreamDefaultController + | undefined; + const stale = () => staleTerminal(terminalId, operation); + const release = () => { + if (released) return; + released = true; + releaseConnection(); + if (reader) { + try { + void reader.cancel().catch(() => undefined); + } catch {} + } else { + try { + void subscription.cancel().catch(() => undefined); + } catch {} + try { + subscription[Symbol.dispose](); + } catch {} + } + }; + const failStale = () => { + const error = stale(); + controller?.error(error); + release(); + }; + return { + async stream() { + const source = await openRemoteSubscription( + Promise.resolve(subscription), + { operation, protocol: 'stream' } + ); + return new ReadableStream({ + start(streamController) { + controller = streamController; + reader = source.getReader(); + }, + async pull(streamController) { + if (released) { + streamController.error(stale()); + return; + } + try { + const result = await reader!.read(); + if (released) { + streamController.error(stale()); + return; + } + if (result.done) { + release(); + streamController.close(); + return; + } + streamController.enqueue(result.value); + if (result.value.type === 'terminal') { + release(); + streamController.close(); + } + } catch (error) { + release(); + streamController.error(error); + } + }, + cancel() { + release(); + } + }); + }, + async cancel() { + release(); + }, + [Symbol.dispose]() { + release(); + }, + interrupt() { + failStale(); + } + }; +} + +function retainedResponse( + response: Response, + lease: RuntimeLease, + operation: string +): Response { + if (!response.body) return response; + return new Response(retainedStream(response.body, lease, operation).stream, { + status: response.status, + statusText: response.statusText, + headers: response.headers + }); +} + +function retainWebSocketResponse( + response: Response, + releaseConnection: () => void +): { interrupt(): void; webSocketFound: boolean } { + const webSocket = ( + response as { + webSocket?: EventTarget & { + close?: (code?: number, reason?: string) => void; + }; + } + ).webSocket; + if (!webSocket) { + releaseConnection(); + return { interrupt: () => {}, webSocketFound: false }; + } + const release = once(releaseConnection); + const closeInterrupted = once(() => { + try { + webSocket.close?.(1012, 'Runtime replaced'); + } catch { + // The runtime may reject close() after handing the socket to its peer. + } finally { + release(); + } + }); + webSocket.addEventListener('close', release, { once: true }); + webSocket.addEventListener('error', release, { once: true }); + return { interrupt: closeInterrupted, webSocketFound: true }; +} + +function once void>(fn: T): T { + let called = false; + return ((...args: Parameters) => { + if (called) return; + called = true; + fn(...(args as never[])); + }) as T; +} + function processCapabilityControl( - client: ContainerControlClient + lease: RuntimeLease ): ProcessCapabilityControl { - const processes = client.processesWithoutActivity(); + const client = lease.control; + const processes = client.processes; return { - retainConnection: () => client.retainConnection(), + retainRuntimeHold: () => lease.retain().release, getProcess: (id) => processes.get(id), openLogs: (id, options) => processes.openLogs(id, options), openPortWatch: (port, options) => client.ports.openWatch(port, options), @@ -166,6 +440,28 @@ function processCapabilityControl( }; } +type TerminalHandleStub = Omit & { + output( + id: string, + options?: Parameters[1] + ): Promise>; + fetch(request: Request): Promise; +}; + +function localizeTerminalHandle( + descriptor: TerminalRPCDescriptor, + sandboxStub: { + fetch(request: Request): Promise; + } +): ReturnType { + return terminalHandleFromRPCDescriptor(descriptor, async (request) => { + const token = await descriptor.capability.authorizeConnection(); + const headers = new Headers(request.headers); + headers.set('x-sandbox-port-route-token', token); + return await sandboxStub.fetch(new Request(request, { headers })); + }); +} + type SandboxConfiguration = { sandboxName?: { name: string; @@ -188,10 +484,6 @@ type PreviewForwardingContainerState = DurableObjectState<{}> & { container?: PreviewForwardingContainer; }; -type PreviewForwardingLifecycleState = { - inflightRequests?: number; -}; - type ConfigurableSandboxStub = { configure?: (configuration: SandboxConfiguration) => Promise; setSandboxName?: (name: string, normalizeId?: boolean) => Promise; @@ -204,14 +496,23 @@ type ConfigurableSandboxStub = { type SandboxProxyStub = ConfigurableSandboxStub & { fetch: (request: Request) => Promise; + containerFetch: ( + requestOrUrl: Request | string | URL, + portOrInit?: number | RequestInit, + portParam?: number + ) => Promise; callExtension: ( extensionName: string, method: string, args: unknown[] ) => Promise; - createTerminal: (options: CreateTerminalOptions) => Promise; - getTerminal: (id: string) => Promise; - listTerminals: () => Promise; + createTerminal: ( + options: CreateTerminalOptions + ) => Promise; + getTerminal: (id: string) => Promise; + listTerminals: () => Promise; + wsConnect: (request: Request, port: number) => Promise; + authorizePortRequest: (port: number, path: string) => Promise; exec: ( command: SandboxCommand, options?: ExecOptions @@ -220,12 +521,22 @@ type SandboxProxyStub = ConfigurableSandboxStub & { listProcesses: () => Promise; }; -export type SandboxClient = Omit & ISandbox; +type InternalSandboxRPCMethod = 'authorizePortRequest'; + +export type SandboxClient = Omit< + T, + keyof ISandbox | InternalSandboxRPCMethod +> & + ISandbox; const sandboxConfigurationCache = new WeakMap< object, Map >(); +const sandboxConfigurationPending = new WeakMap< + object, + Map> +>(); const BACKUP_DEFAULT_TTL_SECONDS = 259200; const BACKUP_MAX_NAME_LENGTH = 256; @@ -243,6 +554,40 @@ const BACKUP_DOWNLOAD_PARALLEL_PARTS = 8; const BACKUP_DOWNLOAD_PARALLEL_MIN_SIZE = 10 * 1024 * 1024; const BACKUP_DOWNLOAD_MAX_PARTS = 64; +type CancellationOptions = { + abort?: AbortSignal; + instanceGetTimeoutMS?: number; + portReadyTimeoutMS?: number; + waitInterval?: number; +}; + +type StartAndWaitForPortsOptions = { + startOptions?: ContainerStartConfigOptions; + ports?: number | number[]; + cancellationOptions?: CancellationOptions; +}; + +function normalizeStartAndWaitForPortsOptions( + portsOrArgs?: number | number[] | StartAndWaitForPortsOptions, + cancellationOptions?: CancellationOptions, + startOptions?: ContainerStartConfigOptions +): StartAndWaitForPortsOptions { + if (typeof portsOrArgs === 'number' || Array.isArray(portsOrArgs)) { + return { ports: portsOrArgs, cancellationOptions, startOptions }; + } + return { + ...(portsOrArgs ?? {}), + cancellationOptions: + portsOrArgs?.cancellationOptions ?? cancellationOptions, + startOptions: portsOrArgs?.startOptions ?? startOptions + }; +} + +function normalizePorts(ports: number | number[] | undefined): number[] { + if (ports === undefined) return []; + return Array.isArray(ports) ? ports : [ports]; +} + function getNamespaceConfigurationCache( namespace: object ): Map { @@ -256,6 +601,17 @@ function getNamespaceConfigurationCache( return created; } +function getNamespacePendingConfiguration( + namespace: object +): Map> { + const existing = sandboxConfigurationPending.get(namespace); + if (existing) return existing; + + const created = new Map>(); + sandboxConfigurationPending.set(namespace, created); + return created; +} + function sameContainerTimeouts( left?: NonNullable, right?: NonNullable @@ -463,32 +819,46 @@ export function getSandbox>( ) as unknown as T & SandboxProxyStub; const namespaceCache = getNamespaceConfigurationCache(ns); - const cachedConfiguration = namespaceCache.get(effectiveId); - const configuration = buildSandboxConfiguration( - effectiveId, - options, - cachedConfiguration - ); - - if (hasSandboxConfiguration(configuration)) { - const nextConfiguration = mergeSandboxConfiguration( - cachedConfiguration, - configuration + const pendingConfigurations = getNamespacePendingConfiguration(ns); + const applyRequestedConfiguration = async (): Promise => { + const cachedConfiguration = namespaceCache.get(effectiveId); + const configuration = buildSandboxConfiguration( + effectiveId, + options, + cachedConfiguration ); - namespaceCache.set(effectiveId, nextConfiguration); + if (!hasSandboxConfiguration(configuration)) return; - void applySandboxConfiguration(stub, configuration).catch(() => { - if (cachedConfiguration) { - namespaceCache.set(effectiveId, cachedConfiguration); - return; + await applySandboxConfiguration(stub, configuration); + namespaceCache.set( + effectiveId, + mergeSandboxConfiguration(cachedConfiguration, configuration) + ); + }; + const previousConfiguration = pendingConfigurations.get(effectiveId); + const configurationReady = previousConfiguration + ? previousConfiguration.then(applyRequestedConfiguration) + : applyRequestedConfiguration(); + pendingConfigurations.set(effectiveId, configurationReady); + void configurationReady + .finally(() => { + if (pendingConfigurations.get(effectiveId) === configurationReady) { + pendingConfigurations.delete(effectiveId); } - - namespaceCache.delete(effectiveId); - }); - } + }) + .catch(() => undefined); const enhancedMethods = { - fetch: (request: Request) => stub.fetch(request), + authorizePortRequest: undefined, + createPortRequestToken: undefined, + fetch: (request: Request) => { + const targetPort = request.headers.get('cf-container-target-port'); + return targetPort === null + ? stub.fetch(request) + : fetchAuthorizedPort(stub, request, Number(targetPort)); + }, + containerFetch: (...args: Parameters) => + stub.containerFetch(...args), exec: async (command: SandboxCommand, execOptions?: ExecOptions) => createSandboxProcess( await stub.exec(command, sanitizeExecOptions(execOptions)) @@ -529,17 +899,26 @@ export function getSandbox>( stub.watch(path, options), checkChanges: (path: string, options: CheckChangesOptions = {}) => stub.checkChanges(path, options), - createTerminal: (options: CreateTerminalOptions) => - stub.createTerminal(options), - getTerminal: (id: string) => stub.getTerminal(id), - listTerminals: () => stub.listTerminals(), + createTerminal: async (options: CreateTerminalOptions) => + localizeTerminalHandle(await stub.createTerminal(options), stub), + getTerminal: async (id: string) => { + const terminal = await stub.getTerminal(id); + return terminal ? localizeTerminalHandle(terminal, stub) : null; + }, + listTerminals: async () => + (await stub.listTerminals()).map((terminal) => + localizeTerminalHandle(terminal, stub) + ), wsConnect: connect(stub), tunnels: new Proxy({} as TunnelsHandler, { get: (_, method) => { if (typeof method !== 'string' || method === 'then') return undefined; return withSandboxOperationContext( `sandbox.tunnels.${method}`, - (...args: unknown[]) => stub.callTunnels(method, args) + async (...args: unknown[]) => { + await configurationReady; + return stub.callTunnels(method, args); + } ); } }) @@ -557,7 +936,10 @@ export function getSandbox>( if (typeof method === 'function') { return withSandboxOperationContext( `sandbox.${prop}`, - method as (...args: unknown[]) => unknown + async (...args: unknown[]) => { + await configurationReady; + return (method as (...args: unknown[]) => unknown)(...args); + } ); } return method; @@ -576,7 +958,8 @@ export function getSandbox>( // forwards to the stub method (sandbox.method(...)), while a nested // access dispatches sandbox..(...) through callExtension. return new Proxy( - (...args: unknown[]) => { + async (...args: unknown[]) => { + await configurationReady; // @ts-expect-error - RPC stub methods are Proxy-trapped, not visible to TypeScript return target[prop](...args); }, @@ -585,8 +968,10 @@ export function getSandbox>( if (typeof method !== 'string' || method === 'then') { return undefined; } - return (...args: unknown[]) => - stub.callExtension(prop, method, args); + return async (...args: unknown[]) => { + await configurationReady; + return stub.callExtension(prop, method, args); + }; } } ); @@ -603,6 +988,26 @@ function sanitizeExecOptions(options?: ExecOptions): ExecOptions | undefined { return sanitized; } +function isPlainObject(value: unknown): value is Record { + return ( + typeof value === 'object' && + value !== null && + Object.getPrototypeOf(value) === Object.prototype + ); +} + +function mergeExecEnvironment( + sandboxEnv: Record, + commandEnv: Record +): Record { + const merged = { ...sandboxEnv }; + for (const [name, value] of Object.entries(commandEnv)) { + if (value === null) delete merged[name]; + else merged[name] = value; + } + return merged; +} + function getConcreteExtensionMethod( extension: SandboxExtension, method: string @@ -630,6 +1035,7 @@ function getConcreteExtensionMethod( export function connect(stub: { fetch: (request: Request) => Promise; + authorizePortRequest: (port: number, path: string) => Promise; }) { return async (request: Request, port: number) => { if (!validatePort(port)) { @@ -637,16 +1043,35 @@ export function connect(stub: { `Invalid port number: ${port}. Must be 1024-65535, excluding 3000 (sandbox control plane).` ); } - const portSwitchedRequest = switchPort(request, port); - return await stub.fetch(portSwitchedRequest); + return await fetchAuthorizedPort(stub, request, port); }; } +async function fetchAuthorizedPort( + stub: { + fetch: (request: Request) => Promise; + authorizePortRequest: (port: number, path: string) => Promise; + }, + request: Request, + port: number +): Promise { + const token = await stub.authorizePortRequest( + port, + new URL(request.url).pathname + ); + const switched = switchPort(request, port); + const headers = new Headers(switched.headers); + headers.set('x-sandbox-port-route-token', token); + return await stub.fetch(new Request(switched, { headers })); +} + export class Sandbox extends Container { defaultPort = 3000; // Default port for the container's Bun server sleepAfter: string | number = '10m'; // Sleep the sandbox if no requests are made in this timeframe - client: ContainerControlClient; + private runtimeSessions: RuntimeSessionManager; + private runtimeLifecycle: SandboxRuntimeLifecycle; + private runtimeRunner: RuntimeOperationRunner; private sandboxName: string | null = null; // Tunnels subsystem handle. Lazily constructed on first access via the @@ -663,17 +1088,22 @@ export class Sandbox extends Container { private readonly controlCallback: SandboxControlCallbackImpl; private normalizeId: boolean = false; envVars: Record = {}; + private readonly portRequestTokens = new Map< + string, + { + port: number; + path: string; + expiresAt: number; + scope: 'port' | 'terminal'; + } + >(); private logger: ReturnType; private keepAliveEnabled: boolean = false; private bucketMounts: BucketMountService; - private currentRuntime: CurrentRuntimeIdentity; private currentLifetime: CurrentSandboxLifetime; private backupService: BackupService; private previewService: PreviewService; private resourceActivityGate: ResourceActivityGate; - private runtimeControlClient: RuntimeControlClient; - private processLifecycle: ProcessLifecycle; - private controlSessionActivity: ResourceActivityOperation | null = null; private r2AccessKeyId: string | null = null; private r2SecretAccessKey: string | null = null; @@ -754,21 +1184,6 @@ export class Sandbox extends Container { return fn.apply(extension, args); } - /** - * Compute the control-channel upgrade retry budget from current container - * timeouts. - * - * The budget covers the full container startup window (instance provisioning - * + port readiness) plus a 30s margin for the maximum single backoff delay. - * The 120s floor preserves the default for short timeout configurations. - */ - private computeRetryTimeoutMs(): number { - const startupBudgetMs = - this.containerTimeouts.instanceGetTimeoutMS + - this.containerTimeouts.portReadyTimeoutMS; - return Math.max(120_000, startupBudgetMs + 30_000); - } - private renewActivityTimeoutIfAvailable(): void { const renewActivityTimeout = this.renewActivityTimeout; if (typeof renewActivityTimeout === 'function') { @@ -776,47 +1191,10 @@ export class Sandbox extends Container { } } - /** - * Create the single control-plane client used for all SDK operations. - */ - private createClient(): ContainerControlClient { - return new ContainerControlClient({ - stub: this, - port: 3000, - logger: this.logger, - retryTimeoutMs: this.computeRetryTimeoutMs(), - // localMain exposes the DO-side control callback (tunnel-exit - // notifications, etc.) to the container side of the session. - localMain: this.controlCallback, - // The control channel multiplexes all work over a single capnweb - // WebSocket, so we can't bracket per-request — and a method that - // returns a ReadableStream resolves its promise long before the - // stream is actually drained. Instead, ContainerControlClient polls - // capnweb's session stats and reports busy/idle *transitions* of the - // current session. The resource activity gate owns that operation - // until the session returns to idle. See the file-level comment in - // container-control/client.ts for details. - onActivity: () => { - this.resourceActivityGate.recordActivity(); - }, - onOperationStarted: () => { - const activity = this.resourceActivityGate.beginOperation(); - return { - beforeCall: activity.beforeCall.then(() => - this.ensureContainerRunning() - ), - finish: activity.finish - }; - }, - onSessionBusy: () => { - this.controlSessionActivity = - this.resourceActivityGate.beginOperation(); - }, - onSessionIdle: () => { - this.controlSessionActivity?.finish(); - this.controlSessionActivity = null; - } - }); + private getRuntimePortStub(port: number) { + const stub = this.ctx.container?.getTcpPort?.(port); + if (!stub) throw new Error('Container runtime port is not available'); + return stub; } constructor(ctx: DurableObjectState<{}>, env: Env) { @@ -838,11 +1216,6 @@ export class Sandbox extends Container { sandboxId: this.ctx.id.toString() }); - this.currentRuntime = new CurrentRuntimeIdentity( - this.ctx.storage, - () => this.getState(), - () => this.ctx.container?.running === true - ); this.currentLifetime = new CurrentSandboxLifetime(this.ctx.storage); this.namedTunnelConfigResolver = new NamedTunnelConfigResolver({ getEnv: () => this.env @@ -851,8 +1224,20 @@ export class Sandbox extends Container { ctx: this.ctx, getEnv: () => this.env, logger: this.logger, - getClient: () => this.client, - currentRuntime: this.currentRuntime, + runBackupAttempt: (operation, call) => + this.runWakingComposite(operation, (lease) => + call({ + runtime: lease.runtime, + control: lease.control, + retain: lease.retain + }) + ), + runtimeReader: { + get: () => this.runtimeLifecycle.get(), + getStored: (storage) => this.runtimeLifecycle.getStored(storage), + isActive: (runtime) => this.runtimeLifecycle.isActive(runtime), + assertActive: (runtime) => this.runtimeLifecycle.assertActive(runtime) + }, currentLifetime: this.currentLifetime }); @@ -865,37 +1250,82 @@ export class Sandbox extends Container { // session is created eagerly in the connection's constructor. this.controlCallback = new SandboxControlCallbackImpl( () => this.tunnelExitHandler, - this.logger + this.logger, + undefined, + () => this.runtimeLifecycle.get() ); this.resourceActivityGate = new ResourceActivityGate( () => this.renewActivityTimeoutIfAvailable(), - () => super.onActivityExpired() + () => this.performRuntimeStop(() => super.stop()) ); - this.runtimeControlClient = new RuntimeControlClient({ - getTcpPort: (port) => { - const container = this.getPreviewForwardingContainer(); - if (!container) throw new Error('Container runtime is not available'); - return container.getTcpPort(port); + this.runtimeSessions = new RuntimeSessionManager({ + getTcpPort: (port) => this.getRuntimePortStub(port), + logger: this.logger, + callbackBinder: (runtime, isSessionCurrent) => + this.controlCallback.bindRuntime(runtime, isSessionCurrent) + }); + this.runtimeLifecycle = new SandboxRuntimeLifecycle({ + storage: this.ctx.storage, + isRuntimeRunning: () => this.ctx.container?.running === true, + startControlPort: async (port, options) => { + await super.startAndWaitForPorts({ + ports: [port], + cancellationOptions: { + instanceGetTimeoutMS: this.containerTimeouts.instanceGetTimeoutMS, + portReadyTimeoutMS: this.containerTimeouts.portReadyTimeoutMS, + waitInterval: this.containerTimeouts.waitIntervalMS, + abort: options?.signal + }, + startOptions: options?.startOptions + }); }, - beginNonWakingOperation: () => - this.resourceActivityGate.beginNonWakingOperation(), - logger: this.logger + waitForControlPort: async () => undefined, + stopControlPort: async () => undefined, + probe: new RuntimeBootstrapProbe({ + getTcpPort: (port) => this.getRuntimePortStub(port) + }), + sessions: this.runtimeSessions, + observeVersionCompatibility: async (_client, _runtime) => undefined, + reconcileReplacement: async () => { + await pruneTunnelsForRestart(this.ctx.storage); + } }); - this.currentRuntime.onChange(() => this.runtimeControlClient.dispose()); - this.processLifecycle = new ProcessLifecycle({ - currentRuntime: this.currentRuntime, - runtimeClient: this.runtimeControlClient, - beginNonWakingOperation: () => - this.resourceActivityGate.beginNonWakingOperation() + this.runtimeRunner = new RuntimeOperationRunner({ + lifecycle: this.runtimeLifecycle, + activityGate: this.resourceActivityGate }); - - this.client = this.createClient(); this.bucketMounts = new BucketMountService({ getEnv: () => this.env, getEnvVars: () => this.envVars, - getClient: () => this.client, + runMountAttempt: (operation, call) => + this.runWakingComposite(operation, (lease) => + call({ + runtime: lease.runtime, + control: lease.control, + retain: lease.retain + }) + ), + runExistingMountAttempt: async (operation, call) => { + const result = await this.runtimeRunner.runExisting( + { kind: 'current' }, + operation, + (lease) => + call({ + runtime: lease.runtime, + control: lease.control, + retain: lease.retain + }) + ); + if (isRuntimeAbsent(result)) return { status: 'absent' }; + return { status: 'completed', value: result }; + }, logger: this.logger, - currentRuntime: this.currentRuntime, + runtimeReader: { + get: () => this.runtimeLifecycle.get(), + getStored: (storage) => this.runtimeLifecycle.getStored(storage), + isActive: (runtime) => this.runtimeLifecycle.isActive(runtime), + assertActive: (runtime) => this.runtimeLifecycle.assertActive(runtime) + }, currentLifetime: this.currentLifetime, getR2AccessKeyID: () => this.r2AccessKeyId, getR2SecretAccessKey: () => this.r2SecretAccessKey, @@ -904,14 +1334,24 @@ export class Sandbox extends Container { this.previewService = new PreviewService({ storage: this.ctx.storage, logger: this.logger, - currentRuntime: this.currentRuntime, + getStoredRuntime: (storage) => this.runtimeLifecycle.getStored(storage), + assertRuntimeActive: (runtime) => + this.runtimeLifecycle.assertActive(runtime), getContainerState: () => this.getState(), getForwardingContainer: () => this.getPreviewForwardingContainer(), - ensureRuntimeActiveForPreview: () => this.ensureRuntimeActiveForPreview(), + runWaking: (operation, call) => + this.runWakingComposite(operation, (lease) => call(lease)), + runExisting: async (operation, call) => { + const result = await this.runtimeRunner.runExisting( + { kind: 'current' }, + operation, + (lease) => call(lease) + ); + if (isRuntimeAbsent(result)) return null; + return result; + }, getSandboxName: () => this.sandboxName, - getNormalizeID: () => this.normalizeId, - beginForward: () => this.beginPreviewForward(), - renewActivity: () => this.renewActivityTimeout() + getNormalizeID: () => this.normalizeId }); this.ctx.blockConcurrencyWhile(async () => { @@ -933,8 +1373,6 @@ export class Sandbox extends Container { ...storedTimeouts }; this.hasStoredContainerTimeouts = true; - // Update the control-channel retry budget to reflect stored timeouts. - this.client.setRetryTimeoutMs(this.computeRetryTimeoutMs()); } // Restore sleep timeout if previously set via RPC @@ -1073,7 +1511,6 @@ export class Sandbox extends Container { await this.ctx.storage.put('containerTimeouts', validated); this.containerTimeouts = validated; this.hasStoredContainerTimeouts = true; - this.client.setRetryTimeoutMs(this.computeRetryTimeoutMs()); this.logger.debug('Container timeouts updated', this.containerTimeouts); } @@ -1210,7 +1647,9 @@ export class Sandbox extends Container { // Assigned synchronously so concurrent callers observe the promise // before any await point inside doDestroy(). - const work = this.doDestroy(); + const work = this.resourceActivityGate.runDestroyTeardown(() => + this.doDestroy() + ); this.inflightDestroy = work; try { await work; @@ -1230,48 +1669,24 @@ export class Sandbox extends Container { let caughtError: Error | undefined; try { + const cleanupRuntime = + await this.runtimeLifecycle.invalidateAndObserveStoredActive(); + // Preview URL auth and activation are cleared before await-heavy // teardown work. Concurrent preview traffic should observe missing // auth or runtime state and fail from DO-owned state without reaching // the container. - await this.previewService.clearPreviewState(); await this.currentLifetime.rotate(); - await this.currentRuntime.clear(); + await this.previewService.clearPreviewState(); - // Unmount all mounted buckets and cleanup before disconnecting the - // control client used by the mount lifecycle RPCs. ({ mountsProcessed, mountFailures } = - await this.bucketMounts.cleanupForDestroy()); - if (mountFailures > 0) { - throw new Error( - `Failed to clean up ${mountFailures} bucket mount${mountFailures === 1 ? '' : 's'} during destroy()` - ); - } - - // Tear down every tunnel this sandbox created — stops the - // container-side cloudflared processes and removes the Cloudflare - // tunnel + DNS resources for named tunnels. Runs before disconnect - // because destroyAll needs the container RPC. Best-effort per port; - // a failure on one doesn't block the rest of teardown. - // - // Lazily build the handler so destroyAll runs even on a sandbox - // that never accessed `tunnels` during its lifetime — storage may - // hold records from a prior lifetime under the same DO id. - try { - this.ensureTunnelsBuilt(); - await this.tunnelServiceHandle?.destroyAll(); - } catch (error) { - this.logger.warn('Failed to tear down tunnels during destroy()', { - error: error instanceof Error ? error.message : String(error) - }); - } - await this.tunnelServiceHandle?.clearDurableStateAfterDestroy(); - - // Disconnect the control client after all cleanup commands complete. - this.client.disconnect(); + await this.runBoundedDestroyRuntimeCleanup(cleanupRuntime)); + const durableTunnels = this.createDurableDestroyTunnelsHandle(); + await durableTunnels.destroyAll(); + await durableTunnels.clearDurableStateAfterDestroy(); - outcome = 'success'; await super.destroy(); + outcome = 'success'; } catch (error) { caughtError = error instanceof Error ? error : new Error(String(error)); throw error; @@ -1287,93 +1702,221 @@ export class Sandbox extends Container { } } - override async onStart() { - this.logger.debug('Sandbox started'); + private async performRuntimeStop( + physicalStop: () => Promise + ): Promise { + await this.runtimeLifecycle.invalidate(); + await this.previewService.clearActivePreviewPorts(); + await physicalStop(); + } - await this.currentRuntime.markStarted(); + private async runBoundedDestroyRuntimeCleanup( + cleanupRuntime: RuntimeIdentity | null + ): Promise<{ + mountsProcessed: number; + mountFailures: number; + }> { + if (!cleanupRuntime) { + return await this.bucketMounts.cleanupForDestroyWithoutRuntime(); + } - // Fire-and-forget: version check is observability, not load-bearing. - this.checkVersionCompatibility().catch((error) => { - this.logger.error( - 'Version compatibility check failed', - error instanceof Error ? error : new Error(String(error)) + let timeoutID: ReturnType | undefined; + const timeout = new Promise<'timeout'>((resolve) => { + timeoutID = setTimeout( + () => resolve('timeout'), + DESTROY_RUNTIME_CLEANUP_TIMEOUT_MS ); }); - - // Reconcile tunnel storage with the fresh container inside - // onStart's blockConcurrencyWhile gate so any get() that arrived - // during startup sees tunnel state for the current runtime. + const cleanup = this.cleanupRuntimeBeforeDestroy(cleanupRuntime); + let result: 'timeout' | { mountsProcessed: number; mountFailures: number }; try { - this.ensureTunnelsBuilt(); - await this.tunnelServiceHandle?.onRuntimeStart(); + result = await Promise.race([cleanup, timeout]); } catch (error) { - this.logger.error( - 'Failed to reconcile tunnel storage after container start', - error instanceof Error ? error : new Error(String(error)) + this.logger.warn('Failed runtime cleanup before destroy()', { + error: error instanceof Error ? error.message : String(error) + }); + return await this.bucketMounts.cleanupForDestroyWithoutRuntime(); + } finally { + if (timeoutID) clearTimeout(timeoutID); + this.runtimeSessions.closeActive(); + } + if (result === 'timeout') { + this.logger.warn('Timed out during runtime cleanup before destroy()', { + timeoutMs: DESTROY_RUNTIME_CLEANUP_TIMEOUT_MS + }); + cleanup.catch((error) => { + this.logger.warn('Late runtime cleanup failed after destroy timeout', { + error: error instanceof Error ? error.message : String(error) + }); + }); + return await this.bucketMounts.cleanupForDestroyWithoutRuntime(); + } + return result; + } + + private async cleanupRuntimeBeforeDestroy( + cleanupRuntime: RuntimeIdentity + ): Promise<{ + mountsProcessed: number; + mountFailures: number; + }> { + const session = await this.runtimeSessions.acquireSession(cleanupRuntime); + const hold = session.retain(); + const runControl = async ( + _operation: string, + call: (control: ContainerControlClient) => Promise + ) => { + if (session.isInterrupted()) return session.interrupted; + return Promise.race([call(session.client), session.interrupted]); + }; + let mountsProcessed = 0; + let mountFailures = 0; + try { + ({ mountsProcessed, mountFailures } = + await this.bucketMounts.cleanupForDestroyUsing(runControl)); + if (mountFailures > 0) { + this.logger.warn('Failed to clean up bucket mounts during destroy()', { + mountFailures + }); + } + } catch (error) { + this.logger.warn('Failed to clean up bucket mounts during destroy()', { + error: error instanceof Error ? error.message : String(error) + }); + } + + try { + const teardownTunnels = this.createTeardownTunnelsHandle( + runControl, + cleanupRuntime ); + await teardownTunnels.destroyAllRuntimeRuns(); + } catch (error) { + this.logger.warn('Failed to tear down tunnels during destroy()', { + error: error instanceof Error ? error.message : String(error) + }); + } finally { + hold.release(); } + + return { mountsProcessed, mountFailures }; } - override async stop( - signal?: Parameters['stop']>[0] - ): Promise { - this.runtimeControlClient.dispose(); - this.client.disconnect(); - await this.currentRuntime.clear(); - await this.previewService.clearActivePreviewPorts(); - await super.stop(signal); + private createTeardownTunnelsHandle( + runControl: ( + operation: string, + call: (control: ContainerControlClient) => Promise + ) => Promise, + cleanupRuntime: RuntimeIdentity + ): TunnelsHandle { + return this.createDestroyTunnelsHandle(async (runtime, operation, call) => { + if ( + runtime.id !== cleanupRuntime.id || + runtime.runtimeIncarnationID !== cleanupRuntime.runtimeIncarnationID + ) { + return null; + } + return await runControl(operation, (control) => + call(control.tunnels as SandboxTunnelsAPI) + ); + }); } - /** - * Check if the container version matches the SDK version - * Logs a warning if there's a mismatch - */ - private async checkVersionCompatibility(): Promise { - const sdkVersion = SDK_VERSION; - let containerVersion: string | undefined; - let outcome: string; + private createDurableDestroyTunnelsHandle(): TunnelsHandle { + return this.createDestroyTunnelsHandle(async () => null); + } - try { - containerVersion = await this.client.utils.getVersion(); + private createDestroyTunnelsHandle( + runExisting: Parameters[0]['runExisting'] + ): TunnelsHandle { + return createTunnelsHandle({ + runProvision: async () => { + throw new Error('Tunnel provisioning is unavailable during teardown'); + }, + runExisting, + getStoredRuntime: (storage) => this.runtimeLifecycle.getStored(storage), + storage: this.ctx.storage, + logger: this.logger, + sandboxId: this.ctx.id.toString(), + currentLifetime: this.currentLifetime, + getNamedTunnelConfig: () => this.namedTunnelConfigResolver.getConfig() + }); + } - if (containerVersion === 'unknown') { - outcome = 'container_version_unknown'; - } else if (containerVersion !== sdkVersion) { - outcome = 'version_mismatch'; - } else { - outcome = 'compatible'; - } - } catch (error) { - outcome = 'check_failed'; - containerVersion = undefined; - } + override async onStart() { + const replacementStartTransitionCompleted = + this.runtimeLifecycle.markRuntimeStarted(); + this.logger.debug('Sandbox started', { + replacementStartTransitionCompleted + }); + } - const successLevel = - outcome === 'compatible' - ? ('debug' as const) - : outcome === 'container_version_unknown' - ? ('info' as const) - : ('warn' as const); // version_mismatch or check_failed + override async start( + options?: Parameters['start']>[0] + ): Promise { + await this.runtimeRunner.runWaking('runtime.start', async () => undefined, { + startOptions: options as ContainerStartConfigOptions | undefined + }); + } - logCanonicalEvent( - this.logger, - { - event: 'version.check', - outcome: 'success', - durationMs: 0, - sdkVersion, - containerVersion: containerVersion ?? 'unknown', - versionOutcome: outcome + override startAndWaitForPorts( + args: StartAndWaitForPortsOptions + ): Promise; + override startAndWaitForPorts( + portsOrArgs?: number | number[] | StartAndWaitForPortsOptions, + cancellationOptions?: CancellationOptions, + startOptions?: ContainerStartConfigOptions + ): Promise; + override async startAndWaitForPorts( + portsOrArgs?: number | number[] | StartAndWaitForPortsOptions, + cancellationOptions?: CancellationOptions, + startOptions?: ContainerStartConfigOptions + ): Promise { + const normalizedOptions = normalizeStartAndWaitForPortsOptions( + portsOrArgs, + cancellationOptions, + startOptions + ); + const ports = normalizePorts(normalizedOptions.ports); + await this.runtimeRunner.runWaking( + 'runtime.port.ready', + async (lease) => { + await Promise.all( + ports + .filter((port) => port !== 3000) + .map((port) => + waitForRuntimePort(lease, port, { + timeout: + normalizedOptions.cancellationOptions?.portReadyTimeoutMS, + interval: normalizedOptions.cancellationOptions?.waitInterval, + signal: normalizedOptions.cancellationOptions?.abort + }) + ) + ); }, - { successLevel } + { + signal: normalizedOptions.cancellationOptions?.abort, + startOptions: normalizedOptions.startOptions + } + ); + } + + override async stop( + signal?: Parameters['stop']>[0] + ): Promise { + await this.resourceActivityGate.runStopTeardown(() => + this.performRuntimeStop(() => super.stop(signal)) ); } override async onStop() { this.logger.debug('Sandbox stopped'); - this.runtimeControlClient.dispose(); - await this.currentRuntime.clear(); + const runtimeStopDisposition = + await this.runtimeLifecycle.reconcileObservedStop(); + this.logger.debug('Sandbox runtime stop reconciled', { + runtimeStopDisposition + }); await this.previewService.clearActivePreviewPorts(); try { @@ -1386,13 +1929,8 @@ export class Sandbox extends Container { ); } - // Stop local sync managers and clear runtime-scoped mount state before - // closing the control client; FUSE cleanup may need container RPC. await this.bucketMounts.cleanupForStop(); - // Disconnect the active client so open sockets do not hold the DO alive. - this.client.disconnect(); - // Port tokens are durable authorization and survive container restarts; // runtime-scoped preview activation is cleared separately above. } @@ -1405,183 +1943,173 @@ export class Sandbox extends Container { } /** - * Override Container.containerFetch to use production-friendly timeouts - * Automatically starts container with longer timeouts if not running + * Override Container.containerFetch to route direct forwarding through runtime admission. */ override async containerFetch( requestOrUrl: Request | string | URL, portOrInit?: number | RequestInit, portParam?: number ): Promise { - // Parse arguments to extract request and port const { request, port } = this.parseContainerFetchArgs( requestOrUrl, portOrInit, portParam ); + const pathname = new URL(request.url).pathname; + if (port === 3000 && pathname === '/rpc') { + throw new SandboxSecurityError( + 'Container RPC connection is not authorized' + ); + } + if (port === 3000 && pathname === '/ws/terminal') { + throw new SandboxSecurityError('Terminal connection is not authorized'); + } + let physicalForwardStarted = false; - const activity = this.resourceActivityGate.beginOperation(); try { - await activity.beforeCall; - - const state = await this.getState(); - const containerRunning = this.ctx.container?.running; - - // Start container if persisted state is not healthy OR if runtime reports container is not running. - // The runtime check catches stale persisted state (e.g., state says 'healthy' after DO recreation - // but Docker container is gone). - const staleStateDetected = - state.status === 'healthy' && containerRunning === false; - if (state.status !== 'healthy' || containerRunning === false) { - try { - await this.startAndWaitForPorts({ - ports: port, - cancellationOptions: { - instanceGetTimeoutMS: this.containerTimeouts.instanceGetTimeoutMS, - portReadyTimeoutMS: this.containerTimeouts.portReadyTimeoutMS, - waitInterval: this.containerTimeouts.waitIntervalMS, - abort: request.signal - } + return await this.runtimeRunner.runWaking( + 'container.fetch', + async (lease) => { + await waitForRuntimePort(lease, port, { + timeout: this.containerTimeouts.portReadyTimeoutMS, + interval: this.containerTimeouts.waitIntervalMS, + signal: request.signal }); - } catch (e) { - // 1. Provisioning: Container VM not yet available - if (this.isNoInstanceError(e)) { - const errorBody: ErrorResponse = { - code: ErrorCode.CONTAINER_UNAVAILABLE, - message: - 'Container is currently provisioning. This can take several minutes on first deployment.', - context: { reason: 'container_starting', retryable: true }, - httpStatus: 503, - timestamp: new Date().toISOString(), - suggestion: - 'The container is still being provisioned. Retry the operation in a moment.' - }; - return new Response(JSON.stringify(errorBody), { - status: 503, - headers: { - 'Content-Type': 'application/json', - 'Retry-After': '10' - } - }); - } - - // 2. Permanent errors: Resource exhaustion, misconfiguration, bad image - // These will never recover on retry — fail fast so the caller gets a clear signal. - // Checked before transient to avoid broad transient patterns (e.g., "container did not - // start") masking specific permanent causes in wrapped error messages. - if (this.isPermanentStartupError(e)) { - this.logger.error( - 'Permanent container startup error, returning 500', - e instanceof Error ? e : new Error(String(e)) - ); - const errorBody: ErrorResponse = { - code: ErrorCode.INTERNAL_ERROR, - message: - 'Container failed to start due to a permanent error. Check your container configuration.', - context: { - phase: 'startup', - error: e instanceof Error ? e.message : String(e) - }, - httpStatus: 500, - timestamp: new Date().toISOString(), - suggestion: - 'This error will not resolve with retries. Check container logs, image name, and resource limits.' - }; - return new Response(JSON.stringify(errorBody), { - status: 500, - headers: { - 'Content-Type': 'application/json' - } - }); - } - - // 3. Transient startup errors: Container starting, port not ready yet - if (this.isTransientStartupError(e)) { - // If startup failed after detecting stale state, the container runtime is likely stuck - // (e.g., workerd can't restart after an unexpected container death). Abort the DO so the - // next request gets a fresh instance with a clean container binding. This mirrors the - // recovery pattern in the base Container class for 'Network connection lost' errors. - if (staleStateDetected) { - this.logger.warn('container.startup', { - outcome: 'stale_state_abort', - staleStateDetected: true, - error: e instanceof Error ? e.message : String(e) - }); - this.ctx.abort(); - } else { - this.logger.debug('container.startup', { - outcome: 'transient_error', - staleStateDetected, - error: e instanceof Error ? e.message : String(e) - }); - } - const errorBody: ErrorResponse = { - code: ErrorCode.CONTAINER_UNAVAILABLE, - message: 'Container is starting. Please retry in a moment.', - context: { reason: 'container_starting', retryable: true }, - httpStatus: 503, - timestamp: new Date().toISOString(), - suggestion: - 'The container is not ready yet. Retry the operation in a moment.' - }; - return new Response(JSON.stringify(errorBody), { - status: 503, - headers: { - 'Content-Type': 'application/json', - 'Retry-After': '3' - } - }); + try { + await this.runtimeLifecycle.assertActive(lease.runtime); + } catch { + throw runtimeInterrupted('container.fetch'); } - - // 4. Unrecognized errors: Treat as transient since retries are safe - // and new platform error messages may not yet be in our pattern list. - this.logger.warn('container.startup', { - outcome: 'unrecognized_error', - staleStateDetected, - error: e instanceof Error ? e.message : String(e) - }); - const errorBody: ErrorResponse = { - code: ErrorCode.CONTAINER_UNAVAILABLE, - message: 'Container is starting. Please retry in a moment.', - context: { reason: 'container_starting', retryable: true }, - httpStatus: 503, - timestamp: new Date().toISOString(), - suggestion: - 'The container is not ready yet. Retry the operation in a moment.' - }; - return new Response(JSON.stringify(errorBody), { - status: 503, - headers: { - 'Content-Type': 'application/json', - 'Retry-After': '5' - } - }); - } + physicalForwardStarted = true; + return retainedResponse( + await this.getRuntimePortStub(port).fetch( + toContainerHTTPRequest(request) + ), + lease, + 'container.fetch.body' + ); + }, + { signal: request.signal } + ); + } catch (error) { + if ( + error instanceof OperationInterruptedError || + physicalForwardStarted + ) { + throw error; } - - // Delegate to parent for the actual fetch (handles TCP port access internally) - return await super.containerFetch(requestOrUrl, portOrInit, portParam); - } finally { - activity.finish(); + if (request.signal.aborted) { + throw request.signal.reason ?? error; + } + const state = await this.getState(); + const staleStateDetected = + state.status === 'healthy' && this.ctx.container?.running === false; + return this.containerStartupErrorResponse(error, staleStateDetected); } } - private async ensureContainerRunning(signal?: AbortSignal): Promise { - const state = await this.getState(); - if (state.status === 'healthy' && this.ctx.container?.running === true) { - return; + private containerStartupErrorResponse( + error: unknown, + staleStateDetected: boolean + ): Response { + if (this.isNoInstanceError(error)) { + const errorBody: ErrorResponse = { + code: ErrorCode.CONTAINER_UNAVAILABLE, + message: + 'Container is currently provisioning. This can take several minutes on first deployment.', + context: { reason: 'container_starting', retryable: true }, + httpStatus: 503, + timestamp: new Date().toISOString(), + suggestion: + 'The container is still being provisioned. Retry the operation in a moment.' + }; + return new Response(JSON.stringify(errorBody), { + status: 503, + headers: { + 'Content-Type': 'application/json', + 'Retry-After': '10' + } + }); } - await this.start({ - envVars: this.envVars, - entrypoint: this.entrypoint, - enableInternet: this.enableInternet, - labels: this.labels - }); + if (this.isPermanentStartupError(error)) { + this.logger.error( + 'Permanent container startup error, returning 500', + error instanceof Error ? error : new Error(String(error)) + ); + const errorBody: ErrorResponse = { + code: ErrorCode.INTERNAL_ERROR, + message: + 'Container failed to start due to a permanent error. Check your container configuration.', + context: { + phase: 'startup', + error: error instanceof Error ? error.message : String(error) + }, + httpStatus: 500, + timestamp: new Date().toISOString(), + suggestion: + 'This error will not resolve with retries. Check container logs, image name, and resource limits.' + }; + return new Response(JSON.stringify(errorBody), { + status: 500, + headers: { 'Content-Type': 'application/json' } + }); + } - if (signal?.aborted) { - throw new Error('Operation was aborted'); + if (this.isTransientStartupError(error)) { + if (staleStateDetected) { + this.logger.warn('container.startup', { + outcome: 'stale_state_abort', + staleStateDetected: true, + error: error instanceof Error ? error.message : String(error) + }); + this.ctx.abort(); + } else { + this.logger.debug('container.startup', { + outcome: 'transient_error', + staleStateDetected, + error: error instanceof Error ? error.message : String(error) + }); + } + const errorBody: ErrorResponse = { + code: ErrorCode.CONTAINER_UNAVAILABLE, + message: 'Container is starting. Please retry in a moment.', + context: { reason: 'container_starting', retryable: true }, + httpStatus: 503, + timestamp: new Date().toISOString(), + suggestion: + 'The container is not ready yet. Retry the operation in a moment.' + }; + return new Response(JSON.stringify(errorBody), { + status: 503, + headers: { + 'Content-Type': 'application/json', + 'Retry-After': '3' + } + }); } + + this.logger.warn('container.startup', { + outcome: 'unrecognized_error', + staleStateDetected, + error: error instanceof Error ? error.message : String(error) + }); + const errorBody: ErrorResponse = { + code: ErrorCode.CONTAINER_UNAVAILABLE, + message: 'Container is starting. Please retry in a moment.', + context: { reason: 'container_starting', retryable: true }, + httpStatus: 503, + timestamp: new Date().toISOString(), + suggestion: + 'The container is not ready yet. Retry the operation in a moment.' + }; + return new Response(JSON.stringify(errorBody), { + status: 503, + headers: { + 'Content-Type': 'application/json', + 'Retry-After': '5' + } + }); } /** @@ -1729,10 +2257,22 @@ export class Sandbox extends Container { if (this.ctx.container?.running !== true) return 'unknown'; return 'available'; }, - processesHasActive: () => - this.client.processesWithoutActivity().hasActive(), - terminalsHasActive: () => - this.client.terminalsWithoutActivity().hasActive() + processesHasActive: async () => { + const result = await this.runtimeRunner.probeExisting( + { kind: 'current' }, + 'processes.hasActive', + (lease) => lease.control.processes.hasActive() + ); + return result === RUNTIME_ABSENT ? false : (result as boolean); + }, + terminalsHasActive: async () => { + const result = await this.runtimeRunner.probeExisting( + { kind: 'current' }, + 'terminals.hasActive', + (lease) => lease.control.terminals.hasActive() + ); + return result === RUNTIME_ABSENT ? false : (result as boolean); + } }, this.keepAliveEnabled ); @@ -1742,27 +2282,6 @@ export class Sandbox extends Container { return (this.ctx as PreviewForwardingContainerState).container; } - private beginPreviewForward(): () => void { - const lifecycle = this as unknown as PreviewForwardingLifecycleState; - lifecycle.inflightRequests = (lifecycle.inflightRequests ?? 0) + 1; - this.renewActivityTimeout(); - - let settled = false; - return () => { - if (settled) { - return; - } - settled = true; - lifecycle.inflightRequests = Math.max( - 0, - (lifecycle.inflightRequests ?? 0) - 1 - ); - if (lifecycle.inflightRequests === 0) { - this.renewActivityTimeout(); - } - }; - } - // Override fetch to route internal container requests to appropriate ports override async fetch(request: Request): Promise { // Extract or generate trace ID from request @@ -1793,14 +2312,13 @@ export class Sandbox extends Container { connectionHeader?.toLowerCase().includes('upgrade'); if (isWebSocket) { - // WebSocket path: Let parent Container class handle WebSocket proxying - // This bypasses containerFetch() which uses JSRPC and cannot handle WebSocket upgrades + const port = this.determinePort(request); try { requestLogger.debug('WebSocket upgrade requested', { path: url.pathname, - port: this.determinePort(url) + port }); - return await super.fetch(request); + return await this.connectWebSocket(request, port); } catch (error) { requestLogger.error( 'WebSocket connection failed', @@ -1812,66 +2330,332 @@ export class Sandbox extends Container { } // Non-WebSocket: Use existing port determination and HTTP routing logic - const port = this.determinePort(url); + const port = this.determinePort(request); // Route to the appropriate port return await this.containerFetch(request, port); } - wsConnect(request: Request, port: number): Promise { - // Stub - actual implementation is attached by getSandbox() on the stub object - throw new Error( - 'wsConnect must be called on the stub returned by getSandbox()' + async wsConnect(request: Request, port: number): Promise { + return await connect(this)(request, port); + } + + async authorizePortRequest(port: number, path: string): Promise { + if (!Number.isInteger(port) || port < 1 || port > 65535) { + throw new SandboxSecurityError( + `Invalid port number: ${port}. Must be between 1 and 65535.` + ); + } + return this.#createPortRequestToken(port, path, 'port'); + } + + #createPortRequestToken( + port: number, + path: string, + scope: 'port' | 'terminal' + ): string { + const now = Date.now(); + for (const [token, route] of this.portRequestTokens) { + if (route.expiresAt <= now) this.portRequestTokens.delete(token); + } + const token = crypto.randomUUID(); + this.portRequestTokens.set(token, { + port, + path, + expiresAt: now + 30_000, + scope + }); + return token; + } + + private connectWebSocket(request: Request, port: number): Promise { + return this.runtimeRunner.runWaking( + 'container.websocket', + async (lease) => { + await waitForRuntimePort(lease, port, { + timeout: this.containerTimeouts.portReadyTimeoutMS, + interval: this.containerTimeouts.waitIntervalMS, + signal: request.signal + }); + try { + await this.runtimeLifecycle.assertActive(lease.runtime); + } catch { + throw runtimeInterrupted('container.websocket'); + } + let interrupted = false; + let retained: ReturnType | undefined; + let releaseHold = () => {}; + const hold = lease.retain(() => { + interrupted = true; + retained?.interrupt(); + releaseHold(); + }); + releaseHold = hold.release; + if (interrupted) { + hold.release(); + throw runtimeInterrupted('container.websocket'); + } + try { + const response = await this.getRuntimePortStub(port).fetch( + toContainerHTTPRequest(request) + ); + retained = retainWebSocketResponse(response, hold.release); + if (interrupted) { + retained.interrupt(); + throw runtimeInterrupted('container.websocket'); + } + return response; + } catch (error) { + hold.release(); + throw error; + } + }, + { signal: request.signal } ); } - async createTerminal(options: CreateTerminalOptions): Promise { - return terminalHandleFromSnapshot( - this.terminalStub(), - await this.client.terminals.create(options) + async createTerminal( + options: CreateTerminalOptions + ): Promise { + return this.runtimeRunner.runWaking('terminal.create', async (lease) => + this.terminalDescriptor( + await lease.control.terminals.create(options), + lease.runtime + ) ); } - async getTerminal(id: string): Promise { - const snapshot = await this.client.terminals.get(id); - return snapshot - ? terminalHandleFromSnapshot(this.terminalStub(), snapshot) - : null; + async getTerminal(id: string): Promise { + const result = await this.runtimeRunner.runExisting( + { kind: 'current' }, + 'terminal.get', + async (lease) => { + const snapshot = await lease.control.terminals.get(id); + return snapshot + ? this.terminalDescriptor(snapshot, lease.runtime) + : null; + } + ); + if (isRuntimeAbsent(result)) return null; + return result; } - async listTerminals(): Promise { - return (await this.client.terminals.list()).map((snapshot) => - terminalHandleFromSnapshot(this.terminalStub(), snapshot) + async listTerminals(): Promise { + const result = await this.runtimeRunner.runExisting( + { kind: 'current' }, + 'terminal.list', + async (lease) => + (await lease.control.terminals.list()).map((snapshot) => + this.terminalDescriptor(snapshot, lease.runtime) + ) ); + if (isRuntimeAbsent(result)) return []; + return result; } - private terminalStub() { - const terminals = this.client.terminals; + private terminalDescriptor( + snapshot: TerminalSnapshot, + runtime: RuntimeIdentity + ): TerminalRPCDescriptor { + const stub = this.terminalStub(runtime); + const control: TerminalCapabilityControl = { + get: (id) => stub.get(id), + openOutput: (id, options) => stub.output(id, options), + write: (id, data) => stub.write(id, data), + resize: (id, cols, rows) => stub.resize(id, cols, rows), + interrupt: (id) => stub.interrupt(id), + terminate: (id) => stub.terminate(id), + authorizeConnection: async () => + this.#createPortRequestToken(3000, '/ws/terminal', 'terminal') + }; return { - create: (options: CreateTerminalOptions) => terminals.create(options), - get: (id: string) => terminals.get(id), - list: () => terminals.list(), - output: (id: string, options?: Parameters[1]) => - terminals.output(id, options), - write: (id: string, data: Uint8Array) => terminals.write(id, data), + snapshot, + runtimeIncarnationID: runtime.runtimeIncarnationID, + capability: new TerminalCapabilityTarget(snapshot.id, control) + }; + } + + private terminalStub(runtime: RuntimeIdentity): TerminalHandleStub { + const runTerminal = async ( + terminalId: string, + operation: string, + call: (lease: RuntimeLease) => Promise + ): Promise => { + let result: T | typeof RUNTIME_ABSENT; + try { + result = await this.runtimeRunner.runExisting( + { kind: 'runtime', runtime }, + operation, + call + ); + } catch (error) { + if (error instanceof OperationInterruptedError) { + throw staleTerminal(terminalId, operation); + } + throw error; + } + if (!isRuntimeAbsent(result)) return result; + throw staleTerminal(terminalId, operation); + }; + return { + create: (options: CreateTerminalOptions) => + this.runtimeRunner.runWaking('terminal.create', (lease) => + lease.control.terminals.create(options) + ), + get: (id: string) => + runTerminal(id, 'terminal.getSnapshot', (lease) => + lease.control.terminals.get(id) + ), + list: async (): Promise => { + const result = await this.runtimeRunner.runExisting( + { kind: 'current' }, + 'terminal.list', + (lease) => lease.control.terminals.list() + ); + if (isRuntimeAbsent(result)) return []; + return result; + }, + output: ( + id: string, + options?: Parameters[1] + ) => + runTerminal(id, 'terminal.output', async (lease) => { + let retained: InterruptibleTerminalSubscription | undefined; + let releaseHold = () => {}; + const hold = lease.retain(() => { + retained?.interrupt(); + releaseHold(); + }); + releaseHold = hold.release; + try { + const subscription = await lease.control.terminals.output( + id, + options + ); + retained = retainedTerminalSubscription( + id, + 'terminal.output', + subscription, + hold.release + ); + return new PullSubscriptionTarget(await retained.stream()); + } catch (error) { + hold.release(); + throw error; + } + }), + write: (id: string, data: Uint8Array) => + runTerminal(id, 'terminal.write', (lease) => + lease.control.terminals.write(id, data) + ), resize: (id: string, cols: number, rows: number) => - terminals.resize(id, cols, rows), - interrupt: (id: string) => terminals.interrupt(id), - terminate: (id: string) => terminals.terminate(id), - hasActive: () => terminals.hasActive(), - fetch: (request: Request) => this.fetch(request) + runTerminal(id, 'terminal.resize', (lease) => + lease.control.terminals.resize(id, cols, rows) + ), + interrupt: (id: string) => + runTerminal(id, 'terminal.interrupt', (lease) => + lease.control.terminals.interrupt(id) + ), + terminate: (id: string) => + runTerminal(id, 'terminal.terminate', (lease) => + lease.control.terminals.terminate(id) + ), + hasActive: () => + this.runtimeRunner + .probeExisting({ kind: 'current' }, 'terminals.hasActive', (lease) => + lease.control.terminals.hasActive() + ) + .then((result) => (isRuntimeAbsent(result) ? false : result)), + fetch: (request: Request) => + runTerminal('', 'terminal.connect', async (lease) => { + let interrupted = false; + let retained: ReturnType | undefined; + let releaseHold = () => {}; + const hold = lease.retain(() => { + interrupted = true; + retained?.interrupt(); + releaseHold(); + }); + releaseHold = hold.release; + if (interrupted) { + hold.release(); + throw staleTerminal('', 'terminal.connect'); + } + try { + const response = await this.getRuntimePortStub(3000).fetch( + toContainerHTTPRequest(request) + ); + retained = retainWebSocketResponse(response, hold.release); + if (interrupted) { + retained.interrupt(); + throw staleTerminal('', 'terminal.connect'); + } + return response; + } catch (error) { + hold.release(); + throw error; + } + }) }; } - private determinePort(url: URL): number { - // Direct DO fetch compatibility path used by switchPort()/wsConnect(). - // Public preview URL traffic enters through proxyPreviewRequest() instead. - const proxyMatch = url.pathname.match(/^\/proxy\/(\d+)/); - if (proxyMatch) { - return parseInt(proxyMatch[1], 10); + private determinePort(request: Request): number { + const targetPort = Number(request.headers.get('cf-container-target-port')); + const routeToken = request.headers.get('x-sandbox-port-route-token'); + const pathname = new URL(request.url).pathname; + const targetsControlPlane = + !request.headers.has('cf-container-target-port') || targetPort === 3000; + + if (routeToken) { + const route = this.portRequestTokens.get(routeToken); + this.portRequestTokens.delete(routeToken); + if ( + route && + route.expiresAt > Date.now() && + route.port === targetPort && + route.path === new URL(request.url).pathname + ) { + if ( + pathname === '/ws/terminal' && + targetsControlPlane && + route.scope !== 'terminal' + ) { + throw new SandboxSecurityError( + 'Terminal connection is not authorized' + ); + } + if (pathname === '/rpc' && targetsControlPlane) { + throw new SandboxSecurityError( + 'Container RPC connection is not authorized' + ); + } + return targetPort; + } + } + + if (pathname === '/ws/terminal' && targetsControlPlane) { + throw new SandboxSecurityError('Terminal connection is not authorized'); + } + if (pathname === '/rpc' && targetsControlPlane) { + throw new SandboxSecurityError( + 'Container RPC connection is not authorized' + ); } - // Direct fetch compatibility defaults to the container server port. + if (request.headers.has('cf-container-target-port')) { + if ( + !Number.isInteger(targetPort) || + targetPort < 1 || + targetPort > 65535 + ) { + throw new SandboxSecurityError( + `Invalid container target port: ${request.headers.get('cf-container-target-port')}` + ); + } + return targetPort; + } + + // Direct fetches target the container server port. // SDK control operations use ContainerControlClient over /rpc instead. return 3000; } @@ -1883,27 +2667,34 @@ export class Sandbox extends Container { const argv = validateExecArgv(command); const startTime = Date.now(); const commandText = argv.join(' '); - const activity = this.resourceActivityGate.beginOperation(); - + const canMergeEnvironment = + options.env === undefined || isPlainObject(options.env); + const launchOptions = + canMergeEnvironment && + (Object.keys(this.envVars).length > 0 || options.env !== undefined) + ? { + ...options, + env: mergeExecEnvironment(this.envVars, options.env ?? {}) + } + : options; try { - await activity.beforeCall; - await this.ensureContainerRunning(); - const owningRuntime = await this.currentRuntime.get(); - if (!owningRuntime) { - throw runtimeInterrupted('process.start', 'none'); - } - await this.assertLaunchRuntime(owningRuntime, 'none'); - - const status = await this.client.processes.start(argv, options); - await this.assertLaunchRuntime(owningRuntime, 'unknown'); - const descriptor = this.processDescriptor(status, owningRuntime); + const descriptor = await this.runtimeRunner.runWaking( + 'process.start', + async (lease) => { + const status = await lease.control.processes.start( + argv, + launchOptions + ); + return this.processDescriptor(status, lease.runtime); + } + ); logCanonicalEvent(this.logger, { event: 'sandbox.exec', outcome: 'success', command: commandText, - processId: status.id, - pid: status.pid, + processId: descriptor.id, + pid: descriptor.pid, durationMs: Date.now() - startTime, origin: 'user' }); @@ -1921,20 +2712,6 @@ export class Sandbox extends Container { errorMessage: execError.message }); throw error; - } finally { - activity.finish(); - } - } - - private async assertLaunchRuntime( - runtime: RuntimeIdentity, - effect: 'none' | 'unknown' - ): Promise { - try { - await this.currentRuntime.assertActive(runtime); - } catch { - this.runtimeControlClient.dispose(); - throw runtimeInterrupted('process.start', effect); } } @@ -1942,10 +2719,7 @@ export class Sandbox extends Container { status: ProcessStatus, runtime: RuntimeIdentity ): ProcessRPCDescriptor { - const lifecycle = this.processCapabilityLifecycle(runtime, { - id: status.id, - pid: status.pid - }); + const lifecycle = this.processCapabilityLifecycle(runtime, status); return { id: status.id, pid: status.pid, @@ -1960,61 +2734,90 @@ export class Sandbox extends Container { private processCapabilityLifecycle( owningRuntime: RuntimeIdentity, - process: { id: string; pid: number } + process: Pick ) { - const lifecycle = new ProcessLifecycle({ - currentRuntime: this.currentRuntime, - runtimeClient: this.runtimeControlClient, - beginNonWakingOperation: () => - this.resourceActivityGate.beginNonWakingOperation(), - process - }); + const stale = (error: unknown) => { + if (error instanceof OperationInterruptedError) { + throw new StaleProcessHandleError({ + code: ErrorCode.STALE_PROCESS_HANDLE, + message: 'Process handle refers to a previous runtime incarnation', + context: { + processId: process.id, + pid: process.pid, + operation: 'process.handle' + }, + httpStatus: 409, + timestamp: new Date().toISOString() + }); + } + throw error; + }; return { - runRead: ( - _runtime: { readonly id: string }, + runRead: async ( + _runtime: import('./processes/process-capability').ProcessCapabilityRuntime, operation: string, call: (control: ProcessCapabilityControl) => Promise - ) => - lifecycle.runRead(owningRuntime, operation, (client) => - call(processCapabilityControl(client)) - ), - runControl: ( - _runtime: { readonly id: string }, + ) => { + const result = await this.runtimeRunner + .runExisting( + { kind: 'runtime', runtime: owningRuntime }, + operation, + (lease) => call(processCapabilityControl(lease)) + ) + .catch(stale); + if (result === RUNTIME_ABSENT) stale(runtimeInterrupted(operation)); + return result as T; + }, + runControl: async ( + _runtime: import('./processes/process-capability').ProcessCapabilityRuntime, operation: string, call: (control: ProcessCapabilityControl) => Promise - ) => - lifecycle.runControl(owningRuntime, operation, (client) => - call(processCapabilityControl(client)) - ) + ) => { + const result = await this.runtimeRunner + .runExisting( + { kind: 'runtime', runtime: owningRuntime }, + operation, + (lease) => call(processCapabilityControl(lease)) + ) + .catch(stale); + if (result === RUNTIME_ABSENT) stale(runtimeInterrupted(operation)); + return result as T; + } }; } /** Internal bridge liveness read that neither starts nor renews a runtime. */ async isRuntimeActive(): Promise { - return (await this.processLifecycle.captureCurrent()) !== null; + return (await this.runtimeLifecycle.get()) !== null; } async getProcess(id: string): Promise { - const runtime = await this.processLifecycle.captureCurrent(); - if (!runtime) return null; - const status = await this.processLifecycle.runRead( - runtime, + const result = await this.runtimeRunner.runExisting( + { kind: 'current' }, 'process.get', - (client) => client.processesWithoutActivity().get(id) + async (lease) => { + const status = await lease.control.processes.get(id); + return status ? this.processDescriptor(status, lease.runtime) : null; + } ); - return status ? this.processDescriptor(status, runtime) : null; + return result === RUNTIME_ABSENT + ? null + : (result as ProcessRPCDescriptor | null); } async listProcesses(): Promise { - const runtime = await this.processLifecycle.captureCurrent(); - if (!runtime) return []; - return this.processLifecycle.runRead(runtime, 'process.list', (client) => - client.processesWithoutActivity().list() + const result = await this.runtimeRunner.runExisting( + { kind: 'current' }, + 'process.list', + (lease) => lease.control.processes.list() ); + return result === RUNTIME_ABSENT ? [] : (result as ProcessStatus[]); } async mkdir(path: string, options: { recursive?: boolean } = {}) { - return this.client.files.mkdir(path, { recursive: options.recursive }); + return this.runtimeRunner.runWaking('files.mkdir', (lease) => + lease.control.files.mkdir(path, { recursive: options.recursive }) + ); } async writeFile( @@ -2022,25 +2825,40 @@ export class Sandbox extends Container { content: string | ReadableStream, options: { encoding?: string } = {} ) { - if (content instanceof ReadableStream) { - return this.client.files.writeFileStream(path, content); - } - - return this.client.files.writeFile(path, content, { - encoding: options.encoding + return this.runtimeRunner.runWaking('files.write', async (lease) => { + if (content instanceof ReadableStream) { + const retained = retainedStream(content, lease); + try { + return await lease.control.files.writeFileStream( + path, + retained.stream + ); + } finally { + retained.cancel('writeFileStream completed'); + } + } + return lease.control.files.writeFile(path, content, { + encoding: options.encoding + }); }); } async deleteFile(path: string) { - return this.client.files.deleteFile(path); + return this.runtimeRunner.runWaking('files.delete', (lease) => + lease.control.files.deleteFile(path) + ); } async renameFile(oldPath: string, newPath: string) { - return this.client.files.renameFile(oldPath, newPath); + return this.runtimeRunner.runWaking('files.rename', (lease) => + lease.control.files.renameFile(oldPath, newPath) + ); } async moveFile(sourcePath: string, destinationPath: string) { - return this.client.files.moveFile(sourcePath, destinationPath); + return this.runtimeRunner.runWaking('files.move', (lease) => + lease.control.files.moveFile(sourcePath, destinationPath) + ); } /** @@ -2065,10 +2883,18 @@ export class Sandbox extends Container { path: string, options: { encoding?: FileEncoding } = {} ): Promise { - if (options.encoding === 'none') { - return this.client.files.readFile(path, { encoding: options.encoding }); - } - return this.client.files.readFile(path, { encoding: options.encoding }); + return this.runtimeRunner.runWaking('files.read', async (lease) => { + if (options.encoding === 'none') { + const result = await lease.control.files.readFile(path, { + encoding: 'none' + }); + return { + ...result, + content: retainStream(result.content, lease) + }; + } + return lease.control.files.readFile(path, { encoding: options.encoding }); + }); } /** @@ -2081,10 +2907,14 @@ export class Sandbox extends Container { root: '/workspace' } ): Promise { - const result = await this.client.workspace.createArchive({ - root: options.root, - excludes: options.excludes ?? [] - }); + const result = await this.runtimeRunner.runWaking( + 'workspace.archive.create', + (lease) => + lease.control.workspace.createArchive({ + root: options.root, + excludes: options.excludes ?? [] + }) + ); return result.archivePath; } @@ -2092,30 +2922,42 @@ export class Sandbox extends Container { root: string; archivePath: string; }): Promise { - await this.client.workspace.extractArchive(options); + await this.runtimeRunner.runWaking('workspace.archive.extract', (lease) => + lease.control.workspace.extractArchive(options) + ); } async cleanupWorkspaceArchive(archivePath: string): Promise { - await this.client.workspace.cleanupArchive(archivePath); + await this.runtimeRunner.runWaking('workspace.archive.cleanup', (lease) => + lease.control.workspace.cleanupArchive(archivePath) + ); } async cleanupMountDirectory(mountPath: string): Promise { - await this.client.mounts.removeMountDirectory({ - path: mountPath, - onlyIfNotMountpoint: true - }); + await this.runtimeRunner.runWaking('mounts.cleanup', (lease) => + lease.control.mounts.removeMountDirectory({ + path: mountPath, + onlyIfNotMountpoint: true + }) + ); } async readFileStream(path: string): Promise> { - return this.client.files.readFileStream(path); + return this.runtimeRunner.runWaking('files.stream.read', async (lease) => + retainStream(await lease.control.files.readFileStream(path), lease) + ); } async listFiles(path: string, options?: ListFilesOptions) { - return this.client.files.listFiles(path, options); + return this.runtimeRunner.runWaking('files.list', (lease) => + lease.control.files.listFiles(path, options) + ); } async exists(path: string) { - return this.client.files.exists(path); + return this.runtimeRunner.runWaking('files.exists', (lease) => + lease.control.files.exists(path) + ); } /** @@ -2135,14 +2977,19 @@ export class Sandbox extends Container { path: string, options: WatchOptions = {} ): Promise> { - return openRemoteSubscription( - this.client.watch.watch({ - path, - recursive: options.recursive, - include: options.include, - exclude: options.exclude - }), - { operation: 'open filesystem watch' } + return this.runtimeRunner.runWaking('watch.open', async (lease) => + retainStream( + await openRemoteSubscription( + lease.control.watch.watch({ + path, + recursive: options.recursive, + include: options.include, + exclude: options.exclude + }), + { operation: 'open filesystem watch', protocol: 'stream' } + ), + lease + ) ); } @@ -2160,33 +3007,15 @@ export class Sandbox extends Container { path: string, options: CheckChangesOptions = {} ): Promise { - return this.client.watch.checkChanges({ - path, - recursive: options.recursive, - include: options.include, - exclude: options.exclude, - since: options.since - }); - } - - private async ensureRuntimeActiveForPreview(): Promise { - const activity = this.resourceActivityGate.beginOperation(); - try { - await activity.beforeCall; - await this.startAndWaitForPorts({ - ports: this.defaultPort, - cancellationOptions: { - instanceGetTimeoutMS: this.containerTimeouts.instanceGetTimeoutMS, - portReadyTimeoutMS: this.containerTimeouts.portReadyTimeoutMS, - waitInterval: this.containerTimeouts.waitIntervalMS - } - }); - - const runtime = await this.currentRuntime.get(); - return runtime ?? (await this.currentRuntime.markStarted()); - } finally { - activity.finish(); - } + return this.runtimeRunner.runWaking('watch.checkChanges', (lease) => + lease.control.watch.checkChanges({ + path, + recursive: options.recursive, + include: options.include, + exclude: options.exclude, + since: options.since + }) + ); } /** @@ -2218,6 +3047,27 @@ export class Sandbox extends Container { * }); * // url: https://8080-sandbox-id-my_token_v1.example.com */ + private async runWakingComposite( + operation: string, + call: (lease: RuntimeLease) => Promise + ): Promise { + let callbackOutcome: Promise | undefined; + try { + return await this.runtimeRunner.runWaking(operation, (lease) => { + callbackOutcome = call(lease); + return callbackOutcome; + }); + } catch (error) { + await callbackOutcome?.catch(() => undefined); + throw error; + } + } + + [sandboxRuntimeCall] = (async (operation, call) => + await this.runWakingComposite(operation, (lease) => + call(extensionRuntimeControl(lease.control)) + )) as ExtensionRuntimeCall; + async exposePort( port: number, options: { name?: string; hostname: string; token?: string } @@ -2296,11 +3146,26 @@ export class Sandbox extends Container { private ensureTunnelsBuilt(): void { if (this.tunnelsHandler) return; const built = createTunnelsHandle({ - client: this.client, + runProvision: (call) => + this.runWakingComposite('tunnel.provision', (lease) => + call({ + runtime: lease.runtime, + tunnels: lease.control.tunnels, + retain: lease.retain + }) + ), + runExisting: async (runtime, operation, call) => { + const result = await this.runtimeRunner.runExisting( + { kind: 'runtime', runtime }, + operation, + (lease) => call(lease.control.tunnels) + ); + return isRuntimeAbsent(result) ? null : result; + }, + getStoredRuntime: (storage) => this.runtimeLifecycle.getStored(storage), storage: this.ctx.storage, logger: this.logger, sandboxId: this.ctx.id.toString(), - currentRuntime: this.currentRuntime, currentLifetime: this.currentLifetime, getNamedTunnelConfig: () => this.namedTunnelConfigResolver.getConfig() }); diff --git a/packages/sandbox/src/storage-mount/lifecycle-cleanup.ts b/packages/sandbox/src/storage-mount/lifecycle-cleanup.ts index ecdc20a3b..9935ebc64 100644 --- a/packages/sandbox/src/storage-mount/lifecycle-cleanup.ts +++ b/packages/sandbox/src/storage-mount/lifecycle-cleanup.ts @@ -28,7 +28,7 @@ export interface BucketMountDestroyCleanupResult { export interface BucketMountLifecycleCleanupHost { registry: MountRegistry; logger: Logger; - getS3FSHost(): S3FSHost; + s3fsHost: S3FSHost | null; getOutboundHost(): MountOutboundHost; runMountOperation(operation: () => Promise): Promise; } @@ -49,7 +49,11 @@ export async function cleanupBucketMountsForDestroy( mountsProcessed++; if (mountInfo.mountType === 'local-sync') { try { - await mountInfo.syncManager.stop(); + if (host.s3fsHost) { + await mountInfo.syncManager.stop(); + } else { + mountInfo.syncManager.interrupt(); + } mountInfo.mounted = false; cleanedMountPaths.push(mountPath); cleanedMountIds.push(mountInfo.mountId); @@ -70,36 +74,38 @@ export async function cleanupBucketMountsForDestroy( } let supportFilesSafeToDelete = false; - try { - supportFilesSafeToDelete = await unmountTrackedFuseMount( - host.getS3FSHost(), - mountPath, - mountInfo - ); - } catch (error) { - mountFailures++; - host.logger.warn( - `Failed to unmount bucket ${mountInfo.bucket} from ${mountPath}: ${error instanceof Error ? error.message : String(error)}` - ); - } - if (supportFilesSafeToDelete) { - await deletePasswordFile( - host.getS3FSHost(), - mountInfo.passwordFilePath - ); - if (mountInfo.additionalHeaderFilePath) { - await deleteAdditionalHeaderFile( - host.getS3FSHost(), - mountInfo.additionalHeaderFilePath + if (host.s3fsHost) { + try { + supportFilesSafeToDelete = await unmountTrackedFuseMount( + host.s3fsHost, + mountPath, + mountInfo + ); + } catch (error) { + mountFailures++; + host.logger.warn( + `Failed to unmount bucket ${mountInfo.bucket} from ${mountPath}: ${error instanceof Error ? error.message : String(error)}` ); } - cleanedMountPaths.push(mountPath); - cleanedMountIds.push(mountInfo.mountId); - - if (mountInfo.mountType === 'fuse' && mountInfo.credentialProxy) { - evictSigV4ClientCacheEntry(mountInfo.mountId); - evictDirectoryMarkerCacheForMount(mountInfo.mountId); + if (supportFilesSafeToDelete) { + await deletePasswordFile(host.s3fsHost, mountInfo.passwordFilePath); + if (mountInfo.additionalHeaderFilePath) { + await deleteAdditionalHeaderFile( + host.s3fsHost, + mountInfo.additionalHeaderFilePath + ); + } } + } else if (mountInfo.mounted) { + mountFailures++; + } + + cleanedMountPaths.push(mountPath); + cleanedMountIds.push(mountInfo.mountId); + + if (mountInfo.mountType === 'fuse' && mountInfo.credentialProxy) { + evictSigV4ClientCacheEntry(mountInfo.mountId); + evictDirectoryMarkerCacheForMount(mountInfo.mountId); } } } @@ -135,10 +141,8 @@ export async function cleanupBucketMountsForDestroy( } } - if (mountFailures === 0) { - for (const mountPath of cleanedMountPaths) { - host.registry.delete(mountPath); - } + for (const mountPath of cleanedMountPaths) { + host.registry.delete(mountPath); } return { mountsProcessed, mountFailures }; @@ -153,7 +157,7 @@ export async function cleanupBucketMountsForStop( let hadCredentialProxyMount = false; for (const [, mountInfo] of host.registry) { if (mountInfo.mountType === 'local-sync') { - await mountInfo.syncManager.stop().catch(() => {}); + mountInfo.syncManager.interrupt(); } else if (mountInfo.mountType === 'r2-egress') { hadR2EgressMount = true; } else if (mountInfo.mountType === 'fuse' && mountInfo.credentialProxy) { diff --git a/packages/sandbox/src/storage-mount/lifecycle.ts b/packages/sandbox/src/storage-mount/lifecycle.ts index 1d02b7888..3df6df701 100644 --- a/packages/sandbox/src/storage-mount/lifecycle.ts +++ b/packages/sandbox/src/storage-mount/lifecycle.ts @@ -1,34 +1,29 @@ -import type { - CurrentRuntimeIdentity, - RuntimeIdentity -} from '../current-runtime-identity'; +import type { RuntimeIdentity, RuntimeIdentityReader } from '../runtime'; import type { CurrentSandboxLifetime, SandboxLifetime } from '../sandbox-lifetime'; export type MountLifecycleSnapshot = { - runtime: RuntimeIdentity | null; + runtime: RuntimeIdentity; lifetime: SandboxLifetime; }; export class MountLifecycle { constructor( - private readonly currentRuntime: CurrentRuntimeIdentity, + private readonly runtimeReader: RuntimeIdentityReader, private readonly currentLifetime: CurrentSandboxLifetime ) {} - async capture(): Promise { + async capture(runtime: RuntimeIdentity): Promise { return { - runtime: await this.currentRuntime.get(), + runtime, lifetime: await this.currentLifetime.getOrCreate() }; } async assertCurrent(snapshot: MountLifecycleSnapshot): Promise { - if (snapshot.runtime) { - await this.currentRuntime.assertActive(snapshot.runtime); - } await this.currentLifetime.assertCurrent(snapshot.lifetime); + await this.runtimeReader.assertActive(snapshot.runtime); } } diff --git a/packages/sandbox/src/storage-mount/operations/context.ts b/packages/sandbox/src/storage-mount/operations/context.ts index f8a94b20c..d2ee9a673 100644 --- a/packages/sandbox/src/storage-mount/operations/context.ts +++ b/packages/sandbox/src/storage-mount/operations/context.ts @@ -1,12 +1,13 @@ -import type { Logger, SandboxMountsAPI } from '@repo/shared'; +import type { Logger } from '@repo/shared'; import type { MountOutboundHost } from '../outbound'; import type { MountRegistry } from '../registry'; +import type { MountRuntimeCall } from '../runtime-call'; import type { S3FSHost } from '../s3fs'; export interface BucketMountOperationContext { registry: MountRegistry; logger: Logger; - getMounts(): SandboxMountsAPI; + runRuntimeCall: MountRuntimeCall; getOutboundHost(): MountOutboundHost; - getS3FSHost(): S3FSHost; + s3fsHost: S3FSHost | null; } diff --git a/packages/sandbox/src/storage-mount/operations/fuse-cleanup.ts b/packages/sandbox/src/storage-mount/operations/fuse-cleanup.ts index 6b25943f6..e9780037e 100644 --- a/packages/sandbox/src/storage-mount/operations/fuse-cleanup.ts +++ b/packages/sandbox/src/storage-mount/operations/fuse-cleanup.ts @@ -4,10 +4,16 @@ export async function unmountFuseIfMountedForCleanup( context: BucketMountOperationContext, mountPath: string ): Promise { - const mounts = context.getMounts(); - if (!(await mounts.isMountpoint(mountPath))) return true; + const isMountpoint = await context.runRuntimeCall( + 'mount.cleanup.isMountpoint', + (control) => control.mounts.isMountpoint(mountPath) + ); + if (!isMountpoint) return true; - const result = await mounts.unmountFuse(mountPath); + const result = await context.runRuntimeCall( + 'mount.cleanup.unmountFuse', + (control) => control.mounts.unmountFuse(mountPath) + ); if (result.success) return true; context.logger.warn('FUSE mount cleanup unmount failed', { diff --git a/packages/sandbox/src/storage-mount/operations/local-sync-mount.ts b/packages/sandbox/src/storage-mount/operations/local-sync-mount.ts index d19d2aa61..32c7cff53 100644 --- a/packages/sandbox/src/storage-mount/operations/local-sync-mount.ts +++ b/packages/sandbox/src/storage-mount/operations/local-sync-mount.ts @@ -1,18 +1,49 @@ import type { LocalMountBucketOptions } from '@repo/shared'; import { logCanonicalEvent } from '@repo/shared'; -import type { ContainerControlClient } from '../../container-control'; import { LocalMountSyncManager } from '../../local-mount-sync'; import { InvalidMountConfigError } from '../errors'; import type { MountLifecycle } from '../lifecycle'; import type { MountRegistry } from '../registry'; +import type { MountRuntimeLease } from '../runtime-call'; +import type { S3FSHost } from '../s3fs'; import type { LocalSyncMountInfo } from '../types'; import { isR2Bucket } from '../validation'; import type { BucketMountOperationContext } from './context'; export interface LocalSyncMountContext extends BucketMountOperationContext { getEnv(): unknown; - getClient(): ContainerControlClient; lifecycle: MountLifecycle; + runtime: MountRuntimeLease['runtime']; + s3fsHost: S3FSHost; + retainRuntime: MountRuntimeLease['retain']; +} + +export function validateLocalSyncMount( + context: Pick, + bucket: string, + mountPath: string +): R2Bucket { + const envObj = context.getEnv() as Record; + const r2Binding = envObj[bucket]; + if (!r2Binding || !isR2Bucket(r2Binding)) { + throw new InvalidMountConfigError( + `R2 binding "${bucket}" not found in env or is not an R2Bucket. ` + + 'Make sure the binding name matches your wrangler.jsonc R2 binding.' + ); + } + + if (!mountPath || !mountPath.startsWith('/')) { + throw new InvalidMountConfigError( + `Invalid mount path: "${mountPath}". Must be an absolute path starting with /` + ); + } + + if (context.registry.has(mountPath)) { + throw new InvalidMountConfigError( + `Mount path already in use: ${mountPath}` + ); + } + return r2Binding; } export async function mountLocalSyncBucket( @@ -25,33 +56,19 @@ export async function mountLocalSyncBucket( let mountOutcome: 'success' | 'error' = 'error'; let mountError: Error | undefined; try { - const envObj = context.getEnv() as Record; - const r2Binding = envObj[bucket]; - if (!r2Binding || !isR2Bucket(r2Binding)) { - throw new InvalidMountConfigError( - `R2 binding "${bucket}" not found in env or is not an R2Bucket. ` + - 'Make sure the binding name matches your wrangler.jsonc R2 binding.' - ); - } - - if (!mountPath || !mountPath.startsWith('/')) { - throw new InvalidMountConfigError( - `Invalid mount path: "${mountPath}". Must be an absolute path starting with /` - ); - } + const r2Binding = validateLocalSyncMount(context, bucket, mountPath); - if (context.registry.has(mountPath)) { - throw new InvalidMountConfigError( - `Mount path already in use: ${mountPath}` - ); - } - - const syncManager = new LocalMountSyncManager({ + let syncManager: LocalMountSyncManager | null = null; + const runtimeHold = context.retainRuntime(() => { + syncManager?.interrupt(); + }); + syncManager = new LocalMountSyncManager({ bucket: r2Binding, mountPath, prefix: options.prefix, readOnly: options.readOnly ?? false, - client: context.getClient(), + runRuntimeCall: context.runRuntimeCall, + runtimeHold, logger: context.logger }); @@ -63,7 +80,7 @@ export async function mountLocalSyncBucket( syncManager, mounted: false }; - const lifecycle = await context.lifecycle.capture(); + const lifecycle = await context.lifecycle.capture(context.runtime); try { await syncManager.start(); await context.lifecycle.assertCurrent(lifecycle); diff --git a/packages/sandbox/src/storage-mount/operations/r2-egress-mount.ts b/packages/sandbox/src/storage-mount/operations/r2-egress-mount.ts index 4b4e122f4..cd4a52de3 100644 --- a/packages/sandbox/src/storage-mount/operations/r2-egress-mount.ts +++ b/packages/sandbox/src/storage-mount/operations/r2-egress-mount.ts @@ -4,6 +4,8 @@ import { InvalidMountConfigError } from '../errors'; import type { MountLifecycle } from '../lifecycle'; import { configureR2EgressOutbound } from '../outbound'; import { buildR2EgressParams } from '../outbound/params'; +import type { MountRuntimeLease } from '../runtime-call'; +import type { S3FSHost } from '../s3fs'; import { createDisableExpectHeaderFile, createPasswordFile, @@ -22,6 +24,43 @@ import { unmountFuseIfMountedForCleanup } from './fuse-cleanup'; export interface R2EgressMountContext extends BucketMountOperationContext { lifecycle: MountLifecycle; + runtime: MountRuntimeLease['runtime']; + s3fsHost: S3FSHost; +} + +export function validateR2EgressMount( + context: Pick, + bucket: string, + mountPath: string, + options: R2BindingMountBucketOptions +): void { + const prefix = options.prefix; + validateBucketBindingName(bucket, mountPath); + validateMountPath(context.registry.activeMounts, mountPath); + validateProtectedS3fsOptions(options.s3fsOptions, 'R2 binding'); + + for (const [existingMountPath, existingMount] of context.registry) { + if ( + existingMount.mountType === 'r2-egress' && + existingMount.bucket === bucket && + existingMount.prefix !== prefix + ) { + throw new InvalidMountConfigError( + `R2 binding "${bucket}" is already mounted at ${existingMountPath} with a different prefix. ` + + 'Mount the same binding only once, or use the same prefix for additional mounts.' + ); + } + if ( + existingMount.mountType === 'r2-egress' && + existingMount.bucket === bucket && + existingMount.readOnly !== (options.readOnly ?? false) + ) { + throw new InvalidMountConfigError( + `R2 binding "${bucket}" is already mounted at ${existingMountPath} with a different readOnly setting. ` + + 'Mount the same binding only once, or use the same readOnly value for additional mounts.' + ); + } + } } export async function mountR2EgressBucket( @@ -40,42 +79,17 @@ export async function mountR2EgressBucket( let mountInfo: R2BindingMountInfo | undefined; try { - validateBucketBindingName(bucket, mountPath); - validateMountPath(context.registry.activeMounts, mountPath); - validateProtectedS3fsOptions(options.s3fsOptions, 'R2 binding'); + validateR2EgressMount(context, bucket, mountPath, options); - for (const [existingMountPath, existingMount] of context.registry) { - if ( - existingMount.mountType === 'r2-egress' && - existingMount.bucket === bucket && - existingMount.prefix !== prefix - ) { - throw new InvalidMountConfigError( - `R2 binding "${bucket}" is already mounted at ${existingMountPath} with a different prefix. ` + - 'Mount the same binding only once, or use the same prefix for additional mounts.' - ); - } - if ( - existingMount.mountType === 'r2-egress' && - existingMount.bucket === bucket && - existingMount.readOnly !== (options.readOnly ?? false) - ) { - throw new InvalidMountConfigError( - `R2 binding "${bucket}" is already mounted at ${existingMountPath} with a different readOnly setting. ` + - 'Mount the same binding only once, or use the same readOnly value for additional mounts.' - ); - } - } - - const lifecycle = await context.lifecycle.capture(); + const lifecycle = await context.lifecycle.capture(context.runtime); passwordFilePath = generatePasswordFilePath(); additionalHeaderFilePath = generateS3FSAdditionalHeaderFilePath(); - await createPasswordFile(context.getS3FSHost(), passwordFilePath, bucket, { + await createPasswordFile(context.s3fsHost, passwordFilePath, bucket, { accessKeyId: 'x', secretAccessKey: 'x' }); await createDisableExpectHeaderFile( - context.getS3FSHost(), + context.s3fsHost, additionalHeaderFilePath ); @@ -100,10 +114,12 @@ export async function mountR2EgressBucket( } }); - await context.getMounts().ensureDirectory(mountPath); + await context.runRuntimeCall('mount.ensureDirectory', (control) => + control.mounts.ensureDirectory(mountPath) + ); s3fsStarted = true; - await executeS3FSMount(context.getS3FSHost(), { + await executeS3FSMount(context.s3fsHost, { bucket, mountPath, provider: 'r2', @@ -158,13 +174,13 @@ export async function mountR2EgressBucket( if (cleanupPasswordFilePath) { await deletePasswordFile( - context.getS3FSHost(), + context.s3fsHost, cleanupPasswordFilePath ).catch(() => {}); } if (cleanupAdditionalHeaderFilePath) { await deleteAdditionalHeaderFile( - context.getS3FSHost(), + context.s3fsHost, cleanupAdditionalHeaderFilePath ).catch(() => {}); } diff --git a/packages/sandbox/src/storage-mount/operations/remote-fuse-mount.ts b/packages/sandbox/src/storage-mount/operations/remote-fuse-mount.ts index fdea1ee83..abf0afad6 100644 --- a/packages/sandbox/src/storage-mount/operations/remote-fuse-mount.ts +++ b/packages/sandbox/src/storage-mount/operations/remote-fuse-mount.ts @@ -8,6 +8,8 @@ import { evictDirectoryMarkerCacheForMount, evictSigV4ClientCacheEntry } from '../outbound/s3-credential-proxy-handler'; +import type { MountRuntimeLease } from '../runtime-call'; +import type { S3FSHost } from '../s3fs'; import { createDisableExpectHeaderFile, createPasswordFile, @@ -32,6 +34,55 @@ export interface RemoteFuseMountContext extends BucketMountOperationContext { getR2AccessKeyID(): string | null; getR2SecretAccessKey(): string | null; lifecycle: MountLifecycle; + runtime: MountRuntimeLease['runtime']; + s3fsHost: S3FSHost; +} + +export function validateRemoteFuseMount( + context: Pick< + RemoteFuseMountContext, + | 'registry' + | 'logger' + | 'getEnv' + | 'getEnvVars' + | 'getR2AccessKeyID' + | 'getR2SecretAccessKey' + >, + bucket: string, + mountPath: string, + options: RemoteMountBucketOptions +): { + provider: BucketProvider | null; + credentials: ReturnType; +} { + const prefix = options.prefix; + validateRemoteMountOptions(context.registry.activeMounts, bucket, mountPath, { + ...options, + prefix + }); + const provider = options.provider || detectProviderFromUrl(options.endpoint); + context.logger.debug(`Detected provider: ${provider || 'unknown'}`, { + explicitProvider: options.provider, + prefix + }); + const envObj = context.getEnv() as Record; + const envCredentials = { + AWS_ACCESS_KEY_ID: getEnvString(envObj, 'AWS_ACCESS_KEY_ID'), + AWS_SECRET_ACCESS_KEY: getEnvString(envObj, 'AWS_SECRET_ACCESS_KEY'), + R2_ACCESS_KEY_ID: context.getR2AccessKeyID() || undefined, + R2_SECRET_ACCESS_KEY: context.getR2SecretAccessKey() || undefined + }; + const credentials = detectCredentials(options, { + ...envCredentials, + ...context.getEnvVars() + }); + if (options.credentialProxy === true) { + validateProtectedS3fsOptions(options.s3fsOptions, 'credential proxy', [ + 'ahbe_conf', + 'use_path_request_style' + ]); + } + return { provider, credentials }; } export async function mountRemoteFuseBucket( @@ -52,43 +103,17 @@ export async function mountRemoteFuseBucket( let credentialProxyMountId: string | undefined; let mountInfo: FuseMountInfo | undefined; try { - validateRemoteMountOptions( - context.registry.activeMounts, + const validation = validateRemoteFuseMount( + context, bucket, mountPath, - { - ...options, - prefix - } + options ); - const s3fsSource = buildS3fsSource(bucket, prefix); - provider = options.provider || detectProviderFromUrl(options.endpoint); - - context.logger.debug(`Detected provider: ${provider || 'unknown'}`, { - explicitProvider: options.provider, - prefix - }); - - const envObj = context.getEnv() as Record; - const envCredentials = { - AWS_ACCESS_KEY_ID: getEnvString(envObj, 'AWS_ACCESS_KEY_ID'), - AWS_SECRET_ACCESS_KEY: getEnvString(envObj, 'AWS_SECRET_ACCESS_KEY'), - R2_ACCESS_KEY_ID: context.getR2AccessKeyID() || undefined, - R2_SECRET_ACCESS_KEY: context.getR2SecretAccessKey() || undefined - }; - const credentials = detectCredentials(options, { - ...envCredentials, - ...context.getEnvVars() - }); + provider = validation.provider; + const credentials = validation.credentials; credentialProxyEnabled = options.credentialProxy === true; - if (credentialProxyEnabled) { - validateProtectedS3fsOptions(options.s3fsOptions, 'credential proxy', [ - 'ahbe_conf', - 'use_path_request_style' - ]); - } passwordFilePath = generatePasswordFilePath(); if (credentialProxyEnabled) { @@ -121,10 +146,10 @@ export async function mountRemoteFuseBucket( } : {}) }; - const lifecycle = await context.lifecycle.capture(); + const lifecycle = await context.lifecycle.capture(context.runtime); await createPasswordFile( - context.getS3FSHost(), + context.s3fsHost, passwordFilePath, bucket, credentialProxyEnabled @@ -134,7 +159,7 @@ export async function mountRemoteFuseBucket( if (credentialProxyEnabled) { if (additionalHeaderFilePath) { await createDisableExpectHeaderFile( - context.getS3FSHost(), + context.s3fsHost, additionalHeaderFilePath ); } @@ -146,8 +171,12 @@ export async function mountRemoteFuseBucket( ); } - dirExisted = await context.getMounts().pathExists(mountPath); - await context.getMounts().ensureDirectory(mountPath); + dirExisted = await context.runRuntimeCall('mount.pathExists', (control) => + control.mounts.pathExists(mountPath) + ); + await context.runRuntimeCall('mount.ensureDirectory', (control) => + control.mounts.ensureDirectory(mountPath) + ); const effectiveOptions: RemoteMountBucketOptions = credentialProxyEnabled ? { @@ -163,7 +192,7 @@ export async function mountRemoteFuseBucket( ] } : options; - await executeS3FSMount(context.getS3FSHost(), { + await executeS3FSMount(context.s3fsHost, { bucket: s3fsSource, mountPath, options: effectiveOptions, @@ -195,11 +224,11 @@ export async function mountRemoteFuseBucket( if (supportFilesSafeToDelete) { if (passwordFilePath) { - await deletePasswordFile(context.getS3FSHost(), passwordFilePath); + await deletePasswordFile(context.s3fsHost, passwordFilePath); } if (additionalHeaderFilePath) { await deleteAdditionalHeaderFile( - context.getS3FSHost(), + context.s3fsHost, additionalHeaderFilePath ); } @@ -207,10 +236,12 @@ export async function mountRemoteFuseBucket( if (!dirExisted) { try { - await context.getMounts().removeMountDirectory({ - path: mountPath, - onlyIfNotMountpoint: false - }); + await context.runRuntimeCall('mount.removeMountDirectory', (control) => + control.mounts.removeMountDirectory({ + path: mountPath, + onlyIfNotMountpoint: false + }) + ); } catch { // best-effort cleanup } diff --git a/packages/sandbox/src/storage-mount/operations/unmount.ts b/packages/sandbox/src/storage-mount/operations/unmount.ts index 97c4c14de..f1f5c0089 100644 --- a/packages/sandbox/src/storage-mount/operations/unmount.ts +++ b/packages/sandbox/src/storage-mount/operations/unmount.ts @@ -32,13 +32,20 @@ export async function unmountBucketOperation( } if (mountInfo.mountType === 'local-sync') { - await mountInfo.syncManager.stop(); + if (context.s3fsHost) { + await mountInfo.syncManager.stop(); + } else { + mountInfo.syncManager.interrupt(); + } mountInfo.mounted = false; context.registry.delete(mountPath); } else { let unmounted = !mountInfo.mounted; - if (mountInfo.mounted) { - const result = await context.getMounts().unmountFuse(mountPath); + if (mountInfo.mounted && context.s3fsHost) { + const result = await context.runRuntimeCall( + 'mount.unmountFuse', + (control) => control.mounts.unmountFuse(mountPath) + ); if (!result.success) { const stderr = result.stderr || 'unknown error'; throw new BucketUnmountError( @@ -47,6 +54,9 @@ export async function unmountBucketOperation( } mountInfo.mounted = false; unmounted = true; + } else if (mountInfo.mounted) { + mountInfo.mounted = false; + unmounted = false; } if (mountInfo.mountType === 'r2-egress') { @@ -70,16 +80,22 @@ export async function unmountBucketOperation( context.registry.delete(mountPath); try { - const cleanup = await context.getMounts().removeMountDirectory({ - path: mountPath, - onlyIfNotMountpoint: true - }); - if (!cleanup.success) { - context.logger.warn('mount directory removal failed', { - mountPath, - exitCode: cleanup.exitCode, - stderr: cleanup.stderr - }); + if (context.s3fsHost) { + const cleanup = await context.runRuntimeCall( + 'mount.removeMountDirectory', + (control) => + control.mounts.removeMountDirectory({ + path: mountPath, + onlyIfNotMountpoint: true + }) + ); + if (!cleanup.success) { + context.logger.warn('mount directory removal failed', { + mountPath, + exitCode: cleanup.exitCode, + stderr: cleanup.stderr + }); + } } } catch (err) { context.logger.warn('mount directory removal failed', { @@ -88,14 +104,11 @@ export async function unmountBucketOperation( }); } - if (unmounted) { - await deletePasswordFile( - context.getS3FSHost(), - mountInfo.passwordFilePath - ); + if (unmounted && context.s3fsHost) { + await deletePasswordFile(context.s3fsHost, mountInfo.passwordFilePath); if (mountInfo.additionalHeaderFilePath) { await deleteAdditionalHeaderFile( - context.getS3FSHost(), + context.s3fsHost, mountInfo.additionalHeaderFilePath ); } diff --git a/packages/sandbox/src/storage-mount/runtime-call.ts b/packages/sandbox/src/storage-mount/runtime-call.ts new file mode 100644 index 000000000..82acb1c47 --- /dev/null +++ b/packages/sandbox/src/storage-mount/runtime-call.ts @@ -0,0 +1,31 @@ +import type { ContainerControlClient } from '../container-control'; +import type { RuntimeIdentity } from '../runtime'; + +export type MountRuntimeHold = { release(): void }; + +export type MountRuntimeLease = { + runtime: RuntimeIdentity; + control: ContainerControlClient; + retain(onInterrupt?: () => void): MountRuntimeHold; +}; + +export type MountRuntimeCall = ( + operation: string, + call: (control: ContainerControlClient) => Promise +) => Promise; + +export type MountRuntimeAttempt = ( + operation: string, + call: (lease: MountRuntimeLease) => Promise +) => Promise; + +export type MountExistingRuntimeAttempt = ( + operation: string, + call: (lease: MountRuntimeLease) => Promise +) => Promise<{ status: 'absent' } | { status: 'completed'; value: T }>; + +export function callWithMountControl( + control: ContainerControlClient +): MountRuntimeCall { + return async (_operation, call) => await call(control); +} diff --git a/packages/sandbox/src/storage-mount/s3fs/host.ts b/packages/sandbox/src/storage-mount/s3fs/host.ts index 116c075d9..51ac1108b 100644 --- a/packages/sandbox/src/storage-mount/s3fs/host.ts +++ b/packages/sandbox/src/storage-mount/s3fs/host.ts @@ -1,7 +1,7 @@ import type { Logger } from '@repo/shared'; -import type { ContainerControlClient } from '../../container-control'; +import type { MountRuntimeCall } from '../runtime-call'; export interface S3FSHost { - client: ContainerControlClient; + runRuntimeCall: MountRuntimeCall; logger: Logger; } diff --git a/packages/sandbox/src/storage-mount/s3fs/index.ts b/packages/sandbox/src/storage-mount/s3fs/index.ts index 7532fd9cc..c5382f6c9 100644 --- a/packages/sandbox/src/storage-mount/s3fs/index.ts +++ b/packages/sandbox/src/storage-mount/s3fs/index.ts @@ -108,11 +108,13 @@ export async function executeS3FSMount( url: params.options.endpoint, ...(params.options.readOnly ? { ro: true } : {}) }; - const result = await host.client.mounts.mountS3FSAndVerify({ - source: params.bucket, - mountPath: params.mountPath, - options: s3fsOptions - }); + const result = await host.runRuntimeCall('mount.s3fs.mount', (control) => + control.mounts.mountS3FSAndVerify({ + source: params.bucket, + mountPath: params.mountPath, + options: s3fsOptions + }) + ); if (result.success) return; const detail = result.stdout?.trim() || result.stderr?.trim() || ''; @@ -136,7 +138,9 @@ export async function unmountTrackedFuseMount( if (!mountInfo.mounted) return true; host.logger.debug(`Unmounting bucket ${mountInfo.bucket} from ${mountPath}`); - const result = await host.client.mounts.unmountFuse(mountPath); + const result = await host.runRuntimeCall('mount.s3fs.unmount', (control) => + control.mounts.unmountFuse(mountPath) + ); if (!result.success) { throw new Error( `fusermount -u failed (exit ${result.exitCode}): ${result.stderr || 'unknown error'}` diff --git a/packages/sandbox/src/storage-mount/s3fs/support-files.ts b/packages/sandbox/src/storage-mount/s3fs/support-files.ts index b8bb417ab..6a77246ed 100644 --- a/packages/sandbox/src/storage-mount/s3fs/support-files.ts +++ b/packages/sandbox/src/storage-mount/s3fs/support-files.ts @@ -15,11 +15,12 @@ export async function createDisableExpectHeaderFile( host: S3FSHost, headerFilePath: string ): Promise { - await host.client.files.writeFile( - headerFilePath, - S3FS_DISABLE_EXPECT_HEADER_CONFIG + await host.runRuntimeCall('mount.s3fs.writeHeaderFile', (control) => + control.files.writeFile(headerFilePath, S3FS_DISABLE_EXPECT_HEADER_CONFIG) + ); + await host.runRuntimeCall('mount.s3fs.chmodHeaderFile', (control) => + control.mounts.chmodOwnerOnly(headerFilePath) ); - await host.client.mounts.chmodOwnerOnly(headerFilePath); } export async function createPasswordFile( @@ -29,8 +30,12 @@ export async function createPasswordFile( credentials: BucketCredentials ): Promise { const content = `${bucket}:${credentials.accessKeyId}:${credentials.secretAccessKey}`; - await host.client.files.writeFile(passwordFilePath, content); - await host.client.mounts.chmodOwnerOnly(passwordFilePath); + await host.runRuntimeCall('mount.s3fs.writePasswordFile', (control) => + control.files.writeFile(passwordFilePath, content) + ); + await host.runRuntimeCall('mount.s3fs.chmodPasswordFile', (control) => + control.mounts.chmodOwnerOnly(passwordFilePath) + ); } export async function deletePasswordFile( @@ -38,7 +43,9 @@ export async function deletePasswordFile( passwordFilePath: string ): Promise { try { - await host.client.mounts.deleteFile(passwordFilePath); + await host.runRuntimeCall('mount.s3fs.deletePasswordFile', (control) => + control.mounts.deleteFile(passwordFilePath) + ); } catch (error) { host.logger.warn('password file cleanup failed', { passwordFilePath, @@ -52,7 +59,9 @@ export async function deleteAdditionalHeaderFile( headerFilePath: string ): Promise { try { - await host.client.mounts.deleteFile(headerFilePath); + await host.runRuntimeCall('mount.s3fs.deleteHeaderFile', (control) => + control.mounts.deleteFile(headerFilePath) + ); } catch (error) { host.logger.warn('s3fs additional header file cleanup failed', { headerFilePath, diff --git a/packages/sandbox/src/storage-mount/service.ts b/packages/sandbox/src/storage-mount/service.ts index d54bf374b..123632a7f 100644 --- a/packages/sandbox/src/storage-mount/service.ts +++ b/packages/sandbox/src/storage-mount/service.ts @@ -5,8 +5,8 @@ import type { R2BindingMountBucketOptions, RemoteMountBucketOptions } from '@repo/shared'; -import type { ContainerControlClient } from '../container-control'; -import type { CurrentRuntimeIdentity } from '../current-runtime-identity'; +import { OperationInterruptedError } from '../errors'; +import type { RuntimeIdentityReader } from '../runtime'; import type { CurrentSandboxLifetime } from '../sandbox-lifetime'; import { InvalidMountConfigError } from './errors'; import { MountLifecycle } from './lifecycle'; @@ -16,21 +16,36 @@ import { cleanupBucketMountsForStop } from './lifecycle-cleanup'; import { MountOperationQueue } from './operation-queue'; -import { mountLocalSyncBucket } from './operations/local-sync-mount'; -import { mountR2EgressBucket } from './operations/r2-egress-mount'; -import { mountRemoteFuseBucket } from './operations/remote-fuse-mount'; +import { + mountLocalSyncBucket, + validateLocalSyncMount +} from './operations/local-sync-mount'; +import { + mountR2EgressBucket, + validateR2EgressMount +} from './operations/r2-egress-mount'; +import { + mountRemoteFuseBucket, + validateRemoteFuseMount +} from './operations/remote-fuse-mount'; import { unmountBucketOperation } from './operations/unmount'; import type { MountOutboundHost } from './outbound'; import { MountRegistry } from './registry'; -import type { S3FSHost } from './s3fs'; +import { + callWithMountControl, + type MountExistingRuntimeAttempt, + type MountRuntimeAttempt, + type MountRuntimeCall +} from './runtime-call'; import { isR2Bucket, validateBucketName, validatePrefix } from './validation'; export interface BucketMountServiceDeps { getEnv(): unknown; getEnvVars(): Record; - getClient(): ContainerControlClient; + runMountAttempt: MountRuntimeAttempt; + runExistingMountAttempt: MountExistingRuntimeAttempt; logger: Logger; - currentRuntime: CurrentRuntimeIdentity; + runtimeReader: RuntimeIdentityReader; currentLifetime: CurrentSandboxLifetime; getR2AccessKeyID(): string | null; getR2SecretAccessKey(): string | null; @@ -41,18 +56,13 @@ export class BucketMountService { private readonly registry = new MountRegistry(); private readonly operations = new MountOperationQueue(); private readonly lifecycle: MountLifecycle; - constructor(private readonly deps: BucketMountServiceDeps) { this.lifecycle = new MountLifecycle( - deps.currentRuntime, + deps.runtimeReader, deps.currentLifetime ); } - private get client(): ContainerControlClient { - return this.deps.getClient(); - } - /** * Mount an S3-compatible bucket as a local directory. * @@ -116,28 +126,33 @@ export class BucketMountService { mountPath: string, options: LocalMountBucketOptions ): Promise { - await mountLocalSyncBucket( + validateLocalSyncMount( { registry: this.registry, - logger: this.deps.logger, - getMounts: () => this.client.mounts, - getOutboundHost: () => this.deps.getOutboundHost(), - getS3FSHost: () => this.getS3FSHost(), - getEnv: () => this.deps.getEnv(), - getClient: () => this.client, - lifecycle: this.lifecycle + getEnv: () => this.deps.getEnv() }, bucket, - mountPath, - options + mountPath ); - } - - private getS3FSHost(): S3FSHost { - return { - client: this.client, - logger: this.deps.logger - }; + await this.deps.runMountAttempt('mount.local', async (lease) => { + const runRuntimeCall = callWithMountControl(lease.control); + await mountLocalSyncBucket( + { + registry: this.registry, + logger: this.deps.logger, + runRuntimeCall, + getOutboundHost: () => this.deps.getOutboundHost(), + s3fsHost: { runRuntimeCall, logger: this.deps.logger }, + getEnv: () => this.deps.getEnv(), + lifecycle: this.lifecycle, + runtime: lease.runtime, + retainRuntime: lease.retain + }, + bucket, + mountPath, + options + ); + }); } private async mountBucketR2Egress( @@ -145,19 +160,29 @@ export class BucketMountService { mountPath: string, options: R2BindingMountBucketOptions ): Promise { - await mountR2EgressBucket( - { - registry: this.registry, - logger: this.deps.logger, - getMounts: () => this.client.mounts, - getOutboundHost: () => this.deps.getOutboundHost(), - getS3FSHost: () => this.getS3FSHost(), - lifecycle: this.lifecycle - }, + validateR2EgressMount( + { registry: this.registry }, bucket, mountPath, options ); + await this.deps.runMountAttempt('mount.r2-egress', async (lease) => { + const runRuntimeCall = callWithMountControl(lease.control); + await mountR2EgressBucket( + { + registry: this.registry, + logger: this.deps.logger, + runRuntimeCall, + getOutboundHost: () => this.deps.getOutboundHost(), + s3fsHost: { runRuntimeCall, logger: this.deps.logger }, + lifecycle: this.lifecycle, + runtime: lease.runtime + }, + bucket, + mountPath, + options + ); + }); } private async mountBucketFuse( @@ -165,23 +190,40 @@ export class BucketMountService { mountPath: string, options: RemoteMountBucketOptions ): Promise { - await mountRemoteFuseBucket( + validateRemoteFuseMount( { registry: this.registry, logger: this.deps.logger, - getMounts: () => this.client.mounts, - getOutboundHost: () => this.deps.getOutboundHost(), - getS3FSHost: () => this.getS3FSHost(), getEnv: () => this.deps.getEnv(), getEnvVars: () => this.deps.getEnvVars(), getR2AccessKeyID: () => this.deps.getR2AccessKeyID(), - getR2SecretAccessKey: () => this.deps.getR2SecretAccessKey(), - lifecycle: this.lifecycle + getR2SecretAccessKey: () => this.deps.getR2SecretAccessKey() }, bucket, mountPath, options ); + await this.deps.runMountAttempt('mount.fuse', async (lease) => { + const runRuntimeCall = callWithMountControl(lease.control); + await mountRemoteFuseBucket( + { + registry: this.registry, + logger: this.deps.logger, + runRuntimeCall, + getOutboundHost: () => this.deps.getOutboundHost(), + s3fsHost: { runRuntimeCall, logger: this.deps.logger }, + getEnv: () => this.deps.getEnv(), + getEnvVars: () => this.deps.getEnvVars(), + getR2AccessKeyID: () => this.deps.getR2AccessKeyID(), + getR2SecretAccessKey: () => this.deps.getR2SecretAccessKey(), + lifecycle: this.lifecycle, + runtime: lease.runtime + }, + bucket, + mountPath, + options + ); + }); } /** @@ -197,33 +239,86 @@ export class BucketMountService { } private async unmountBucketUnlocked(mountPath: string): Promise { + try { + const result = await this.deps.runExistingMountAttempt( + 'mount.unmount', + async (lease) => { + const runRuntimeCall = callWithMountControl(lease.control); + await unmountBucketOperation( + { + registry: this.registry, + logger: this.deps.logger, + runRuntimeCall, + getOutboundHost: () => this.deps.getOutboundHost(), + s3fsHost: { runRuntimeCall, logger: this.deps.logger } + }, + mountPath + ); + } + ); + if (result.status === 'completed') return; + } catch (error) { + if (!(error instanceof OperationInterruptedError)) throw error; + if (!this.registry.has(mountPath)) return; + } + await this.unmountBucketWithoutRuntime(mountPath); + } + + private async unmountBucketWithoutRuntime(mountPath: string): Promise { await unmountBucketOperation( { registry: this.registry, logger: this.deps.logger, - getMounts: () => this.client.mounts, + runRuntimeCall: async () => { + throw new Error('runtime is not active'); + }, getOutboundHost: () => this.deps.getOutboundHost(), - getS3FSHost: () => this.getS3FSHost() + s3fsHost: null }, mountPath ); } async cleanupForDestroy(): Promise { + const result = await this.deps.runExistingMountAttempt( + 'mount.destroyCleanup', + async (lease) => + await this.cleanupForDestroyUsing(callWithMountControl(lease.control)) + ); + if (result.status === 'completed') return result.value; + return this.cleanupForDestroyWithoutRuntime(); + } + + async cleanupForDestroyUsing( + runRuntimeCall: MountRuntimeCall + ): Promise { return cleanupBucketMountsForDestroy({ registry: this.registry, logger: this.deps.logger, - getS3FSHost: () => this.getS3FSHost(), + s3fsHost: { + runRuntimeCall, + logger: this.deps.logger + }, getOutboundHost: () => this.deps.getOutboundHost(), runMountOperation: (operation) => this.operations.run(operation) }); } + async cleanupForDestroyWithoutRuntime(): Promise { + return cleanupBucketMountsForDestroy({ + registry: this.registry, + logger: this.deps.logger, + s3fsHost: null, + getOutboundHost: () => this.deps.getOutboundHost(), + runMountOperation: (operation) => operation() + }); + } + async cleanupForStop(): Promise { return cleanupBucketMountsForStop({ registry: this.registry, logger: this.deps.logger, - getS3FSHost: () => this.getS3FSHost(), + s3fsHost: null, getOutboundHost: () => this.deps.getOutboundHost(), runMountOperation: (operation) => this.operations.run(operation) }); diff --git a/packages/sandbox/src/tunnels/lifecycle.ts b/packages/sandbox/src/tunnels/lifecycle.ts index 74f811e01..ee5fd4f37 100644 --- a/packages/sandbox/src/tunnels/lifecycle.ts +++ b/packages/sandbox/src/tunnels/lifecycle.ts @@ -1,129 +1,6 @@ -import type { - CurrentRuntimeIdentity, - RuntimeIdentity -} from '../current-runtime-identity'; -import { RuntimeIdentityInactiveError } from '../current-runtime-identity'; import { ErrorCode, OperationInterruptedError } from '../errors'; -import type { - CurrentSandboxLifetime, - SandboxLifetime -} from '../sandbox-lifetime'; -import { SandboxLifetimeChangedError } from '../sandbox-lifetime'; -export interface TunnelLifecycleHost { - currentRuntime?: Pick; - currentLifetime?: Pick< - CurrentSandboxLifetime, - 'getOrCreate' | 'assertCurrent' - >; -} - -export interface TunnelLifecycleSnapshot { - runtime?: RuntimeIdentity; - lifetime?: SandboxLifetime; -} - -const TUNNEL_GET_MAX_RECOVERY_ATTEMPTS = 2; - -export class TunnelOperationLifecycle { - readonly #host: TunnelLifecycleHost; - - constructor(host: TunnelLifecycleHost) { - this.#host = host; - } - - async capture(): Promise { - return { - runtime: await this.#captureRuntime(), - lifetime: await this.#host.currentLifetime?.getOrCreate() - }; - } - - async requireRuntime( - snapshot: TunnelLifecycleSnapshot, - phase: string, - admitted: true | 'unknown' - ): Promise { - if (snapshot.runtime) return snapshot; - const runtime = await this.#captureRuntime(); - if (runtime) { - return { ...snapshot, runtime }; - } - if (this.#host.currentRuntime) { - throw createTunnelInterruptedError({ - reason: 'runtime_replaced', - phase, - admitted, - retryable: true, - message: 'Tunnel operation was interrupted by a runtime replacement' - }); - } - return snapshot; - } - - async assertActive( - snapshot: TunnelLifecycleSnapshot, - phase: string, - admitted: true | 'unknown' - ): Promise { - try { - if (snapshot.runtime) { - await this.#host.currentRuntime?.assertActive(snapshot.runtime); - } - if (snapshot.lifetime) { - await this.#host.currentLifetime?.assertCurrent(snapshot.lifetime); - } - } catch (error) { - if (error instanceof RuntimeIdentityInactiveError) { - throw createTunnelInterruptedError({ - reason: 'runtime_replaced', - phase, - admitted, - retryable: true, - message: 'Tunnel operation was interrupted by a runtime replacement' - }); - } - if (error instanceof SandboxLifetimeChangedError) { - throw createTunnelInterruptedError({ - reason: 'sandbox_lifetime_changed', - phase, - admitted, - retryable: false, - message: - 'Tunnel operation was interrupted by a sandbox lifetime change' - }); - } - throw error; - } - } - - async runGetWithRecovery(attempt: () => Promise): Promise { - let recoveryAttempts = 0; - while (true) { - try { - return await attempt(); - } catch (error) { - if (!(error instanceof OperationInterruptedError)) throw error; - if (!error.context.retryable) throw error; - if (recoveryAttempts >= TUNNEL_GET_MAX_RECOVERY_ATTEMPTS) { - throw createTunnelRecoveryExhaustedError(error, recoveryAttempts); - } - recoveryAttempts += 1; - } - } - } - - async #captureRuntime(): Promise { - const currentRuntime = this.#host.currentRuntime; - if (!currentRuntime) return undefined; - const runtime = await currentRuntime.get(); - if (!runtime) return undefined; - await currentRuntime.assertActive(runtime); - return runtime; - } -} - -function createTunnelInterruptedError(params: { +export function createTunnelInterruptedError(params: { reason: 'runtime_replaced' | 'sandbox_lifetime_changed'; phase: string; admitted: true | 'unknown'; @@ -145,23 +22,3 @@ function createTunnelInterruptedError(params: { suggestion: 'Retry tunnels.get() with the same port and options.' }); } - -function createTunnelRecoveryExhaustedError( - error: OperationInterruptedError, - recoveryAttempts: number -): OperationInterruptedError { - return new OperationInterruptedError({ - message: 'Tunnel operation recovery attempts were exhausted', - code: ErrorCode.OPERATION_INTERRUPTED, - httpStatus: 409, - context: { - ...error.context, - reason: 'recovery_exhausted', - retryable: true, - recoveryAttempts, - maxRecoveryAttempts: TUNNEL_GET_MAX_RECOVERY_ATTEMPTS - }, - timestamp: new Date().toISOString(), - suggestion: 'Retry tunnels.get() with the same port and options.' - }); -} diff --git a/packages/sandbox/src/tunnels/provisioner.ts b/packages/sandbox/src/tunnels/provisioner.ts index af09ff8de..c4b5de730 100644 --- a/packages/sandbox/src/tunnels/provisioner.ts +++ b/packages/sandbox/src/tunnels/provisioner.ts @@ -5,7 +5,6 @@ import type { QuickTunnelInfo, SandboxTunnelsAPI } from '@repo/shared'; -import { RPCTransportError } from '../errors'; import { createTunnel, findTunnelByName, @@ -21,12 +20,7 @@ import { type TunnelMetaEntry } from './storage'; -interface TunnelsRPCClient { - tunnels: SandboxTunnelsAPI; -} - export interface TunnelProvisionerHost { - client: TunnelsRPCClient; sandboxId?: string; getNamedTunnelConfig?: () => Promise<{ token: string; @@ -53,10 +47,6 @@ function createQuickTunnelId(): string { return `quick-${randomId()}`; } -// Replays use the same request so container runId idempotency can resolve -// an ambiguous transport failure. -const TUNNEL_RUN_TRANSPORT_REPLAY_ATTEMPTS = 1; - export class TunnelProvisioner { readonly #host: TunnelProvisionerHost; #zoneNamePromise: Promise | null = null; @@ -66,11 +56,12 @@ export class TunnelProvisioner { } async provisionQuickTunnel( + tunnels: SandboxTunnelsAPI, port: number, tunnelRunId: string, tunnelId = createQuickTunnelId() ): Promise { - const result = await this.#ensureTunnelRun({ + const result = await this.#ensureTunnelRun(tunnels, { mode: 'quick', tunnelId, runId: tunnelRunId, @@ -196,10 +187,11 @@ export class TunnelProvisioner { } async startNamedTunnelRun( + tunnels: SandboxTunnelsAPI, prepared: PreparedNamedTunnel, tunnelRunId: string ): Promise { - const result = await this.#ensureTunnelRun({ + const result = await this.#ensureTunnelRun(tunnels, { mode: 'named', tunnelId: prepared.tunnelId, runId: tunnelRunId, @@ -216,22 +208,10 @@ export class TunnelProvisioner { } async #ensureTunnelRun( + tunnels: SandboxTunnelsAPI, request: EnsureTunnelRunRequest ): Promise { - let replays = 0; - while (true) { - try { - return await this.#host.client.tunnels.ensureTunnelRun(request); - } catch (error) { - if ( - !(error instanceof RPCTransportError) || - replays >= TUNNEL_RUN_TRANSPORT_REPLAY_ATTEMPTS - ) { - throw error; - } - replays += 1; - } - } + return await tunnels.ensureTunnelRun(request); } async #getZoneName(config: { diff --git a/packages/sandbox/src/tunnels/restart.ts b/packages/sandbox/src/tunnels/restart.ts index a4a884365..478f168fd 100644 --- a/packages/sandbox/src/tunnels/restart.ts +++ b/packages/sandbox/src/tunnels/restart.ts @@ -13,9 +13,8 @@ import { /** * Reconcile storage with a fresh container. * - * Called from `Sandbox.onStart()` after every container restart. The - * `cloudflared` processes the container was running all died with it, so - * any stored record is not currently backed by a running tunnel. + * Called after a proven runtime replacement or stop. The old runtime's + * `cloudflared` processes no longer back any stored tunnel record. * * Quick tunnels are dropped because the `*.trycloudflare.com` URL is bound * to the dead process. Named tunnels keep private metadata so a later diff --git a/packages/sandbox/src/tunnels/rpc-target.ts b/packages/sandbox/src/tunnels/rpc-target.ts index b655eea9d..88d640abc 100644 --- a/packages/sandbox/src/tunnels/rpc-target.ts +++ b/packages/sandbox/src/tunnels/rpc-target.ts @@ -46,11 +46,25 @@ export function createTunnelsHandle(host: TunnelServiceHost): TunnelsHandle { return { tunnels, - handleTunnelExit: (id, port, exitCode, tunnelRunId) => - service.onTunnelExit(id, port, exitCode, tunnelRunId), + handleTunnelExit: ( + id, + port, + exitCode, + tunnelRunId, + runtime, + isSessionCurrent + ) => + service.onTunnelExit( + id, + port, + exitCode, + tunnelRunId, + runtime, + isSessionCurrent + ), destroyAll: () => service.destroyAll(), + destroyAllRuntimeRuns: () => service.destroyAllRuntimeRuns(), resumeCleanup: () => service.resumeCleanup(), - onRuntimeStart: () => service.onRuntimeStart(), onRuntimeStop: () => service.onRuntimeStop(), clearDurableStateAfterDestroy: () => service.clearDurableStateAfterDestroy() }; diff --git a/packages/sandbox/src/tunnels/sandbox-control-callback.ts b/packages/sandbox/src/tunnels/sandbox-control-callback.ts index 14387d74a..f2fd43032 100644 --- a/packages/sandbox/src/tunnels/sandbox-control-callback.ts +++ b/packages/sandbox/src/tunnels/sandbox-control-callback.ts @@ -11,6 +11,7 @@ import type { TunnelRunExitEvent } from '@repo/shared'; import { RpcTarget } from 'capnweb'; +import type { RuntimeIdentity } from '../runtime'; import type { TunnelExitHandler } from './rpc-target'; export class SandboxControlCallbackImpl @@ -25,12 +26,50 @@ export class SandboxControlCallbackImpl * that window. */ private readonly getHandler: () => TunnelExitHandler | null, - private readonly logger: Logger + private readonly logger: Logger, + private readonly expectedRuntime?: RuntimeIdentity, + private readonly getCurrentRuntime?: () => + | RuntimeIdentity + | null + | Promise, + private readonly isSessionCurrent?: () => boolean ) { super(); } + bindRuntime( + runtime: RuntimeIdentity, + isSessionCurrent: () => boolean + ): SandboxControlCallbackImpl { + return new SandboxControlCallbackImpl( + this.getHandler, + this.logger, + runtime, + this.getCurrentRuntime, + isSessionCurrent + ); + } + async onTunnelRunExit(event: TunnelRunExitEvent): Promise { + const currentRuntime = this.getCurrentRuntime + ? await this.getCurrentRuntime() + : null; + if ( + this.isSessionCurrent?.() !== true || + !this.expectedRuntime || + !currentRuntime || + currentRuntime.id !== this.expectedRuntime.id || + currentRuntime.runtimeIncarnationID !== + this.expectedRuntime.runtimeIncarnationID + ) { + this.logger.debug('onTunnelRunExit: stale runtime callback ignored', { + tunnelId: event.tunnelId, + runId: event.runId, + mode: event.mode, + port: event.port + }); + return; + } const handler = this.getHandler(); if (!handler) { this.logger.debug('onTunnelRunExit: no handler bound; ignoring', { @@ -42,6 +81,13 @@ export class SandboxControlCallbackImpl }); return; } - await handler(event.tunnelId, event.port, event.exitCode, event.runId); + await handler( + event.tunnelId, + event.port, + event.exitCode, + event.runId, + this.expectedRuntime, + this.isSessionCurrent + ); } } diff --git a/packages/sandbox/src/tunnels/storage.ts b/packages/sandbox/src/tunnels/storage.ts index daae524a6..8a1019dab 100644 --- a/packages/sandbox/src/tunnels/storage.ts +++ b/packages/sandbox/src/tunnels/storage.ts @@ -1,5 +1,6 @@ import type { NamedTunnelInfo, TunnelInfo, TunnelOptions } from '@repo/shared'; -import type { RuntimeIdentityID } from '../current-runtime-identity'; +import type { RuntimeIncarnationID } from '../runtime'; +import type { RuntimeIdentityID } from '../runtime/types'; import type { SandboxLifetimeID } from '../sandbox-lifetime'; /** DO storage key for the `port → TunnelInfo` map. */ @@ -22,6 +23,8 @@ export interface TunnelMetaEntry { dnsRecordId?: string; /** Runtime identity that owns the current cloudflared process. */ runtimeIdentityID?: RuntimeIdentityID; + /** Exact runtime incarnation that owns the current cloudflared process. */ + runtimeIncarnationID?: RuntimeIncarnationID; /** Sandbox lifetime that owns this tunnel record. */ sandboxLifetimeID?: SandboxLifetimeID; /** Runtime-local cloudflared run that owns current process callbacks. */ @@ -146,7 +149,8 @@ export async function updatePortState( storage: TunnelsStorage, port: number, updater: ( - state: Readonly + state: Readonly, + txn: TunnelsStorageTxn ) => | TunnelPortStatePatch | undefined @@ -164,7 +168,7 @@ export async function updatePortState( meta: meta[portKey], cleanup: cleanup[portKey] }; - const patch = await updater(state); + const patch = await updater(state, txn); const writes: Array> = []; if (patch && 'info' in patch && patch.info !== state.info) { diff --git a/packages/sandbox/src/tunnels/tunnel-service.ts b/packages/sandbox/src/tunnels/tunnel-service.ts index 59699ff84..eafa850d2 100644 --- a/packages/sandbox/src/tunnels/tunnel-service.ts +++ b/packages/sandbox/src/tunnels/tunnel-service.ts @@ -12,8 +12,21 @@ import type { TunnelOptions } from '@repo/shared'; import { logCanonicalEvent } from '@repo/shared'; -import type { CurrentRuntimeIdentity } from '../current-runtime-identity'; -import type { CurrentSandboxLifetime } from '../sandbox-lifetime'; +import { + OperationInterruptedError, + RuntimeControlProtocolError +} from '../errors'; +import { + RuntimeIdentity, + type RuntimeLease, + type RuntimeRecordStorage +} from '../runtime'; +import { RuntimeIdentityInactiveError } from '../runtime/types'; +import type { + CurrentSandboxLifetime, + SandboxLifetime +} from '../sandbox-lifetime'; +import { SandboxLifetimeChangedError } from '../sandbox-lifetime'; import { SandboxSecurityError, validatePort, @@ -24,11 +37,15 @@ import { resumeNamedTunnelCleanupEntry, resumeNamedTunnelCleanupRecords } from './cleanup'; -import { TunnelOperationLifecycle } from './lifecycle'; +import { createTunnelInterruptedError } from './lifecycle'; import { TunnelProvisioner } from './provisioner'; import { randomId } from './random-id'; import { pruneTunnelsForRestart } from './restart'; -import type { TunnelCleanupEntry, TunnelsStorage } from './storage'; +import type { + TunnelCleanupEntry, + TunnelMetaEntry, + TunnelsStorage +} from './storage'; import { CLEANUP_STORAGE_KEY, computeOptionsHash, @@ -50,14 +67,25 @@ import { export type { TunnelsStorage, TunnelsStorageTxn } from './storage'; -/** Subset of the RPC client this service depends on. */ -interface TunnelsRPCClient { +/** Subset of the Sandbox DO the service reads from. */ +export type TunnelProvisionLease = Pick & { tunnels: SandboxTunnelsAPI; -} +}; + +export type TunnelExistingRuntimeCall = ( + runtime: RuntimeIdentity, + operation: string, + call: (tunnels: SandboxTunnelsAPI) => Promise +) => Promise; -/** Subset of the Sandbox DO the service reads from. */ export interface TunnelServiceHost { - client: TunnelsRPCClient; + runProvision( + call: (lease: TunnelProvisionLease) => Promise + ): Promise; + runExisting: TunnelExistingRuntimeCall; + getStoredRuntime( + storage?: RuntimeRecordStorage + ): Promise; storage: TunnelsStorage; logger: Logger; /** @@ -85,8 +113,6 @@ export interface TunnelServiceHost { * to the global `fetch`. Tests inject a mock here. */ fetcher?: typeof fetch; - /** Runtime fence for records backed by a current container process. */ - currentRuntime?: Pick; /** Sandbox lifetime fence for operations that must not cross destroy(). */ currentLifetime?: Pick< CurrentSandboxLifetime, @@ -111,7 +137,9 @@ export type TunnelExitHandler = ( id: string, port: number, exitCode: number | null, - tunnelRunId?: string + tunnelRunId: string, + runtime: RuntimeIdentity, + isSessionCurrent: () => boolean ) => Promise; export interface TunnelsHandle { @@ -127,10 +155,10 @@ export interface TunnelsHandle { * call this; they call `destroy(port)` for an individual tunnel. */ destroyAll: () => Promise; + /** Stop stored container-side tunnel runs without mutating durable state. */ + destroyAllRuntimeRuns: () => Promise; /** Resume retained Cloudflare-side cleanup records. Internal lifecycle hook. */ resumeCleanup: () => Promise; - /** Reconcile durable tunnel state after a fresh container runtime starts. */ - onRuntimeStart: () => Promise; /** Reconcile durable tunnel state after the container runtime stops. */ onRuntimeStop: () => Promise; /** Clear public tunnel state after sandbox destroy processing completes. */ @@ -155,6 +183,27 @@ function validateTunnelPort(port: number): void { } } +function tunnelMetaOwnsRuntime( + meta: TunnelMetaEntry | undefined, + runtime: RuntimeIdentity | null +): boolean { + return Boolean( + runtime && + meta?.runtimeIdentityID === runtime.id && + meta.runtimeIncarnationID === runtime.runtimeIncarnationID + ); +} + +function tunnelOwningRuntime( + meta: TunnelMetaEntry | undefined +): RuntimeIdentity | null { + if (!meta?.runtimeIdentityID || !meta.runtimeIncarnationID) return null; + return new RuntimeIdentity({ + id: meta.runtimeIdentityID, + runtimeIncarnationID: meta.runtimeIncarnationID + }); +} + /** * Match a structured SandboxError code anywhere on the error — translated * SandboxErrors expose the code both as a top-level `code` field and on @@ -179,6 +228,13 @@ function isTunnelNotFoundError(error: unknown): boolean { return hasErrorCode(error, 'TUNNEL_NOT_FOUND'); } +function isRuntimeActivationMismatch(error: unknown): boolean { + return ( + error instanceof RuntimeControlProtocolError && + error.context.reason === 'activation-mismatch' + ); +} + interface TunnelGetRecoveryState { quickRun?: { tunnelId: string; @@ -194,12 +250,10 @@ function createTunnelRunId(): string { export class TunnelService implements TunnelsHandler { readonly #host: TunnelServiceHost; readonly #portLocks = new Map>(); - readonly #lifecycle: TunnelOperationLifecycle; readonly #provisioner: TunnelProvisioner; constructor(host: TunnelServiceHost) { this.#host = host; - this.#lifecycle = new TunnelOperationLifecycle(host); this.#provisioner = new TunnelProvisioner(host); } @@ -228,16 +282,18 @@ export class TunnelService implements TunnelsHandler { const recovery: TunnelGetRecoveryState = {}; const result = await this.#withPortLock(port, () => - this.#lifecycle.runGetWithRecovery(() => - this.#getLocked(port, options, requestedHash, recovery) - ) + this.#getLocked(port, options, requestedHash, recovery) ); cacheState = result.cacheState; outcome = 'success'; return result.info; } catch (error) { caughtError = error instanceof Error ? error : new Error(String(error)); - throw error; + if (caughtError instanceof OperationInterruptedError) { + // Ensure retryable is false since we do not recover + caughtError.context.retryable = false; + } + throw caughtError; } finally { logCanonicalEvent(this.#host.logger, { event: 'tunnel.get', @@ -285,19 +341,17 @@ export class TunnelService implements TunnelsHandler { } } - if (this.#host.currentRuntime) { - const currentRuntime = await this.#host.currentRuntime.get(); - if ( - !metaEntry?.runtimeIdentityID || - currentRuntime?.id !== metaEntry.runtimeIdentityID - ) { - return { - info: existing.name - ? await this.#provisionNamedTunnel(port, existing.name) - : await this.#provisionQuickTunnel(port, recovery), - cacheState: 'miss' - }; - } + const currentRuntime = await this.#host.getStoredRuntime(); + if ( + !tunnelMetaOwnsRuntime(metaEntry, currentRuntime) || + !(await this.#validateCachedRuntime(currentRuntime)) + ) { + return { + info: existing.name + ? await this.#provisionNamedTunnel(port, existing.name) + : await this.#provisionQuickTunnel(port, recovery), + cacheState: 'miss' + }; } return { info: existing, cacheState: 'hit' }; @@ -315,6 +369,30 @@ export class TunnelService implements TunnelsHandler { }; } + async #validateCachedRuntime( + runtime: RuntimeIdentity | null + ): Promise { + if (!runtime) return false; + try { + return ( + (await this.#host.runExisting( + runtime, + 'tunnel.lookup', + async () => true + )) === true + ); + } catch (error) { + if ( + error instanceof OperationInterruptedError || + (error instanceof RuntimeControlProtocolError && + error.context.reason === 'activation-mismatch') + ) { + return false; + } + throw error; + } + } + #assertSameOptions( port: number, existingHash: string, @@ -347,11 +425,14 @@ export class TunnelService implements TunnelsHandler { async #updatePortCleanup( port: number, - next: (entry: TunnelCleanupEntry | undefined) => TunnelCleanupEntry + runtime: RuntimeIdentity, + next: (entry: TunnelCleanupEntry | undefined) => TunnelCleanupEntry, + isInterrupted: () => boolean = () => false ): Promise { - await updatePortState(this.#host.storage, port, (state) => ({ - cleanup: next(state.cleanup) - })); + await updatePortState(this.#host.storage, port, async (state, txn) => { + await this.#assertStoredRuntime(txn, runtime, isInterrupted); + return { cleanup: next(state.cleanup) }; + }); } #requireCleanupEntry( @@ -367,105 +448,257 @@ export class TunnelService implements TunnelsHandler { port: number, recovery: TunnelGetRecoveryState ): Promise { - let lifecycle = await this.#lifecycle.capture(); + const lifetime = await this.#host.currentLifetime?.getOrCreate(); recovery.quickRun ??= { tunnelId: `quick-${randomId()}`, runId: createTunnelRunId() }; const { tunnelId, runId } = recovery.quickRun; - const spawned = await this.#provisioner.provisionQuickTunnel( - port, - runId, - tunnelId - ); - lifecycle = await this.#lifecycle.requireRuntime( - lifecycle, - 'process_ready', - true - ); - await this.#lifecycle.assertActive(lifecycle, 'process_ready', true); - await updatePortState(this.#host.storage, port, () => ({ - info: spawned, - meta: { - optionsHash: 'v1:quick', - ...(lifecycle.runtime && { - runtimeIdentityID: lifecycle.runtime.id - }), - ...(lifecycle.lifetime && { - sandboxLifetimeID: lifecycle.lifetime.id - }), - tunnelRunId: runId + + return await this.#host.runProvision(async (lease) => { + let interrupted = false; + let committedInfo: TunnelInfo | undefined; + const hold = lease.retain(() => { + interrupted = true; + }); + const isInterrupted = () => interrupted; + try { + await this.#assertStoredRuntime( + undefined, + lease.runtime, + isInterrupted + ); + await this.#assertSandboxLifetime(lifetime, 'starting'); + const spawned = await this.#provisioner.provisionQuickTunnel( + lease.tunnels, + port, + runId, + tunnelId + ); + await this.#assertSandboxLifetime(lifetime, 'process_ready'); + await updatePortState(this.#host.storage, port, async (_state, txn) => { + await this.#assertStoredRuntime(txn, lease.runtime, isInterrupted); + return { + info: spawned, + meta: { + optionsHash: 'v1:quick', + runtimeIdentityID: lease.runtime.id, + runtimeIncarnationID: lease.runtime.runtimeIncarnationID, + ...(lifetime && { sandboxLifetimeID: lifetime.id }), + tunnelRunId: runId + } + }; + }); + committedInfo = spawned; + await this.#assertStoredRuntime( + undefined, + lease.runtime, + isInterrupted + ); + await this.#assertSandboxLifetime(lifetime, 'committing'); + return spawned; + } catch (error) { + if (committedInfo) { + await this.#invalidateCommittedPortIfUnchanged( + port, + committedInfo, + lease.runtime, + runId + ); + } + throw error; + } finally { + hold.release(); } - })); - await this.#lifecycle.assertActive(lifecycle, 'committing', true); - return spawned; + }); } async #provisionNamedTunnel( port: number, name: string ): Promise { - let lifecycle = await this.#lifecycle.capture(); - - await this.#resumePortCleanup(port); - - const prepared = await this.#provisioner.prepareNamedTunnel(port, name, { - onIntentReady: (entry) => this.#updatePortCleanup(port, () => entry), - onTunnelReady: (tunnelId) => - this.#updatePortCleanup(port, (entry) => - markCleanupTunnelReady( - this.#requireCleanupEntry(port, entry), - tunnelId - ) - ), - onDNSReady: (dnsRecordId) => - this.#updatePortCleanup(port, (entry) => - markCleanupDNSReady( - this.#requireCleanupEntry(port, entry), - dnsRecordId - ) - ) + const lifetime = await this.#host.currentLifetime?.getOrCreate(); + + return await this.#host.runProvision(async (lease) => { + let interrupted = false; + let committedInfo: TunnelInfo | undefined; + let committedRunID: string | undefined; + const hold = lease.retain(() => { + interrupted = true; + }); + const isInterrupted = () => interrupted; + try { + await this.#assertStoredRuntime( + undefined, + lease.runtime, + isInterrupted + ); + await this.#assertSandboxLifetime(lifetime, 'starting'); + await this.#resumePortCleanup(port); + await this.#assertStoredRuntime( + undefined, + lease.runtime, + isInterrupted + ); + + const prepared = await this.#provisioner.prepareNamedTunnel( + port, + name, + { + onIntentReady: (entry) => + this.#updatePortCleanup( + port, + lease.runtime, + () => entry, + isInterrupted + ), + onTunnelReady: (tunnelID) => + this.#updatePortCleanup( + port, + lease.runtime, + (entry) => + markCleanupTunnelReady( + this.#requireCleanupEntry(port, entry), + tunnelID + ), + isInterrupted + ), + onDNSReady: (dnsRecordID) => + this.#updatePortCleanup( + port, + lease.runtime, + (entry) => + markCleanupDNSReady( + this.#requireCleanupEntry(port, entry), + dnsRecordID + ), + isInterrupted + ) + } + ); + const cleanupEntry = createNamedTunnelCleanupEntry( + prepared.info, + prepared.meta + ); + if (cleanupEntry) { + await this.#updatePortCleanup( + port, + lease.runtime, + () => cleanupEntry, + isInterrupted + ); + } + await this.#assertSandboxLifetime(lifetime, 'cloudflare_ready'); + + const tunnelRunID = createTunnelRunId(); + await this.#provisioner.startNamedTunnelRun( + lease.tunnels, + prepared, + tunnelRunID + ); + await this.#assertSandboxLifetime(lifetime, 'process_ready'); + + await updatePortState(this.#host.storage, port, async (_state, txn) => { + await this.#assertStoredRuntime(txn, lease.runtime, isInterrupted); + return { + info: prepared.info, + meta: { + ...prepared.meta, + runtimeIdentityID: lease.runtime.id, + runtimeIncarnationID: lease.runtime.runtimeIncarnationID, + ...(lifetime && { sandboxLifetimeID: lifetime.id }), + tunnelRunId: tunnelRunID + }, + cleanup: undefined + }; + }); + committedInfo = prepared.info; + committedRunID = tunnelRunID; + await this.#assertStoredRuntime( + undefined, + lease.runtime, + isInterrupted + ); + await this.#assertSandboxLifetime(lifetime, 'committing'); + return prepared.info; + } catch (error) { + if (committedInfo && committedRunID) { + await this.#invalidateCommittedPortIfUnchanged( + port, + committedInfo, + lease.runtime, + committedRunID + ); + } + throw error; + } finally { + hold.release(); + } }); - const cleanupEntry = createNamedTunnelCleanupEntry( - prepared.info, - prepared.meta - ); - if (cleanupEntry) { - await updatePortState(this.#host.storage, port, () => ({ - cleanup: cleanupEntry - })); + } + + async #invalidateCommittedPortIfUnchanged( + port: number, + expectedInfo: TunnelInfo, + runtime: RuntimeIdentity, + runID: string + ): Promise { + await updatePortState(this.#host.storage, port, (state) => { + if ( + state.info?.id !== expectedInfo.id || + state.meta?.runtimeIdentityID !== runtime.id || + state.meta.runtimeIncarnationID !== runtime.runtimeIncarnationID || + state.meta.tunnelRunId !== runID + ) { + return undefined; + } + if (state.info.name) { + return { + info: undefined, + meta: namedRespawnMeta(state.info, state.meta) + }; + } + return { info: undefined, meta: undefined }; + }); + } + + async #assertStoredRuntime( + storage: Parameters[0], + expected: RuntimeIdentity, + isInterrupted: () => boolean = () => false + ): Promise { + if (isInterrupted()) throw new RuntimeIdentityInactiveError(); + const current = await this.#host.getStoredRuntime(storage); + if ( + isInterrupted() || + !current || + current.id !== expected.id || + current.runtimeIncarnationID !== expected.runtimeIncarnationID + ) { + throw new RuntimeIdentityInactiveError(); } - await this.#lifecycle.assertActive( - lifecycle, - 'cloudflare_ready', - 'unknown' - ); + } - const tunnelRunId = createTunnelRunId(); - await this.#provisioner.startNamedTunnelRun(prepared, tunnelRunId); - lifecycle = await this.#lifecycle.requireRuntime( - lifecycle, - 'process_ready', - true - ); - await this.#lifecycle.assertActive(lifecycle, 'process_ready', true); - - await updatePortState(this.#host.storage, port, () => ({ - info: prepared.info, - meta: { - ...prepared.meta, - ...(lifecycle.runtime && { - runtimeIdentityID: lifecycle.runtime.id - }), - ...(lifecycle.lifetime && { - sandboxLifetimeID: lifecycle.lifetime.id - }), - tunnelRunId - }, - cleanup: undefined - })); - await this.#lifecycle.assertActive(lifecycle, 'committing', true); - return prepared.info; + async #assertSandboxLifetime( + lifetime: SandboxLifetime | undefined, + phase: string + ): Promise { + if (!lifetime || !this.#host.currentLifetime) return; + try { + await this.#host.currentLifetime.assertCurrent(lifetime); + } catch (error) { + if (error instanceof SandboxLifetimeChangedError) { + throw createTunnelInterruptedError({ + reason: 'sandbox_lifetime_changed', + phase, + admitted: true, + retryable: false, + message: + 'Tunnel operation was interrupted by a sandbox lifetime change' + }); + } + throw error; + } } async destroy(portOrInfo: number | TunnelInfo): Promise { @@ -509,15 +742,26 @@ export class TunnelService implements TunnelsHandler { })); try { - if (current.meta?.tunnelRunId) { - await this.#host.client.tunnels.stopTunnelRun({ - tunnelId: existing.id, - runId: current.meta.tunnelRunId - }); + const owningRuntime = tunnelOwningRuntime(current.meta); + if (owningRuntime && current.meta?.tunnelRunId) { + const tunnelRunId = current.meta.tunnelRunId; + await this.#host.runExisting( + owningRuntime, + 'tunnel.destroy', + (tunnels) => + tunnels.stopTunnelRun({ + tunnelId: existing.id, + runId: tunnelRunId + }) + ); } } catch (error) { - if (isTunnelNotFoundError(error)) { - // Container already forgot — fall through to CF cleanup. + if ( + isTunnelNotFoundError(error) || + error instanceof OperationInterruptedError || + isRuntimeActivationMismatch(error) + ) { + // The owning runtime is already absent — continue durable cleanup. } else if (current.meta?.dnsRecordId) { this.#host.logger.warn( 'tunnel.destroy: container tunnel cleanup failed', @@ -567,10 +811,22 @@ export class TunnelService implements TunnelsHandler { } async list(): Promise { - const map = await readMap(this.#host.storage); - const meta = await readMetaMap(this.#host.storage); + const { map, meta, runtime } = await this.#host.storage.transaction( + async (txn) => { + const [map, meta, runtime] = await Promise.all([ + readMap(txn), + readMetaMap(txn), + this.#host.getStoredRuntime(txn) + ]); + return { map, meta, runtime }; + } + ); return Object.entries(map) - .filter(([port]) => !meta[port]?.needsRespawn) + .filter( + ([port]) => + !meta[port]?.needsRespawn && + tunnelMetaOwnsRuntime(meta[port], runtime) + ) .map(([, info]) => info); } @@ -578,18 +834,29 @@ export class TunnelService implements TunnelsHandler { id: string, port: number, exitCode: number | null, - tunnelRunId?: string + tunnelRunId: string, + runtime: RuntimeIdentity, + isSessionCurrent: () => boolean ): Promise { const startTime = Date.now(); let outcome: 'success' | 'error' = 'error'; let caughtError: Error | undefined; try { await this.#withPortLock(port, async () => { - await updatePortState(this.#host.storage, port, (state) => { + await updatePortState(this.#host.storage, port, async (state, txn) => { const existing = state.info; const meta = state.meta; - if (existing?.id !== id) return undefined; - if (meta?.tunnelRunId && tunnelRunId !== meta.tunnelRunId) { + const activeRuntime = await this.#host.getStoredRuntime(txn); + if (!isSessionCurrent() || existing?.id !== id) return undefined; + if ( + !activeRuntime || + activeRuntime.id !== runtime.id || + activeRuntime.runtimeIncarnationID !== + runtime.runtimeIncarnationID || + meta?.runtimeIdentityID !== runtime.id || + meta.runtimeIncarnationID !== runtime.runtimeIncarnationID || + meta.tunnelRunId !== tunnelRunId + ) { return undefined; } @@ -621,15 +888,7 @@ export class TunnelService implements TunnelsHandler { } async destroyAll(): Promise { - const map = await readMap(this.#host.storage); - const meta = await readMetaMap(this.#host.storage); - const ports = new Set(Object.keys(map).map((p) => Number(p))); - for (const [portKey, entry] of Object.entries(meta)) { - const port = Number(portKey); - if (Number.isFinite(port) && namedTunnelInfoFromMeta(port, entry)) { - ports.add(port); - } - } + const ports = await this.destroyAllPorts(); for (const port of ports) { try { @@ -645,9 +904,43 @@ export class TunnelService implements TunnelsHandler { await this.resumeCleanup(); } - async onRuntimeStart(): Promise { - await pruneTunnelsForRestart(this.#host.storage); - await this.resumeCleanup(); + async destroyAllRuntimeRuns(): Promise { + const map = await readMap(this.#host.storage); + const meta = await readMetaMap(this.#host.storage); + const ports = await this.destroyAllPorts(); + for (const port of ports) { + const info = map[String(port)]; + const runId = meta[String(port)]?.tunnelRunId; + if (!info || !runId) continue; + const owningRuntime = tunnelOwningRuntime(meta[String(port)]); + if (!owningRuntime) continue; + try { + await this.#host.runExisting( + owningRuntime, + 'tunnel.destroy', + (tunnels) => tunnels.stopTunnelRun({ tunnelId: info.id, runId }) + ); + } catch (error) { + if (isTunnelNotFoundError(error)) continue; + this.#host.logger.warn('tunnels.destroyAllRuntimeRuns: stop failed', { + port, + error: error instanceof Error ? error.message : String(error) + }); + } + } + } + + private async destroyAllPorts(): Promise> { + const map = await readMap(this.#host.storage); + const meta = await readMetaMap(this.#host.storage); + const ports = new Set(Object.keys(map).map((p) => Number(p))); + for (const [portKey, entry] of Object.entries(meta)) { + const port = Number(portKey); + if (Number.isFinite(port) && namedTunnelInfoFromMeta(port, entry)) { + ports.add(port); + } + } + return ports; } async onRuntimeStop(): Promise { diff --git a/packages/sandbox/tests-harness/README.md b/packages/sandbox/tests-harness/README.md new file mode 100644 index 000000000..c1a6d5d41 --- /dev/null +++ b/packages/sandbox/tests-harness/README.md @@ -0,0 +1,20 @@ +# Sandbox reconstruction harness + +This opt-in suite uses Wrangler's `createTestHarness()` with the real Sandbox +container image: + +```bash +npm run test:harness -w @cloudflare/sandbox +``` + +It requires a running Docker daemon. The suite characterizes Durable Object +in-memory state across explicit eviction and a coordinated local Worker reload, +contrasts dynamic environment variables with storage-backed `sleepAfter`, and +verifies that a fresh container process can execute afterward. + +The suite is intentionally not part of the ordinary unit-test job because it +builds the full container image. It also does not model production rollout +propagation, an in-flight container operation, runtime-incarnation fencing, or +stale-handle behavior. Miniflare cannot evict a Sandbox DO after its local +container has active references, so privileged E2E remains authoritative for +those lifecycle paths. diff --git a/packages/sandbox/tests-harness/reconstruction.test.ts b/packages/sandbox/tests-harness/reconstruction.test.ts new file mode 100644 index 000000000..c7bdad960 --- /dev/null +++ b/packages/sandbox/tests-harness/reconstruction.test.ts @@ -0,0 +1,129 @@ +import { execFileSync } from 'node:child_process'; +import type { DurableObjectNamespace as WorkersDurableObjectNamespace } from '@cloudflare/workers-types'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { createTestHarness, type TestHarness } from 'wrangler'; + +const dockerAvailable = (() => { + try { + execFileSync('docker', ['info'], { stdio: 'ignore' }); + return true; + } catch { + return false; + } +})(); + +const skipHarness = process.platform !== 'linux' && process.env.CI === 'true'; +const skipWithoutDocker = process.env.CI !== 'true' && !dockerAvailable; + +describe.skipIf(skipHarness || skipWithoutDocker)( + 'Sandbox DO eviction and Worker reload characterization', + () => { + let server: TestHarness; + + beforeEach(async () => { + server = createTestHarness({ + root: import.meta.dirname, + workers: [{ configPath: './wrangler.jsonc' }] + }); + await server.listen(); + }); + + afterEach(async () => { + await server?.close(); + }); + + it('applies dynamic environment to a fresh container process', async () => { + const sandboxId = crypto.randomUUID(); + const configured = await server.fetch( + `/configure?sandboxId=${sandboxId}`, + { method: 'POST' } + ); + expect(configured.status).toBe(200); + + const execution = await server.fetch(`/snapshot?sandboxId=${sandboxId}`); + expect(execution.status).toBe(200); + expect(await execution.json()).toMatchObject({ + exitCode: 0, + stderr: '', + stdout: 'configured', + timedOut: false, + truncated: false + }); + }); + + it('contrasts durable and in-memory state after DO eviction', async () => { + const sandboxId = crypto.randomUUID(); + const configured = await server.fetch( + `/configure?sandboxId=${sandboxId}`, + { method: 'POST' } + ); + expect(configured.status).toBe(200); + expect(await configured.json()).toMatchObject({ + envVars: { HARNESS_MARKER: 'configured' }, + sleepAfter: '30m' + }); + + await server + .getWorker<{ Sandbox: WorkersDurableObjectNamespace }>() + .evictDurableObject('Sandbox', { name: sandboxId }); + + const response = await server.fetch(`/state?sandboxId=${sandboxId}`); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ + envVars: {}, + sleepAfter: '30m' + }); + + // Starting the target container before eviction would leave active local + // references that Miniflare cannot evict. The preceding test is the + // positive control for the same setEnvVars-to-exec path. + const execution = await server.fetch(`/snapshot?sandboxId=${sandboxId}`); + expect(execution.status).toBe(200); + expect(await execution.json()).toMatchObject({ + exitCode: 0, + stderr: '', + stdout: '', + timedOut: false, + truncated: false + }); + }); + + it('clears in-memory DO state after a coordinated Worker reload', async () => { + const sandboxId = crypto.randomUUID(); + const configured = await server.fetch( + `/configure?sandboxId=${sandboxId}`, + { method: 'POST' } + ); + expect(configured.status).toBe(200); + + await server.update((options) => ({ + ...options, + workers: options.workers.map((worker) => + 'configPath' in worker + ? { + ...worker, + vars: { ...worker.vars, HARNESS_RELOAD: 'after' } + } + : worker + ) + })); + + const response = await server.fetch(`/state?sandboxId=${sandboxId}`); + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ envVars: {} }); + + const execution = await server.fetch(`/snapshot?sandboxId=${sandboxId}`); + const executionBody = await execution.json(); + expect(execution.status, JSON.stringify(executionBody, null, 2)).toBe( + 200 + ); + expect(executionBody).toMatchObject({ + exitCode: 0, + stderr: '', + stdout: '', + timedOut: false, + truncated: false + }); + }); + } +); diff --git a/packages/sandbox/tests-harness/worker.ts b/packages/sandbox/tests-harness/worker.ts new file mode 100644 index 000000000..7cd05cb83 --- /dev/null +++ b/packages/sandbox/tests-harness/worker.ts @@ -0,0 +1,84 @@ +import { getSandbox, Sandbox } from '../src'; + +export class ReconstructionSandbox extends Sandbox { + getStateForTest(): { + envVars: Record; + sleepAfter: string | number; + } { + return { envVars: { ...this.envVars }, sleepAfter: this.sleepAfter }; + } +} + +interface Env { + Sandbox: DurableObjectNamespace; +} + +interface CommandSnapshot { + exitCode: number; + stderr: string; + stdout: string; + timedOut: boolean; + truncated: boolean; +} + +function getSandboxID(request: Request): string { + return new URL(request.url).searchParams.get('sandboxId') ?? 'reconstruction'; +} + +async function runEnvironmentSnapshot( + sandbox: ReturnType> +): Promise { + const process = await sandbox.exec([ + '/bin/bash', + '-lc', + // biome-ignore lint/suspicious/noTemplateCurlyInString: Bash parameter expansion. + 'printf %s "${HARNESS_MARKER-}"' + ]); + return process.output({ encoding: 'utf8' }); +} + +function serializeError(error: unknown): Record { + if (!(error instanceof Error)) return { value: error }; + + const structured = error as Error & { + code?: unknown; + context?: unknown; + httpStatus?: unknown; + }; + return { + name: error.name, + message: error.message, + code: structured.code, + context: structured.context, + httpStatus: structured.httpStatus + }; +} + +export default { + async fetch(request: Request, env: Env): Promise { + const url = new URL(request.url); + const sandbox = getSandbox(env.Sandbox, getSandboxID(request), { + keepAlive: true, + sleepAfter: '30m' + }); + + try { + if (url.pathname === '/configure' && request.method === 'POST') { + await sandbox.setEnvVars({ HARNESS_MARKER: 'configured' }); + return Response.json(await sandbox.getStateForTest()); + } + + if (url.pathname === '/state' && request.method === 'GET') { + return Response.json(await sandbox.getStateForTest()); + } + + if (url.pathname === '/snapshot' && request.method === 'GET') { + return Response.json(await runEnvironmentSnapshot(sandbox)); + } + + return new Response('not found', { status: 404 }); + } catch (error) { + return Response.json(serializeError(error), { status: 500 }); + } + } +}; diff --git a/packages/sandbox/tests-harness/wrangler.jsonc b/packages/sandbox/tests-harness/wrangler.jsonc new file mode 100644 index 000000000..32a0f13e2 --- /dev/null +++ b/packages/sandbox/tests-harness/wrangler.jsonc @@ -0,0 +1,24 @@ +{ + "$schema": "../../../node_modules/wrangler/config-schema.json", + "name": "sandbox-reconstruction-harness", + "main": "worker.ts", + "compatibility_date": "2026-07-01", + "compatibility_flags": ["nodejs_compat"], + "containers": [ + { + "class_name": "ReconstructionSandbox", + "image": "../Dockerfile", + "image_build_context": "../../..", + "name": "sandbox-reconstruction-harness", + "max_instances": 2 + } + ], + "durable_objects": { + "bindings": [ + { "name": "Sandbox", "class_name": "ReconstructionSandbox" } + ] + }, + "migrations": [ + { "tag": "v1", "new_sqlite_classes": ["ReconstructionSandbox"] } + ] +} diff --git a/packages/sandbox/tests/backup-restore-lifecycle.test.ts b/packages/sandbox/tests/backup-restore-lifecycle.test.ts index 3db97804b..3be7ee571 100644 --- a/packages/sandbox/tests/backup-restore-lifecycle.test.ts +++ b/packages/sandbox/tests/backup-restore-lifecycle.test.ts @@ -1,5 +1,6 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import type { BackupRestoreOperationRecord } from '../src/backup/restore-operation-store'; +import type { ContainerControlClient } from '../src/container-control'; import { ErrorCode, OperationInterruptedError, @@ -7,7 +8,10 @@ import { } from '../src/errors'; import { Sandbox } from '../src/sandbox'; import type { SandboxLifetimeID } from '../src/sandbox-lifetime'; -import { createMockControlClient } from './helpers/mock-control-client'; +import { + asSandboxWithClient, + createMockControlClient +} from './helpers/mock-control-client'; vi.mock('@cloudflare/containers', () => { class MockContainer { @@ -94,7 +98,7 @@ function createNonRetryableInterruptedError( message: 'Restore was interrupted after unknown admission', httpStatus: 409, context: { - reason: 'runtime_replaced', + reason: 'sandbox_lifetime_changed', operation: 'backup.restore', operationId: crypto.randomUUID(), operationKey: `restore:${backupId}:${dir}`, @@ -117,7 +121,11 @@ async function createBackupSandbox(params?: { }; }) { const storageMap = params?.storageMap ?? new Map(); - storageMap.set('currentRuntimeIdentity', { id: 'runtime-1' }); + storageMap.set('currentRuntimeIdentity', { + schemaVersion: 1, + id: 'runtime-1', + runtimeIncarnationID: 'incarnation-1' + }); storageMap.set('sandbox:lifetime', { id: 'lifetime-1', generation: 1, @@ -151,13 +159,44 @@ async function createBackupSandbox(params?: { expect(ctx.blockConcurrencyWhile).toHaveBeenCalled(); }); - sandbox.client = createMockControlClient(); - vi.spyOn(sandbox.client.backup, 'restoreArchive').mockResolvedValue({ - success: true, - dir: '/workspace/project' - } as never); + const sandboxWithClient = asSandboxWithClient(sandbox); + sandboxWithClient.client = createMockControlClient(); + const runWakingSpy = vi + .spyOn( + ( + sandbox as unknown as { + runtimeRunner: { + runWaking( + operation: string, + call: (lease: { + runtime: { id: string; runtimeIncarnationID: string }; + control: ContainerControlClient; + retain(): { release(): void }; + }) => Promise + ): Promise; + }; + } + ).runtimeRunner, + 'runWaking' + ) + .mockImplementation(async (_operation, call) => + call({ + runtime: { + id: 'runtime-1', + runtimeIncarnationID: 'incarnation-1' + }, + control: sandboxWithClient.client, + retain: () => ({ release: () => {} }) + }) + ); + vi.spyOn(sandboxWithClient.client.backup, 'restoreArchive').mockResolvedValue( + { + success: true, + dir: '/workspace/project' + } as never + ); - return { sandbox, storageMap, bucket }; + return { sandbox, storageMap, bucket, runWakingSpy }; } describe('backup restore lifecycle', () => { @@ -167,7 +206,7 @@ describe('backup restore lifecycle', () => { }); it('writes a verified operation record after restore succeeds under the current runtime', async () => { - const { sandbox, storageMap } = await createBackupSandbox(); + const { sandbox, storageMap, runWakingSpy } = await createBackupSandbox(); const backupId = crypto.randomUUID(); await sandbox.restoreBackup({ id: backupId, dir: '/workspace/project' }); @@ -175,11 +214,17 @@ describe('backup restore lifecycle', () => { const record = storageMap.get( `operations:restore:${backupId}:/workspace/project` ) as BackupRestoreOperationRecord; + expect(runWakingSpy).toHaveBeenCalledTimes(1); + expect(runWakingSpy).toHaveBeenCalledWith( + 'backup.restore', + expect.any(Function) + ); expect(record).toMatchObject({ operationKey: `restore:${backupId}:/workspace/project`, kind: 'backup.restore', sandboxLifetimeID: 'lifetime-1', runtimeIdentityID: 'runtime-1', + runtimeIncarnationID: 'incarnation-1', phase: 'verified', status: 'committed', payload: { @@ -196,27 +241,21 @@ describe('backup restore lifecycle', () => { }); }); - it('captures cold-start runtime identity before restore transfer', async () => { - const order: string[] = []; - const storageMap = new Map(); + it('records the supplied restore runtime without marking a new one', async () => { + const writes: string[] = []; const { sandbox } = await createBackupSandbox({ - storageMap, storageHooks: { - onPut: (key) => { - if (key === 'currentRuntimeIdentity') { - order.push('runtimeReady'); - } - } + onPut: (key) => writes.push(key) } }); - storageMap.delete('currentRuntimeIdentity'); + const backupId = crypto.randomUUID(); await sandbox.restoreBackup({ - id: crypto.randomUUID(), + id: backupId, dir: '/workspace/project' }); - expect(order.slice(0, 1)).toEqual(['runtimeReady']); + expect(writes).not.toContain('currentRuntimeIdentity'); }); it('does not retry non-retryable restore interruptions', async () => { @@ -226,77 +265,56 @@ describe('backup restore lifecycle', () => { backupId, '/workspace/project' ); + const sandboxWithClient = asSandboxWithClient(sandbox); const restoreArchiveSpy = vi - .spyOn(sandbox.client.backup, 'restoreArchive') + .spyOn(sandboxWithClient.client.backup, 'restoreArchive') .mockRejectedValueOnce(interruption) .mockResolvedValueOnce({ success: true, dir: '/workspace/project' }); - await expect( - sandbox.restoreBackup({ id: backupId, dir: '/workspace/project' }) - ).rejects.toBe(interruption); - expect(restoreArchiveSpy).toHaveBeenCalledTimes(1); - }); - - it('recovers internally when the first restore attempt loses the RPC transport', async () => { - const { sandbox, storageMap } = await createBackupSandbox(); - const backupId = crypto.randomUUID(); - const restoreArchiveSpy = vi - .spyOn(sandbox.client.backup, 'restoreArchive') - .mockRejectedValueOnce(createDisposedRPCError()) - .mockResolvedValueOnce({ success: true, dir: '/workspace/project' }); - - const result = await sandbox.restoreBackup({ - id: backupId, - dir: '/workspace/project' - }); - - expect(result).toEqual({ - success: true, - id: backupId, - dir: '/workspace/project' + const error = await sandbox + .restoreBackup({ id: backupId, dir: '/workspace/project' }) + .catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(OperationInterruptedError); + expect((error as OperationInterruptedError).context).toMatchObject({ + operationId: expect.any(String), + backupId, + retryable: false }); - expect(restoreArchiveSpy).toHaveBeenCalledTimes(2); - - const record = storageMap.get( - `operations:restore:${backupId}:/workspace/project` - ) as BackupRestoreOperationRecord; - expect(record.status).toBe('committed'); - expect(record.phase).toBe('verified'); - expect(record.result).toEqual(result); + expect(restoreArchiveSpy).toHaveBeenCalledTimes(1); }); - it('surfaces OPERATION_INTERRUPTED after exhausting restore recovery attempts', async () => { - const { sandbox, storageMap } = await createBackupSandbox(); + it('records runtime-runner interruption with restore context', async () => { + const { sandbox, storageMap, runWakingSpy } = await createBackupSandbox(); const backupId = crypto.randomUUID(); - const restoreArchiveSpy = vi - .spyOn(sandbox.client.backup, 'restoreArchive') - .mockRejectedValue(createDisposedRPCError()); + runWakingSpy.mockRejectedValueOnce( + new OperationInterruptedError({ + code: ErrorCode.OPERATION_INTERRUPTED, + message: 'Runtime changed', + httpStatus: 409, + context: { + reason: 'runtime_replaced', + operation: 'backup.restore', + admitted: true, + retryable: false + }, + timestamp: '2026-06-15T12:00:00.000Z' + }) + ); - let thrown: unknown; - try { - await sandbox.restoreBackup({ id: backupId, dir: '/workspace/project' }); - } catch (error) { - thrown = error; - } + const error = await sandbox + .restoreBackup({ id: backupId, dir: '/workspace/project' }) + .catch((caught: unknown) => caught); - expect(restoreArchiveSpy).toHaveBeenCalledTimes(3); - expect(thrown).toBeInstanceOf(OperationInterruptedError); - const interrupted = thrown as OperationInterruptedError; - expect(interrupted.context).toEqual({ - reason: 'recovery_exhausted', - operation: 'backup.restore', + expect(error).toBeInstanceOf(OperationInterruptedError); + expect((error as OperationInterruptedError).context).toMatchObject({ + reason: 'runtime_replaced', operationId: expect.any(String), + retryable: true, operationKey: `restore:${backupId}:/workspace/project`, - idempotencyKey: `restore:${backupId}:/workspace/project`, backupId, dir: '/workspace/project', - phase: 'interrupted', - admitted: 'unknown', - retryable: true, - recoveryAttempts: 2, - maxRecoveryAttempts: 2 + phase: 'validating' }); - const record = storageMap.get( `operations:restore:${backupId}:/workspace/project` ) as BackupRestoreOperationRecord; @@ -304,48 +322,34 @@ describe('backup restore lifecycle', () => { expect(record.phase).toBe('interrupted'); }); - it('preserves the same operation id across internal restore recovery attempts', async () => { - const operationIds: string[] = []; - const { sandbox, storageMap } = await createBackupSandbox({ - storageHooks: { - onPut: (_key, value) => { - const record = value as Partial; - if (record.kind === 'backup.restore' && record.operationId) { - operationIds.push(record.operationId); - } - } - } - }); + it('surfaces transport interruption without replaying restore work', async () => { + const { sandbox, storageMap } = await createBackupSandbox(); const backupId = crypto.randomUUID(); - vi.spyOn(sandbox.client.backup, 'restoreArchive') + const sandboxWithClient = asSandboxWithClient(sandbox); + const restoreArchiveSpy = vi + .spyOn(sandboxWithClient.client.backup, 'restoreArchive') .mockRejectedValueOnce(createDisposedRPCError()) - .mockResolvedValueOnce({ - success: true, - dir: '/workspace/project' - } as never); + .mockResolvedValueOnce({ success: true, dir: '/workspace/project' }); - await sandbox.restoreBackup({ id: backupId, dir: '/workspace/project' }); + await expect( + sandbox.restoreBackup({ id: backupId, dir: '/workspace/project' }) + ).rejects.toBeInstanceOf(OperationInterruptedError); - expect(new Set(operationIds).size).toBe(1); + expect(restoreArchiveSpy).toHaveBeenCalledTimes(1); const record = storageMap.get( `operations:restore:${backupId}:/workspace/project` ) as BackupRestoreOperationRecord; - expect(record.operationId).toBe(operationIds[0]); - expect(record.status).toBe('committed'); + expect(record.status).toBe('interrupted'); + expect(record.phase).toBe('interrupted'); }); it('reuses an interrupted operation id when the caller retries restoreBackup with the same backup handle', async () => { const { sandbox, storageMap } = await createBackupSandbox(); const backupId = crypto.randomUUID(); - // Three consecutive failures exhaust the two internal recovery attempts - // so the first restoreBackup() call surfaces OperationInterruptedError to - // the caller instead of being swallowed by internal retry. - vi.spyOn(sandbox.client.backup, 'restoreArchive') - .mockRejectedValueOnce(createDisposedRPCError()) + const sandboxWithClient = asSandboxWithClient(sandbox); + vi.spyOn(sandboxWithClient.client.backup, 'restoreArchive') .mockRejectedValueOnce(createDisposedRPCError()) - .mockRejectedValueOnce(createDisposedRPCError()) - // Second restoreBackup() call: succeeds on the first attempt. .mockResolvedValueOnce({ success: true, dir: '/workspace/project' @@ -369,37 +373,31 @@ describe('backup restore lifecycle', () => { expect(committedRecord.status).toBe('committed'); }); - it('recovers internally when the runtime changes after restoreArchive returns', async () => { + it('surfaces interruption when the supplied restore runtime is no longer active', async () => { const { sandbox, storageMap } = await createBackupSandbox(); const backupId = crypto.randomUUID(); + const sandboxWithClient = asSandboxWithClient(sandbox); const restoreArchiveSpy = vi - .spyOn(sandbox.client.backup, 'restoreArchive') + .spyOn(sandboxWithClient.client.backup, 'restoreArchive') .mockImplementationOnce(async () => { - storageMap.delete('currentRuntimeIdentity'); + storageMap.set('currentRuntimeIdentity', { + schemaVersion: 1, + id: 'runtime-1', + runtimeIncarnationID: 'incarnation-2' + }); return { success: true, dir: '/workspace/project' } as never; - }) - .mockResolvedValueOnce({ success: true, dir: '/workspace/project' }); - - const result = await sandbox.restoreBackup({ - id: backupId, - dir: '/workspace/project' - }); + }); - expect(result).toEqual({ - success: true, - id: backupId, - dir: '/workspace/project' - }); - expect(restoreArchiveSpy).toHaveBeenCalledTimes(2); + await expect( + sandbox.restoreBackup({ id: backupId, dir: '/workspace/project' }) + ).rejects.toBeInstanceOf(OperationInterruptedError); + expect(restoreArchiveSpy).toHaveBeenCalledTimes(1); const record = storageMap.get( `operations:restore:${backupId}:/workspace/project` ) as BackupRestoreOperationRecord; - expect(record.status).toBe('committed'); - expect(record.phase).toBe('verified'); - expect(record.runtimeIdentityID).toEqual(expect.any(String)); - expect(record.runtimeIdentityID).not.toBe('runtime-1'); - expect(record.result).toEqual(result); + expect(record.status).toBe('interrupted'); + expect(record.runtimeIncarnationID).toBe('incarnation-1'); }); it('surfaces interruption when the sandbox lifetime changes before runtime work starts', async () => { @@ -408,7 +406,7 @@ describe('backup restore lifecycle', () => { storageMap, storageHooks: { onPut: (key) => { - if (key === 'currentRuntimeIdentity') { + if (key.startsWith('operations:restore:')) { storageMap.set('sandbox:lifetime', { id: 'lifetime-2', generation: 2, @@ -419,9 +417,12 @@ describe('backup restore lifecycle', () => { } } }); - storageMap.delete('currentRuntimeIdentity'); const backupId = crypto.randomUUID(); - const restoreArchiveSpy = vi.spyOn(sandbox.client.backup, 'restoreArchive'); + const sandboxWithClient = asSandboxWithClient(sandbox); + const restoreArchiveSpy = vi.spyOn( + sandboxWithClient.client.backup, + 'restoreArchive' + ); let thrown: unknown; try { @@ -451,7 +452,7 @@ describe('backup restore lifecycle', () => { expect(record.error?.retryable).toBe(false); }); - it('returns a committed restore result from durable operation state without restoring again', async () => { + it('starts a new restore after a previously committed restore', async () => { const backupId = crypto.randomUUID(); const storageMap = new Map(); storageMap.set(`operations:restore:${backupId}:/workspace/project`, { @@ -470,7 +471,11 @@ describe('backup restore lifecycle', () => { } satisfies BackupRestoreOperationRecord); const { sandbox } = await createBackupSandbox({ storageMap }); - const restoreArchiveSpy = vi.spyOn(sandbox.client.backup, 'restoreArchive'); + const sandboxWithClient = asSandboxWithClient(sandbox); + const restoreArchiveSpy = vi.spyOn( + sandboxWithClient.client.backup, + 'restoreArchive' + ); await expect( sandbox.restoreBackup({ id: backupId, dir: '/workspace/project' }) @@ -479,14 +484,20 @@ describe('backup restore lifecycle', () => { id: backupId, dir: '/workspace/project' }); - expect(restoreArchiveSpy).not.toHaveBeenCalled(); + expect(restoreArchiveSpy).toHaveBeenCalledTimes(1); + const record = storageMap.get( + `operations:restore:${backupId}:/workspace/project` + ) as BackupRestoreOperationRecord; + expect(record.status).toBe('committed'); + expect(record.operationId).not.toBe('operation-1'); }); it('does not retry restore across a sandbox lifetime change', async () => { const { sandbox, storageMap } = await createBackupSandbox(); const backupId = crypto.randomUUID(); + const sandboxWithClient = asSandboxWithClient(sandbox); const restoreArchiveSpy = vi - .spyOn(sandbox.client.backup, 'restoreArchive') + .spyOn(sandboxWithClient.client.backup, 'restoreArchive') .mockImplementationOnce(async () => { storageMap.set('sandbox:lifetime', { id: 'lifetime-2', diff --git a/packages/sandbox/tests/container-connection.test.ts b/packages/sandbox/tests/container-connection.test.ts index 6836efcc3..d03d736a6 100644 --- a/packages/sandbox/tests/container-connection.test.ts +++ b/packages/sandbox/tests/container-connection.test.ts @@ -7,10 +7,12 @@ import { ContainerUnavailableError, ErrorCode } from '../src/errors'; function deferred() { let resolve!: (value: T) => void; - const promise = new Promise((res) => { + let reject!: (reason: unknown) => void; + const promise = new Promise((res, rej) => { resolve = res; + reject = rej; }); - return { promise, resolve }; + return { promise, resolve, reject }; } /** @@ -28,11 +30,11 @@ describe('ContainerControlConnection', () => { expect(conn.isConnected()).toBe(false); }); - it('should have a stub available immediately after construction', () => { + it('should not expose the RPC stub before activation', () => { const conn = new ContainerControlConnection({ stub: { fetch: vi.fn() } }); - expect(conn.rpc()).toBeDefined(); + expect(() => conn.rpc()).toThrow(/not activated/); }); }); @@ -66,9 +68,13 @@ describe('ContainerControlConnection', () => { }) } }); - const internals = conn as unknown as { transport: DeferredTransport }; + const internals = conn as unknown as { + transport: DeferredTransport; + state: 'active'; + }; const activate = vi.spyOn(internals.transport, 'activate'); const connecting = conn.connect(); + internals.state = 'active'; const connectRejected = vi.fn(); const connectOutcome = connecting.then( () => 'resolved', @@ -167,7 +173,9 @@ describe('ContainerControlConnection', () => { } }); - // rpc() triggers connect() in the background and returns the stub. + const connecting = conn.connect().catch(() => undefined); + const internals = conn as unknown as { state: 'active' }; + internals.state = 'active'; const stub = conn.rpc(); // Calling a method on the stub queues a send and starts a receive(). @@ -176,11 +184,12 @@ describe('ContainerControlConnection', () => { const rpcCall = stub.utils.ping(); await expect(rpcCall).rejects.toThrow(); + await connecting; }, 5000); }); describe('rpc', () => { - it('should trigger connect lazily when calling rpc()', () => { + it('should reject rpc access before activation', () => { const fetchMock = vi .fn() .mockResolvedValue(new Response('Not Found', { status: 404 })); @@ -188,10 +197,8 @@ describe('ContainerControlConnection', () => { stub: { fetch: fetchMock } }); - // rpc() returns the stub immediately and triggers connect in the background - const stub = conn.rpc(); - expect(stub).toBeDefined(); - expect(fetchMock).toHaveBeenCalled(); + expect(() => conn.rpc()).toThrow(/not activated/); + expect(fetchMock).not.toHaveBeenCalled(); }); }); @@ -201,13 +208,13 @@ describe('ContainerControlConnection', () => { stub: { fetch: vi.fn() } }); const internals = conn as unknown as { - connected: boolean; + state: 'connected'; ws: unknown; doConnect: () => Promise; }; vi.spyOn(internals, 'doConnect').mockImplementation(async () => { - internals.connected = true; + internals.state = 'connected'; internals.ws = { close: vi.fn(), removeEventListener: vi.fn() }; }); @@ -215,34 +222,88 @@ describe('ContainerControlConnection', () => { expect(conn.isConnected()).toBe(true); }); - it('should return the same stub before and after connect', async () => { + it('should return the same stub after activation', async () => { const conn = new ContainerControlConnection({ stub: { fetch: vi.fn() } }); const internals = conn as unknown as { - connected: boolean; + state: 'active' | 'connected'; ws: unknown; doConnect: () => Promise; }; vi.spyOn(internals, 'doConnect').mockImplementation(async () => { - internals.connected = true; + internals.state = 'connected'; internals.ws = { close: vi.fn(), removeEventListener: vi.fn() }; }); - // rpc() returns the stub immediately — same reference before and after connect - const stubBefore = conn.rpc(); await conn.connect(); + internals.state = 'active'; + const stubBefore = conn.rpc(); const stubAfter = conn.rpc(); expect(stubAfter).toBe(stubBefore); }); + it('does not overwrite disconnected state when activation fails after peer close', async () => { + const conn = new ContainerControlConnection({ + stub: { fetch: vi.fn() } + }); + const activation = deferred(); + const internals = conn as unknown as { + state: 'connected' | 'activating' | 'disconnected'; + stub: { + utils: { activateControlSession: (id: string) => Promise }; + }; + onWebSocketClose: () => void; + }; + internals.state = 'connected'; + internals.stub = { + utils: { activateControlSession: () => activation.promise } + }; + + const activating = conn.activateControlSession('incarnation-1'); + await Promise.resolve(); + expect(internals.state).toBe('activating'); + internals.onWebSocketClose(); + activation.reject(new Error('Peer closed WebSocket: 1006')); + + await expect(activating).rejects.toThrow('Peer closed WebSocket'); + expect(internals.state).toBe('disconnected'); + }); + + it('does not overwrite disconnected state when activation fails after peer error', async () => { + const conn = new ContainerControlConnection({ + stub: { fetch: vi.fn() } + }); + const activation = deferred(); + const internals = conn as unknown as { + state: 'connected' | 'activating' | 'disconnected'; + stub: { + utils: { activateControlSession: (id: string) => Promise }; + }; + onWebSocketError: () => void; + }; + internals.state = 'connected'; + internals.stub = { + utils: { activateControlSession: () => activation.promise } + }; + + const activating = conn.activateControlSession('incarnation-1'); + await Promise.resolve(); + expect(internals.state).toBe('activating'); + internals.onWebSocketError(); + activation.reject(new Error('WebSocket connection failed.')); + + await expect(activating).rejects.toThrow('WebSocket connection failed'); + expect(internals.state).toBe('disconnected'); + }); + it('should permanently revoke an explicitly disconnected connection', async () => { const conn = new ContainerControlConnection({ stub: { fetch: vi.fn() } }); const internals = conn as unknown as { - connected: boolean; + state: 'connected'; ws: unknown; doConnect: () => Promise; }; @@ -250,7 +311,7 @@ describe('ContainerControlConnection', () => { const doConnect = vi .spyOn(internals, 'doConnect') .mockImplementation(async () => { - internals.connected = true; + internals.state = 'connected'; internals.ws = { close: vi.fn(), removeEventListener: vi.fn() }; }); @@ -269,7 +330,7 @@ describe('ContainerControlConnection', () => { stub: { fetch: vi.fn() } }); const internals = conn as unknown as { - connected: boolean; + state: 'connected'; ws: unknown; doConnect: () => Promise; }; @@ -277,7 +338,7 @@ describe('ContainerControlConnection', () => { const doConnect = vi .spyOn(internals, 'doConnect') .mockImplementation(async () => { - internals.connected = true; + internals.state = 'connected'; internals.ws = { close: vi.fn(), removeEventListener: vi.fn() }; }); @@ -381,107 +442,7 @@ describe('ContainerControlConnection', () => { }); }); - describe('WebSocket upgrade retry', () => { - /** - * Build a fake successful upgrade Response. Mirrors what - * Cloudflare's Container base class returns from `stub.fetch()`: - * a Response with `status === 101` and a non-standard `webSocket` - * property carrying the WebSocket instance. - */ - function makeUpgradeResponse(): Response { - const target = new EventTarget(); - const ws = Object.assign(target, { - send: () => {}, - close: () => {}, - accept: () => {} - }) as unknown as WebSocket; - // The workerd test runtime rejects new Response(null, { status: 101 }), - // so synthesize a Response-shaped object exposing only the fields - // ContainerControlConnection actually reads (status, statusText, - // and the non-standard `webSocket` accessor). - return { - status: 101, - statusText: 'Switching Protocols', - webSocket: ws - } as unknown as Response; - } - - function makeUpgradeFailure(status: number): Response { - return new Response('Container upgrade unavailable.', { - status, - statusText: 'Service Unavailable' - }); - } - - it('retries retryable upgrade responses until success', async () => { - vi.useFakeTimers(); - try { - const fetchMock = vi - .fn<(req: Request) => Promise>() - .mockResolvedValueOnce(makeUpgradeFailure(500)) - .mockResolvedValueOnce(makeUpgradeFailure(500)) - .mockResolvedValueOnce(makeUpgradeResponse()); - - const conn = new ContainerControlConnection({ - stub: { fetch: fetchMock }, - retryTimeoutMs: 60_000 - }); - - const connectPromise = conn.connect(); - // First attempt fires synchronously. - await vi.advanceTimersByTimeAsync(0); - expect(fetchMock).toHaveBeenCalledTimes(1); - - // Backoff is 3s for attempt 1, 6s for attempt 2. - await vi.advanceTimersByTimeAsync(3_000); - expect(fetchMock).toHaveBeenCalledTimes(2); - - await vi.advanceTimersByTimeAsync(6_000); - await connectPromise; - - expect(fetchMock).toHaveBeenCalledTimes(3); - expect(conn.isConnected()).toBe(true); - } finally { - vi.useRealTimers(); - } - }); - - it('surfaces container unavailability once upgrade retry budget is exhausted', async () => { - vi.useFakeTimers(); - try { - const fetchMock = vi - .fn<(req: Request) => Promise>() - .mockResolvedValue(makeUpgradeFailure(500)); - - const conn = new ContainerControlConnection({ - stub: { fetch: fetchMock }, - // 20s budget. Walk-through with MIN_TIME_FOR_RETRY_MS = 15s and - // 3s/6s/12s exponential backoff: - // attempt 1 at t=0 (remaining 20s, retry after 3s) - // attempt 2 at t=3s (remaining 17s, retry after 6s) - // attempt 3 at t=9s (remaining 11s < 15s, give up) - retryTimeoutMs: 20_000 - }); - - const connectPromise = conn.connect(); - const assertion = expect(connectPromise).rejects.toMatchObject({ - name: 'ContainerUnavailableError', - code: ErrorCode.CONTAINER_UNAVAILABLE, - context: { reason: 'rpc_upgrade_failed', retryable: true } - }); - - // Run all timers — connect() must settle even with fake timers. - await vi.advanceTimersByTimeAsync(60_000); - await assertion; - - expect(conn.isConnected()).toBe(false); - // Three attempts before remaining < MIN_TIME_FOR_RETRY_MS. - expect(fetchMock).toHaveBeenCalledTimes(3); - } finally { - vi.useRealTimers(); - } - }); - + describe('WebSocket upgrade errors', () => { function makeContainerUnavailableResponse(): Response { return new Response( JSON.stringify({ @@ -505,8 +466,7 @@ describe('ContainerControlConnection', () => { .mockResolvedValue(makeContainerUnavailableResponse()); const conn = new ContainerControlConnection({ - stub: { fetch: fetchMock }, - retryTimeoutMs: 0 + stub: { fetch: fetchMock } }); let thrown: unknown; @@ -542,8 +502,7 @@ describe('ContainerControlConnection', () => { ); const conn = new ContainerControlConnection({ - stub: { fetch: fetchMock }, - retryTimeoutMs: 0 + stub: { fetch: fetchMock } }); await expect(conn.connect()).rejects.toMatchObject({ @@ -560,10 +519,12 @@ describe('ContainerControlConnection', () => { .mockResolvedValue(makeContainerUnavailableResponse()); const conn = new ContainerControlConnection({ - stub: { fetch: fetchMock }, - retryTimeoutMs: 0 + stub: { fetch: fetchMock } }); + const connecting = conn.connect().catch(() => undefined); + const internals = conn as unknown as { state: 'active' }; + internals.state = 'active'; const rpcCall = conn.rpc().utils.ping(); await expect(rpcCall).rejects.toMatchObject({ @@ -572,6 +533,7 @@ describe('ContainerControlConnection', () => { context: { reason: 'container_starting', retryable: true } }); expect(fetchMock).toHaveBeenCalledTimes(1); + await connecting; }); it('does not retry terminal upgrade failures', async () => { @@ -580,8 +542,7 @@ describe('ContainerControlConnection', () => { .mockResolvedValue(new Response('Not retryable', { status: 404 })); const conn = new ContainerControlConnection({ - stub: { fetch: fetchMock }, - retryTimeoutMs: 120_000 + stub: { fetch: fetchMock } }); await expect(conn.connect()).rejects.toThrow( @@ -589,117 +550,6 @@ describe('ContainerControlConnection', () => { ); expect(fetchMock).toHaveBeenCalledTimes(1); }); - - it('disables retries when retryTimeoutMs is set to 0', async () => { - const fetchMock = vi - .fn<(req: Request) => Promise>() - .mockResolvedValue(makeUpgradeFailure(500)); - - const conn = new ContainerControlConnection({ - stub: { fetch: fetchMock }, - retryTimeoutMs: 0 - }); - - await expect(conn.connect()).rejects.toMatchObject({ - name: 'ContainerUnavailableError', - code: ErrorCode.CONTAINER_UNAVAILABLE, - context: { reason: 'rpc_upgrade_failed', retryable: true } - }); - expect(fetchMock).toHaveBeenCalledTimes(1); - }); - - it('uses a default retry budget when retryTimeoutMs is omitted', async () => { - vi.useFakeTimers(); - try { - const fetchMock = vi - .fn<(req: Request) => Promise>() - .mockResolvedValueOnce(makeUpgradeFailure(503)) - .mockResolvedValueOnce(makeUpgradeResponse()); - - const conn = new ContainerControlConnection({ - stub: { fetch: fetchMock } - }); - - const connectPromise = conn.connect(); - await vi.advanceTimersByTimeAsync(0); - await vi.advanceTimersByTimeAsync(3_000); - await connectPromise; - - expect(fetchMock).toHaveBeenCalledTimes(2); - expect(conn.isConnected()).toBe(true); - } finally { - vi.useRealTimers(); - } - }); - - it('respects setRetryTimeoutMs() updates made before connect()', async () => { - vi.useFakeTimers(); - try { - const fetchMock = vi - .fn<(req: Request) => Promise>() - .mockResolvedValue(makeUpgradeFailure(503)); - - const conn = new ContainerControlConnection({ - stub: { fetch: fetchMock }, - // Start with a very large budget — would normally allow many retries. - retryTimeoutMs: 600_000 - }); - - // Lower the budget so the very first elapsed-time check gives up. - conn.setRetryTimeoutMs(1_000); - - const connectPromise = conn.connect(); - const assertion = expect(connectPromise).rejects.toMatchObject({ - name: 'ContainerUnavailableError', - code: ErrorCode.CONTAINER_UNAVAILABLE, - context: { reason: 'rpc_upgrade_failed', retryable: true } - }); - - await vi.advanceTimersByTimeAsync(60_000); - await assertion; - - // Budget too small to satisfy MIN_TIME_FOR_RETRY_MS — no retries. - expect(fetchMock).toHaveBeenCalledTimes(1); - } finally { - vi.useRealTimers(); - } - }); - - it('passes a fresh, non-aborted Request to each retry attempt', async () => { - vi.useFakeTimers(); - try { - const seenRequests: Request[] = []; - const fetchMock = vi - .fn<(req: Request) => Promise>() - .mockImplementationOnce(async (req) => { - seenRequests.push(req); - return makeUpgradeFailure(503); - }) - .mockImplementationOnce(async (req) => { - seenRequests.push(req); - if (req.signal.aborted) { - throw new Error('retry reused an aborted signal'); - } - return makeUpgradeResponse(); - }); - - const conn = new ContainerControlConnection({ - stub: { fetch: fetchMock }, - retryTimeoutMs: 60_000 - }); - - const connectPromise = conn.connect(); - await vi.advanceTimersByTimeAsync(0); - await vi.advanceTimersByTimeAsync(3_000); - await connectPromise; - - expect(seenRequests).toHaveLength(2); - expect(seenRequests[0]).not.toBe(seenRequests[1]); - expect(seenRequests[1].signal.aborted).toBe(false); - } finally { - vi.useRealTimers(); - } - }); }); /** diff --git a/packages/sandbox/tests/container-runtime-client.test.ts b/packages/sandbox/tests/container-runtime-client.test.ts deleted file mode 100644 index 0c2e3ab27..000000000 --- a/packages/sandbox/tests/container-runtime-client.test.ts +++ /dev/null @@ -1,150 +0,0 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest'; -import type { RuntimeIdentityID } from '../src/current-runtime-identity'; -import type { ResourceActivityOperation } from '../src/resource-activity-gate'; - -const constructed: Array<{ - options: { - stub: { fetch(request: Request): Promise }; - retryTimeoutMs?: number; - onOperationStarted?: () => ResourceActivityOperation; - onConnectionClose?: () => void; - onDispatch?: () => void; - translateTransportErrorsAsInterruptions?: boolean; - }; - disconnect: ReturnType; -}> = []; - -vi.mock('../src/container-control/client', () => ({ - ContainerControlClient: class { - disconnect = vi.fn(); - constructor( - readonly options: { - stub: { fetch(request: Request): Promise }; - retryTimeoutMs?: number; - onOperationStarted?: () => ResourceActivityOperation; - onConnectionClose?: () => void; - onDispatch?: () => void; - translateTransportErrorsAsInterruptions?: boolean; - } - ) { - constructed.push({ options, disconnect: this.disconnect }); - } - } -})); - -import { RuntimeControlClient } from '../src/container-control/runtime-client'; - -function id(value: string): RuntimeIdentityID { - return value as RuntimeIdentityID; -} - -function noActivity(): ResourceActivityOperation { - return { beforeCall: Promise.resolve(), finish: () => undefined }; -} - -describe('RuntimeControlClient', () => { - beforeEach(() => constructed.splice(0)); - - it('uses only the direct TCP port fetcher and never a waking path', async () => { - const directFetch = vi.fn(async () => new Response(null, { status: 200 })); - const start = vi.fn(); - const containerFetch = vi.fn(); - const fetch = vi.fn(); - const startAndWaitForPorts = vi.fn(); - const ensureContainerRunning = vi.fn(); - const getTcpPort = vi.fn(() => ({ fetch: directFetch })); - const runtimeClient = new RuntimeControlClient({ - getTcpPort, - beginNonWakingOperation: noActivity - }); - - runtimeClient.get(id('runtime-a')); - await constructed[0].options.stub.fetch( - new Request('http://localhost/rpc') - ); - - expect(getTcpPort).toHaveBeenCalledWith(3000); - expect(directFetch).toHaveBeenCalledTimes(1); - expect(constructed[0].options.retryTimeoutMs).toBe(0); - expect(constructed[0].options.onOperationStarted).toBe(noActivity); - expect(constructed[0].options.translateTransportErrorsAsInterruptions).toBe( - false - ); - expect(start).not.toHaveBeenCalled(); - expect(containerFetch).not.toHaveBeenCalled(); - expect(fetch).not.toHaveBeenCalled(); - expect(startAndWaitForPorts).not.toHaveBeenCalled(); - expect(ensureContainerRunning).not.toHaveBeenCalled(); - }); - - it('disconnects runtime A before constructing runtime B', () => { - const runtimeClient = new RuntimeControlClient({ - getTcpPort: () => ({ fetch: vi.fn() }), - beginNonWakingOperation: noActivity - }); - - runtimeClient.get(id('runtime-a')); - const first = constructed[0]; - runtimeClient.get(id('runtime-b')); - - expect(first.disconnect).toHaveBeenCalledTimes(1); - expect(constructed).toHaveLength(2); - }); - - it('permanently revokes a captured client before stale dispatch', async () => { - const directFetch = vi.fn(async () => new Response(null, { status: 200 })); - const runtimeClient = new RuntimeControlClient({ - getTcpPort: () => ({ fetch: directFetch }), - beginNonWakingOperation: noActivity - }); - - runtimeClient.get(id('runtime-a')); - const captured = constructed[0]; - let resume!: () => void; - const delayed = new Promise((resolve) => { - resume = resolve; - }); - const staleOperation = (async () => { - await delayed; - captured.options.onDispatch?.(); - return captured.options.stub.fetch(new Request('http://localhost/rpc')); - })(); - - runtimeClient.get(id('runtime-b')); - resume(); - - await expect(staleOperation).rejects.toThrow(); - expect(directFetch).not.toHaveBeenCalled(); - expect(() => constructed[1].options.onDispatch?.()).not.toThrow(); - await constructed[1].options.stub.fetch( - new Request('http://localhost/rpc') - ); - expect(directFetch).toHaveBeenCalledTimes(1); - }); - - it('translates direct port acquisition failures', async () => { - const runtimeClient = new RuntimeControlClient({ - getTcpPort: () => { - throw new Error('direct port unavailable'); - }, - beginNonWakingOperation: noActivity - }); - - expect(() => runtimeClient.get(id('runtime-a'))).toThrowError( - expect.objectContaining({ name: 'RPCTransportError' }) - ); - }); - - it('drops a direct client when its connection closes', () => { - const runtimeClient = new RuntimeControlClient({ - getTcpPort: () => ({ fetch: vi.fn() }), - beginNonWakingOperation: noActivity - }); - - runtimeClient.get(id('runtime-a')); - constructed[0].options.onConnectionClose?.(); - runtimeClient.get(id('runtime-a')); - - expect(constructed).toHaveLength(2); - }); -}); diff --git a/packages/sandbox/tests/current-runtime-identity.test.ts b/packages/sandbox/tests/current-runtime-identity.test.ts deleted file mode 100644 index 887e74a26..000000000 --- a/packages/sandbox/tests/current-runtime-identity.test.ts +++ /dev/null @@ -1,403 +0,0 @@ -import { describe, expect, it, vi } from 'vitest'; -import { - CurrentRuntimeIdentity, - RuntimeIdentityInactiveError -} from '../src/current-runtime-identity'; - -function createStorage(initial = new Map()) { - return { - get: vi.fn(async (key: string) => initial.get(key)), - put: vi.fn(async (key: string, value: unknown) => { - initial.set(key, value); - }), - delete: vi.fn(async (key: string) => { - initial.delete(key); - }) - } as unknown as DurableObjectState['storage']; -} - -function deferred() { - let resolve!: (value: T) => void; - let reject!: (error: Error) => void; - const promise = new Promise((res, rej) => { - resolve = res; - reject = rej; - }); - return { promise, resolve, reject }; -} - -describe('CurrentRuntimeIdentity', () => { - it('returns inactive when the container is not healthy', async () => { - const storage = createStorage( - new Map([['currentRuntimeIdentity', { id: 'runtime-1' }]]) - ); - const currentRuntime = new CurrentRuntimeIdentity( - storage, - async () => ({ status: 'stopped' }), - () => true - ); - - await expect(currentRuntime.getStatus()).resolves.toMatchObject({ - status: 'inactive', - reason: 'runtime-not-healthy', - containerStatus: 'stopped' - }); - }); - - it('returns inactive when the container is not running', async () => { - const storage = createStorage( - new Map([['currentRuntimeIdentity', { id: 'runtime-1' }]]) - ); - const currentRuntime = new CurrentRuntimeIdentity( - storage, - async () => ({ status: 'healthy' }), - () => false - ); - - await expect(currentRuntime.getStatus()).resolves.toMatchObject({ - status: 'inactive', - reason: 'runtime-not-running', - containerStatus: 'healthy' - }); - }); - - it('returns inactive when the runtime identity is missing', async () => { - const currentRuntime = new CurrentRuntimeIdentity( - createStorage(), - async () => ({ status: 'healthy' }), - () => true - ); - - await expect(currentRuntime.getStatus()).resolves.toMatchObject({ - status: 'inactive', - reason: 'missing-runtime-id', - containerStatus: 'healthy' - }); - }); - - it('returns active when storage, health, and running state agree', async () => { - const storage = createStorage( - new Map([['currentRuntimeIdentity', { id: 'runtime-1' }]]) - ); - const currentRuntime = new CurrentRuntimeIdentity( - storage, - async () => ({ status: 'healthy' }), - () => true - ); - - const status = await currentRuntime.getStatus(); - - expect(status.status).toBe('active'); - if (status.status === 'active') { - expect(status.runtime.id).toBe('runtime-1'); - expect(status.containerStatus).toBe('healthy'); - } - }); - - it('marks and clears the current runtime identity', async () => { - const map = new Map(); - const storage = createStorage(map); - const currentRuntime = new CurrentRuntimeIdentity( - storage, - async () => ({ status: 'healthy' }), - () => true - ); - - const runtime = await currentRuntime.markStarted(); - - expect(map.get('currentRuntimeIdentity')).toEqual({ id: runtime.id }); - - await currentRuntime.clear(); - - expect(map.has('currentRuntimeIdentity')).toBe(false); - }); - - it('invalidates a status read awaiting container state when markStarted begins', async () => { - const map = new Map([ - ['currentRuntimeIdentity', { id: 'runtime-1' }] - ]); - const state = deferred<{ status: string }>(); - const putMutation = deferred(); - const get = vi.fn(async (key: string) => map.get(key)); - const put = vi.fn(async (key: string, value: unknown) => { - await putMutation.promise; - map.set(key, value); - }); - const storage = { - get, - put, - delete: vi.fn() - } as unknown as DurableObjectState['storage']; - const currentRuntime = new CurrentRuntimeIdentity( - storage, - async () => state.promise, - () => true - ); - - const captured = currentRuntime.getStatus(); - const starting = currentRuntime.markStarted(); - state.resolve({ status: 'healthy' }); - - await expect(captured).resolves.toEqual({ - status: 'inactive', - reason: 'missing-runtime-id' - }); - expect(get).not.toHaveBeenCalled(); - - putMutation.resolve(); - await starting; - }); - - it('invalidates a status read awaiting storage when clear begins', async () => { - const stored = deferred(); - const deletion = deferred(); - const get = vi.fn(async () => stored.promise); - const deleteIdentity = vi.fn(async () => deletion.promise); - const storage = { - get, - put: vi.fn(), - delete: deleteIdentity - } as unknown as DurableObjectState['storage']; - const currentRuntime = new CurrentRuntimeIdentity( - storage, - async () => ({ status: 'healthy' }), - () => true - ); - - const captured = currentRuntime.getStatus(); - await vi.waitFor(() => expect(get).toHaveBeenCalledTimes(1)); - const clearing = currentRuntime.clear(); - stored.resolve({ id: 'runtime-1' }); - - await expect(captured).resolves.toEqual({ - status: 'inactive', - reason: 'missing-runtime-id' - }); - - deletion.resolve(true); - await clearing; - }); - - it('serializes overlapping transitions without dropping the transition fence', async () => { - const map = new Map([ - ['currentRuntimeIdentity', { id: 'runtime-1' }] - ]); - const putMutation = deferred(); - const deletion = deferred(); - const get = vi.fn(async (key: string) => map.get(key)); - const put = vi.fn(async (key: string, value: unknown) => { - await putMutation.promise; - map.set(key, value); - }); - const deleteIdentity = vi.fn(async (key: string) => { - await deletion.promise; - map.delete(key); - return true; - }); - const storage = { - get, - put, - delete: deleteIdentity - } as unknown as DurableObjectState['storage']; - const currentRuntime = new CurrentRuntimeIdentity( - storage, - async () => ({ status: 'healthy' }), - () => true - ); - - const starting = currentRuntime.markStarted(); - const clearing = currentRuntime.clear(); - - expect(put).toHaveBeenCalledTimes(1); - expect(deleteIdentity).not.toHaveBeenCalled(); - putMutation.resolve(); - await starting; - await vi.waitFor(() => expect(deleteIdentity).toHaveBeenCalledTimes(1)); - - await expect(currentRuntime.getStatus()).resolves.toEqual({ - status: 'inactive', - reason: 'missing-runtime-id' - }); - expect(get).not.toHaveBeenCalled(); - - deletion.resolve(true); - await clearing; - expect(map.has('currentRuntimeIdentity')).toBe(false); - }); - - it('fences the old identity before a clear reaches storage', async () => { - let releaseDelete!: (deleted: number) => void; - const deletePending = new Promise((resolve) => { - releaseDelete = resolve; - }); - const storage = createStorage( - new Map([['currentRuntimeIdentity', { id: 'runtime-1' }]]) - ); - vi.mocked(storage.delete).mockImplementation(async () => deletePending); - const currentRuntime = new CurrentRuntimeIdentity( - storage, - async () => ({ status: 'healthy' }), - () => true - ); - const runtime = await currentRuntime.get(); - if (!runtime) throw new Error('expected active runtime'); - - const clearing = currentRuntime.clear(); - await expect(currentRuntime.isActive(runtime)).resolves.toBe(false); - - releaseDelete(1); - await clearing; - }); - - it('resets the transition fence when storage.put rejects', async () => { - const storage = createStorage( - new Map([['currentRuntimeIdentity', { id: 'runtime-1' }]]) - ); - const put = deferred(); - vi.mocked(storage.put).mockImplementation(async () => put.promise); - const currentRuntime = new CurrentRuntimeIdentity( - storage, - async () => ({ status: 'healthy' }), - () => true - ); - const listenerStatuses: Promise[] = []; - const listenerStorageCalls: number[] = []; - currentRuntime.onChange(() => { - listenerStorageCalls.push(vi.mocked(storage.put).mock.calls.length); - listenerStatuses.push(currentRuntime.getStatus()); - }); - - const starting = currentRuntime.markStarted(); - - expect(listenerStorageCalls).toEqual([0]); - expect(storage.put).toHaveBeenCalledTimes(1); - await expect(listenerStatuses[0]).resolves.toEqual({ - status: 'inactive', - reason: 'missing-runtime-id' - }); - - const error = new Error('put failed'); - put.reject(error); - await expect(starting).rejects.toBe(error); - - await expect(currentRuntime.getStatus()).resolves.toMatchObject({ - status: 'active', - runtime: { id: 'runtime-1' }, - containerStatus: 'healthy' - }); - }); - - it('resets the transition fence when storage.delete rejects', async () => { - const storage = createStorage( - new Map([['currentRuntimeIdentity', { id: 'runtime-1' }]]) - ); - const deletion = deferred(); - vi.mocked(storage.delete).mockImplementation(async () => deletion.promise); - const currentRuntime = new CurrentRuntimeIdentity( - storage, - async () => ({ status: 'healthy' }), - () => true - ); - const listenerStatuses: Promise[] = []; - const listenerStorageCalls: number[] = []; - currentRuntime.onChange(() => { - listenerStorageCalls.push(vi.mocked(storage.delete).mock.calls.length); - listenerStatuses.push(currentRuntime.getStatus()); - }); - - const clearing = currentRuntime.clear(); - - expect(listenerStorageCalls).toEqual([0]); - expect(storage.delete).toHaveBeenCalledTimes(1); - await expect(listenerStatuses[0]).resolves.toEqual({ - status: 'inactive', - reason: 'missing-runtime-id' - }); - - const error = new Error('delete failed'); - deletion.reject(error); - await expect(clearing).rejects.toBe(error); - - await expect(currentRuntime.get()).resolves.toMatchObject({ - id: 'runtime-1' - }); - }); - - it('resets the mark-started transition fence when a listener throws', async () => { - const storage = createStorage( - new Map([['currentRuntimeIdentity', { id: 'runtime-1' }]]) - ); - const currentRuntime = new CurrentRuntimeIdentity( - storage, - async () => ({ status: 'healthy' }), - () => true - ); - const error = new Error('listener failed'); - currentRuntime.onChange(() => { - expect(storage.put).not.toHaveBeenCalled(); - throw error; - }); - - await expect(currentRuntime.markStarted()).rejects.toBe(error); - - expect(storage.put).not.toHaveBeenCalled(); - await expect(currentRuntime.get()).resolves.toMatchObject({ - id: 'runtime-1' - }); - }); - - it('resets the clear transition fence when a listener throws', async () => { - const storage = createStorage( - new Map([['currentRuntimeIdentity', { id: 'runtime-1' }]]) - ); - const currentRuntime = new CurrentRuntimeIdentity( - storage, - async () => ({ status: 'healthy' }), - () => true - ); - const error = new Error('listener failed'); - currentRuntime.onChange(() => { - expect(storage.delete).not.toHaveBeenCalled(); - throw error; - }); - - await expect(currentRuntime.clear()).rejects.toBe(error); - - expect(storage.delete).not.toHaveBeenCalled(); - await expect(currentRuntime.get()).resolves.toMatchObject({ - id: 'runtime-1' - }); - }); - - it('notifies lifecycle owners when runtime identity changes', async () => { - const currentRuntime = new CurrentRuntimeIdentity( - createStorage(), - async () => ({ status: 'healthy' }), - () => true - ); - const changed = vi.fn(); - const unsubscribe = currentRuntime.onChange(changed); - - await currentRuntime.markStarted(); - await currentRuntime.clear(); - unsubscribe(); - await currentRuntime.markStarted(); - - expect(changed).toHaveBeenCalledTimes(2); - }); - - it('throws a typed error when asserting an inactive runtime', async () => { - const currentRuntime = new CurrentRuntimeIdentity( - createStorage(), - async () => ({ status: 'healthy' }), - () => true - ); - const runtime = await currentRuntime.markStarted(); - - await currentRuntime.clear(); - - await expect(currentRuntime.assertActive(runtime)).rejects.toBeInstanceOf( - RuntimeIdentityInactiveError - ); - }); -}); diff --git a/packages/sandbox/tests/extensions.test.ts b/packages/sandbox/tests/extensions.test.ts index 3fa30664f..34a930b40 100644 --- a/packages/sandbox/tests/extensions.test.ts +++ b/packages/sandbox/tests/extensions.test.ts @@ -1,8 +1,8 @@ /** * SDK-side extensions unit tests. * - * Exercises `SandboxExtension`: lazy construction, the hash-first connect - * dance against `client.extensions`, and reconnect-on-use sidecar semantics. + * Exercises `SandboxExtension`: lazy construction, scoped runtime access, and + * the hash-first sidecar connect dance against the extension host. * * Container-side end-to-end coverage lives in * `packages/sandbox-container/tests/extensions/extension-host.test.ts`. @@ -18,7 +18,15 @@ import type { import { EXTENSION_TARBALL_REQUIRED } from '@repo/shared'; import type { Mock } from 'vitest'; import { afterEach, describe, expect, it, vi } from 'vitest'; -import { SandboxExtension, type SandboxLike } from '../src/extensions'; +import type { + ExtensionRuntimeCall, + ExtensionRuntimeControl +} from '../src/extensions'; +import { + SandboxExtension, + type SandboxLike, + sandboxRuntimeCall +} from '../src/extensions'; type ExtensionsAPIMock = { connect: Mock; @@ -28,13 +36,18 @@ type ExtensionsAPIMock = { type UtilsAPIMock = { ping: Mock; - getVersion: Mock; +}; + +type RuntimeScope = { + operation: string; + control: ExtensionRuntimeControl; }; function makeSandbox(): { sandbox: SandboxLike; api: ExtensionsAPIMock; utils: UtilsAPIMock; + scopes: RuntimeScope[]; } { const api: ExtensionsAPIMock = { connect: vi.fn(async () => ({}) as unknown), @@ -42,21 +55,30 @@ function makeSandbox(): { stop: vi.fn(async () => {}) }; const utils: UtilsAPIMock = { - ping: vi.fn(async () => 'pong'), - getVersion: vi.fn(async () => '1.0.0') + ping: vi.fn(async () => 'pong') }; - // The tests only ever exercise `client.extensions` and `client.utils`; - // widening to SandboxAPI would force us to stub every sub-API for no benefit. - const sandbox = { - client: { + const scopes: RuntimeScope[] = []; + const runtimeCall = (async (operation, call) => { + const control = { + files: { domain: 'files' }, + ports: { domain: 'ports' }, + backup: { domain: 'backup' }, + watch: { domain: 'watch' }, + tunnels: { domain: 'tunnels' }, + terminals: { domain: 'terminals' }, extensions: api as unknown as SandboxExtensionsAPI, utils: utils as unknown as SandboxUtilsAPI - } - } as unknown as SandboxLike; - return { sandbox, api, utils }; + } as unknown as ExtensionRuntimeControl; + scopes.push({ operation, control }); + return await call(control); + }) as ExtensionRuntimeCall; + const sandbox: SandboxLike = { + [sandboxRuntimeCall]: runtimeCall + }; + return { sandbox, api, utils, scopes }; } -const TARBALL = new Uint8Array([0x1f, 0x8b, 0x08, 0x00]); // gzip magic + flags, plenty enough to hash +const TARBALL = new Uint8Array([0x1f, 0x8b, 0x08, 0x00]); const PKG: ExtensionPackage = { tarball: TARBALL }; @@ -71,16 +93,92 @@ describe('SandboxExtension', () => { super(sandbox); } health(packageHash: string) { - return this.client.extensions.health(packageHash); + return this.withRuntime('dummy.health', (control) => + control.extensions.health(packageHash) + ); + } + + pingTwice() { + return this.withRuntime('dummy.pingTwice', async (control) => { + const first = await control.utils.ping(); + const second = await control.utils.ping(); + return `${first}:${second}`; + }); + } + } + + class TypeContractExtension extends SandboxExtension { + // biome-ignore lint/complexity/noUselessConstructor: widens the protected base constructor + constructor(sandbox: SandboxLike) { + super(sandbox); + } + + escapeControl() { + return this.withRuntime( + 'type.escape.control', + // @ts-expect-error runtime control must not be returned from withRuntime + async (control) => control + ); + } + + escapeFiles() { + return this.withRuntime( + 'type.escape.files', + // @ts-expect-error runtime control domains must not be returned from withRuntime + async (control) => control.files + ); + } + + escapeExtensions() { + return this.withRuntime( + 'type.escape.extensions', + // @ts-expect-error runtime control domains must not be returned from withRuntime + async (control) => control.extensions + ); + } + } + + void TypeContractExtension; + + class EscapeExtension extends SandboxExtension { + capturedControl: ExtensionRuntimeControl | undefined; + capturedFiles: ExtensionRuntimeControl['files'] | undefined; + + constructor(sandbox: SandboxLike) { + super(sandbox, PKG); + } + + captureControl() { + return this.withRuntime('escape.capture', async (control) => { + this.capturedControl = control; + this.capturedFiles = control.files; + return { files: control.files }; + }); } - ping() { - return this.client.utils.ping(); + escapeControl() { + return this.withRuntime( + 'escape.control', + async (control) => control as unknown as object + ); + } + + escapeDomain(domain: keyof ExtensionRuntimeControl) { + return this.withRuntime( + 'escape.domain', + async (control) => control[domain] as unknown as object + ); + } + + ordinaryObject() { + return this.withRuntime('escape.ordinaryObject', async () => ({ + ok: true + })); } } it('captures the sandbox without exposing it as an own property (RPC-safe)', () => { - const sandbox = { client: {} } as unknown as SandboxLike; + const { sandbox } = makeSandbox(); const ext = new DummyExtension(sandbox); expect(Object.getOwnPropertyNames(ext)).not.toContain('sandbox'); @@ -88,8 +186,8 @@ describe('SandboxExtension', () => { expect(Object.keys(ext)).toHaveLength(0); }); - it('exposes the extension control client to subclasses lazily', async () => { - const { sandbox, api, utils } = makeSandbox(); + it('uses scoped runtime callbacks for direct control access', async () => { + const { sandbox, api, utils, scopes } = makeSandbox(); api.health.mockResolvedValue({ packageHash: 'abc123', id: 'ext', @@ -106,9 +204,84 @@ describe('SandboxExtension', () => { running: true, pid: 123 }); + await expect(ext.pingTwice()).resolves.toBe('pong:pong'); + expect(api.health).toHaveBeenCalledWith('abc123'); - await expect(ext.ping()).resolves.toBe('pong'); - expect(utils.ping).toHaveBeenCalled(); + expect(utils.ping).toHaveBeenCalledTimes(2); + expect(scopes.map((scope) => scope.operation)).toEqual([ + 'dummy.health', + 'dummy.pingTwice' + ]); + expect(scopes[0].control).not.toBe(scopes[1].control); + }); + + it('keeps multiple control calls inside one explicit runtime scope', async () => { + const { sandbox, utils, scopes } = makeSandbox(); + const ext = new DummyExtension(sandbox); + + await expect(ext.pingTwice()).resolves.toBe('pong:pong'); + + expect(utils.ping).toHaveBeenCalledTimes(2); + expect(scopes).toHaveLength(1); + expect(scopes[0].operation).toBe('dummy.pingTwice'); + }); + + it('rejects cast attempts to escape runtime control handles', async () => { + const { sandbox } = makeSandbox(); + const ext = new EscapeExtension(sandbox); + + await expect(ext.escapeControl()).rejects.toThrow( + /must not return runtime control handles/ + ); + await expect(ext.escapeDomain('files')).rejects.toThrow( + /must not return runtime control handles/ + ); + await expect(ext.escapeDomain('ports')).rejects.toThrow( + /must not return runtime control handles/ + ); + await expect(ext.escapeDomain('backup')).rejects.toThrow( + /must not return runtime control handles/ + ); + await expect(ext.escapeDomain('watch')).rejects.toThrow( + /must not return runtime control handles/ + ); + await expect(ext.escapeDomain('tunnels')).rejects.toThrow( + /must not return runtime control handles/ + ); + await expect(ext.escapeDomain('terminals')).rejects.toThrow( + /must not return runtime control handles/ + ); + await expect(ext.escapeDomain('extensions')).rejects.toThrow( + /must not return runtime control handles/ + ); + await expect(ext.escapeDomain('utils')).rejects.toThrow( + /must not return runtime control handles/ + ); + }); + + it('revokes captured and nested runtime control handles', async () => { + const { sandbox, utils } = makeSandbox(); + const ext = new EscapeExtension(sandbox); + + const nested = await ext.captureControl(); + + await expect(ext.capturedControl?.utils.ping()).rejects.toThrow( + /no longer valid outside its runtime callback/ + ); + await expect(ext.capturedFiles?.readFile('/tmp/file')).rejects.toThrow( + /no longer valid outside its runtime callback/ + ); + await expect(nested.files.readFile('/tmp/file')).rejects.toThrow( + /no longer valid outside its runtime callback/ + ); + expect(utils.ping).not.toHaveBeenCalled(); + }); + + it('allows ordinary objects to leave runtime callbacks', async () => { + const { sandbox } = makeSandbox(); + const ext = new EscapeExtension(sandbox); + + await expect(ext.ordinaryObject()).resolves.toEqual({ ok: true }); }); it('throws a helpful error if sidecar methods are used without a package', async () => { @@ -118,7 +291,10 @@ describe('SandboxExtension', () => { super(sandbox); } run() { - return this.sidecar(); + return this.withSidecar( + 'no-package.run', + async (api) => api + ); } health() { return this.sidecarHealth(); @@ -137,7 +313,7 @@ describe('SandboxExtension', () => { }); it('does not touch the sandbox during construction (lazy)', () => { - const { sandbox, api } = makeSandbox(); + const { sandbox, api, scopes } = makeSandbox(); class Ext extends SandboxExtension { constructor(s: SandboxLike) { @@ -149,23 +325,69 @@ describe('SandboxExtension', () => { expect(api.connect).not.toHaveBeenCalled(); expect(api.health).not.toHaveBeenCalled(); expect(api.stop).not.toHaveBeenCalled(); + expect(scopes).toHaveLength(0); }); }); describe('SandboxExtension (sidecar mode)', () => { + interface NestedFakeAPI { + doNested(input: string): Promise; + } + interface FakeAPI { do(input: string): Promise; + nested?: NestedFakeAPI; + getNested?(): Promise; + plainData?(): Promise<{ nested: { value: string } }>; } function buildExt() { - const { sandbox, api } = makeSandbox(); + const { sandbox, api, scopes } = makeSandbox(); class Ext extends SandboxExtension { + capturedSidecar: FakeAPI | undefined; + capturedNestedSidecar: NestedFakeAPI | undefined; + constructor(s: SandboxLike) { super(s, PKG); } - async run(input: string) { - const stub = await this.sidecar(); - return stub.do(input); + run(input: string) { + return this.withSidecar('fake.run', (stub) => + stub.do(input) + ); + } + captureSidecarCall() { + return this.withSidecar( + 'fake.capture', + async (stub) => { + this.capturedSidecar = stub; + await stub.do('inside'); + await expect(stub.do('inside-again')).resolves.toBe( + 'first:inside-again' + ); + return 'captured'; + } + ); + } + nestedCall() { + return this.withSidecar( + 'fake.nested', + async (stub) => { + const nested = await stub.getNested?.(); + if (!nested) throw new Error('missing nested api'); + this.capturedNestedSidecar = nested; + return await nested.doNested('inside'); + } + ); + } + plainData() { + return this.withSidecar( + 'fake.plainData', + async (stub) => { + const data = await stub.plainData?.(); + if (!data) throw new Error('missing plain data'); + return data; + } + ); } health() { return this.sidecarHealth(); @@ -174,16 +396,15 @@ describe('SandboxExtension (sidecar mode)', () => { return this.stopSidecar(); } } - return { ext: new Ext(sandbox), api }; + return { ext: new Ext(sandbox), api, scopes }; } - it('sends the hash alone on first connect; retries with tarball on ExtensionTarballRequired', async () => { - const { ext, api } = buildExt(); + it('sends the hash alone on first connect; retries with tarball on ExtensionTarballRequired in one scope', async () => { + const { ext, api, scopes } = buildExt(); const fakeStub = { do: vi.fn(async (s: string) => `did:${s}`) }; api.connect .mockImplementationOnce(async () => { - // Host has not provisioned this hash yet. const err = new Error('need tarball'); (err as { name: string }).name = EXTENSION_TARBALL_REQUIRED; throw err; @@ -194,6 +415,7 @@ describe('SandboxExtension (sidecar mode)', () => { expect(result).toBe('did:hi'); expect(api.connect).toHaveBeenCalledTimes(2); + expect(scopes.map((scope) => scope.operation)).toEqual(['fake.run']); const first = api.connect.mock.calls[0][0] as ExtensionConnectRequest; const second = api.connect.mock.calls[1][0] as ExtensionConnectRequest; @@ -204,7 +426,7 @@ describe('SandboxExtension (sidecar mode)', () => { }); it('retries when capnweb wraps ExtensionTarballRequired as RPCTransportError', async () => { - const { ext, api } = buildExt(); + const { ext, api, scopes } = buildExt(); const fakeStub = { do: vi.fn(async (s: string) => `did:${s}`) }; api.connect @@ -217,12 +439,13 @@ describe('SandboxExtension (sidecar mode)', () => { await expect(ext.run('hi')).resolves.toBe('did:hi'); expect(api.connect).toHaveBeenCalledTimes(2); + expect(scopes).toHaveLength(1); const second = api.connect.mock.calls[1][0] as ExtensionConnectRequest; expect(second.tarball).toBeInstanceOf(Uint8Array); }); it('adds a diagnostic helper when sidecar provisioning fails after tarball retry', async () => { - const { ext, api } = buildExt(); + const { ext, api, scopes } = buildExt(); api.connect .mockRejectedValueOnce( new Error( @@ -235,26 +458,64 @@ describe('SandboxExtension (sidecar mode)', () => { /Failed to provision sandbox sidecar package.*valid npm-style \.tgz.*bun add failed/ ); expect(api.connect).toHaveBeenCalledTimes(2); + expect(scopes).toHaveLength(1); }); - it('reconnects through the host on each sidecar call', async () => { - const { ext, api } = buildExt(); - const firstStub = { do: vi.fn(async (s: string) => `first:${s}`) }; + it('opens a new runtime scope for each sidecar call without reconnecting captured stubs', async () => { + const { ext, api, scopes } = buildExt(); + let firstRuntimeActive = true; + const firstStub = { + do: vi.fn(async (s: string) => { + if (!firstRuntimeActive) throw new Error('stale sidecar'); + return `first:${s}`; + }) + }; const secondStub = { do: vi.fn(async (s: string) => `second:${s}`) }; api.connect .mockResolvedValueOnce(firstStub) .mockResolvedValueOnce(secondStub); - await expect(ext.run('a')).resolves.toBe('first:a'); + await expect(ext.captureSidecarCall()).resolves.toBe('captured'); + await expect(ext.capturedSidecar?.do('outside')).rejects.toThrow( + /no longer valid/ + ); + firstRuntimeActive = false; await expect(ext.run('b')).resolves.toBe('second:b'); expect(api.connect).toHaveBeenCalledTimes(2); - expect(firstStub.do).toHaveBeenCalledTimes(1); + expect(firstStub.do).toHaveBeenCalledTimes(2); expect(secondStub.do).toHaveBeenCalledTimes(1); + expect(scopes.map((scope) => scope.operation)).toEqual([ + 'fake.capture', + 'fake.run' + ]); }); - it('retries cleanly after a failed connect', async () => { + it('revokes nested sidecar remotes and detaches plain data results', async () => { const { ext, api } = buildExt(); + const sourceData = { nested: { value: 'before' } }; + const nestedStub = { + doNested: vi.fn(async (input: string) => `nested:${input}`) + }; + const fakeStub: FakeAPI = { + do: vi.fn(async (s: string) => `did:${s}`), + getNested: vi.fn(async () => nestedStub), + plainData: vi.fn(async () => sourceData) + }; + api.connect.mockResolvedValue(fakeStub); + + await expect(ext.nestedCall()).resolves.toBe('nested:inside'); + await expect( + ext.capturedNestedSidecar?.doNested('outside') + ).rejects.toThrow(/no longer valid/); + + const detached = await ext.plainData(); + sourceData.nested.value = 'after'; + expect(detached).toEqual({ nested: { value: 'before' } }); + }); + + it('retries cleanly after a failed connect', async () => { + const { ext, api, scopes } = buildExt(); const fakeStub = { do: vi.fn(async (s: string) => `did:${s}`) }; api.connect .mockRejectedValueOnce(new Error('connect failed')) @@ -263,27 +524,35 @@ describe('SandboxExtension (sidecar mode)', () => { await expect(ext.run('a')).rejects.toThrow(/connect failed/); await expect(ext.run('b')).resolves.toBe('did:b'); expect(api.connect).toHaveBeenCalledTimes(2); + expect(scopes.map((scope) => scope.operation)).toEqual([ + 'fake.run', + 'fake.run' + ]); }); it('does not retry on a non-ExtensionTarballRequired error', async () => { - const { ext, api } = buildExt(); + const { ext, api, scopes } = buildExt(); api.connect.mockRejectedValueOnce(new Error('something else')); await expect(ext.run('a')).rejects.toThrow(/something else/); expect(api.connect).toHaveBeenCalledTimes(1); + expect(scopes).toHaveLength(1); }); - it('forwards health by package hash', async () => { - const { ext, api } = buildExt(); + it('forwards health by package hash in its own scope', async () => { + const { ext, api, scopes } = buildExt(); await ext.health(); expect(api.health).toHaveBeenCalledTimes(1); + expect(scopes.map((scope) => scope.operation)).toEqual([ + 'extension.health' + ]); const arg = api.health.mock.calls[0][0]; expect(typeof arg).toBe('string'); expect(arg).toMatch(/^[0-9a-f]{64}$/); }); it('stopSidecar stops the host-side sidecar and the next call reconnects', async () => { - const { ext, api } = buildExt(); + const { ext, api, scopes } = buildExt(); const fakeStub = { do: vi.fn(async (s: string) => `did:${s}`) }; api.connect.mockResolvedValue(fakeStub); @@ -293,5 +562,10 @@ describe('SandboxExtension (sidecar mode)', () => { expect(api.connect).toHaveBeenCalledTimes(2); expect(api.stop).toHaveBeenCalledTimes(1); + expect(scopes.map((scope) => scope.operation)).toEqual([ + 'fake.run', + 'extension.stop', + 'fake.run' + ]); }); }); diff --git a/packages/sandbox/tests/fixtures/process-capability-rpc.ts b/packages/sandbox/tests/fixtures/process-capability-rpc.ts index 5e3e4653d..8115e8bd4 100644 --- a/packages/sandbox/tests/fixtures/process-capability-rpc.ts +++ b/packages/sandbox/tests/fixtures/process-capability-rpc.ts @@ -44,7 +44,7 @@ class CleanLogSubscription } class CapabilityControl implements ProcessCapabilityControl { - retainConnection(): () => void { + retainRuntimeHold(): () => void { return () => undefined; } @@ -103,7 +103,10 @@ export class ProcessCapabilityRPCTestDO extends DurableObject { capability: new ProcessCapabilityTarget({ id: status.id, pid: status.pid, - runtime: { id: 'runtime-test' }, + runtime: { + id: 'runtime-test', + runtimeIncarnationID: 'incarnation-test' + }, lifecycle: new CapabilityLifecycle() }) }); diff --git a/packages/sandbox/tests/get-sandbox.test.ts b/packages/sandbox/tests/get-sandbox.test.ts index c4c38588b..7bb7ceeb0 100644 --- a/packages/sandbox/tests/get-sandbox.test.ts +++ b/packages/sandbox/tests/get-sandbox.test.ts @@ -6,9 +6,9 @@ import { getSandbox, type Sandbox } from '../src/sandbox'; // Mock the Container module vi.mock('@cloudflare/containers', () => ({ switchPort: vi.fn((request: Request, port: number) => { - const url = new URL(request.url); - url.pathname = `/proxy/${port}${url.pathname}`; - return new Request(url, request); + const headers = new Headers(request.headers); + headers.set('cf-container-target-port', String(port)); + return new Request(request, { headers }); }), Container: class Container { ctx: any; @@ -54,6 +54,8 @@ describe('getSandbox', () => { return Promise.resolve(); } ), + containerFetch: vi.fn(async () => new Response('container response')), + authorizePortRequest: vi.fn(async () => 'route-token'), setSandboxName: vi.fn(), setSleepAfter: vi.fn((value: string | number) => { mockStub.sleepAfter = value; @@ -80,6 +82,154 @@ describe('getSandbox', () => { }); }); + it('exposes containerFetch but not internal port authorization', async () => { + const sandbox = getSandbox({} as any, 'test-sandbox'); + const internal = sandbox as unknown as Record; + + expect(internal.containerFetch).toBeTypeOf('function'); + expect(internal.authorizePortRequest).toBeUndefined(); + + const request = new Request('https://example.com/data'); + await sandbox.containerFetch(request, 8080); + await sandbox.containerFetch( + 'https://example.com/data', + { method: 'POST' }, + 8081 + ); + + expect(mockStub.containerFetch).toHaveBeenNthCalledWith(1, request, 8080); + expect(mockStub.containerFetch).toHaveBeenNthCalledWith( + 2, + 'https://example.com/data', + { method: 'POST' }, + 8081 + ); + }); + + it('authorizes switchPort requests across the Sandbox RPC boundary', async () => { + mockStub.fetch = vi.fn(async () => new Response('forwarded')); + const sandbox = getSandbox({} as any, 'test-sandbox'); + const request = new Request('https://example.com/ws', { + headers: { 'cf-container-target-port': '8080' } + }); + + await sandbox.fetch(request); + + expect(mockStub.authorizePortRequest).toHaveBeenCalledWith(8080, '/ws'); + const forwarded = mockStub.fetch.mock.calls[0][0] as Request; + expect(forwarded.headers.get('cf-container-target-port')).toBe('8080'); + expect(forwarded.headers.get('x-sandbox-port-route-token')).toBe( + 'route-token' + ); + }); + + it('preserves switchPort routing to the inherited default port', async () => { + mockStub.fetch = vi.fn(async () => new Response('forwarded')); + const sandbox = getSandbox({} as any, 'test-sandbox'); + const request = new Request('https://example.com/app', { + headers: { 'cf-container-target-port': '3000' } + }); + + await sandbox.fetch(request); + + expect(mockStub.authorizePortRequest).toHaveBeenCalledWith(3000, '/app'); + }); + + it('does not expose token creation internals', () => { + const sandbox = getSandbox({} as any, 'test-sandbox'); + const internal = sandbox as unknown as Record; + + expect(internal.createPortRequestToken).toBeUndefined(); + }); + + it('forwards native watch streams', async () => { + const bytes = new TextEncoder().encode('data: watching\n\n'); + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(bytes); + controller.close(); + } + }); + mockStub.watch = vi.fn().mockResolvedValue(stream); + + const sandbox = getSandbox({} as any, 'watch-sandbox'); + const result = await sandbox.watch('/workspace'); + const reader = result.getReader(); + + await expect(reader.read()).resolves.toEqual({ done: false, value: bytes }); + await expect(reader.read()).resolves.toEqual({ + done: true, + value: undefined + }); + expect(mockStub.watch).toHaveBeenCalledWith('/workspace', {}); + }); + + it('applies configuration before forwarding operations', async () => { + let resolveConfiguration!: () => void; + mockStub.configure = vi.fn( + () => + new Promise((resolve) => { + resolveConfiguration = resolve; + }) + ); + mockStub.writeFile = vi.fn(async () => {}); + + const sandbox = getSandbox({} as any, 'configured-sandbox', { + keepAlive: true + }); + const write = sandbox.writeFile('/workspace/file.txt', 'content'); + + expect(mockStub.writeFile).not.toHaveBeenCalled(); + resolveConfiguration(); + await write; + expect(mockStub.writeFile).toHaveBeenCalledOnce(); + }); + + it('does not forward operations when configuration fails', async () => { + mockStub.configure = vi.fn().mockRejectedValue(new Error('config failed')); + mockStub.writeFile = vi.fn(async () => {}); + + const sandbox = getSandbox({} as any, 'configured-sandbox', { + keepAlive: true + }); + + await expect( + sandbox.writeFile('/workspace/file.txt', 'content') + ).rejects.toThrow('config failed'); + expect(mockStub.writeFile).not.toHaveBeenCalled(); + }); + + it('shares pending configuration failures across clients', async () => { + let rejectConfiguration!: (error: Error) => void; + mockStub.configure = vi.fn( + () => + new Promise((_, reject) => { + rejectConfiguration = reject; + }) + ); + mockStub.writeFile = vi.fn(async () => {}); + const mockNamespace = {} as any; + + const first = getSandbox(mockNamespace, 'configured-sandbox', { + keepAlive: true + }); + const second = getSandbox(mockNamespace, 'configured-sandbox', { + keepAlive: true + }); + const firstWrite = first.writeFile('/workspace/first.txt', 'first'); + const secondWrite = second.writeFile('/workspace/second.txt', 'second'); + + expect(mockStub.configure).toHaveBeenCalledOnce(); + expect(mockStub.writeFile).not.toHaveBeenCalled(); + const rejectedWrites = Promise.all([ + expect(firstWrite).rejects.toThrow('config failed'), + expect(secondWrite).rejects.toThrow('config failed') + ]); + rejectConfiguration(new Error('config failed')); + await rejectedWrites; + expect(mockStub.writeFile).not.toHaveBeenCalled(); + }); + it('maps Durable Object code-update resets to OperationInterruptedError for enhanced methods', async () => { const mockNamespace = {} as any; mockStub.exec = vi.fn(async () => { @@ -235,11 +385,15 @@ describe('getSandbox', () => { it('should only configure fields that changed on later calls', async () => { const mockNamespace = {} as any; + mockStub.listProcesses = vi.fn(async () => []); getSandbox(mockNamespace, 'test-sandbox'); await Promise.resolve(); - getSandbox(mockNamespace, 'test-sandbox', { sleepAfter: '5m' }); + const sandbox = getSandbox(mockNamespace, 'test-sandbox', { + sleepAfter: '5m' + }); + await sandbox.listProcesses(); expect(mockStub.configure).toHaveBeenNthCalledWith(1, { sandboxName: { @@ -286,7 +440,7 @@ describe('getSandbox', () => { expect(response).toBe(expectedResponse); }); - it('should pass through non-enhanced methods to the stub', () => { + it('should pass through non-enhanced methods to the stub', async () => { // RPC methods like exec, writeFile, etc. are accessed via target[prop] // and dispatched through JSRPC which doesn't need this binding. mockStub.validatePortToken = vi.fn().mockResolvedValue(true); @@ -294,7 +448,7 @@ describe('getSandbox', () => { const mockNamespace = {} as any; const sandbox = getSandbox(mockNamespace, 'test-sandbox'); - sandbox.validatePortToken(8080, 'token123'); + await sandbox.validatePortToken(8080, 'token123'); expect(mockStub.validatePortToken).toHaveBeenCalledWith(8080, 'token123'); }); @@ -425,8 +579,15 @@ describe('getSandbox', () => { return new Response(null, { status: 200 }); }); mockStub.createTerminal = vi.fn(async () => ({ - id: 'terminal-a', - connect: (request: Request) => mockStub.fetch(request) + snapshot: { + id: 'terminal-a', + command: ['bash'], + status: 'running' + }, + runtimeIncarnationID: 'runtime-a', + capability: { + authorizeConnection: vi.fn(async () => 'terminal-route-token') + } })); const mockNamespace = {} as any; @@ -443,17 +604,97 @@ describe('getSandbox', () => { command: ['bash'] }); expect(mockStub.fetch).toHaveBeenCalledOnce(); - expect(proxiedRequest?.url).toBe('https://example.com/terminal'); + const proxiedURL = new URL(proxiedRequest!.url); + expect(proxiedURL.pathname).toBe('/ws/terminal'); + expect(proxiedRequest?.headers.get('cf-container-target-port')).toBe( + '3000' + ); + expect(proxiedURL.searchParams.get('terminalId')).toBe('terminal-a'); + expect(proxiedURL.searchParams.get('runtimeIncarnationID')).toBe( + 'runtime-a' + ); + }); + + it('reconstructs terminal output from a pull subscription', async () => { + const event = { + type: 'data' as const, + terminalId: 'terminal-a', + cursor: '1', + timestamp: new Date().toISOString(), + data: new Uint8Array([65]) + }; + const next = vi + .fn() + .mockResolvedValueOnce({ done: false, value: event }) + .mockResolvedValueOnce({ done: true, value: undefined }); + const cancel = vi.fn().mockResolvedValue(undefined); + const dispose = vi.fn(); + mockStub.createTerminal = vi.fn(async () => ({ + snapshot: { + id: 'terminal-a', + command: ['bash'], + status: 'running' + }, + runtimeIncarnationID: 'runtime-a', + capability: { + openOutput: vi.fn(async () => ({ + next, + cancel, + [Symbol.dispose]: dispose + })) + } + })); + + const sandbox = getSandbox({} as any, 'test-sandbox'); + const terminal = await sandbox.createTerminal({ command: ['bash'] }); + const reader = (await terminal.output()).getReader(); + + await expect(reader.read()).resolves.toEqual({ + done: false, + value: event + }); + await expect(reader.read()).resolves.toEqual({ + done: true, + value: undefined + }); + expect(next).toHaveBeenCalledTimes(2); + expect(cancel).toHaveBeenCalledOnce(); + expect(dispose).toHaveBeenCalledOnce(); + expect(Object.keys(terminal)).toEqual([ + 'id', + 'getSnapshot', + 'write', + 'resize', + 'output', + 'waitForExit', + 'interrupt', + 'terminate', + 'connect' + ]); + expect('openOutput' in terminal).toBe(false); + expect('runtimeIncarnationID' in terminal).toBe(false); }); it('gets and lists terminal handles by snapshot', async () => { mockStub.getTerminal = vi.fn(async () => ({ - id: 'terminal-a', - command: ['bash'], - status: 'running' + snapshot: { + id: 'terminal-a', + command: ['bash'], + status: 'running' + }, + runtimeIncarnationID: 'runtime-a', + capability: {} })); mockStub.listTerminals = vi.fn(async () => [ - { id: 'terminal-a', command: ['bash'], status: 'running' } + { + snapshot: { + id: 'terminal-a', + command: ['bash'], + status: 'running' + }, + runtimeIncarnationID: 'runtime-a', + capability: {} + } ]); const mockNamespace = {} as any; @@ -470,11 +711,16 @@ describe('getSandbox', () => { it('forwards terminal interrupt and terminate through handle methods', async () => { mockStub.getTerminal = vi.fn(async () => ({ - id: 'terminal-a', - command: ['bash'], - status: 'running', - interrupt: vi.fn(), - terminate: vi.fn() + snapshot: { + id: 'terminal-a', + command: ['bash'], + status: 'running' + }, + runtimeIncarnationID: 'runtime-a', + capability: { + interrupt: vi.fn(), + terminate: vi.fn() + } })); const mockNamespace = {} as any; diff --git a/packages/sandbox/tests/helpers/mock-control-client.ts b/packages/sandbox/tests/helpers/mock-control-client.ts index 219d1efb8..2ea0f2cd8 100644 --- a/packages/sandbox/tests/helpers/mock-control-client.ts +++ b/packages/sandbox/tests/helpers/mock-control-client.ts @@ -1,13 +1,24 @@ import { vi } from 'vitest'; +import type { ContainerControlClient } from '../../src/container-control'; import type { Sandbox } from '../../src/sandbox'; +export type SandboxWithClient = Sandbox & { + client: ContainerControlClient; +}; + +export function asSandboxWithClient( + sandbox: Sandbox +): SandboxWithClient { + return sandbox as SandboxWithClient; +} + /** * Create a test double for Sandbox's container control client. * * Keep this aligned with ContainerControlClient's public surface so tests can * override only the methods relevant to the scenario under test. */ -export function createMockControlClient(): Sandbox['client'] { +export function createMockControlClient(): ContainerControlClient { return { files: { readFile: vi.fn(), @@ -22,7 +33,19 @@ export function createMockControlClient(): Sandbox['client'] { exists: vi.fn() }, ports: { - openWatch: vi.fn() + openWatch: vi.fn(async () => ({ + stream: vi.fn( + async () => + new ReadableStream({ + start(controller) { + controller.enqueue({ type: 'ready' }); + controller.close(); + } + }) + ), + cancel: vi.fn(async () => undefined), + [Symbol.dispose]: vi.fn() + })) }, processes: { start: vi.fn(async (command) => ({ @@ -94,8 +117,7 @@ export function createMockControlClient(): Sandbox['client'] { })) }, utils: { - ping: vi.fn(), - getVersion: vi.fn() + ping: vi.fn() }, workspace: { createArchive: vi.fn(async () => ({ archivePath: '/tmp/archive.tar' })), @@ -131,9 +153,8 @@ export function createMockControlClient(): Sandbox['client'] { terminate: vi.fn(), hasActive: vi.fn() }, - setRetryTimeoutMs: vi.fn(), isWebSocketConnected: vi.fn(), connect: vi.fn(), disconnect: vi.fn() - } as unknown as Sandbox['client']; + } as unknown as ContainerControlClient; } diff --git a/packages/sandbox/tests/local-backup-restore-lifecycle.test.ts b/packages/sandbox/tests/local-backup-restore-lifecycle.test.ts index 087266c03..89dbb178e 100644 --- a/packages/sandbox/tests/local-backup-restore-lifecycle.test.ts +++ b/packages/sandbox/tests/local-backup-restore-lifecycle.test.ts @@ -1,8 +1,16 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import type { BackupRestoreOperationRecord } from '../src/backup/restore-operation-store'; -import { ErrorCode, RPCTransportError } from '../src/errors'; +import type { ContainerControlClient } from '../src/container-control'; +import { + ErrorCode, + OperationInterruptedError, + RPCTransportError +} from '../src/errors'; import { Sandbox } from '../src/sandbox'; -import { createMockControlClient } from './helpers/mock-control-client'; +import { + asSandboxWithClient, + createMockControlClient +} from './helpers/mock-control-client'; vi.mock('@cloudflare/containers', () => { class MockContainer { @@ -109,7 +117,11 @@ async function createLocalRestoreSandbox(params?: { storageHooks?: { onPut?: (key: string, value: StoredValue) => void }; }) { const storageMap = new Map(); - storageMap.set('currentRuntimeIdentity', { id: 'runtime-1' }); + storageMap.set('currentRuntimeIdentity', { + schemaVersion: 1, + id: 'runtime-1', + runtimeIncarnationID: 'incarnation-1' + }); storageMap.set('sandbox:lifetime', { id: 'lifetime-1', generation: 1, @@ -151,7 +163,34 @@ async function createLocalRestoreSandbox(params?: { expect(ctx.blockConcurrencyWhile).toHaveBeenCalled(); }); - sandbox.client = createMockControlClient(); + const sandboxWithClient = asSandboxWithClient(sandbox); + sandboxWithClient.client = createMockControlClient(); + vi.spyOn( + ( + sandbox as unknown as { + runtimeRunner: { + runWaking( + operation: string, + call: (lease: { + runtime: { id: string; runtimeIncarnationID: string }; + control: ContainerControlClient; + retain(): { release(): void }; + }) => Promise + ): Promise; + }; + } + ).runtimeRunner, + 'runWaking' + ).mockImplementation(async (_operation, call) => + call({ + runtime: { + id: 'runtime-1', + runtimeIncarnationID: 'incarnation-1' + }, + control: sandboxWithClient.client, + retain: () => ({ release: () => {} }) + }) + ); return { sandbox, storageMap, backupId }; } @@ -174,6 +213,7 @@ describe('local backup restore lifecycle', () => { status: 'committed', phase: 'verified', runtimeIdentityID: 'runtime-1', + runtimeIncarnationID: 'incarnation-1', payload: { backupId, dir: '/workspace/project', @@ -183,18 +223,11 @@ describe('local backup restore lifecycle', () => { }); }); - it('captures cold-start runtime identity before restore transfer', async () => { - const order: string[] = []; - const { sandbox, storageMap, backupId } = await createLocalRestoreSandbox({ - storageHooks: { - onPut: (key) => { - if (key === 'currentRuntimeIdentity') { - order.push('runtimeReady'); - } - } - } + it('records the supplied runtime without marking a new one', async () => { + const writes: string[] = []; + const { sandbox, backupId } = await createLocalRestoreSandbox({ + storageHooks: { onPut: (key) => writes.push(key) } }); - storageMap.delete('currentRuntimeIdentity'); await sandbox.restoreBackup({ id: backupId, @@ -202,25 +235,26 @@ describe('local backup restore lifecycle', () => { localBucket: true }); - expect(order.slice(0, 1)).toEqual(['runtimeReady']); + expect(writes).not.toContain('currentRuntimeIdentity'); }); it('does not mark the local archive ready before stream upload completes', async () => { const { sandbox, storageMap, backupId } = await createLocalRestoreSandbox(); - vi.spyOn(sandbox.client.files, 'writeFileStream').mockImplementationOnce( - async () => { - const record = storageMap.get( - `operations:restore:${backupId}:/workspace/project` - ) as BackupRestoreOperationRecord; - expect(record.phase).toBe('runtime_ready'); - return { - success: true, - path: `/var/backups/${backupId}.sqsh`, - bytesWritten: 4, - timestamp: '2026-06-15T12:00:00.000Z' - } as never; - } - ); + vi.spyOn( + asSandboxWithClient(sandbox).client.files, + 'writeFileStream' + ).mockImplementationOnce(async () => { + const record = storageMap.get( + `operations:restore:${backupId}:/workspace/project` + ) as BackupRestoreOperationRecord; + expect(record.phase).toBe('runtime_ready'); + return { + success: true, + path: `/var/backups/${backupId}.sqsh`, + bytesWritten: 4, + timestamp: '2026-06-15T12:00:00.000Z' + } as never; + }); await sandbox.restoreBackup({ id: backupId, @@ -229,10 +263,10 @@ describe('local backup restore lifecycle', () => { }); }); - it('recovers internally when local archive streaming loses the RPC transport once', async () => { + it('does not replay local archive streaming after transport loss', async () => { const { sandbox, storageMap, backupId } = await createLocalRestoreSandbox(); const writeFileStreamSpy = vi - .spyOn(sandbox.client.files, 'writeFileStream') + .spyOn(asSandboxWithClient(sandbox).client.files, 'writeFileStream') .mockRejectedValueOnce(createDisposedRPCError()) .mockResolvedValueOnce({ success: true, @@ -241,22 +275,19 @@ describe('local backup restore lifecycle', () => { timestamp: '2026-06-15T12:00:00.000Z' }); - const result = await sandbox.restoreBackup({ - id: backupId, - dir: '/workspace/project', - localBucket: true - }); + await expect( + sandbox.restoreBackup({ + id: backupId, + dir: '/workspace/project', + localBucket: true + }) + ).rejects.toBeInstanceOf(OperationInterruptedError); - expect(result).toEqual({ - success: true, - id: backupId, - dir: '/workspace/project' - }); - expect(writeFileStreamSpy).toHaveBeenCalledTimes(2); + expect(writeFileStreamSpy).toHaveBeenCalledTimes(1); const record = storageMap.get( `operations:restore:${backupId}:/workspace/project` ) as BackupRestoreOperationRecord; - expect(record.status).toBe('committed'); - expect(record.phase).toBe('verified'); + expect(record.status).toBe('interrupted'); + expect(record.phase).toBe('interrupted'); }); }); diff --git a/packages/sandbox/tests/local-backup.test.ts b/packages/sandbox/tests/local-backup.test.ts index 226779e67..05106ee3c 100644 --- a/packages/sandbox/tests/local-backup.test.ts +++ b/packages/sandbox/tests/local-backup.test.ts @@ -1,7 +1,11 @@ import type { SandboxCommand } from '@repo/shared'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { connect, Sandbox } from '../src/sandbox'; -import { createMockControlClient } from './helpers/mock-control-client'; +import type { ContainerControlClient } from '../src/container-control'; +import { Sandbox } from '../src/sandbox'; +import { + asSandboxWithClient, + createMockControlClient +} from './helpers/mock-control-client'; vi.mock('@cloudflare/containers', () => { const mockSwitchPort = vi.fn((request: Request, port: number) => { @@ -50,6 +54,64 @@ vi.mock('@cloudflare/containers', () => { }); // Mock R2 bucket binding +function createSSEFileStream(content: Uint8Array): ReadableStream { + const ssePayload = [ + `data: ${JSON.stringify({ type: 'metadata', mimeType: 'application/octet-stream', size: content.length, isBinary: true, encoding: 'base64' })}\n\n`, + `data: ${JSON.stringify({ type: 'chunk', data: btoa(String.fromCharCode(...content)) })}\n\n`, + `data: ${JSON.stringify({ type: 'complete' })}\n\n` + ].join(''); + + return new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(ssePayload)); + controller.close(); + } + }); +} + +type TestBackupLease = { + runtime: { id: string; runtimeIncarnationID: string }; + control: ContainerControlClient; + retain(onInterrupt?: () => void): { release(): void }; +}; + +function testBackupLease(control: ContainerControlClient): TestBackupLease { + return { + runtime: { id: 'runtime-1', runtimeIncarnationID: 'incarnation-1' }, + control, + retain: () => ({ release: () => {} }) + }; +} + +function installRuntimeCallRecorder( + sandbox: Sandbox, + controls: ContainerControlClient[] +): { operations: string[]; controls: ContainerControlClient[] } { + const calls: { operations: string[]; controls: ContainerControlClient[] } = { + operations: [], + controls: [] + }; + let index = 0; + const target = sandbox as unknown as { + runWakingComposite( + operation: string, + call: (lease: { + runtime: { id: string; runtimeIncarnationID: string }; + control: ContainerControlClient; + retain(onInterrupt?: () => void): { release(): void }; + }) => Promise + ): Promise; + }; + target.runWakingComposite = async (operation, call) => { + const control = controls[index++]; + if (!control) throw new Error(`Missing test control for ${operation}`); + calls.operations.push(operation); + calls.controls.push(control); + return await call(testBackupLease(control)); + }; + return calls; +} + function createMockR2Bucket() { const store = new Map(); return { @@ -142,6 +204,11 @@ describe('Local Backup & Restore', () => { mockBucket = createMockR2Bucket(); const storageMap = new Map(); + storageMap.set('currentRuntimeIdentity', { + schemaVersion: 1, + id: 'runtime-1', + runtimeIncarnationID: 'incarnation-1' + }); mockCtx = { storage: { @@ -181,10 +248,13 @@ describe('Local Backup & Restore', () => { expect(mockCtx.blockConcurrencyWhile).toHaveBeenCalled(); }); - sandbox = Object.assign(stub, { - wsConnect: connect(stub) - }); - sandbox.client = createMockControlClient(); + sandbox = stub; + const sandboxWithClient = asSandboxWithClient(sandbox); + sandboxWithClient.client = createMockControlClient(); + installRuntimeCallRecorder( + sandbox, + Array.from({ length: 100 }, () => sandboxWithClient.client) + ); }); afterEach(() => { @@ -196,7 +266,10 @@ describe('Local Backup & Restore', () => { // Mock createArchive const archiveContent = new Uint8Array([0x68, 0x73, 0x71, 0x73]); // "hsqs" squashfs magic - vi.spyOn(sandbox.client.backup, 'createArchive').mockResolvedValue({ + vi.spyOn( + asSandboxWithClient(sandbox).client.backup, + 'createArchive' + ).mockResolvedValue({ success: true, archivePath: '/var/backups/test.sqsh', sizeBytes: archiveContent.length @@ -214,9 +287,10 @@ describe('Local Backup & Restore', () => { } }); - vi.spyOn(sandbox.client.files, 'readFileStream').mockResolvedValue( - stream - ); + vi.spyOn( + asSandboxWithClient(sandbox).client.files, + 'readFileStream' + ).mockResolvedValue(stream); const result = await sandbox.createBackup({ dir: '/workspace/myapp', @@ -230,7 +304,9 @@ describe('Local Backup & Restore', () => { expect(result.localBucket).toBe(true); // Verify archive was created in the container - expect(sandbox.client.backup.createArchive).toHaveBeenCalledWith( + expect( + asSandboxWithClient(sandbox).client.backup.createArchive + ).toHaveBeenCalledWith( '/workspace/myapp', expect.stringContaining('/var/backups/'), { @@ -259,6 +335,169 @@ describe('Local Backup & Restore', () => { expect(mockBucket.head).toHaveBeenCalled(); }); + it('uses one runtime lease for the complete backup attempt', async () => { + const archiveContent = new Uint8Array([0x68, 0x73, 0x71, 0x73]); + const control = createMockControlClient(); + const runtimeCalls = installRuntimeCallRecorder(sandbox, [control]); + + vi.spyOn(control.backup, 'createArchive').mockResolvedValue({ + success: true, + archivePath: '/var/backups/test.sqsh', + sizeBytes: archiveContent.length + }); + vi.spyOn(control.files, 'readFileStream').mockResolvedValue( + createSSEFileStream(archiveContent) + ); + vi.spyOn(control.backup, 'cleanupArchive').mockResolvedValue(undefined); + + await sandbox.createBackup({ + dir: '/workspace/myapp', + localBucket: true + }); + + expect(runtimeCalls.operations).toEqual(['backup.create']); + expect(runtimeCalls.controls).toEqual([control]); + expect(control.backup.createArchive).toHaveBeenCalledTimes(1); + expect(control.files.readFileStream).toHaveBeenCalledTimes(1); + expect(control.backup.cleanupArchive).toHaveBeenCalledTimes(1); + }); + + it('keeps the backup lease open until archive upload completes', async () => { + const archiveContent = new Uint8Array([0x68, 0x73, 0x71, 0x73]); + const control = createMockControlClient(); + let attemptSettled = false; + const target = sandbox as unknown as { + runWakingComposite( + operation: string, + call: (lease: TestBackupLease) => Promise + ): Promise; + }; + target.runWakingComposite = async (_operation, call) => { + try { + return await call(testBackupLease(control)); + } finally { + attemptSettled = true; + } + }; + + vi.spyOn(control.backup, 'createArchive').mockResolvedValue({ + success: true, + archivePath: '/var/backups/test.sqsh', + sizeBytes: archiveContent.length + }); + vi.spyOn(control.backup, 'cleanupArchive').mockResolvedValue(undefined); + + let streamController: ReadableStreamDefaultController; + const stream = new ReadableStream({ + start(controller) { + streamController = controller; + controller.enqueue( + new TextEncoder().encode( + `data: ${JSON.stringify({ type: 'metadata', mimeType: 'application/octet-stream', size: archiveContent.length, isBinary: true, encoding: 'base64' })}\n\n` + ) + ); + controller.enqueue( + new TextEncoder().encode( + `data: ${JSON.stringify({ type: 'chunk', data: btoa(String.fromCharCode(...archiveContent)) })}\n\n` + ) + ); + } + }); + vi.spyOn(control.files, 'readFileStream').mockResolvedValue(stream); + + const backupPromise = sandbox.createBackup({ + dir: '/workspace/myapp', + localBucket: true + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(attemptSettled).toBe(false); + expect(mockBucket.put).toHaveBeenCalledWith( + expect.stringMatching(/^backups\/.*\/data\.sqsh$/), + expect.any(ReadableStream) + ); + expect(control.backup.cleanupArchive).not.toHaveBeenCalled(); + + streamController!.enqueue( + new TextEncoder().encode( + `data: ${JSON.stringify({ type: 'complete' })}\n\n` + ) + ); + streamController!.close(); + + await backupPromise; + expect(attemptSettled).toBe(true); + expect(control.backup.cleanupArchive).toHaveBeenCalledTimes(1); + }); + + it('cleans up before the failed backup lease settles', async () => { + const archiveContent = new Uint8Array([0x68, 0x73, 0x71, 0x73]); + const control = createMockControlClient(); + const settlementOrder: string[] = []; + const target = sandbox as unknown as { + runWakingComposite( + operation: string, + call: (lease: TestBackupLease) => Promise + ): Promise; + }; + target.runWakingComposite = async (_operation, call) => { + try { + return await call(testBackupLease(control)); + } finally { + settlementOrder.push('lease'); + } + }; + + vi.spyOn(control.backup, 'createArchive').mockResolvedValue({ + success: true, + archivePath: '/var/backups/test.sqsh', + sizeBytes: archiveContent.length + }); + vi.spyOn(control.files, 'readFileStream').mockResolvedValue( + createSSEFileStream(archiveContent) + ); + vi.spyOn(control.backup, 'cleanupArchive').mockImplementation( + async () => { + settlementOrder.push('cleanup'); + } + ); + mockBucket.put.mockRejectedValueOnce(new Error('r2 put failed')); + + await expect( + sandbox.createBackup({ + dir: '/workspace/myapp', + localBucket: true + }) + ).rejects.toThrow('r2 put failed'); + + expect(settlementOrder).toEqual(['cleanup', 'lease']); + expect(control.backup.cleanupArchive).toHaveBeenCalledTimes(1); + }); + + it('uses the admitted control for rejection cleanup', async () => { + const control = createMockControlClient(); + const runtimeCalls = installRuntimeCallRecorder(sandbox, [control]); + + vi.spyOn(control.backup, 'createArchive').mockResolvedValue({ + success: false, + archivePath: '', + sizeBytes: 0 + }); + vi.spyOn(control.backup, 'cleanupArchive').mockResolvedValue(undefined); + + await expect( + sandbox.createBackup({ + dir: '/workspace/myapp', + localBucket: true + }) + ).rejects.toThrow('Container failed to create backup archive'); + + expect(runtimeCalls.operations).toEqual(['backup.create']); + expect(runtimeCalls.controls).toEqual([control]); + expect(control.backup.createArchive).toHaveBeenCalledTimes(1); + expect(control.backup.cleanupArchive).toHaveBeenCalledTimes(1); + }); + it('should throw if BACKUP_BUCKET binding is missing', async () => { // Remove the BACKUP_BUCKET binding (sandbox as any).env = {}; @@ -299,7 +538,10 @@ describe('Local Backup & Restore', () => { }); it('should clean up on archive creation failure', async () => { - vi.spyOn(sandbox.client.backup, 'createArchive').mockResolvedValue({ + vi.spyOn( + asSandboxWithClient(sandbox).client.backup, + 'createArchive' + ).mockResolvedValue({ success: false, archivePath: '', sizeBytes: 0 @@ -313,11 +555,16 @@ describe('Local Backup & Restore', () => { ).rejects.toThrow('Container failed to create backup archive'); // Verify archive cleanup uses specialized backup RPC. - expect(sandbox.client.backup.cleanupArchive).toHaveBeenCalled(); + expect( + asSandboxWithClient(sandbox).client.backup.cleanupArchive + ).toHaveBeenCalled(); }); it('should not require presigned URL credentials', async () => { - vi.spyOn(sandbox.client.backup, 'createArchive').mockResolvedValue({ + vi.spyOn( + asSandboxWithClient(sandbox).client.backup, + 'createArchive' + ).mockResolvedValue({ success: true, archivePath: '/var/backups/test.sqsh', sizeBytes: 4 @@ -337,9 +584,10 @@ describe('Local Backup & Restore', () => { } }); - vi.spyOn(sandbox.client.files, 'readFileStream').mockResolvedValue( - stream - ); + vi.spyOn( + asSandboxWithClient(sandbox).client.files, + 'readFileStream' + ).mockResolvedValue(stream); // Should succeed without R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY, etc. const result = await sandbox.createBackup({ @@ -352,7 +600,10 @@ describe('Local Backup & Restore', () => { }); it('should normalize globstar excludes before calling createArchive', async () => { - vi.spyOn(sandbox.client.backup, 'createArchive').mockResolvedValue({ + vi.spyOn( + asSandboxWithClient(sandbox).client.backup, + 'createArchive' + ).mockResolvedValue({ success: true, archivePath: '/var/backups/test.sqsh', sizeBytes: 4 @@ -365,7 +616,10 @@ describe('Local Backup & Restore', () => { `data: ${JSON.stringify({ type: 'complete' })}\n\n` ].join(''); - vi.spyOn(sandbox.client.files, 'readFileStream').mockResolvedValue( + vi.spyOn( + asSandboxWithClient(sandbox).client.files, + 'readFileStream' + ).mockResolvedValue( new ReadableStream({ start(controller) { controller.enqueue(new TextEncoder().encode(ssePayload)); @@ -380,7 +634,9 @@ describe('Local Backup & Restore', () => { excludes: ['**/node_modules/.cache', '**/.next/cache', 'dist/**', '**'] }); - expect(sandbox.client.backup.createArchive).toHaveBeenCalledWith( + expect( + asSandboxWithClient(sandbox).client.backup.createArchive + ).toHaveBeenCalledWith( '/workspace/myapp', expect.stringContaining('/var/backups/'), { @@ -418,7 +674,10 @@ describe('Local Backup & Restore', () => { ); // Mock writeFileStream for writing archive to container - vi.spyOn(sandbox.client.files, 'writeFileStream').mockResolvedValue({ + vi.spyOn( + asSandboxWithClient(sandbox).client.files, + 'writeFileStream' + ).mockResolvedValue({ success: true, path: '/var/backups/test.sqsh', bytesWritten: archiveData.length, @@ -441,13 +700,17 @@ describe('Local Backup & Restore', () => { ); // Verify archive was streamed to container - expect(sandbox.client.files.writeFileStream).toHaveBeenCalledWith( + expect( + asSandboxWithClient(sandbox).client.files.writeFileStream + ).toHaveBeenCalledWith( expect.stringContaining('/var/backups/'), expect.any(ReadableStream) ); // Verify extraction used the specialized backup RPC. - expect(sandbox.client.backup.extractArchive).toHaveBeenCalledWith( + expect( + asSandboxWithClient(sandbox).client.backup.extractArchive + ).toHaveBeenCalledWith( '/workspace/myapp', expect.stringContaining('/var/backups/') ); @@ -529,7 +792,10 @@ describe('Local Backup & Restore', () => { archiveData ); - vi.spyOn(sandbox.client.files, 'writeFile').mockResolvedValue({ + vi.spyOn( + asSandboxWithClient(sandbox).client.files, + 'writeFile' + ).mockResolvedValue({ success: true, path: '/var/backups/test.sqsh', timestamp: new Date().toISOString() @@ -542,9 +808,9 @@ describe('Local Backup & Restore', () => { }); // Verify cleanup used the specialized backup RPC. - expect(sandbox.client.backup.cleanupArchive).toHaveBeenCalledWith( - expect.stringContaining('.sqsh') - ); + expect( + asSandboxWithClient(sandbox).client.backup.cleanupArchive + ).toHaveBeenCalledWith(expect.stringContaining('.sqsh')); }); it('should handle unsquashfs failure', async () => { @@ -567,14 +833,20 @@ describe('Local Backup & Restore', () => { archiveData ); - vi.spyOn(sandbox.client.files, 'writeFile').mockResolvedValue({ + vi.spyOn( + asSandboxWithClient(sandbox).client.files, + 'writeFile' + ).mockResolvedValue({ success: true, path: '/var/backups/test.sqsh', timestamp: new Date().toISOString() } as any); // Make specialized extraction fail - vi.spyOn(sandbox.client.backup, 'extractArchive').mockRejectedValue( + vi.spyOn( + asSandboxWithClient(sandbox).client.backup, + 'extractArchive' + ).mockRejectedValue( new Error('unsquashfs extraction failed: unsquashfs: bad archive') ); @@ -607,7 +879,10 @@ describe('Local Backup & Restore', () => { archiveData ); - vi.spyOn(sandbox.client.files, 'writeFileStream').mockRejectedValue( + vi.spyOn( + asSandboxWithClient(sandbox).client.files, + 'writeFileStream' + ).mockRejectedValue( new Error("Failed to write file '/var/backups/test.sqsh': disk full") ); @@ -619,13 +894,18 @@ describe('Local Backup & Restore', () => { }) ).rejects.toThrow('disk full'); - expect(sandbox.client.backup.extractArchive).not.toHaveBeenCalled(); + expect( + asSandboxWithClient(sandbox).client.backup.extractArchive + ).not.toHaveBeenCalled(); }); }); describe('localBucket round-trip', () => { it('should round-trip localBucket through DirectoryBackup', async () => { - vi.spyOn(sandbox.client.backup, 'createArchive').mockResolvedValue({ + vi.spyOn( + asSandboxWithClient(sandbox).client.backup, + 'createArchive' + ).mockResolvedValue({ success: true, archivePath: '/var/backups/test.sqsh', sizeBytes: 4 @@ -638,7 +918,10 @@ describe('Local Backup & Restore', () => { `data: ${JSON.stringify({ type: 'complete' })}\n\n` ].join(''); - vi.spyOn(sandbox.client.files, 'readFileStream').mockResolvedValue( + vi.spyOn( + asSandboxWithClient(sandbox).client.files, + 'readFileStream' + ).mockResolvedValue( new ReadableStream({ start(controller) { controller.enqueue(new TextEncoder().encode(ssePayload)); @@ -647,7 +930,10 @@ describe('Local Backup & Restore', () => { }) ); - vi.spyOn(sandbox.client.files, 'writeFile').mockResolvedValue({ + vi.spyOn( + asSandboxWithClient(sandbox).client.files, + 'writeFile' + ).mockResolvedValue({ success: true, path: '/var/backups/test.sqsh', timestamp: new Date().toISOString() @@ -667,11 +953,15 @@ describe('Local Backup & Restore', () => { expect(result.success).toBe(true); // Verify local extraction was used, not production transfer commands. - expect(sandbox.client.backup.extractArchive).toHaveBeenCalledWith( + expect( + asSandboxWithClient(sandbox).client.backup.extractArchive + ).toHaveBeenCalledWith( '/workspace/myapp', expect.stringContaining('/var/backups/') ); - expect(sandbox.client.backup.downloadArchive).not.toHaveBeenCalled(); + expect( + asSandboxWithClient(sandbox).client.backup.downloadArchive + ).not.toHaveBeenCalled(); }); it('should use production path when localBucket is not set', async () => { diff --git a/packages/sandbox/tests/local-mount-sync.test.ts b/packages/sandbox/tests/local-mount-sync.test.ts index 94bbbaf08..606879994 100644 --- a/packages/sandbox/tests/local-mount-sync.test.ts +++ b/packages/sandbox/tests/local-mount-sync.test.ts @@ -144,6 +144,9 @@ function createControllableWatchClient() { const close = () => { controller!.close(); }; + const fail = (error: Error) => { + controller!.error(error); + }; return { client: { @@ -155,6 +158,7 @@ function createControllableWatchClient() { }, emit, close, + fail, cancel, dispose }; @@ -170,6 +174,16 @@ function createMockControlClient( } as any; } +function deferred() { + let resolve!: (value: T | PromiseLike) => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- @@ -187,6 +201,269 @@ describe('LocalMountSyncManager', () => { vi.restoreAllMocks(); }); + describe('runtime callback scoping', () => { + it('releases retained runtime authority exactly once', async () => { + const release = vi.fn(); + const manager = new LocalMountSyncManager({ + bucket: createMockR2Bucket(new Map()) as unknown as R2Bucket, + mountPath: '/mnt/data', + prefix: undefined, + readOnly: true, + runRuntimeCall: async (_operation, call) => + await call( + createMockControlClient( + createMockFileClient(), + createMockWatchClient() + ) + ), + runtimeHold: { release }, + logger + }); + + manager.interrupt(); + await manager.stop(); + + expect(release).toHaveBeenCalledTimes(1); + }); + + it('delegates sequential file RPCs through the provided scope', async () => { + const r2Objects = new Map([ + ['file1.txt', { body: 'hello', etag: 'etag1' }] + ]); + const bucket = createMockR2Bucket(r2Objects); + const watchClient = createMockWatchClient(); + const controls: ReturnType[] = []; + + const manager = new LocalMountSyncManager({ + bucket: bucket as unknown as R2Bucket, + mountPath: '/mnt/data', + prefix: undefined, + readOnly: true, + runRuntimeCall: async (_operation, call) => { + const control = createMockControlClient( + createMockFileClient(), + watchClient + ); + controls.push(control); + return await call(control); + }, + runtimeHold: { release: () => {} }, + logger + }); + + await manager.start(); + + expect(controls.length).toBeGreaterThanOrEqual(2); + expect(new Set(controls).size).toBe(controls.length); + }); + + it('stop remains pending until the active watch callback settles', async () => { + const bucket = createMockR2Bucket(new Map()); + const fileClient = createMockFileClient(); + const watch = createControllableWatchClient(); + const client = createMockControlClient(fileClient, watch.client); + const watchRelease = deferred(); + let watchEntered = false; + let stopped = false; + + const manager = new LocalMountSyncManager({ + bucket: bucket as unknown as R2Bucket, + mountPath: '/mnt/data', + prefix: undefined, + readOnly: false, + runRuntimeCall: async (operation, call) => { + if (operation === 'mount.local.watch') { + watchEntered = true; + } + const result = await call(client); + if (operation === 'mount.local.watch') { + await watchRelease.promise; + } + return result; + }, + runtimeHold: { release: () => {} }, + logger + }); + + await manager.start(); + await vi.waitFor(() => expect(watchEntered).toBe(true)); + + const stopPromise = manager.stop().then(() => { + stopped = true; + }); + await Promise.resolve(); + + expect(stopped).toBe(false); + watchRelease.resolve(); + await stopPromise; + expect(stopped).toBe(true); + }); + + it('does not reconnect after a stopped watch callback rejects', async () => { + const bucket = createMockR2Bucket(new Map()); + const fileClient = createMockFileClient(); + const watch = createControllableWatchClient(); + const client = createMockControlClient(fileClient, watch.client); + + const manager = new LocalMountSyncManager({ + bucket: bucket as unknown as R2Bucket, + mountPath: '/mnt/data', + prefix: undefined, + readOnly: false, + runRuntimeCall: async (operation, call) => { + if (operation === 'mount.local.watch') { + throw new Error('runtime replaced'); + } + return await call(client); + }, + runtimeHold: { release: () => {} }, + logger, + pollIntervalMs: 1000 + }); + + await manager.start(); + await manager.stop(); + await vi.advanceTimersByTimeAsync(5000); + + expect(watch.client.watch).not.toHaveBeenCalled(); + }); + + it('stop joins already-admitted poll work and prevents later poll RPCs', async () => { + const r2Objects = new Map(); + const bucket = createMockR2Bucket(r2Objects); + const fileClient = createMockFileClient(); + const watchClient = createMockWatchClient(); + const client = createMockControlClient(fileClient, watchClient); + const writeRelease = deferred(); + const operations: string[] = []; + let stopped = false; + + const manager = new LocalMountSyncManager({ + bucket: bucket as unknown as R2Bucket, + mountPath: '/mnt/data', + prefix: undefined, + readOnly: true, + runRuntimeCall: async (operation, call) => { + operations.push(operation); + if (operation === 'mount.local.writeFile') { + await writeRelease.promise; + } + return await call(client); + }, + runtimeHold: { release: () => {} }, + logger, + pollIntervalMs: 1000 + }); + + await manager.start(); + operations.length = 0; + r2Objects.set('new-file.txt', { body: 'new', etag: 'etag-new' }); + await vi.advanceTimersByTimeAsync(1000); + await vi.waitFor(() => + expect(operations).toContain('mount.local.writeFile') + ); + + const stopPromise = manager.stop().then(() => { + stopped = true; + }); + await Promise.resolve(); + expect(stopped).toBe(false); + writeRelease.resolve(); + await stopPromise; + + operations.length = 0; + r2Objects.set('later.txt', { body: 'later', etag: 'etag-later' }); + await vi.advanceTimersByTimeAsync(5000); + expect(operations).toEqual([]); + }); + + it('watch events read files through a fresh control outside the watch control', async () => { + const bucket = createMockR2Bucket(new Map()); + const watch = createControllableWatchClient(); + const watchControl = createMockControlClient( + createMockFileClient(), + watch.client + ); + const readControl = createMockControlClient( + createMockFileClient(), + createMockWatchClient() + ); + const controlsByOperation = new Map(); + + const manager = new LocalMountSyncManager({ + bucket: bucket as unknown as R2Bucket, + mountPath: '/mnt/data', + prefix: undefined, + readOnly: false, + runRuntimeCall: async (operation, call) => { + const control = + operation === 'mount.local.readFile' ? readControl : watchControl; + controlsByOperation.set(operation, [ + ...(controlsByOperation.get(operation) ?? []), + control + ]); + return await call(control); + }, + runtimeHold: { release: () => {} }, + logger + }); + + await manager.start(); + await vi.waitFor(() => expect(watch.client.watch).toHaveBeenCalled()); + watch.emit({ + type: 'event', + path: '/mnt/data/file.txt', + eventType: 'modify', + isDirectory: false + }); + await vi.waitFor(() => + expect(readControl.files.readFile).toHaveBeenCalled() + ); + + expect(controlsByOperation.get('mount.local.watch')).toEqual([ + watchControl + ]); + expect(controlsByOperation.get('mount.local.readFile')).toEqual([ + readControl + ]); + expect(readControl).not.toBe(watchControl); + watch.close(); + await manager.stop(); + }); + + it('keeps the local watch runtime callback pending until the stream closes', async () => { + const bucket = createMockR2Bucket(new Map()); + const fileClient = createMockFileClient(); + const watch = createControllableWatchClient(); + const client = createMockControlClient(fileClient, watch.client); + const settled: string[] = []; + + const manager = new LocalMountSyncManager({ + bucket: bucket as unknown as R2Bucket, + mountPath: '/mnt/data', + prefix: undefined, + readOnly: false, + runRuntimeCall: async (operation, call) => { + const result = await call(client); + settled.push(operation); + return result; + }, + runtimeHold: { release: () => {} }, + logger + }); + + await manager.start(); + await vi.waitFor(() => expect(watch.client.watch).toHaveBeenCalled()); + await Promise.resolve(); + + expect(settled).not.toContain('mount.local.watch'); + + watch.close(); + await vi.waitFor(() => expect(settled).toContain('mount.local.watch')); + await manager.stop(); + }); + }); + describe('initial full sync (R2 → Container)', () => { it('should sync all R2 objects to the container on start', async () => { const r2Objects = new Map([ @@ -203,7 +480,7 @@ describe('LocalMountSyncManager', () => { mountPath: '/mnt/data', prefix: undefined, readOnly: true, - client, + runRuntimeCall: async (_operation, call) => call(client), logger }); @@ -253,7 +530,7 @@ describe('LocalMountSyncManager', () => { mountPath: '/mnt/data', prefix: undefined, readOnly: true, - client, + runRuntimeCall: async (_operation, call) => call(client), logger }); @@ -276,7 +553,7 @@ describe('LocalMountSyncManager', () => { mountPath: '/mnt/data', prefix: undefined, readOnly: false, - client, + runRuntimeCall: async (_operation, call) => call(client), logger }); @@ -305,8 +582,9 @@ describe('LocalMountSyncManager', () => { mountPath: '/mnt/data', prefix: undefined, readOnly: true, - client, + runRuntimeCall: async (_operation, call) => call(client), logger, + runtimeHold: { release: () => {} }, pollIntervalMs: 1000 }); @@ -347,8 +625,9 @@ describe('LocalMountSyncManager', () => { mountPath: '/mnt/data', prefix: undefined, readOnly: true, - client, + runRuntimeCall: async (_operation, call) => call(client), logger, + runtimeHold: { release: () => {} }, pollIntervalMs: 1000 }); @@ -385,8 +664,9 @@ describe('LocalMountSyncManager', () => { mountPath: '/mnt/data', prefix: undefined, readOnly: true, - client, + runRuntimeCall: async (_operation, call) => call(client), logger, + runtimeHold: { release: () => {} }, pollIntervalMs: 1000 }); @@ -421,8 +701,9 @@ describe('LocalMountSyncManager', () => { mountPath: '/mnt/data', prefix: undefined, readOnly: true, - client, + runRuntimeCall: async (_operation, call) => call(client), logger, + runtimeHold: { release: () => {} }, pollIntervalMs: 1000 }); @@ -458,7 +739,7 @@ describe('LocalMountSyncManager', () => { mountPath: '/mnt/data', prefix: '/data/', readOnly: true, - client, + runRuntimeCall: async (_operation, call) => call(client), logger }); @@ -493,7 +774,7 @@ describe('LocalMountSyncManager', () => { mountPath: '/mnt/data', prefix: '/some/prefix/', readOnly: true, - client, + runRuntimeCall: async (_operation, call) => call(client), logger }); @@ -530,8 +811,9 @@ describe('LocalMountSyncManager', () => { mountPath: '/mnt/data', prefix: '/some/prefix/', readOnly: false, - client, + runRuntimeCall: async (_operation, call) => call(client), logger, + runtimeHold: { release: () => {} }, pollIntervalMs: 60_000 }); @@ -572,8 +854,9 @@ describe('LocalMountSyncManager', () => { mountPath: '/mnt/data', prefix: undefined, readOnly: false, - client, + runRuntimeCall: async (_operation, call) => call(client), logger, + runtimeHold: { release: () => {} }, pollIntervalMs: 60_000 }); @@ -606,7 +889,7 @@ describe('LocalMountSyncManager', () => { mountPath: '/mnt/data', prefix: '/', readOnly: true, - client, + runRuntimeCall: async (_operation, call) => call(client), logger }); @@ -637,7 +920,7 @@ describe('LocalMountSyncManager', () => { mountPath: '/mnt/data', prefix: 'data/', readOnly: true, - client, + runRuntimeCall: async (_operation, call) => call(client), logger }) ).toThrow(/Prefix must start with/); @@ -657,7 +940,7 @@ describe('LocalMountSyncManager', () => { mountPath: '/mnt/data', prefix: '/uploads', readOnly: true, - client, + runRuntimeCall: async (_operation, call) => call(client), logger }); @@ -694,8 +977,9 @@ describe('LocalMountSyncManager', () => { mountPath: '/mnt/data', prefix: undefined, readOnly: false, - client, + runRuntimeCall: async (_operation, call) => call(client), logger, + runtimeHold: { release: () => {} }, pollIntervalMs: 60_000 }); @@ -746,8 +1030,9 @@ describe('LocalMountSyncManager', () => { mountPath: '/mnt/data', prefix: undefined, readOnly: false, - client, + runRuntimeCall: async (_operation, call) => call(client), logger, + runtimeHold: { release: () => {} }, pollIntervalMs: 60_000 }); @@ -792,8 +1077,9 @@ describe('LocalMountSyncManager', () => { mountPath: '/mnt/data', prefix: undefined, readOnly: false, - client, + runRuntimeCall: async (_operation, call) => call(client), logger, + runtimeHold: { release: () => {} }, pollIntervalMs: 60_000 }); @@ -834,8 +1120,9 @@ describe('LocalMountSyncManager', () => { mountPath: '/mnt/data', prefix: undefined, readOnly: false, - client, + runRuntimeCall: async (_operation, call) => call(client), logger, + runtimeHold: { release: () => {} }, pollIntervalMs: 60_000 }); @@ -888,8 +1175,9 @@ describe('LocalMountSyncManager', () => { mountPath: '/mnt/data', prefix: undefined, readOnly: false, - client, + runRuntimeCall: async (_operation, call) => call(client), logger, + runtimeHold: { release: () => {} }, pollIntervalMs: 60_000 }); @@ -928,8 +1216,9 @@ describe('LocalMountSyncManager', () => { mountPath: '/mnt/data', prefix: undefined, readOnly: false, - client, + runRuntimeCall: async (_operation, call) => call(client), logger, + runtimeHold: { release: () => {} }, pollIntervalMs: 60_000 }); @@ -968,8 +1257,9 @@ describe('LocalMountSyncManager', () => { mountPath: '/mnt/data', prefix: '/uploads/', readOnly: false, - client, + runRuntimeCall: async (_operation, call) => call(client), logger, + runtimeHold: { release: () => {} }, pollIntervalMs: 60_000 }); @@ -1008,8 +1298,9 @@ describe('LocalMountSyncManager', () => { mountPath: '/mnt/data', prefix: undefined, readOnly: true, - client, + runRuntimeCall: async (_operation, call) => call(client), logger, + runtimeHold: { release: () => {} }, pollIntervalMs: 1000 }); diff --git a/packages/sandbox/tests/preview-forwarding.test.ts b/packages/sandbox/tests/preview-forwarding.test.ts index 1da5a6e60..bdcc8184c 100644 --- a/packages/sandbox/tests/preview-forwarding.test.ts +++ b/packages/sandbox/tests/preview-forwarding.test.ts @@ -1,44 +1,90 @@ import { describe, expect, it, vi } from 'vitest'; +import { OperationInterruptedError } from '../src/errors'; import { forwardPreviewRequest, - type PreviewForwardingLifecycle + type PreviewForwardingLease } from '../src/preview/forwarding'; -function createLifecycle() { - const settle = vi.fn(); - const lifecycle: PreviewForwardingLifecycle = { - beginForward: vi.fn(() => settle), - renewActivity: vi.fn() +function createLease() { + let interrupt: (() => void) | undefined; + const release = vi.fn(); + const retain = vi.fn((onInterrupt?: () => void) => { + interrupt = onInterrupt; + return { release }; + }); + const lease: PreviewForwardingLease = { retain }; + return { + lease, + retain, + release, + interrupt: () => interrupt?.() }; - return { lifecycle, settle }; +} + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + +function nextTick(): Promise { + return new Promise((resolve) => setTimeout(resolve, 0)); } describe('forwardPreviewRequest', () => { - it('forwards HTTP requests through the provided TCP port', async () => { - const { lifecycle, settle } = createLifecycle(); - const tcpFetch = vi.fn().mockResolvedValue(new Response('ok')); + it('does not fetch when retain synchronously interrupts', async () => { + const release = vi.fn(); + const lease: PreviewForwardingLease = { + retain: (onInterrupt) => { + onInterrupt?.(); + return { release }; + } + }; + const tcpFetch = vi.fn().mockResolvedValue(new Response('late')); + + await expect( + forwardPreviewRequest( + { fetch: tcpFetch }, + new Request('http://localhost:8080/path'), + lease + ) + ).rejects.toBeInstanceOf(OperationInterruptedError); + + expect(tcpFetch).not.toHaveBeenCalled(); + expect(release).toHaveBeenCalledTimes(1); + }); + + it('forwards HTTP requests through the provided TCP port and releases bodyless responses', async () => { + const lease = createLease(); + const tcpFetch = vi + .fn() + .mockResolvedValue(new Response(null, { status: 204 })); const result = await forwardPreviewRequest( { fetch: tcpFetch }, new Request('http://localhost:8080/path?x=1'), - lifecycle + lease.lease ); - expect(result.status).toBe('response'); + expect(result).toMatchObject({ status: 'response' }); if (result.status === 'response') { - expect(await result.response.text()).toBe('ok'); + expect(result.response.status).toBe(204); } expect(tcpFetch).toHaveBeenCalledWith( 'http://localhost:8080/path?x=1', expect.any(Request) ); - expect(lifecycle.beginForward).toHaveBeenCalledTimes(1); - expect(settle).toHaveBeenCalledTimes(1); + expect(lease.retain).toHaveBeenCalledTimes(1); + expect(lease.release).toHaveBeenCalledTimes(1); }); - it('settles after streamed HTTP body completion', async () => { - const { lifecycle, settle } = createLifecycle(); - const stream = new ReadableStream({ + it('retains HTTP response bodies until EOF', async () => { + const lease = createLease(); + const stream = new ReadableStream({ start(controller) { controller.enqueue(new TextEncoder().encode('streamed')); controller.close(); @@ -49,20 +95,76 @@ describe('forwardPreviewRequest', () => { const result = await forwardPreviewRequest( { fetch: tcpFetch }, new Request('http://localhost:8080/stream'), - lifecycle + lease.lease ); expect(result.status).toBe('response'); - expect(settle).not.toHaveBeenCalled(); + expect(lease.release).not.toHaveBeenCalled(); if (result.status === 'response') { expect(await result.response.text()).toBe('streamed'); } - await Promise.resolve(); - expect(settle).toHaveBeenCalledTimes(1); + expect(lease.release).toHaveBeenCalledTimes(1); + }); + + it('releases and cancels HTTP response bodies on caller cancellation', async () => { + const lease = createLease(); + const cancel = vi.fn(); + const stream = new ReadableStream({ + pull(controller) { + controller.enqueue(new TextEncoder().encode('chunk')); + }, + cancel + }); + const tcpFetch = vi.fn().mockResolvedValue(new Response(stream)); + + const result = await forwardPreviewRequest( + { fetch: tcpFetch }, + new Request('http://localhost:8080/stream'), + lease.lease + ); + + if (result.status !== 'response' || !result.response.body) { + throw new Error('Expected streaming response'); + } + await result.response.body.cancel('caller done'); + + expect(cancel).toHaveBeenCalledWith('caller done'); + expect(lease.release).toHaveBeenCalledTimes(1); + }); + + it('errors HTTP response bodies on runtime invalidation', async () => { + const lease = createLease(); + const bodyGate = deferred(); + const cancel = vi.fn(); + const stream = new ReadableStream({ + async pull(controller) { + controller.enqueue(await bodyGate.promise); + }, + cancel + }); + const tcpFetch = vi.fn().mockResolvedValue(new Response(stream)); + + const result = await forwardPreviewRequest( + { fetch: tcpFetch }, + new Request('http://localhost:8080/stream'), + lease.lease + ); + + if (result.status !== 'response' || !result.response.body) { + throw new Error('Expected streaming response'); + } + const reader = result.response.body.getReader(); + const read = reader.read(); + lease.interrupt(); + bodyGate.resolve(new Uint8Array([1])); + + await expect(read).rejects.toBeInstanceOf(OperationInterruptedError); + expect(cancel).toHaveBeenCalledTimes(1); + expect(lease.release).toHaveBeenCalledTimes(1); }); - it('classifies network-loss errors', async () => { - const { lifecycle, settle } = createLifecycle(); + it('classifies network-loss errors and releases the hold', async () => { + const lease = createLease(); const tcpFetch = vi .fn() .mockRejectedValue(new Error('Network connection lost.')); @@ -70,29 +172,55 @@ describe('forwardPreviewRequest', () => { const result = await forwardPreviewRequest( { fetch: tcpFetch }, new Request('http://localhost:8080/'), - lifecycle + lease.lease ); expect(result).toEqual({ status: 'network-lost' }); - expect(settle).toHaveBeenCalledTimes(1); + expect(lease.release).toHaveBeenCalledTimes(1); }); - it('settles and rethrows generic errors', async () => { - const { lifecycle, settle } = createLifecycle(); + it('releases and rethrows generic errors', async () => { + const lease = createLease(); const tcpFetch = vi.fn().mockRejectedValue(new Error('boom')); await expect( forwardPreviewRequest( { fetch: tcpFetch }, new Request('http://localhost:8080/'), - lifecycle + lease.lease ) ).rejects.toThrow('boom'); - expect(settle).toHaveBeenCalledTimes(1); + expect(lease.release).toHaveBeenCalledTimes(1); }); - it('bridges WebSocket responses and settles once on close', async () => { - const { lifecycle, settle } = createLifecycle(); + it('closes a late WebSocket response when invalidated during fetch assignment', async () => { + const lease = createLease(); + const pair = new WebSocketPair(); + const [containerClient] = Object.values(pair); + const close = vi.spyOn(containerClient, 'close'); + const response = new Response(null, { + status: 101, + webSocket: containerClient + }); + const pendingResponse = deferred(); + const tcpFetch = vi.fn(() => pendingResponse.promise); + + const forwarded = forwardPreviewRequest( + { fetch: tcpFetch }, + new Request('http://localhost:8080/ws'), + lease.lease + ); + await vi.waitFor(() => expect(lease.retain).toHaveBeenCalledOnce()); + lease.interrupt(); + pendingResponse.resolve(response); + + await expect(forwarded).rejects.toBeInstanceOf(OperationInterruptedError); + expect(close).toHaveBeenCalledWith(1012, 'Runtime replaced'); + expect(lease.release).toHaveBeenCalledTimes(1); + }); + + it('bridges WebSocket messages without manual renewal and releases on close', async () => { + const lease = createLease(); const pair = new WebSocketPair(); const [containerClient, containerServer] = Object.values(pair); const tcpFetch = vi @@ -106,24 +234,74 @@ describe('forwardPreviewRequest', () => { new Request('http://localhost:8080/ws', { headers: { Upgrade: 'websocket', Connection: 'Upgrade' } }), - lifecycle + lease.lease ); expect(result.status).toBe('response'); if (result.status !== 'response') { throw new Error('Expected WebSocket response'); } - expect(result.response.status).toBe(101); - expect(result.response.webSocket).not.toBeNull(); - const clientSocket = result.response.webSocket; if (!clientSocket) { throw new Error('Expected client WebSocket'); } + const clientMessages: unknown[] = []; + const containerMessages: unknown[] = []; clientSocket.accept(); containerServer.accept(); + clientSocket.addEventListener('message', (event) => { + clientMessages.push(event.data); + }); + containerServer.addEventListener('message', (event) => { + containerMessages.push(event.data); + }); + + clientSocket.send('to-container'); + containerServer.send('to-client'); + await nextTick(); + + expect(containerMessages).toEqual(['to-container']); + expect(clientMessages).toEqual(['to-client']); + expect(lease.release).not.toHaveBeenCalled(); + clientSocket.close(1000, 'done'); - await new Promise((resolve) => setTimeout(resolve, 0)); - expect(settle).toHaveBeenCalledTimes(1); + await nextTick(); + expect(lease.release).toHaveBeenCalledTimes(1); + }); + + it('closes bridged WebSockets on runtime invalidation', async () => { + const lease = createLease(); + const pair = new WebSocketPair(); + const [containerClient] = Object.values(pair); + const containerClose = vi.spyOn(containerClient, 'close'); + const tcpFetch = vi + .fn() + .mockResolvedValue( + new Response(null, { status: 101, webSocket: containerClient }) + ); + + const result = await forwardPreviewRequest( + { fetch: tcpFetch }, + new Request('http://localhost:8080/ws'), + lease.lease + ); + + if (result.status !== 'response' || !result.response.webSocket) { + throw new Error('Expected WebSocket response'); + } + const clientClose = new Promise((resolve) => { + result.response.webSocket!.accept(); + result.response.webSocket!.addEventListener('close', resolve, { + once: true + }); + }); + lease.interrupt(); + + expect(containerClose).toHaveBeenCalledWith(1012, 'Runtime replaced'); + await expect(clientClose).resolves.toMatchObject({ + code: 1012, + reason: 'Runtime replaced' + }); + expect(lease.release).toHaveBeenCalledTimes(1); }); }); diff --git a/packages/sandbox/tests/process-readiness.test.ts b/packages/sandbox/tests/process-readiness.test.ts index c341cfe39..043ca27fa 100644 --- a/packages/sandbox/tests/process-readiness.test.ts +++ b/packages/sandbox/tests/process-readiness.test.ts @@ -11,8 +11,8 @@ import { } from '../src'; import { createSandboxProcess } from '../src/processes'; import type { - ProcessRPCDescriptor, - ProcessSubscriptionRPC + ProcessPullSubscriptionRPC, + ProcessRPCDescriptor } from '../src/processes/rpc-types'; const now = new Date().toISOString(); @@ -35,18 +35,16 @@ function deferred(): { return { promise, resolve }; } -function remote(events?: T[]): ProcessSubscriptionRPC { +function remote(events?: T[]): ProcessPullSubscriptionRPC { + const remaining = events ? [...events] : undefined; return { - stream: vi.fn( - async () => - new ReadableStream({ - start(controller) { - if (events === undefined) return; - for (const event of events) controller.enqueue(event); - controller.close(); - } - }) - ), + next: vi.fn(async (): Promise> => { + if (remaining === undefined) return new Promise(() => {}); + const value = remaining.shift(); + return value === undefined + ? { done: true, value: undefined } + : { done: false, value }; + }), cancel: vi.fn(async () => undefined), [Symbol.dispose]: vi.fn() }; @@ -63,8 +61,8 @@ function exited(code: number): ProcessLogEvent { } function descriptor( - portRemote: ProcessSubscriptionRPC, - logRemote: ProcessSubscriptionRPC + portRemote: ProcessPullSubscriptionRPC, + logRemote: ProcessPullSubscriptionRPC ): ProcessRPCDescriptor { return { id: running.id, @@ -79,7 +77,7 @@ function descriptor( } function expectReleased( - remoteSubscription: ProcessSubscriptionRPC + remoteSubscription: ProcessPullSubscriptionRPC ): void { expect(remoteSubscription.cancel).toHaveBeenCalledTimes(1); expect(remoteSubscription[Symbol.dispose]).toHaveBeenCalledTimes(1); @@ -124,7 +122,7 @@ describe('process readiness', () => { }); it('times out while readiness subscription acquisition is pending', async () => { - const pending = deferred>(); + const pending = deferred>(); const port = remote(); const logs = remote(); const processDescriptor = descriptor(port, logs); diff --git a/packages/sandbox/tests/processes/process-capability.test.ts b/packages/sandbox/tests/processes/process-capability.test.ts index 96c2a9fdc..32aa69152 100644 --- a/packages/sandbox/tests/processes/process-capability.test.ts +++ b/packages/sandbox/tests/processes/process-capability.test.ts @@ -49,7 +49,7 @@ function host(status: ProcessStatus | null = running) { const ports = subscription() satisfies PortWatchSubscriptionAPI; const releaseConnection = vi.fn(); const control: ProcessCapabilityControl = { - retainConnection: vi.fn(() => releaseConnection), + retainRuntimeHold: vi.fn(() => releaseConnection), getProcess: vi.fn(async () => status), openLogs: vi.fn(async () => logs), openPortWatch: vi.fn(async () => ports), @@ -68,7 +68,7 @@ describe('ProcessCapabilityTarget', () => { const capability = new ProcessCapabilityTarget({ id: 'p1', pid: 123, - runtime: { id: 'runtime-a' }, + runtime: { id: 'runtime-a', runtimeIncarnationID: 'incarnation-a' }, lifecycle: testHost.lifecycle }); @@ -84,7 +84,7 @@ describe('ProcessCapabilityTarget', () => { const capability = new ProcessCapabilityTarget({ id: 'p1', pid: 123, - runtime: { id: 'runtime-a' }, + runtime: { id: 'runtime-a', runtimeIncarnationID: 'incarnation-a' }, lifecycle: testHost.lifecycle }); @@ -108,7 +108,7 @@ describe('ProcessCapabilityTarget', () => { const capability = new ProcessCapabilityTarget({ id: 'p1', pid: 123, - runtime: { id: 'runtime-a' }, + runtime: { id: 'runtime-a', runtimeIncarnationID: 'incarnation-a' }, lifecycle: testHost.lifecycle }); @@ -129,8 +129,7 @@ describe('ProcessCapabilityTarget', () => { reason: 'runtime_replaced', operation: 'process.kill', admitted: true, - retryable: false, - effect: 'unknown' + retryable: false }, httpStatus: 409, timestamp: new Date().toISOString() @@ -139,7 +138,7 @@ describe('ProcessCapabilityTarget', () => { const capability = new ProcessCapabilityTarget({ id: 'p1', pid: 123, - runtime: { id: 'runtime-a' }, + runtime: { id: 'runtime-a', runtimeIncarnationID: 'incarnation-a' }, lifecycle: testHost.lifecycle }); @@ -164,7 +163,7 @@ describe('ProcessCapabilityTarget', () => { const capability = new ProcessCapabilityTarget({ id: 'p1', pid: 123, - runtime: { id: 'runtime-a' }, + runtime: { id: 'runtime-a', runtimeIncarnationID: 'incarnation-a' }, lifecycle: testHost.lifecycle }); @@ -214,7 +213,7 @@ describe('ProcessCapabilityTarget', () => { const capability = new ProcessCapabilityTarget({ id: 'p1', pid: 123, - runtime: { id: 'runtime-a' }, + runtime: { id: 'runtime-a', runtimeIncarnationID: 'incarnation-a' }, lifecycle: testHost.lifecycle }); @@ -242,7 +241,7 @@ describe('ProcessCapabilityTarget', () => { const capability = new ProcessCapabilityTarget({ id: 'p1', pid: 123, - runtime: { id: 'runtime-a' }, + runtime: { id: 'runtime-a', runtimeIncarnationID: 'incarnation-a' }, lifecycle: testHost.lifecycle }); @@ -271,7 +270,7 @@ describe('ProcessCapabilityTarget', () => { const capability = new ProcessCapabilityTarget({ id: 'p1', pid: 123, - runtime: { id: 'runtime-a' }, + runtime: { id: 'runtime-a', runtimeIncarnationID: 'incarnation-a' }, lifecycle: testHost.lifecycle }); @@ -290,7 +289,7 @@ describe('ProcessCapabilityTarget', () => { const capability = new ProcessCapabilityTarget({ id: 'p1', pid: 123, - runtime: { id: 'runtime-a' }, + runtime: { id: 'runtime-a', runtimeIncarnationID: 'incarnation-a' }, lifecycle: testHost.lifecycle }); diff --git a/packages/sandbox/tests/processes/process-lifecycle-transport.test.ts b/packages/sandbox/tests/processes/process-lifecycle-transport.test.ts deleted file mode 100644 index e1f5d6cc3..000000000 --- a/packages/sandbox/tests/processes/process-lifecycle-transport.test.ts +++ /dev/null @@ -1,122 +0,0 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { - RuntimeIdentity, - type RuntimeIdentityID -} from '../../src/current-runtime-identity'; -import { OperationInterruptedError, RPCTransportError } from '../../src/errors'; -import { ProcessLifecycle } from '../../src/processes/process-lifecycle'; -import type { ResourceActivityOperation } from '../../src/resource-activity-gate'; - -let listProcesses: () => Promise; - -vi.mock('../../src/container-control/connection', () => ({ - ContainerControlConnection: class { - isConnected() { - return false; - } - getStats() { - return { imports: 1, exports: 1 }; - } - disconnect() {} - rpc() { - return { - processes: { - list: () => listProcesses() - } - }; - } - } -})); - -import { RuntimeControlClient } from '../../src/container-control/runtime-client'; - -function runtime(id: string): RuntimeIdentity { - return new RuntimeIdentity({ id: id as RuntimeIdentityID }); -} - -function operation(): ResourceActivityOperation { - return { beforeCall: Promise.resolve(), finish: vi.fn() }; -} - -function deferred() { - let reject!: (error: Error) => void; - const promise = new Promise((_resolve, rej) => { - reject = rej; - }); - return { promise, reject }; -} - -function createHost(initial: RuntimeIdentity) { - let current = initial; - const runtimeClient = new RuntimeControlClient({ - getTcpPort: () => ({ fetch: vi.fn() }), - beginNonWakingOperation: operation - }); - const lifecycle = new ProcessLifecycle({ - currentRuntime: { - get: async () => current, - assertActive: async (expected: RuntimeIdentity) => { - if (current.id !== expected.id) throw new Error('inactive'); - } - }, - runtimeClient, - beginNonWakingOperation: operation - }); - return { - lifecycle, - replaceRuntime(next: RuntimeIdentity) { - current = next; - } - }; -} - -describe('ProcessLifecycle direct transport errors', () => { - beforeEach(() => { - listProcesses = async () => []; - }); - - it('surfaces a typed transport error while the runtime remains active', async () => { - const expected = runtime('runtime-a'); - const host = createHost(expected); - listProcesses = async () => { - throw new Error('WebSocket connection failed.'); - }; - - const error = await host.lifecycle - .runRead(expected, 'process.list', (client) => client.processes.list()) - .catch((caught: Error) => caught); - - expect(error).toBeInstanceOf(RPCTransportError); - expect(error).not.toBeInstanceOf(OperationInterruptedError); - expect((error as RPCTransportError).kind).toBe('connection_failed'); - expect((error as RPCTransportError).context).not.toHaveProperty('phase'); - }); - - it('reclassifies the same transport failure after runtime replacement', async () => { - const expected = runtime('runtime-a'); - const host = createHost(expected); - const pending = deferred(); - listProcesses = vi.fn(() => pending.promise); - - const call = host.lifecycle.runRead(expected, 'process.list', (client) => - client.processes.list() - ); - await vi.waitFor(() => expect(listProcesses).toHaveBeenCalled()); - - host.replaceRuntime(runtime('runtime-b')); - pending.reject(new Error('WebSocket connection failed.')); - - const error = await call.catch((caught: Error) => caught); - expect(error).toBeInstanceOf(OperationInterruptedError); - expect((error as OperationInterruptedError).context).toMatchObject({ - reason: 'runtime_replaced', - operation: 'process.list', - admitted: true, - retryable: false, - effect: 'none' - }); - expect((error as OperationInterruptedError).context).not.toHaveProperty( - 'phase' - ); - }); -}); diff --git a/packages/sandbox/tests/processes/process-lifecycle.test.ts b/packages/sandbox/tests/processes/process-lifecycle.test.ts deleted file mode 100644 index a0303ea43..000000000 --- a/packages/sandbox/tests/processes/process-lifecycle.test.ts +++ /dev/null @@ -1,236 +0,0 @@ -import { describe, expect, it, vi } from 'vitest'; -import type { ContainerControlClient } from '../../src/container-control/client'; -import { - CurrentRuntimeIdentity, - RuntimeIdentity, - type RuntimeIdentityID -} from '../../src/current-runtime-identity'; -import { - ErrorCode, - OperationInterruptedError, - RPCTransportError, - StaleProcessHandleError -} from '../../src/errors'; -import { ProcessLifecycle } from '../../src/processes/process-lifecycle'; -import type { ResourceActivityOperation } from '../../src/resource-activity-gate'; - -function runtime(id: string): RuntimeIdentity { - return new RuntimeIdentity({ id: id as RuntimeIdentityID }); -} - -function deferred() { - let resolve!: (value: T) => void; - let reject!: (error: Error) => void; - const promise = new Promise((res, rej) => { - resolve = res; - reject = rej; - }); - return { promise, resolve, reject }; -} - -function operation(): ResourceActivityOperation { - return { beforeCall: Promise.resolve(), finish: vi.fn() }; -} - -function createStorage(initial = new Map()) { - return { - get: vi.fn(async (key: string) => initial.get(key)), - put: vi.fn(async (key: string, value: unknown) => { - initial.set(key, value); - }), - delete: vi.fn(async (key: string) => { - initial.delete(key); - }) - } as unknown as DurableObjectState['storage']; -} - -function createLifecycle(options: { - current?: RuntimeIdentity | null; - client?: ContainerControlClient; - process?: { id: string; pid: number }; - acquisitionError?: Error; -}) { - let current = options.current ?? null; - const currentRuntime = { - get: vi.fn(async () => current), - assertActive: vi.fn(async (expected: RuntimeIdentity) => { - if (current?.id !== expected.id) { - throw new Error('inactive'); - } - }) - } as Pick; - const directClient = options.client ?? ({} as ContainerControlClient); - const runtimeClient = { - get: vi.fn(() => { - if (options.acquisitionError) throw options.acquisitionError; - return directClient; - }), - dispose: vi.fn() - }; - const admission = operation(); - const lifecycle = new ProcessLifecycle({ - currentRuntime, - runtimeClient, - beginNonWakingOperation: () => admission, - process: options.process - }); - return { - lifecycle, - currentRuntime, - runtimeClient, - admission, - setCurrent(value: RuntimeIdentity | null) { - current = value; - } - }; -} - -describe('ProcessLifecycle', () => { - it('captures inactive runtime absence without creating a direct client', async () => { - const host = createLifecycle({ current: null }); - - await expect(host.lifecycle.captureCurrent()).resolves.toBeNull(); - - expect(host.runtimeClient.get).not.toHaveBeenCalled(); - }); - - it('captures the current runtime after a failed identity clear', async () => { - const storage = createStorage( - new Map([['currentRuntimeIdentity', { id: 'runtime-1' }]]) - ); - const currentRuntime = new CurrentRuntimeIdentity( - storage, - async () => ({ status: 'healthy' }), - () => true - ); - const runtimeClient = { - get: vi.fn(), - dispose: vi.fn() - }; - const lifecycle = new ProcessLifecycle({ - currentRuntime, - runtimeClient, - beginNonWakingOperation: operation - }); - const error = new Error('delete failed'); - vi.mocked(storage.delete).mockRejectedValueOnce(error); - - await expect(currentRuntime.clear()).rejects.toBe(error); - - await expect(lifecycle.captureCurrent()).resolves.toMatchObject({ - id: 'runtime-1' - }); - expect(runtimeClient.get).not.toHaveBeenCalled(); - }); - - it('rejects a stale handle before direct contact', async () => { - const expected = runtime('runtime-a'); - const direct = vi.fn(); - const host = createLifecycle({ - current: runtime('runtime-b'), - process: { id: 'process-1', pid: 42 } - }); - - await expect( - host.lifecycle.runRead(expected, 'process.status', async () => direct()) - ).rejects.toBeInstanceOf(StaleProcessHandleError); - - expect(direct).not.toHaveBeenCalled(); - expect(host.runtimeClient.get).not.toHaveBeenCalled(); - expect(host.admission.finish).toHaveBeenCalledTimes(1); - }); - - it.each([ - ['process.get', 'read', 'none'] as const, - ['process.list', 'read', 'none'] as const, - ['process.status', 'read', 'none'] as const, - ['process.kill', 'control', 'unknown'] as const - ])( - 'post-fences an in-flight %s operation', - async (operationName, kind, effect) => { - const expected = runtime('runtime-a'); - const pending = deferred(); - const host = createLifecycle({ - current: expected, - process: { id: 'process-1', pid: 42 } - }); - const call = - kind === 'read' - ? host.lifecycle.runRead( - expected, - operationName, - () => pending.promise - ) - : host.lifecycle.runControl( - expected, - operationName, - () => pending.promise - ); - - await vi.waitFor(() => expect(host.runtimeClient.get).toHaveBeenCalled()); - host.setCurrent(runtime('runtime-b')); - pending.resolve('ack'); - - const error = await call.catch((caught: Error) => caught); - expect(error).toBeInstanceOf(OperationInterruptedError); - expect((error as OperationInterruptedError).context).toMatchObject({ - operation: operationName, - effect, - admitted: true - }); - expect( - (error as OperationInterruptedError).context.phase - ).toBeUndefined(); - expect(host.admission.finish).toHaveBeenCalledTimes(1); - } - ); - - it('preserves a typed expected-live client acquisition failure', async () => { - const expected = runtime('runtime-a'); - const transport = new RPCTransportError({ - code: ErrorCode.RPC_TRANSPORT_ERROR, - message: 'direct port unavailable', - context: { - kind: 'connection_failed', - originalMessage: 'direct port unavailable', - errorName: 'Error' - }, - httpStatus: 503, - timestamp: new Date().toISOString() - }); - const host = createLifecycle({ - current: expected, - acquisitionError: transport - }); - - await expect( - host.lifecycle.runRead(expected, 'process.list', async () => []) - ).rejects.toBe(transport); - expect(host.currentRuntime.assertActive).toHaveBeenCalledTimes(2); - expect(host.admission.finish).toHaveBeenCalledTimes(1); - }); - - it('preserves an expected-live transport failure after a successful post-fence', async () => { - const expected = runtime('runtime-a'); - const transport = new RPCTransportError({ - code: ErrorCode.RPC_TRANSPORT_ERROR, - message: 'direct transport failed', - context: { - kind: 'connection_failed', - originalMessage: 'direct transport failed', - errorName: 'Error' - }, - httpStatus: 503, - timestamp: new Date().toISOString() - }); - const host = createLifecycle({ current: expected }); - - await expect( - host.lifecycle.runRead(expected, 'process.list', async () => { - throw transport; - }) - ).rejects.toBe(transport); - - expect(host.admission.finish).toHaveBeenCalledTimes(1); - }); -}); diff --git a/packages/sandbox/tests/processes/sandbox-process.test.ts b/packages/sandbox/tests/processes/sandbox-process.test.ts index c298d9ad5..b4eb6eace 100644 --- a/packages/sandbox/tests/processes/sandbox-process.test.ts +++ b/packages/sandbox/tests/processes/sandbox-process.test.ts @@ -15,8 +15,8 @@ import { import { ProcessError } from '../../src/errors'; import { createSandboxProcess } from '../../src/processes'; import type { - ProcessRPCDescriptor, - ProcessSubscriptionRPC + ProcessPullSubscriptionRPC, + ProcessRPCDescriptor } from '../../src/processes/rpc-types'; const now = new Date().toISOString(); @@ -39,17 +39,15 @@ function deferred(): { return { promise, resolve }; } -function subscription(events: T[]): ProcessSubscriptionRPC { +function subscription(events: T[]): ProcessPullSubscriptionRPC { + const remaining = [...events]; return { - stream: vi.fn( - async () => - new ReadableStream({ - start(controller) { - for (const event of events) controller.enqueue(event); - controller.close(); - } - }) - ), + next: vi.fn(async (): Promise> => { + const value = remaining.shift(); + return value === undefined + ? { done: true, value: undefined } + : { done: false, value }; + }), cancel: vi.fn(async () => undefined), [Symbol.dispose]: vi.fn() }; @@ -204,8 +202,8 @@ describe('SandboxProcessImpl', () => { it('waits for capability-scoped port readiness', async () => { const descriptor = processDescriptor(); descriptor.capability.openLogs = vi.fn(async () => ({ - stream: vi.fn( - async () => new ReadableStream({ start() {} }) + next: vi.fn( + () => new Promise>(() => {}) ), cancel: vi.fn(async () => undefined), [Symbol.dispose]: vi.fn() @@ -230,8 +228,8 @@ describe('SandboxProcessImpl', () => { it('uses typed local timeout and abort errors without killing the process', async () => { const remote = subscription([]); - remote.stream = vi.fn( - async () => new ReadableStream({ start() {} }) + remote.next = vi.fn( + () => new Promise>(() => {}) ); const descriptor = processDescriptor(); descriptor.capability.openLogs = vi.fn(async () => remote); @@ -263,7 +261,7 @@ describe('SandboxProcessImpl', () => { ])( 'times out while %s subscription acquisition is pending', async (_, wait) => { - const pending = deferred>(); + const pending = deferred>(); const remote = subscription([]); const descriptor = processDescriptor(); descriptor.capability.openLogs = vi.fn(() => pending.promise); @@ -279,24 +277,23 @@ describe('SandboxProcessImpl', () => { } ); - it('aborts logs while stream setup is pending and releases it later', async () => { - const pending = deferred>(); + it('aborts logs while the first pull is pending', async () => { const remote = subscription([]); - remote.stream = vi.fn(() => pending.promise); + remote.next = vi.fn( + () => new Promise>(() => {}) + ); const descriptor = processDescriptor(); descriptor.capability.openLogs = vi.fn(async () => remote); const abort = new AbortController(); - const opening = createSandboxProcess(descriptor).logs({ + const stream = await createSandboxProcess(descriptor).logs({ signal: abort.signal }); + const reading = stream.getReader().read(); abort.abort(); - await expect(opening).rejects.toBeInstanceOf(ProcessAbortedError); - pending.resolve(new ReadableStream()); - await vi.waitFor(() => { - expect(remote.cancel).toHaveBeenCalledOnce(); - expect(remote[Symbol.dispose]).toHaveBeenCalledOnce(); - }); + await expect(reading).rejects.toBeInstanceOf(ProcessAbortedError); + expect(remote.cancel).toHaveBeenCalledOnce(); + expect(remote[Symbol.dispose]).toHaveBeenCalledOnce(); }); it.each([ @@ -412,8 +409,8 @@ describe('SandboxProcessImpl', () => { it('locally aborts logs and cleans up consumer cancellation exactly once', async () => { const remote = subscription([]); - remote.stream = vi.fn( - async () => new ReadableStream({ start() {} }) + remote.next = vi.fn( + () => new Promise>(() => {}) ); const descriptor = processDescriptor(); descriptor.capability.openLogs = vi.fn(async () => remote); @@ -432,15 +429,9 @@ describe('SandboxProcessImpl', () => { it('translates late stream errors and releases their subscription', async () => { const remote = subscription([]); - remote.stream = vi.fn( - async () => - new ReadableStream({ - start(controller) { - controller.enqueue(stdout('data', '1')); - controller.error(new Error('late transport failure')); - } - }) - ); + remote.next = vi + .fn() + .mockRejectedValue(new Error('late transport failure')); const descriptor = processDescriptor(); descriptor.capability.openLogs = vi.fn(async () => remote); const reader = (await createSandboxProcess(descriptor).logs()).getReader(); @@ -450,17 +441,14 @@ describe('SandboxProcessImpl', () => { expect(remote[Symbol.dispose]).toHaveBeenCalledTimes(1); }); - it('releases a subscription when local reader setup fails', async () => { - const source = new ReadableStream(); - source.getReader(); + it('releases a subscription when its first pull fails', async () => { const remote = subscription([]); - remote.stream = vi.fn(async () => source); + remote.next = vi.fn().mockRejectedValue(new Error('pull failed')); const descriptor = processDescriptor(); descriptor.capability.openLogs = vi.fn(async () => remote); + const reader = (await createSandboxProcess(descriptor).logs()).getReader(); - await expect( - createSandboxProcess(descriptor).logs() - ).rejects.toBeInstanceOf(RPCTransportError); + await expect(reader.read()).rejects.toBeInstanceOf(RPCTransportError); expect(remote.cancel).toHaveBeenCalledTimes(1); expect(remote[Symbol.dispose]).toHaveBeenCalledTimes(1); }); diff --git a/packages/sandbox/tests/pty-proxy.test.ts b/packages/sandbox/tests/pty-proxy.test.ts index 560400d28..0e640e089 100644 --- a/packages/sandbox/tests/pty-proxy.test.ts +++ b/packages/sandbox/tests/pty-proxy.test.ts @@ -117,7 +117,11 @@ describe('terminal proxy', () => { headers: { Upgrade: 'websocket' } }); - const terminal = terminalHandle(stub, snapshot('terminal-123')); + const terminal = terminalHandle( + stub, + snapshot('terminal-123'), + 'runtime-incarnation-1' + ); await terminal.connect(request, { cursor: 'cursor-1', cols: 120, @@ -131,6 +135,9 @@ describe('terminal proxy', () => { expect(url.searchParams.get('cursor')).toBe('cursor-1'); expect(url.searchParams.get('cols')).toBe('120'); expect(url.searchParams.get('rows')).toBe('40'); + expect(url.searchParams.get('runtimeIncarnationID')).toBe( + 'runtime-incarnation-1' + ); expect(url.searchParams.get('shell')).toBeNull(); expect(url.searchParams.get('id')).toBeNull(); }); @@ -176,6 +183,72 @@ describe('terminal proxy', () => { expect(subscription[Symbol.dispose]).toHaveBeenCalledTimes(1); }); + it('aborts while terminal output subscription acquisition is pending', async () => { + const stub = createStub(); + stub.output.mockImplementationOnce(() => new Promise(() => {})); + const terminal = terminalHandle(stub, snapshot('terminal-123')); + const abortController = new AbortController(); + + const opening = terminal.output({ signal: abortController.signal }); + abortController.abort(new Error('caller aborted')); + const outcome = await Promise.race([ + opening.then( + () => 'resolved', + (error: Error) => error.message + ), + new Promise((resolve) => + setTimeout(() => resolve('still pending'), 20) + ) + ]); + + expect(outcome).toBe('caller aborted'); + }); + + it('times out while terminal output subscription acquisition is pending', async () => { + const stub = createStub(); + stub.output.mockImplementationOnce(() => new Promise(() => {})); + const terminal = terminalHandle(stub, snapshot('terminal-123')); + + const waiting = terminal.waitForExit({ timeout: 5 }); + const outcome = await Promise.race([ + waiting.then( + () => 'resolved', + (error: Error) => error.message + ), + new Promise((resolve) => + setTimeout(() => resolve('still pending'), 20) + ) + ]); + + expect(outcome).toBe('Terminal wait timed out'); + }); + + it('cleans up wait controls when subscription acquisition fails', async () => { + vi.useFakeTimers(); + const stub = createStub(); + stub.output.mockRejectedValueOnce(new Error('subscription failed')); + const terminal = terminalHandle(stub, snapshot('terminal-123')); + const abortController = new AbortController(); + const removeEventListener = vi.spyOn( + abortController.signal, + 'removeEventListener' + ); + + await expect( + terminal.waitForExit({ + timeout: 1000, + signal: abortController.signal + }) + ).rejects.toThrow('subscription failed'); + + expect(vi.getTimerCount()).toBe(0); + expect(removeEventListener).toHaveBeenCalledWith( + 'abort', + expect.any(Function) + ); + vi.useRealTimers(); + }); + it('reports AbortSignal reason instead of timeout when signal wins', async () => { const stub = createStub(); const terminal = terminalHandle(stub, snapshot('terminal-123')); diff --git a/packages/sandbox/tests/r2-egress-mount.test.ts b/packages/sandbox/tests/r2-egress-mount.test.ts index 50bb6c4f9..74344ae66 100644 --- a/packages/sandbox/tests/r2-egress-mount.test.ts +++ b/packages/sandbox/tests/r2-egress-mount.test.ts @@ -1,13 +1,17 @@ import { getContainer } from '@cloudflare/containers'; -import { describe, expect, it, vi } from 'vitest'; +import { afterAll, describe, expect, it, vi } from 'vitest'; import { createBridgeApp } from '../src/bridge/routes'; +import { RuntimeIdentity, RuntimeOperationRunner } from '../src/runtime'; import { ContainerProxy, Sandbox } from '../src/sandbox'; import { type R2EgressParams, r2EgressHandler } from '../src/storage-mount/outbound/r2-egress-handler'; import type { S3CredentialProxyParams } from '../src/storage-mount/types'; -import { createMockControlClient } from './helpers/mock-control-client'; +import { + asSandboxWithClient, + createMockControlClient +} from './helpers/mock-control-client'; type MockFetcher = { fetch: ReturnType; @@ -208,6 +212,7 @@ function createMockCtx(options?: { name: 'test-sandbox' }, container: { + running: true, interceptOutboundHttp: vi.fn().mockResolvedValue(undefined) }, exports: options?.includeContainerProxy === false ? {} : { ContainerProxy }, @@ -274,6 +279,86 @@ function createMountResult( }; } +type SandboxWithClient = Sandbox & { + client: ReturnType & { + mounts: ReturnType; + }; +}; + +const originalRunExisting = RuntimeOperationRunner.prototype.runExisting; +RuntimeOperationRunner.prototype.runExisting = async function ( + this: { testSandbox?: SandboxWithClient }, + _target: unknown, + _operation: string, + call: (lease: { + runtime: RuntimeIdentity; + control: SandboxWithClient['client']; + retain(): { release(): void }; + }) => Promise +) { + if (!this.testSandbox) return { status: 'absent' }; + return await call({ + runtime: new RuntimeIdentity({ + id: 'runtime-1' as never, + runtimeIncarnationID: 'incarnation-1' as never + }), + control: this.testSandbox.client, + retain: () => ({ release: () => {} }) + }); +}; + +const originalRunWakingCompositeDescriptor = Object.getOwnPropertyDescriptor( + Sandbox.prototype, + 'runWakingComposite' +); + +Object.defineProperty(Sandbox.prototype, 'runWakingComposite', { + configurable: true, + value: async function ( + this: SandboxWithClient, + _operation: string, + call: (lease: { + runtime: RuntimeIdentity; + control: SandboxWithClient['client']; + retain(): { release(): void }; + }) => Promise + ) { + ( + this as unknown as { + runtimeRunner: RuntimeOperationRunner & { + testSandbox?: SandboxWithClient; + }; + } + ).runtimeRunner.testSandbox = this; + await ( + this as unknown as { ctx: { storage: DurableObjectStorage } } + ).ctx.storage.put('currentRuntimeIdentity', { + schemaVersion: 1, + id: 'runtime-1', + runtimeIncarnationID: 'incarnation-1' + }); + return await call({ + runtime: new RuntimeIdentity({ + id: 'runtime-1' as never, + runtimeIncarnationID: 'incarnation-1' as never + }), + control: this.client, + retain: () => ({ release: () => {} }) + }); + } +}); + +afterAll(() => { + RuntimeOperationRunner.prototype.runExisting = originalRunExisting; + if (originalRunWakingCompositeDescriptor) { + Object.defineProperty( + Sandbox.prototype, + 'runWakingComposite', + originalRunWakingCompositeDescriptor + ); + } +}); + function createMountMocks() { return { pathExists: vi.fn(async () => true), @@ -395,7 +480,9 @@ describe('Sandbox credential proxy mounts', () => { credentialProxy: true }); - expect(sandbox.client.files.writeFile).toHaveBeenCalledWith( + expect( + asSandboxWithClient(sandbox).client.files.writeFile + ).toHaveBeenCalledWith( expect.stringContaining('/tmp/.passwd-s3fs-'), 'my-bucket:x:x' ); @@ -922,7 +1009,7 @@ describe('Sandbox R2 egress mounts', () => { { MY_BUCKET: bucket } ); const client = createMockControlClient(); - sandbox.client = client; + asSandboxWithClient(sandbox).client = client; vi.mocked(client.files.mkdir).mockResolvedValue({ success: true, diff --git a/packages/sandbox/tests/resource-activity-gate.test.ts b/packages/sandbox/tests/resource-activity-gate.test.ts index 6253f35f7..9dde5a67e 100644 --- a/packages/sandbox/tests/resource-activity-gate.test.ts +++ b/packages/sandbox/tests/resource-activity-gate.test.ts @@ -127,7 +127,7 @@ describe('ResourceActivityGate', () => { test('in-flight operation prevents stop without leaked count', async () => { const { gate, stop } = createGate(); - const operation = gate.beginOperation(); + const operation = gate.beginActivity(); await gate.runExpiry( { availability: async () => 'available' as const, @@ -151,7 +151,7 @@ describe('ResourceActivityGate', () => { test('expiry does not renew for an in-flight non-waking operation', async () => { const { gate, stop, renew } = createGate(); - const operation = gate.beginNonWakingOperation(); + const operation = gate.beginExistingHold(); const probe = { availability: vi.fn(async () => 'available' as const), processesHasActive: vi.fn(async () => false), @@ -186,7 +186,7 @@ describe('ResourceActivityGate', () => { await Promise.resolve(); await Promise.resolve(); await Promise.resolve(); - const operation = gate.beginOperation(); + const operation = gate.beginActivity(); let unblocked = false; const beforeCall = operation.beforeCall.then(() => { unblocked = true; @@ -214,7 +214,7 @@ describe('ResourceActivityGate', () => { ); await Promise.resolve(); const renewsBeforeObservation = renew.mock.calls.length; - const operation = gate.beginNonWakingOperation(); + const operation = gate.beginProbe(); let admitted = false; const admission = operation.beforeCall.then(() => { admitted = true; @@ -272,6 +272,101 @@ describe('ResourceActivityGate', () => { expect(stop).toHaveBeenCalledTimes(1); }); + test('destroy teardown chains after a committed stop and still runs', async () => { + const stopDone = deferred(); + const events: string[] = []; + const gate = new ResourceActivityGate(vi.fn(), async () => { + events.push('stop:start'); + await stopDone.promise; + events.push('stop:end'); + }); + const expiry = gate.runExpiry( + { + availability: async () => 'absent' as const, + processesHasActive: async () => false, + terminalsHasActive: async () => false + }, + false + ); + await Promise.resolve(); + const destroy = gate.runDestroyTeardown(async () => { + events.push('destroy'); + }); + await Promise.resolve(); + + expect(events).toEqual(['stop:start']); + stopDone.resolve(); + await Promise.all([expiry, destroy]); + + expect(events).toEqual(['stop:start', 'stop:end', 'destroy']); + }); + + test('operation waiting on stop also waits for a later committed destroy', async () => { + const stopDone = deferred(); + const destroyDone = deferred(); + const events: string[] = []; + const gate = new ResourceActivityGate(vi.fn(), async () => { + events.push('stop:start'); + await stopDone.promise; + events.push('stop:end'); + }); + const expiry = gate.runExpiry( + { + availability: async () => 'absent' as const, + processesHasActive: async () => false, + terminalsHasActive: async () => false + }, + false + ); + await Promise.resolve(); + const operation = gate.beginActivity(); + const admitted = operation.beforeCall.then(() => events.push('operation')); + const destroy = gate.runDestroyTeardown(async () => { + events.push('destroy:start'); + await destroyDone.promise; + events.push('destroy:end'); + }); + + stopDone.resolve(); + await vi.waitFor(() => expect(events).toContain('destroy:start')); + expect(events).not.toContain('operation'); + destroyDone.resolve(); + await Promise.all([expiry, destroy, admitted]); + operation.finish(); + + expect(events).toEqual([ + 'stop:start', + 'stop:end', + 'destroy:start', + 'destroy:end', + 'operation' + ]); + }); + + test('destroy teardown still runs after a failed committed stop', async () => { + const events: string[] = []; + const gate = new ResourceActivityGate(vi.fn(), async () => { + events.push('stop'); + throw new Error('stop failed'); + }); + await expect( + gate.runExpiry( + { + availability: async () => 'absent' as const, + processesHasActive: async () => false, + terminalsHasActive: async () => false + }, + false + ) + ).rejects.toThrow('stop failed'); + + await gate.runDestroyTeardown(async () => { + events.push('destroy'); + }); + + expect(events).toEqual(['stop', 'destroy']); + }); + test('absent runtime commits stop without probes', async () => { const { gate, stop } = createGate(); const processesHasActive = vi.fn(async () => false); diff --git a/packages/sandbox/tests/response-retry.test.ts b/packages/sandbox/tests/response-retry.test.ts deleted file mode 100644 index b3e8c77f0..000000000 --- a/packages/sandbox/tests/response-retry.test.ts +++ /dev/null @@ -1,106 +0,0 @@ -import { createNoOpLogger } from '@repo/shared'; -import { describe, expect, it, vi } from 'vitest'; -import { - fetchWithResponseRetry, - isRetryableWebSocketUpgradeResponse -} from '../src/response-retry'; - -function responseWithStatus(status: number): Response { - return new Response(null, { status }); -} - -describe('response retry helpers', () => { - describe('isRetryableWebSocketUpgradeResponse', () => { - it.each([500, 502, 503, 504])( - 'treats %i as a retryable upgrade response', - (status) => { - expect( - isRetryableWebSocketUpgradeResponse(responseWithStatus(status)) - ).toBe(true); - } - ); - - it.each([400, 401, 403, 404])( - 'treats %i as a terminal upgrade response', - (status) => { - expect( - isRetryableWebSocketUpgradeResponse(responseWithStatus(status)) - ).toBe(false); - } - ); - }); - - describe('fetchWithResponseRetry', () => { - it('retries matching responses until success', async () => { - vi.useFakeTimers(); - - try { - const fetchResponse = vi - .fn<() => Promise>() - .mockResolvedValueOnce(responseWithStatus(503)) - .mockResolvedValueOnce(responseWithStatus(200)); - - const retrying = fetchWithResponseRetry(fetchResponse, { - retryTimeoutMs: 20_000, - minTimeForRetryMs: 15_000, - logger: createNoOpLogger(), - retryLogMessage: 'retrying test response', - shouldRetry: (response) => response.status === 503 - }); - - await vi.advanceTimersByTimeAsync(0); - expect(fetchResponse).toHaveBeenCalledTimes(1); - - await vi.advanceTimersByTimeAsync(3_000); - const response = await retrying; - - expect(fetchResponse).toHaveBeenCalledTimes(2); - expect(response.status).toBe(200); - } finally { - vi.useRealTimers(); - } - }); - - it('returns terminal responses without retrying', async () => { - const fetchResponse = vi - .fn<() => Promise>() - .mockResolvedValue(responseWithStatus(404)); - - const response = await fetchWithResponseRetry(fetchResponse, { - retryTimeoutMs: 20_000, - minTimeForRetryMs: 15_000, - logger: createNoOpLogger(), - retryLogMessage: 'retrying test response', - shouldRetry: (candidate) => candidate.status === 503 - }); - - expect(response.status).toBe(404); - expect(fetchResponse).toHaveBeenCalledTimes(1); - }); - - it('returns retryable responses when the retry budget is exhausted', async () => { - const response503 = responseWithStatus(503); - const fetchResponse = vi - .fn<() => Promise>() - .mockResolvedValue(response503); - const onRetryExhausted = vi.fn(); - - const response = await fetchWithResponseRetry(fetchResponse, { - retryTimeoutMs: 0, - minTimeForRetryMs: 15_000, - logger: createNoOpLogger(), - retryLogMessage: 'retrying test response', - shouldRetry: (candidate) => candidate.status === 503, - onRetryExhausted - }); - - expect(response).toBe(response503); - expect(fetchResponse).toHaveBeenCalledTimes(1); - expect(onRetryExhausted).toHaveBeenCalledWith({ - attempts: 1, - elapsedMs: expect.any(Number), - response: response503 - }); - }); - }); -}); diff --git a/packages/sandbox/tests/restore-operation-store.test.ts b/packages/sandbox/tests/restore-operation-store.test.ts index d61138235..e2c68c3fb 100644 --- a/packages/sandbox/tests/restore-operation-store.test.ts +++ b/packages/sandbox/tests/restore-operation-store.test.ts @@ -79,6 +79,15 @@ describe('BackupRestoreOperationStore', () => { phase: 'interrupted', status: 'interrupted', attempt: 2, + runtimeIdentityID: + 'runtime-1' as BackupRestoreOperationRecord['runtimeIdentityID'], + runtimeIncarnationID: + 'incarnation-1' as BackupRestoreOperationRecord['runtimeIncarnationID'], + result: { + success: true, + id: 'backup-1', + dir: '/workspace/project' + }, error: { code: 'OPERATION_INTERRUPTED', message: 'Transport disposed', @@ -93,8 +102,12 @@ describe('BackupRestoreOperationStore', () => { ...record, phase: 'validating', status: 'running', + runtimeIdentityID: undefined, + runtimeIncarnationID: undefined, + result: undefined, error: undefined, completedAt: undefined, + lastInterruptedAt: undefined, updatedAt: '2026-06-15T12:01:00.000Z', attempt: 3 }); diff --git a/packages/sandbox/tests/rpc-client-retry.test.ts b/packages/sandbox/tests/rpc-client-retry.test.ts deleted file mode 100644 index 188617691..000000000 --- a/packages/sandbox/tests/rpc-client-retry.test.ts +++ /dev/null @@ -1,155 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; - -/** - * Verifies that `ContainerControlClient` plumbs the 503 retry budget - * through to the underlying `ContainerControlConnection`. The actual retry - * loop is exercised in `container-connection.test.ts`; here we only assert - * the wiring. - */ - -interface CapturedOptions { - retryTimeoutMs?: number; -} - -const captured: { - options: CapturedOptions[]; - setRetryTimeoutCalls: number[]; -} = { - options: [], - setRetryTimeoutCalls: [] -}; - -vi.mock('../src/container-control/connection', () => ({ - ContainerControlConnection: class { - constructor(options: CapturedOptions) { - captured.options.push(options); - } - setRetryTimeoutMs(ms: number) { - captured.setRetryTimeoutCalls.push(ms); - } - isConnected() { - return false; - } - getStats() { - return { imports: 1, exports: 1 }; - } - disconnect() {} - rpc() { - return new Proxy({}, { get: () => ({}) }); - } - async connect() {} - } -})); - -import { - ContainerControlClient, - translateRPCError -} from '../src/container-control/client'; - -describe('translateRPCError operation interruption mapping', () => { - function translateWithOperation(error: Error): never { - return ( - translateRPCError as ( - error: unknown, - context: { operation: string } - ) => never - )(error, { operation: 'processes.start' }); - } - - it.each([ - ['Peer closed WebSocket: 1006 runtime replaced', 'runtime_replaced'], - ['WebSocket connection failed.', 'runtime_replaced'], - [ - 'RPC session was shut down by disposing the main stub', - 'transport_disposed' - ] - ])( - 'maps in-flight transport loss %s to OPERATION_INTERRUPTED', - (message, reason) => { - let thrown: unknown; - try { - translateWithOperation(new Error(message)); - } catch (error) { - thrown = error; - } - - expect(thrown).toMatchObject({ - name: 'OperationInterruptedError', - code: 'OPERATION_INTERRUPTED', - context: expect.objectContaining({ - reason, - operation: 'processes.start', - admitted: 'unknown', - retryable: false - }) - }); - expect( - (thrown as { context: Record }).context - ).not.toHaveProperty('phase'); - } - ); -}); - -describe('ContainerControlClient retry timeout wiring', () => { - beforeEach(() => { - captured.options.length = 0; - captured.setRetryTimeoutCalls.length = 0; - vi.useFakeTimers(); - }); - - afterEach(() => { - vi.useRealTimers(); - }); - - it('passes retryTimeoutMs through to ContainerControlConnection', async () => { - const client = new ContainerControlClient({ - stub: { fetch: vi.fn() }, - retryTimeoutMs: 75_000 - }); - - await client.connect(); - - expect(captured.options).toHaveLength(1); - expect(captured.options[0].retryTimeoutMs).toBe(75_000); - }); - - it('omits retryTimeoutMs when not configured (lets the connection apply its default)', async () => { - const client = new ContainerControlClient({ - stub: { fetch: vi.fn() } - }); - - await client.connect(); - - expect(captured.options).toHaveLength(1); - expect(captured.options[0].retryTimeoutMs).toBeUndefined(); - }); - - it('forwards setRetryTimeoutMs() to the active connection', async () => { - const client = new ContainerControlClient({ - stub: { fetch: vi.fn() }, - retryTimeoutMs: 60_000 - }); - - await client.connect(); - - client.setRetryTimeoutMs(45_000); - - expect(captured.setRetryTimeoutCalls).toEqual([45_000]); - }); - - it('caches setRetryTimeoutMs() calls made before any connection is created', async () => { - const client = new ContainerControlClient({ - stub: { fetch: vi.fn() } - }); - - // No connection exists yet. The setter should still take effect once the - // connection is created — either by stashing the value and applying it on - // construction, or by applying it immediately if a connection is present. - client.setRetryTimeoutMs(15_000); - - await client.connect(); - - expect(captured.options).toHaveLength(1); - expect(captured.options[0].retryTimeoutMs).toBe(15_000); - }); -}); diff --git a/packages/sandbox/tests/rpc-sandbox-client.test.ts b/packages/sandbox/tests/rpc-sandbox-client.test.ts index 874f87e1c..b5e73d683 100644 --- a/packages/sandbox/tests/rpc-sandbox-client.test.ts +++ b/packages/sandbox/tests/rpc-sandbox-client.test.ts @@ -121,7 +121,7 @@ describe('ContainerControlClient busy/idle tracking', () => { }); await client.connect(); - const release = client.retainConnection(); + const release = client.retainRuntimeHold(); vi.advanceTimersByTime(3_000); expect(disconnects).toHaveLength(0); @@ -183,7 +183,7 @@ describe('ContainerControlClient busy/idle tracking', () => { expect(disconnects).toHaveLength(1); }); - it('waits for a committed stop before acquiring fresh RPC stubs', async () => { + it('does not gate domain calls on operation activity hooks', async () => { let releaseStop!: () => void; const stopSettled = new Promise((resolve) => { releaseStop = resolve; @@ -192,10 +192,6 @@ describe('ContainerControlClient busy/idle tracking', () => { const client = new ContainerControlClient({ stub: { fetch: vi.fn() }, - onOperationStarted: () => ({ - beforeCall: stopCommitted ? stopSettled : Promise.resolve(), - finish: () => undefined - }), busyPollIntervalMs: 1_000, idleDisconnectMs: 60_000 }); @@ -207,25 +203,16 @@ describe('ContainerControlClient busy/idle tracking', () => { const start = client.processes.start(['echo', 'ok']); const terminal = client.terminals.create({ command: ['sh'] }); - await Promise.resolve(); - expect(rpcGenerations).toHaveLength(0); - expect(processStarts).toHaveLength(0); - expect(terminalCreates).toHaveLength(0); - - client.disconnect(); - expect(disconnects).toEqual([0]); - connected = true; - stopCommitted = false; - releaseStop(); await Promise.all([start, terminal]); - expect(connectionGenerations).toEqual([0, 1]); - expect(rpcGenerations).toEqual([1, 1]); - expect(processStarts).toEqual([1]); - expect(terminalCreates).toEqual([1]); + expect(connectionGenerations).toEqual([0]); + expect(rpcGenerations).toEqual([0, 0]); + expect(processStarts).toEqual([0]); + expect(terminalCreates).toEqual([0]); + releaseStop(); }); - it('reconnects safely after a committed stop rejects', async () => { + it('ignores rejected operation activity hooks for domain dispatch', async () => { let rejectStop!: (error: Error) => void; const stopSettled = new Promise((_resolve, reject) => { rejectStop = reject; @@ -234,10 +221,6 @@ describe('ContainerControlClient busy/idle tracking', () => { const client = new ContainerControlClient({ stub: { fetch: vi.fn() }, - onOperationStarted: () => ({ - beforeCall: stopCommitted ? stopSettled : Promise.resolve(), - finish: () => undefined - }), busyPollIntervalMs: 1_000, idleDisconnectMs: 60_000 }); @@ -247,19 +230,10 @@ describe('ContainerControlClient busy/idle tracking', () => { stopCommitted = true; const start = client.processes.start(['echo', 'ok']); - await Promise.resolve(); - expect(processStarts).toHaveLength(0); + await expect(start).resolves.toBeDefined(); - client.disconnect(); - expect(disconnects).toEqual([0]); - connected = true; - stopCommitted = false; - rejectStop(new Error('stop failed')); - await expect(start).rejects.toThrow('stop failed'); - - await client.processes.start(['echo', 'again']); - expect(connectionGenerations).toEqual([0, 1]); - expect(processStarts).toEqual([1]); + expect(connectionGenerations).toEqual([0]); + expect(processStarts).toEqual([0]); }); it('fires onSessionIdle on explicit disconnect to avoid leaking inflight', async () => { diff --git a/packages/sandbox/tests/runtime/bootstrap-probe.test.ts b/packages/sandbox/tests/runtime/bootstrap-probe.test.ts new file mode 100644 index 000000000..9964b067c --- /dev/null +++ b/packages/sandbox/tests/runtime/bootstrap-probe.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it, vi } from 'vitest'; +import { ContainerControlConnection } from '../../src/container-control/connection'; +import { RuntimeControlProtocolError } from '../../src/errors'; +import { RuntimeBootstrapProbe } from '../../src/runtime/bootstrap-probe'; + +const metadata = { + runtimeIncarnationID: 'incarnation-1', + sandboxVersion: '0.0.0', + controlProtocolVersion: 1 as const +}; + +describe('RuntimeBootstrapProbe', () => { + it('uses direct port access, reads metadata, and always disconnects', async () => { + const directStub = { fetch: vi.fn() }; + const getTcpPort = vi.fn(() => directStub); + const connect = vi + .spyOn(ContainerControlConnection.prototype, 'connect') + .mockResolvedValue(undefined); + const readMetadata = vi + .spyOn(ContainerControlConnection.prototype, 'getRuntimeMetadata') + .mockResolvedValue(metadata); + const activate = vi.spyOn( + ContainerControlConnection.prototype, + 'activateControlSession' + ); + const disconnect = vi + .spyOn(ContainerControlConnection.prototype, 'disconnect') + .mockImplementation(() => undefined); + + await expect( + new RuntimeBootstrapProbe({ getTcpPort }).probe() + ).resolves.toEqual(metadata); + + expect(getTcpPort).toHaveBeenCalledWith(3000); + expect(connect).toHaveBeenCalledTimes(1); + expect(readMetadata).toHaveBeenCalledTimes(1); + expect(activate).not.toHaveBeenCalled(); + expect(disconnect).toHaveBeenCalledTimes(1); + }); + + it('adapts unsupported metadata to RuntimeControlProtocolError', async () => { + vi.spyOn(ContainerControlConnection.prototype, 'connect').mockResolvedValue( + undefined + ); + vi.spyOn( + ContainerControlConnection.prototype, + 'getRuntimeMetadata' + ).mockResolvedValue({ + ...metadata, + controlProtocolVersion: 2 as 1 + }); + vi.spyOn( + ContainerControlConnection.prototype, + 'disconnect' + ).mockImplementation(() => undefined); + + await expect( + new RuntimeBootstrapProbe({ + getTcpPort: () => ({ fetch: vi.fn() }) + }).probe() + ).rejects.toBeInstanceOf(RuntimeControlProtocolError); + }); +}); diff --git a/packages/sandbox/tests/runtime/lifecycle.test.ts b/packages/sandbox/tests/runtime/lifecycle.test.ts new file mode 100644 index 000000000..5eda4234e --- /dev/null +++ b/packages/sandbox/tests/runtime/lifecycle.test.ts @@ -0,0 +1,767 @@ +import type { RuntimeMetadata } from '@repo/shared'; +import { describe, expect, it, vi } from 'vitest'; +import type { ContainerControlClient } from '../../src/container-control/client'; +import { ResourceActivityGate } from '../../src/resource-activity-gate'; +import { + RuntimeIdentity, + type RuntimeIncarnationID, + RuntimeOperationRunner, + type RuntimeRecord, + SandboxRuntimeLifecycle +} from '../../src/runtime'; +import type { RuntimeIdentityID } from '../../src/runtime/types'; + +type Stored = RuntimeRecord | { id: RuntimeIdentityID } | undefined; +type FailurePoint = + | 'start' + | 'wait' + | 'probe' + | 'acquire' + | 'compat' + | 'reconcile' + | 'put'; + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (reason: unknown) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + +class MemoryStorage { + value: Stored; + failPut = false; + calls?: string[]; + deferredPut?: ReturnType>; + deferredPutAfterVisible?: ReturnType>; + deferredGetAfterRead?: ReturnType>; + + async get(): Promise { + const value = this.value as T | undefined; + if (this.deferredGetAfterRead) await this.deferredGetAfterRead.promise; + return value; + } + async put(key: string, value: T): Promise; + async put(entries: Record): Promise; + async put( + keyOrEntries: string | Record, + value?: T + ): Promise { + if (typeof keyOrEntries === 'string') { + this.calls?.push('put'); + if (this.failPut) throw new Error('put failed'); + if (this.deferredPut) await this.deferredPut.promise; + this.value = value as Stored; + if (this.deferredPutAfterVisible) + await this.deferredPutAfterVisible.promise; + } + } + async delete(key: string): Promise; + async delete(keys: string[]): Promise; + async delete(keyOrKeys: string | string[]): Promise { + this.value = undefined; + return Array.isArray(keyOrKeys) ? keyOrKeys.length : true; + } +} + +const metadata = ( + runtimeIncarnationID: string, + sandboxVersion = '0.0.0' +): RuntimeMetadata => ({ + runtimeIncarnationID, + sandboxVersion, + controlProtocolVersion: 1 +}); + +const runtime = (id: string, incarnation: string) => + new RuntimeIdentity({ + id: id as RuntimeIdentityID, + runtimeIncarnationID: incarnation as RuntimeIncarnationID + }); + +function record(identity: RuntimeIdentity): RuntimeRecord { + return { + schemaVersion: 1, + id: identity.id, + runtimeIncarnationID: identity.runtimeIncarnationID + }; +} + +type StartControlPortContext = { + lifecycle: SandboxRuntimeLifecycle; + setRunning(value: boolean): void; +}; + +type StartControlPort = (context: StartControlPortContext) => Promise; + +function host( + options: { + stored?: Stored; + running?: boolean; + incarnation?: string; + sandboxVersion?: string; + fail?: FailurePoint; + deferredCompat?: ReturnType>; + startControlPort?: StartControlPort; + } = {} +) { + const storage = new MemoryStorage(); + storage.value = options.stored; + storage.failPut = options.fail === 'put'; + storage.calls = []; + let running = options.running ?? false; + const calls: string[] = []; + let lifecycle!: SandboxRuntimeLifecycle; + const featureReplay = { + restore: vi.fn(), + mountLocalSync: vi.fn(), + namedTunnelProvision: vi.fn(), + previewActivation: vi.fn() + }; + const sessions = { + acquire: vi.fn(async () => { + calls.push('acquire'); + if (options.fail === 'acquire') throw new Error('session failed'); + return {} as ContainerControlClient; + }), + acquireSession: vi.fn(async () => ({ + client: {} as ContainerControlClient, + interrupted: new Promise(() => undefined), + isInterrupted: () => false, + retain: () => ({ release: vi.fn() }) + })), + closeActive: vi.fn(() => { + calls.push('closeActive'); + }), + dispose: vi.fn(() => { + calls.push('dispose'); + }) + }; + const lifecycleOptions = { + storage, + isRuntimeRunning: () => running, + startControlPort: vi.fn(async () => { + calls.push('start'); + if (options.fail === 'start') throw new Error('start failed'); + if (options.startControlPort) { + await options.startControlPort({ + lifecycle, + setRunning(value: boolean) { + running = value; + } + }); + return; + } + running = true; + lifecycle.markRuntimeStarted(); + }), + waitForControlPort: vi.fn(async () => { + calls.push('wait'); + if (options.fail === 'wait') throw new Error('readiness failed'); + }), + stopControlPort: vi.fn(async () => { + calls.push('stop'); + running = false; + }), + probe: { + probe: vi.fn(async () => { + calls.push('probe'); + if (options.fail === 'probe') throw new Error('probe failed'); + return metadata( + options.incarnation ?? 'inc-1', + options.sandboxVersion ?? '0.0.0' + ); + }) + }, + sessions, + observeVersionCompatibility: vi.fn(async () => { + calls.push('compat'); + if (options.deferredCompat) await options.deferredCompat.promise; + if (options.fail === 'compat') throw new Error('version failed'); + }), + reconcileReplacement: vi.fn(async () => { + calls.push('reconcile'); + if (options.fail === 'reconcile') throw new Error('reconcile failed'); + }) + }; + lifecycle = new SandboxRuntimeLifecycle(lifecycleOptions); + const runner = new RuntimeOperationRunner({ + lifecycle, + activityGate: new ResourceActivityGate( + vi.fn(), + vi.fn(async () => undefined) + ) + }); + const api = { + storage, + calls, + featureReplay, + sessions, + lifecycleOptions, + setRunning(value: boolean) { + running = value; + }, + lifecycle, + runner + }; + storage.calls = calls; + return api; +} + +describe('SandboxRuntimeLifecycle', () => { + it('cold concurrent establishment starts once, reconciles once, and publishes one active record', async () => { + const h = host(); + + const [first, second] = await Promise.all([ + h.lifecycle.establish(), + h.lifecycle.establish() + ]); + + expect(first).toBe(second); + expect(h.calls).toEqual([ + 'start', + 'wait', + 'probe', + 'acquire', + 'compat', + 'reconcile', + 'put' + ]); + expect(h.storage.value).toEqual(record(first)); + }); + + it('reuses stored identity for the same incarnation after a non-starting handshake', async () => { + const existing = runtime('runtime-1', 'inc-1'); + const h = host({ stored: record(existing), running: true }); + + await expect(h.lifecycle.establish()).resolves.toMatchObject({ + id: existing.id, + runtimeIncarnationID: existing.runtimeIncarnationID + }); + + expect(h.calls).toEqual(['probe', 'acquire', 'compat', 'put']); + }); + + it('creates a new identity and reconciles before publishing when incarnation changes', async () => { + const existing = runtime('runtime-1', 'inc-old'); + const h = host({ + stored: record(existing), + running: true, + incarnation: 'inc-new' + }); + + const adopted = await h.lifecycle.establish(); + + expect(adopted.id).not.toBe(existing.id); + expect(adopted.runtimeIncarnationID).toBe('inc-new'); + expect(h.calls).toEqual(['probe', 'acquire', 'compat', 'reconcile', 'put']); + expect(h.storage.value).toEqual(record(adopted)); + }); + + it('adopts a running control process when no active record exists', async () => { + const h = host({ running: true }); + + const adopted = await h.lifecycle.establish(); + + expect(h.calls).toEqual(['probe', 'acquire', 'compat', 'reconcile', 'put']); + expect(h.storage.value).toEqual(record(adopted)); + }); + + it('retries idempotent adoption after a crash before the active write', async () => { + const crashed = host({ running: true, fail: 'put' }); + await expect(crashed.lifecycle.establish()).rejects.toThrow('put failed'); + expect(crashed.calls).toEqual([ + 'probe', + 'acquire', + 'compat', + 'reconcile', + 'put' + ]); + expect(crashed.storage.value).toBeUndefined(); + + const retry = host({ stored: crashed.storage.value, running: true }); + const adopted = await retry.lifecycle.establish(); + + expect(retry.calls).toEqual([ + 'probe', + 'acquire', + 'compat', + 'reconcile', + 'put' + ]); + expect(retry.storage.value).toEqual(record(adopted)); + }); + + it('retries adoption after interruption after compatibility before reconciliation', async () => { + const compat = deferred(); + const first = host({ running: true, deferredCompat: compat }); + const establishing = first.lifecycle.establish(); + while (!first.calls.includes('compat')) await Promise.resolve(); + await first.lifecycle.invalidate(); + compat.resolve(); + + await expect(establishing).rejects.toMatchObject({ + name: 'RuntimeIdentityInactiveError' + }); + expect(first.calls).not.toContain('reconcile'); + expect(first.storage.value).toBeUndefined(); + + const retry = host({ running: true, incarnation: 'inc-1' }); + await retry.lifecycle.establish(); + + expect(retry.calls).toEqual([ + 'probe', + 'acquire', + 'compat', + 'reconcile', + 'put' + ]); + }); + + it('fences establishment before returning cleanup authority during invalidation', async () => { + const active = runtime('runtime-1', 'inc-1'); + const compat = deferred(); + const h = host({ + stored: record(active), + running: true, + incarnation: 'inc-2', + deferredCompat: compat + }); + const establishing = h.lifecycle.establish(); + while (!h.calls.includes('compat')) await Promise.resolve(); + + const invalidating = h.lifecycle.invalidateAndObserveStoredActive(); + compat.resolve(); + + await expect(establishing).rejects.toMatchObject({ + name: 'RuntimeIdentityInactiveError' + }); + await expect(invalidating).resolves.toEqual(active); + expect(h.storage.value).toBeUndefined(); + }); + + it('retries reconciliation for the same incarnation after interrupted reconciliation', async () => { + const first = host({ running: true, fail: 'reconcile' }); + await expect(first.lifecycle.establish()).rejects.toThrow( + 'reconcile failed' + ); + expect(first.storage.value).toBeUndefined(); + + const retry = host({ running: true, incarnation: 'inc-1' }); + await retry.lifecycle.establish(); + + expect(retry.calls).toEqual([ + 'probe', + 'acquire', + 'compat', + 'reconcile', + 'put' + ]); + }); + + it('rejects legacy and empty schema records for observations', async () => { + const cases: Stored[] = [ + { id: 'legacy' as RuntimeIdentityID }, + { + schemaVersion: 1, + id: '' as RuntimeIdentityID, + runtimeIncarnationID: 'inc-1' as RuntimeIncarnationID + }, + { + schemaVersion: 1, + id: 'runtime-1' as RuntimeIdentityID, + runtimeIncarnationID: '' as RuntimeIncarnationID + } + ]; + + for (const stored of cases) { + const h = host({ stored, running: true }); + await expect(h.lifecycle.observeStoredActive()).resolves.toBeNull(); + await expect(h.lifecycle.get()).resolves.toBeNull(); + } + }); + + it('surfaces start, readiness, metadata, session, version, and reconciliation failures without publishing', async () => { + const expectations: Array<[FailurePoint, string[]]> = [ + ['start', ['start']], + ['wait', ['start', 'wait']], + ['probe', ['probe']], + ['acquire', ['probe', 'acquire']], + ['compat', ['probe', 'acquire', 'compat']], + ['reconcile', ['probe', 'acquire', 'compat', 'reconcile']] + ]; + + for (const [fail, calls] of expectations) { + const h = host({ fail, running: !['start', 'wait'].includes(fail) }); + await expect(h.lifecycle.establish()).rejects.toThrow(); + expect(h.calls).toEqual(calls); + expect(h.storage.value).toBeUndefined(); + } + }); + + it('rejects empty bootstrap metadata before identity selection or publish', async () => { + for (const options of [{ incarnation: '' }, { sandboxVersion: '' }]) { + const h = host({ running: true, ...options }); + + await expect(h.lifecycle.establish()).rejects.toMatchObject({ + name: 'RuntimeControlProtocolError' + }); + expect(h.calls).toEqual(['probe']); + expect(h.storage.value).toBeUndefined(); + } + }); + + it('does not publish stale authority when invalidated during establishment', async () => { + const h = host(); + const seen: string[] = []; + h.lifecycle.onChange(() => seen.push('changed')); + const establishing = h.lifecycle.establish(); + const rejected = expect(establishing).rejects.toMatchObject({ + name: 'RuntimeIdentityInactiveError' + }); + await h.lifecycle.invalidate(); + + await rejected; + expect(h.storage.value).toBeUndefined(); + expect(seen.length).toBeGreaterThan(0); + }); + + it('removes a stale active record if invalidation wins while durable put is in flight', async () => { + const h = host({ running: true }); + h.storage.deferredPut = deferred(); + + const establishing = h.lifecycle.establish(); + while (!h.calls.includes('put')) await Promise.resolve(); + const invalidating = h.lifecycle.invalidate(); + h.storage.deferredPut.resolve(); + + await expect(establishing).rejects.toMatchObject({ + name: 'RuntimeIdentityInactiveError' + }); + await invalidating; + expect(h.storage.value).toBeUndefined(); + }); + + it('preserves replacement R2 when invalidate(R1) linearizes after pending R2 publish', async () => { + const r1 = runtime('runtime-1', 'inc-old'); + const h = host({ + stored: record(r1), + running: true, + incarnation: 'inc-new' + }); + h.storage.deferredPut = deferred(); + + const establishing = h.lifecycle.establish(); + while (!h.calls.includes('put')) await Promise.resolve(); + const invalidating = h.lifecycle.invalidate(r1); + h.storage.deferredPut.resolve(); + + const established = await establishing; + await invalidating; + expect(h.storage.value).toEqual(record(established)); + }); + + it('linearizes stale-read invalidate(R1) against poised R2 publish under the mutation gate', async () => { + const r1 = runtime('runtime-1', 'inc-old'); + const compat = deferred(); + const h = host({ + stored: record(r1), + running: true, + incarnation: 'inc-new', + deferredCompat: compat + }); + + const establishing = h.lifecycle.establish(); + while (!h.calls.includes('compat')) await Promise.resolve(); + h.storage.deferredGetAfterRead = deferred(); + const invalidating = h.lifecycle.invalidate(r1); + await Promise.resolve(); + compat.resolve(); + await Promise.resolve(); + + expect(h.calls).not.toContain('put'); + h.storage.deferredGetAfterRead.resolve(); + + await invalidating; + await expect(establishing).rejects.toMatchObject({ + name: 'RuntimeIdentityInactiveError' + }); + expect(h.calls).not.toContain('put'); + expect(h.storage.value).toBeUndefined(); + }); + + it('keeps R2 when stale invalidate(R1) runs after R2 put is visible', async () => { + const r1 = runtime('runtime-1', 'inc-old'); + const h = host({ + stored: record(r1), + running: true, + incarnation: 'inc-new' + }); + h.storage.deferredPutAfterVisible = deferred(); + + const establishing = h.lifecycle.establish(); + while (!h.calls.includes('put')) await Promise.resolve(); + while (h.storage.value === undefined || h.storage.value.id === r1.id) + await Promise.resolve(); + const r2Record = h.storage.value as RuntimeRecord; + const invalidating = h.lifecycle.invalidate(r1); + h.storage.deferredPutAfterVisible.resolve(); + + await expect(establishing).resolves.toMatchObject({ id: r2Record.id }); + await invalidating; + expect(h.storage.value).toEqual(r2Record); + }); + + it('removes R2 when qualified invalidate(R2) runs after R2 publish linearizes', async () => { + const r1 = runtime('runtime-1', 'inc-old'); + const h = host({ + stored: record(r1), + running: true, + incarnation: 'inc-new' + }); + h.storage.deferredPutAfterVisible = deferred(); + + const establishing = h.lifecycle.establish(); + while (!h.calls.includes('put')) await Promise.resolve(); + while (h.storage.value === undefined || h.storage.value.id === r1.id) + await Promise.resolve(); + const r2Record = h.storage.value as RuntimeRecord; + const invalidating = h.lifecycle.invalidate(new RuntimeIdentity(r2Record)); + h.storage.deferredPutAfterVisible.resolve(); + + await expect(establishing).resolves.toMatchObject({ id: r2Record.id }); + await invalidating; + expect(h.storage.value).toBeUndefined(); + }); + + it('isolates listener exceptions during publish and invalidate cleanup', async () => { + const h = host({ running: true }); + h.lifecycle.onChange(() => { + throw new Error('listener failed'); + }); + + await expect(h.lifecycle.establish()).resolves.toBeInstanceOf( + RuntimeIdentity + ); + await expect(h.lifecycle.invalidate()).resolves.toBeUndefined(); + + expect(h.calls).toContain('stop'); + expect(h.storage.value).toBeUndefined(); + }); + + it('physically stops and notifies listeners synchronously before awaited cleanup', async () => { + const existing = runtime('runtime-1', 'inc-1'); + const h = host({ stored: record(existing), running: true }); + const seen: string[] = []; + h.lifecycle.onChange(() => seen.push(h.calls.join(','))); + + await h.lifecycle.invalidate(existing); + + expect(seen[0]).toBe(''); + expect(h.calls).toEqual(['closeActive', 'stop']); + expect(h.storage.value).toBeUndefined(); + }); + + it('deletes only matching authority on invalidate(expected)', async () => { + const first = runtime('runtime-1', 'inc-1'); + const second = runtime('runtime-2', 'inc-2'); + const h = host({ stored: record(second), running: true }); + + await h.lifecycle.invalidate(first); + + expect(h.storage.value).toEqual(record(second)); + expect(h.calls).toEqual([]); + }); + + it('invalidates sessions without permanently disposing later establishment', async () => { + const h = host({ running: true }); + + const first = await h.lifecycle.establish(); + await h.lifecycle.invalidate(first); + const second = await h.lifecycle.establish(); + + expect(second.id).not.toBe(first.id); + expect(h.sessions.acquire).toHaveBeenCalledTimes(2); + expect(h.sessions.closeActive).toHaveBeenCalledTimes(1); + expect(h.sessions.dispose).not.toHaveBeenCalled(); + }); + + it('supports repeated establish and qualified invalidate cycles without retained invalidation state', async () => { + const h = host({ running: true }); + + for (let index = 0; index < 5; index++) { + h.setRunning(true); + const active = await h.lifecycle.establish(); + await h.lifecycle.invalidate(active); + expect(h.storage.value).toBeUndefined(); + } + + h.setRunning(true); + const final = await h.lifecycle.establish(); + + expect(h.storage.value).toEqual(record(final)); + expect(h.sessions.acquire).toHaveBeenCalledTimes(6); + expect(h.sessions.closeActive).toHaveBeenCalledTimes(5); + }); + + it('invokes only replacement reconciliation and leaves feature-owned adoption records untouched', async () => { + const h = host({ running: true }); + const featureOwnedRecords = { + restoreAttempt: { status: 'running' }, + mountLocalSync: { path: '/workspace' }, + namedTunnel: { needsRespawn: false }, + preview: { active: true } + }; + + await h.lifecycle.establish(); + + expect(h.calls).toContain('reconcile'); + expect(Object.keys(h.lifecycleOptions)).not.toEqual( + expect.arrayContaining([ + 'restore', + 'mountLocalSync', + 'namedTunnelProvision', + 'previewActivation' + ]) + ); + expect(featureOwnedRecords).toEqual({ + restoreAttempt: { status: 'running' }, + mountLocalSync: { path: '/workspace' }, + namedTunnel: { needsRespawn: false }, + preview: { active: true } + }); + }); + + it('reconciles a delayed previous stop while starting its replacement', async () => { + const previous = runtime('runtime-1', 'inc-old'); + const h = host({ + stored: record(previous), + incarnation: 'inc-new', + startControlPort: async ({ lifecycle, setRunning }) => { + await expect(lifecycle.reconcileObservedStop()).resolves.toBe( + 'reconciled-previous-stop' + ); + setRunning(true); + lifecycle.markRuntimeStarted(); + } + }); + + const dispatch = vi.fn(async (lease) => lease.runtime); + const replacement = await h.runner.runWaking('process.start', dispatch); + + expect(dispatch).toHaveBeenCalledTimes(1); + expect(replacement.id).not.toBe(previous.id); + expect(replacement.runtimeIncarnationID).toBe('inc-new'); + expect(h.sessions.closeActive).toHaveBeenCalledTimes(1); + expect(h.calls).toEqual([ + 'start', + 'closeActive', + 'stop', + 'wait', + 'probe', + 'acquire', + 'compat', + 'reconcile', + 'put' + ]); + expect(h.storage.value).toEqual(record(replacement)); + }); + + it('keeps repeated previous-stop reconciliation idempotent', async () => { + const previous = runtime('runtime-1', 'inc-old'); + const h = host({ + stored: record(previous), + incarnation: 'inc-new', + startControlPort: async ({ lifecycle, setRunning }) => { + await expect(lifecycle.reconcileObservedStop()).resolves.toBe( + 'reconciled-previous-stop' + ); + await expect(lifecycle.reconcileObservedStop()).resolves.toBe( + 'reconciled-previous-stop' + ); + setRunning(true); + lifecycle.markRuntimeStarted(); + } + }); + const replacement = await h.lifecycle.establish(); + expect(replacement.id).not.toBe(previous.id); + expect(h.sessions.closeActive).toHaveBeenCalledTimes(2); + expect(h.calls.filter((call) => call === 'put')).toHaveLength(1); + expect(h.storage.value).toEqual(record(replacement)); + }); + + it('rejects hard invalidation during replacement startup', async () => { + const h = host({ + startControlPort: async ({ lifecycle, setRunning }) => { + await lifecycle.invalidate(); + setRunning(true); + lifecycle.markRuntimeStarted(); + } + }); + await expect(h.lifecycle.establish()).rejects.toMatchObject({ + name: 'RuntimeIdentityInactiveError' + }); + expect(h.calls).not.toContain('probe'); + expect(h.storage.value).toBeUndefined(); + }); + + it('treats an observed stop after replacement start as hard invalidation', async () => { + const h = host({ + startControlPort: async ({ lifecycle, setRunning }) => { + setRunning(true); + lifecycle.markRuntimeStarted(); + await expect(lifecycle.reconcileObservedStop()).resolves.toBe( + 'hard-invalidation' + ); + } + }); + await expect(h.lifecycle.establish()).rejects.toMatchObject({ + name: 'RuntimeIdentityInactiveError' + }); + expect(h.storage.value).toBeUndefined(); + }); + + it('rejects replacement startup that completes without its start hook', async () => { + const h = host({ + startControlPort: async ({ setRunning }) => setRunning(true) + }); + await expect(h.lifecycle.establish()).rejects.toMatchObject({ + name: 'RuntimeIdentityInactiveError' + }); + expect(h.calls).not.toContain('probe'); + expect(h.storage.value).toBeUndefined(); + }); + + it('clears a failed replacement transition before a later establish', async () => { + let attempts = 0; + const previous = runtime('runtime-1', 'inc-old'); + const h = host({ + stored: record(previous), + incarnation: 'inc-new', + startControlPort: async ({ lifecycle, setRunning }) => { + attempts += 1; + if (attempts === 1) { + await expect(lifecycle.reconcileObservedStop()).resolves.toBe( + 'reconciled-previous-stop' + ); + throw new Error('replacement start failed'); + } + setRunning(true); + lifecycle.markRuntimeStarted(); + } + }); + + await expect(h.lifecycle.establish()).rejects.toThrow( + 'replacement start failed' + ); + expect(h.storage.value).toBeUndefined(); + expect(h.calls).not.toContain('put'); + await expect(h.lifecycle.establish()).resolves.toMatchObject({ + runtimeIncarnationID: 'inc-new' + }); + expect(attempts).toBe(2); + }); +}); diff --git a/packages/sandbox/tests/runtime/operation-runner.test.ts b/packages/sandbox/tests/runtime/operation-runner.test.ts new file mode 100644 index 000000000..d696a245d --- /dev/null +++ b/packages/sandbox/tests/runtime/operation-runner.test.ts @@ -0,0 +1,362 @@ +import { describe, expect, test, vi } from 'vitest'; +import type { ContainerControlClient } from '../../src/container-control/client'; +import { ErrorCode, OperationInterruptedError } from '../../src/errors'; +import { ResourceActivityGate } from '../../src/resource-activity-gate'; +import type { + RuntimeConnectionHold, + RuntimeIncarnationID +} from '../../src/runtime'; +import { RuntimeIdentity, RuntimeOperationRunner } from '../../src/runtime'; +import type { RuntimeSession } from '../../src/runtime/types'; +import { + type RuntimeIdentityID, + RuntimeIdentityInactiveError +} from '../../src/runtime/types'; + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (error: unknown) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + +function runtime(id = 'runtime', incarnation = 'incarnation') { + return new RuntimeIdentity({ + id: id as RuntimeIdentityID, + runtimeIncarnationID: incarnation as RuntimeIncarnationID + }); +} + +class FakeSession implements RuntimeSession { + readonly client = {} as ContainerControlClient; + private poisoned = false; + private rejectInterrupted!: (error: Error) => void; + readonly interrupted = new Promise((_, reject) => { + this.rejectInterrupted = reject; + }); + readonly holds = new Set(); + readonly interruptCallbacks = new Map void>(); + + constructor() { + this.interrupted.catch(() => undefined); + } + + isInterrupted(): boolean { + return this.poisoned; + } + + retain(onInterrupt?: () => void): RuntimeConnectionHold { + if (this.poisoned) { + onInterrupt?.(); + return { release: () => {} }; + } + const hold = { + release: vi.fn(() => { + this.holds.delete(hold); + this.interruptCallbacks.delete(hold); + }) + }; + this.holds.add(hold); + if (onInterrupt) this.interruptCallbacks.set(hold, onInterrupt); + return hold; + } + + interrupt(): void { + if (this.poisoned) return; + this.poisoned = true; + this.rejectInterrupted(new Error('closed')); + for (const hold of [...this.holds]) { + this.interruptCallbacks.get(hold)?.(); + hold.release(); + } + } +} + +function setup(current = runtime()) { + const renew = vi.fn(); + const stop = vi.fn(async () => undefined); + const gate = new ResourceActivityGate(renew, stop); + const listeners = new Set<() => void>(); + const session = new FakeSession(); + let active: RuntimeIdentity | null = current; + let establishCalls = 0; + let acquireCalls = 0; + const lifecycle = { + sessions: { + acquireSession: vi.fn(async () => { + acquireCalls += 1; + return session; + }) + }, + establish: vi.fn(async () => { + establishCalls += 1; + active = current; + return current; + }), + get: vi.fn(async () => active), + isActive: vi.fn(async (expected: RuntimeIdentity) => + same(active, expected) + ), + assertActive: vi.fn(async (expected: RuntimeIdentity) => { + if (!same(active, expected)) throw new Error('inactive'); + }), + onChange: vi.fn((listener: () => void) => { + listeners.add(listener); + return () => listeners.delete(listener); + }), + replace(next: RuntimeIdentity | null) { + active = next; + session.interrupt(); + for (const listener of [...listeners]) listener(); + } + }; + const runner = new RuntimeOperationRunner({ + lifecycle: lifecycle as never, + activityGate: gate + }); + return { + runner, + lifecycle, + gate, + renew, + stop, + session, + get establishCalls() { + return establishCalls; + }, + get acquireCalls() { + return acquireCalls; + }, + get listenerCount() { + return listeners.size; + } + }; +} + +describe('RuntimeOperationRunner', () => { + test('runWaking admits once after establishment and renews once', async () => { + const ctx = setup(); + const dispatch = vi.fn(async (lease) => lease.runtime.id); + + await ctx.runner.runWaking('files.read', dispatch); + + expect(dispatch).toHaveBeenCalledTimes(1); + expect(ctx.establishCalls).toBe(1); + expect(ctx.acquireCalls).toBe(1); + expect(ctx.renew).toHaveBeenCalledTimes(2); + }); + + test('replacement during waking establishment is a uniform interruption', async () => { + const ctx = setup(); + ctx.lifecycle.establish.mockRejectedValueOnce( + new RuntimeIdentityInactiveError() + ); + const dispatch = vi.fn(async () => 'forwarded'); + + await expect( + ctx.runner.runWaking('container.fetch', dispatch) + ).rejects.toMatchObject({ + name: 'OperationInterruptedError', + context: { operation: 'container.fetch', retryable: false } + }); + expect(dispatch).not.toHaveBeenCalled(); + expect(ctx.acquireCalls).toBe(0); + }); + + test('runExisting and probeExisting never establish', async () => { + const ctx = setup(); + await ctx.runner.runExisting( + { kind: 'current' }, + 'process.list', + async () => 'ok' + ); + await ctx.runner.probeExisting( + { kind: 'current' }, + 'expiry.probe', + async () => 'ok' + ); + expect(ctx.establishCalls).toBe(0); + expect(ctx.acquireCalls).toBe(2); + expect(ctx.renew).not.toHaveBeenCalled(); + }); + + test('expected runtime mismatch returns absent before dispatch', async () => { + const ctx = setup(runtime('current', 'one')); + const dispatch = vi.fn(async () => 'dispatched'); + const result = await ctx.runner.runExisting( + { kind: 'runtime', runtime: runtime('old', 'one') }, + 'process.get', + dispatch + ); + expect(result).toEqual({ status: 'absent' }); + expect(dispatch).not.toHaveBeenCalled(); + expect(ctx.acquireCalls).toBe(0); + }); + + test('replacement between lookup and session acquisition interrupts before dispatch', async () => { + const current = runtime('current', 'one'); + const ctx = setup(current); + const dispatch = vi.fn(async () => 'dispatched'); + ctx.lifecycle.sessions.acquireSession.mockImplementationOnce(async () => { + ctx.lifecycle.replace(runtime('next', 'two')); + return ctx.session; + }); + await expect( + ctx.runner.runExisting({ kind: 'current' }, 'race.operation', dispatch) + ).rejects.toMatchObject({ name: 'OperationInterruptedError' }); + expect(dispatch).not.toHaveBeenCalled(); + expect(ctx.listenerCount).toBe(0); + }); + + test('preserves structured interruption while the runtime stays active', async () => { + const ctx = setup(runtime('current', 'one')); + const interruption = new OperationInterruptedError({ + code: ErrorCode.OPERATION_INTERRUPTED, + message: 'Sandbox lifetime changed', + httpStatus: 409, + context: { + reason: 'sandbox_lifetime_changed', + operation: 'backup.restore', + admitted: true, + retryable: false + }, + timestamp: '2026-06-15T12:00:00.000Z' + }); + + await expect( + ctx.runner.runWaking('backup.restore', async () => { + throw interruption; + }) + ).rejects.toBe(interruption); + expect(ctx.listenerCount).toBe(0); + }); + + test('post replacement throws uniform OperationInterruptedError', async () => { + const ctx = setup(runtime('current', 'one')); + await expect( + ctx.runner.runExisting({ kind: 'current' }, 'label.only', async () => { + ctx.lifecycle.replace(runtime('next', 'two')); + return 'stale'; + }) + ).rejects.toMatchObject({ + name: 'OperationInterruptedError', + context: { operation: 'label.only', retryable: false } + }); + expect(ctx.listenerCount).toBe(0); + }); + + test('retained waking holds extend activity until last release with a real gate', async () => { + const ctx = setup(); + let first!: RuntimeConnectionHold; + let second!: RuntimeConnectionHold; + await ctx.runner.runWaking('stream.open', async (lease) => { + first = lease.retain(); + second = lease.retain(); + return 'ok'; + }); + + await ctx.gate.runExpiry(activeProbe(false), false); + expect(ctx.stop).not.toHaveBeenCalled(); + + first.release(); + first.release(); + await ctx.gate.runExpiry(activeProbe(false), false); + expect(ctx.stop).not.toHaveBeenCalled(); + + second.release(); + await ctx.gate.runExpiry(activeProbe(false), false); + expect(ctx.stop).toHaveBeenCalledTimes(1); + }); + + test('retained existing holds block committed expiry until last release', async () => { + const ctx = setup(); + let hold!: RuntimeConnectionHold; + await ctx.runner.runExisting( + { kind: 'current' }, + 'watch.open', + async (lease) => { + hold = lease.retain(); + return 'ok'; + } + ); + await ctx.gate.runExpiry(activeProbe(false), false); + expect(ctx.stop).not.toHaveBeenCalled(); + hold.release(); + await ctx.gate.runExpiry(activeProbe(false), false); + expect(ctx.stop).toHaveBeenCalledTimes(1); + }); + + test('session interruption wins and late retain cannot leak after callback continues', async () => { + const ctx = setup(runtime('current', 'one')); + const continueCallback = deferred(); + const lateRejection = new Error('late callback failure'); + const call = ctx.runner.runExisting( + { kind: 'current' }, + 'rpc.late', + async (lease) => { + await continueCallback.promise; + lease.retain().release(); + throw lateRejection; + } + ); + await vi.waitFor(() => expect(ctx.session.holds.size).toBe(1)); + ctx.lifecycle.replace(runtime('next', 'two')); + + await expect(call).rejects.toMatchObject({ + name: 'OperationInterruptedError', + context: { operation: 'rpc.late' } + }); + continueCallback.resolve(); + await Promise.resolve(); + await Promise.resolve(); + await ctx.gate.runExpiry(activeProbe(false), false); + + expect(ctx.session.holds.size).toBe(0); + expect(ctx.stop).toHaveBeenCalledTimes(1); + expect(ctx.listenerCount).toBe(0); + }); + + test('session interruption fanout rejects in-flight RPC and releases retained activity', async () => { + const ctx = setup(runtime('current', 'one')); + let retained!: RuntimeConnectionHold; + const pending = deferred(); + const call = ctx.runner.runExisting( + { kind: 'current' }, + 'rpc.wait', + async (lease) => { + retained = lease.retain(); + return pending.promise; + } + ); + await vi.waitFor(() => expect(ctx.session.holds.size).toBe(2)); + ctx.lifecycle.replace(runtime('next', 'two')); + + await expect(call).rejects.toMatchObject({ + name: 'OperationInterruptedError', + context: { operation: 'rpc.wait' } + }); + retained.release(); + await ctx.gate.runExpiry(activeProbe(false), false); + expect(ctx.stop).toHaveBeenCalledTimes(1); + expect(ctx.session.holds.size).toBe(0); + expect(ctx.listenerCount).toBe(0); + }); +}); + +function activeProbe(processesActive: boolean) { + return { + availability: async () => 'available' as const, + processesHasActive: async () => processesActive, + terminalsHasActive: async () => false + }; +} + +function same(left: RuntimeIdentity | null, right: RuntimeIdentity): boolean { + return ( + left?.id === right.id && + left.runtimeIncarnationID === right.runtimeIncarnationID + ); +} diff --git a/packages/sandbox/tests/runtime/port-readiness.test.ts b/packages/sandbox/tests/runtime/port-readiness.test.ts new file mode 100644 index 000000000..21409ea06 --- /dev/null +++ b/packages/sandbox/tests/runtime/port-readiness.test.ts @@ -0,0 +1,144 @@ +import type { PortWatchEvent, PortWatchSubscriptionAPI } from '@repo/shared'; +import { describe, expect, it, vi } from 'vitest'; +import type { RuntimeLease } from '../../src/runtime'; +import { waitForRuntimePort } from '../../src/runtime/port-readiness'; + +function subscription( + stream: Promise>, + cancel: () => Promise = async () => undefined +): PortWatchSubscriptionAPI { + return { + stream: vi.fn(() => stream), + cancel: vi.fn(cancel), + [Symbol.dispose]: vi.fn() + }; +} + +function leaseWith(subscriptionPromise: Promise) { + const release = vi.fn(); + const lease = { + control: { + ports: { + openWatch: vi.fn(() => subscriptionPromise) + } + }, + retain: vi.fn(() => ({ release })) + } as unknown as RuntimeLease; + return { lease, release }; +} + +function timeoutReject(message: string): Promise { + return new Promise((_, reject) => { + setTimeout(() => reject(new Error(message)), 50); + }); +} + +describe('waitForRuntimePort', () => { + it('times out a hanging readiness read and cancels the subscription', async () => { + const sub = subscription( + Promise.resolve( + new ReadableStream({ + pull: () => new Promise(() => undefined) + }) + ) + ); + const { lease, release } = leaseWith(Promise.resolve(sub)); + + await expect( + waitForRuntimePort(lease, 8080, { timeout: 1 }) + ).rejects.toThrow(/Timed out waiting for runtime port/); + + expect(sub.cancel).toHaveBeenCalledTimes(1); + expect(sub[Symbol.dispose]).toHaveBeenCalledTimes(1); + expect(release).toHaveBeenCalledTimes(1); + }); + + it('aborts while opening the watch and releases the lease', async () => { + const controller = new AbortController(); + const { lease, release } = leaseWith(new Promise(() => undefined)); + controller.abort(new Error('caller aborted')); + + await expect( + waitForRuntimePort(lease, 8080, { signal: controller.signal }) + ).rejects.toThrow(/caller aborted/); + + expect(release).toHaveBeenCalledTimes(1); + }); + + it('aborts while creating the stream and cancels the subscription', async () => { + const controller = new AbortController(); + const sub = subscription(new Promise(() => undefined)); + const { lease, release } = leaseWith(Promise.resolve(sub)); + + const waiting = waitForRuntimePort(lease, 8080, { + signal: controller.signal + }); + controller.abort(new Error('external abort')); + + await expect(waiting).rejects.toThrow(/external abort/); + expect(sub.cancel).toHaveBeenCalledTimes(1); + expect(sub[Symbol.dispose]).toHaveBeenCalledTimes(1); + expect(release).toHaveBeenCalledTimes(1); + }); + + it('does not wait for hanging reader and subscription cancellation on timeout', async () => { + const sub = subscription( + Promise.resolve( + new ReadableStream({ + pull: () => new Promise(() => undefined), + cancel: () => new Promise(() => undefined) + }) + ), + () => new Promise(() => undefined) + ); + const { lease, release } = leaseWith(Promise.resolve(sub)); + + await expect( + Promise.race([ + waitForRuntimePort(lease, 8080, { timeout: 1 }), + timeoutReject('cleanup blocked') + ]) + ).rejects.toThrow(/Timed out waiting for runtime port/); + + expect(sub.cancel).toHaveBeenCalledTimes(1); + expect(sub[Symbol.dispose]).toHaveBeenCalledTimes(1); + expect(release).toHaveBeenCalledTimes(1); + }); + + it('does not wait for hanging reader and subscription cancellation on abort', async () => { + const controller = new AbortController(); + const sub = subscription( + Promise.resolve( + new ReadableStream({ + pull: () => new Promise(() => undefined), + cancel: () => new Promise(() => undefined) + }) + ), + () => new Promise(() => undefined) + ); + const { lease, release } = leaseWith(Promise.resolve(sub)); + + const waiting = waitForRuntimePort(lease, 8080, { + signal: controller.signal + }); + controller.abort(new Error('external abort')); + + await expect( + Promise.race([waiting, timeoutReject('cleanup blocked')]) + ).rejects.toThrow(/external abort/); + + expect(sub.cancel).toHaveBeenCalledTimes(1); + expect(sub[Symbol.dispose]).toHaveBeenCalledTimes(1); + expect(release).toHaveBeenCalledTimes(1); + }); + + it('removes abort listeners without creating an unhandled finally rejection', async () => { + const failure = new Error('open failed'); + const { lease, release } = leaseWith(Promise.reject(failure)); + + await expect(waitForRuntimePort(lease, 8080)).rejects.toThrow( + /open failed/ + ); + expect(release).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/sandbox/tests/runtime/session-manager.test.ts b/packages/sandbox/tests/runtime/session-manager.test.ts new file mode 100644 index 000000000..a8cf169a9 --- /dev/null +++ b/packages/sandbox/tests/runtime/session-manager.test.ts @@ -0,0 +1,435 @@ +import type { SandboxControlCallback, TunnelRunExitEvent } from '@repo/shared'; +import type { RpcTarget } from 'capnweb'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { ContainerControlConnection } from '../../src/container-control/connection'; +import { RuntimeControlProtocolError } from '../../src/errors'; +import type { RuntimeIncarnationID } from '../../src/runtime'; +import { RuntimeIdentity, RuntimeSessionManager } from '../../src/runtime'; +import type { RuntimeIdentityID } from '../../src/runtime/types'; +import { SandboxControlCallbackImpl } from '../../src/tunnels/sandbox-control-callback'; + +const runtime = (id: string, incarnation: string) => + new RuntimeIdentity({ + id: id as RuntimeIdentityID, + runtimeIncarnationID: incarnation as RuntimeIncarnationID + }); + +const metadata = (incarnation: string) => ({ + runtimeIncarnationID: incarnation, + sandboxVersion: '0.0.0', + controlProtocolVersion: 1 as const +}); + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (reason: unknown) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + +beforeEach(() => { + vi.restoreAllMocks(); +}); + +describe('RuntimeSessionManager', () => { + it('sequential acquire after retain/release reuses cached activation', async () => { + const activate = vi + .spyOn(ContainerControlConnection.prototype, 'activateControlSession') + .mockResolvedValue(metadata('incarnation-1')); + const disconnect = vi + .spyOn(ContainerControlConnection.prototype, 'disconnect') + .mockImplementation(() => undefined); + const manager = new RuntimeSessionManager({ + getTcpPort: () => ({ fetch: vi.fn() }) + }); + const expected = runtime('runtime-1', 'incarnation-1'); + + const first = await manager.acquireSession(expected); + const hold = first.retain(); + hold.release(); + hold.release(); + const second = await manager.acquireSession(expected); + + expect(second).toBe(first); + expect(activate).toHaveBeenCalledTimes(1); + expect(disconnect).not.toHaveBeenCalled(); + }); + + it('reuses the same activated session for the same runtime identity and incarnation', async () => { + const activate = vi + .spyOn(ContainerControlConnection.prototype, 'activateControlSession') + .mockResolvedValue(metadata('incarnation-1')); + vi.spyOn( + ContainerControlConnection.prototype, + 'disconnect' + ).mockImplementation(() => undefined); + const manager = new RuntimeSessionManager({ + getTcpPort: () => ({ fetch: vi.fn() }) + }); + const expected = runtime('runtime-1', 'incarnation-1'); + + const first = await manager.acquire(expected); + const second = await manager.acquire(expected); + + expect(first).toBe(second); + expect(activate).toHaveBeenCalledTimes(1); + }); + + it('force-releases multiple holds and callbacks once on supersession', async () => { + vi.spyOn( + ContainerControlConnection.prototype, + 'activateControlSession' + ).mockImplementation(async (incarnation) => metadata(incarnation)); + vi.spyOn( + ContainerControlConnection.prototype, + 'disconnect' + ).mockImplementation(() => undefined); + const manager = new RuntimeSessionManager({ + getTcpPort: () => ({ fetch: vi.fn() }) + }); + const first = await manager.acquireSession( + runtime('runtime-1', 'incarnation-1') + ); + const firstInterrupted = vi.fn(); + const secondInterrupted = vi.fn(); + const firstHold = first.retain(firstInterrupted); + const secondHold = first.retain(secondInterrupted); + + await manager.acquireSession(runtime('runtime-2', 'incarnation-2')); + firstHold.release(); + secondHold.release(); + + expect(firstInterrupted).toHaveBeenCalledTimes(1); + expect(secondInterrupted).toHaveBeenCalledTimes(1); + expect(first.isInterrupted()).toBe(true); + }); + + it('force-releases holds and callbacks once on closeActive and transport close', async () => { + vi.spyOn( + ContainerControlConnection.prototype, + 'activateControlSession' + ).mockResolvedValue(metadata('incarnation-1')); + vi.spyOn( + ContainerControlConnection.prototype, + 'disconnect' + ).mockImplementation(() => undefined); + const manager = new RuntimeSessionManager({ + getTcpPort: () => ({ fetch: vi.fn() }) + }); + const first = await manager.acquireSession( + runtime('runtime-1', 'incarnation-1') + ); + const closeInterrupted = vi.fn(); + first.retain(closeInterrupted); + manager.closeActive(); + expect(closeInterrupted).toHaveBeenCalledTimes(1); + + const second = await manager.acquireSession( + runtime('runtime-1', 'incarnation-1') + ); + const transportInterrupted = vi.fn(); + second.retain(transportInterrupted); + const cached = manager as unknown as { + cached: { connection: { onClose?: () => void } } | null; + }; + cached.cached?.connection.onClose?.(); + expect(transportInterrupted).toHaveBeenCalledTimes(1); + }); + + it('interrupted sessions fail closed for later retain without leaking holds', async () => { + vi.spyOn( + ContainerControlConnection.prototype, + 'activateControlSession' + ).mockResolvedValue(metadata('incarnation-1')); + vi.spyOn( + ContainerControlConnection.prototype, + 'disconnect' + ).mockImplementation(() => undefined); + const manager = new RuntimeSessionManager({ + getTcpPort: () => ({ fetch: vi.fn() }) + }); + const session = await manager.acquireSession( + runtime('runtime-1', 'incarnation-1') + ); + manager.closeActive(); + const interrupted = vi.fn(); + const lateHold = session.retain(interrupted); + lateHold.release(); + + expect(interrupted).toHaveBeenCalledTimes(1); + }); + + it('disconnects and reactivates when either runtime identity dimension changes', async () => { + const activate = vi + .spyOn(ContainerControlConnection.prototype, 'activateControlSession') + .mockImplementation(async (incarnation) => metadata(incarnation)); + const disconnect = vi + .spyOn(ContainerControlConnection.prototype, 'disconnect') + .mockImplementation(() => undefined); + const manager = new RuntimeSessionManager({ + getTcpPort: () => ({ fetch: vi.fn() }) + }); + + const first = await manager.acquire(runtime('runtime-1', 'incarnation-1')); + const second = await manager.acquire(runtime('runtime-1', 'incarnation-2')); + const third = await manager.acquire(runtime('runtime-2', 'incarnation-2')); + + expect(first).not.toBe(second); + expect(second).not.toBe(third); + expect(activate).toHaveBeenCalledTimes(3); + expect(disconnect).toHaveBeenCalledTimes(2); + }); + + it('globally fences concurrent openings so superseded R1 cannot publish after R2', async () => { + const r1 = deferred>(); + const activate = vi + .spyOn(ContainerControlConnection.prototype, 'activateControlSession') + .mockImplementation((incarnation) => + incarnation === 'incarnation-1' + ? r1.promise + : Promise.resolve(metadata('incarnation-2')) + ); + const disconnect = vi + .spyOn(ContainerControlConnection.prototype, 'disconnect') + .mockImplementation(() => undefined); + const manager = new RuntimeSessionManager({ + getTcpPort: () => ({ fetch: vi.fn() }) + }); + + const stale = manager.acquire(runtime('runtime-1', 'incarnation-1')); + expect(disconnect).not.toHaveBeenCalled(); + const freshPromise = manager.acquire(runtime('runtime-2', 'incarnation-2')); + expect(disconnect).toHaveBeenCalledTimes(1); + const fresh = await freshPromise; + r1.resolve(metadata('incarnation-1')); + + await expect(stale).rejects.toMatchObject({ + name: 'RuntimeIdentityInactiveError' + }); + await expect( + manager.acquire(runtime('runtime-2', 'incarnation-2')) + ).resolves.toBe(fresh); + expect(activate).toHaveBeenCalledTimes(2); + expect(disconnect).toHaveBeenCalled(); + }); + + it('closeActive poisons current sessions but allows later acquire', async () => { + const pending = deferred>(); + const activate = vi + .spyOn(ContainerControlConnection.prototype, 'activateControlSession') + .mockReturnValueOnce(pending.promise) + .mockResolvedValue(metadata('incarnation-1')); + const disconnect = vi + .spyOn(ContainerControlConnection.prototype, 'disconnect') + .mockImplementation(() => undefined); + const manager = new RuntimeSessionManager({ + getTcpPort: () => ({ fetch: vi.fn() }) + }); + + const opening = manager.acquire(runtime('runtime-1', 'incarnation-1')); + manager.closeActive(); + pending.resolve(metadata('incarnation-1')); + + await expect(opening).rejects.toMatchObject({ + name: 'RuntimeIdentityInactiveError' + }); + await expect( + manager.acquire(runtime('runtime-1', 'incarnation-1')) + ).resolves.toBeDefined(); + expect(activate).toHaveBeenCalledTimes(2); + expect(disconnect).toHaveBeenCalled(); + }); + + it('dispose poisons an in-flight opening so it cannot publish or return', async () => { + const pending = deferred>(); + vi.spyOn( + ContainerControlConnection.prototype, + 'activateControlSession' + ).mockReturnValue(pending.promise); + const disconnect = vi + .spyOn(ContainerControlConnection.prototype, 'disconnect') + .mockImplementation(() => undefined); + const manager = new RuntimeSessionManager({ + getTcpPort: () => ({ fetch: vi.fn() }) + }); + + const opening = manager.acquire(runtime('runtime-1', 'incarnation-1')); + expect(disconnect).not.toHaveBeenCalled(); + manager.dispose(); + expect(disconnect).toHaveBeenCalledTimes(1); + pending.resolve(metadata('incarnation-1')); + + await expect(opening).rejects.toMatchObject({ + name: 'RuntimeIdentityInactiveError' + }); + await expect( + manager.acquire(runtime('runtime-1', 'incarnation-1')) + ).rejects.toThrow(/disposed/); + expect(disconnect).toHaveBeenCalled(); + }); + + it('does not arm idle teardown for manager-owned activated sessions', async () => { + vi.useFakeTimers(); + const activate = vi + .spyOn(ContainerControlConnection.prototype, 'activateControlSession') + .mockResolvedValue(metadata('incarnation-1')); + const disconnect = vi + .spyOn(ContainerControlConnection.prototype, 'disconnect') + .mockImplementation(() => undefined); + const manager = new RuntimeSessionManager({ + getTcpPort: () => ({ fetch: vi.fn() }) + }); + + const client = await manager.acquire(runtime('runtime-1', 'incarnation-1')); + const cached = manager as unknown as { + cached: { connection: { state: 'active' } } | null; + }; + if (cached.cached) cached.cached.connection.state = 'active'; + await client.connect(); + vi.advanceTimersByTime(10_000); + + expect(activate).toHaveBeenCalledTimes(1); + expect(disconnect).not.toHaveBeenCalled(); + vi.useRealTimers(); + }); + + it('invalidates cache on connection close and reactivates next acquire', async () => { + const activate = vi + .spyOn(ContainerControlConnection.prototype, 'activateControlSession') + .mockResolvedValue(metadata('incarnation-1')); + vi.spyOn( + ContainerControlConnection.prototype, + 'disconnect' + ).mockImplementation(() => undefined); + const manager = new RuntimeSessionManager({ + getTcpPort: () => ({ fetch: vi.fn() }) + }); + const expected = runtime('runtime-1', 'incarnation-1'); + + const first = await manager.acquire(expected); + const cached = manager as unknown as { + cached: { connection: { onClose?: () => void } } | null; + }; + cached.cached?.connection.onClose?.(); + const second = await manager.acquire(expected); + + expect(second).not.toBe(first); + expect(activate).toHaveBeenCalledTimes(2); + }); + + it('closes a mismatched activation before exposing a client', async () => { + vi.spyOn( + ContainerControlConnection.prototype, + 'activateControlSession' + ).mockResolvedValue(metadata('incarnation-2')); + const disconnect = vi + .spyOn(ContainerControlConnection.prototype, 'disconnect') + .mockImplementation(() => undefined); + const manager = new RuntimeSessionManager({ + getTcpPort: () => ({ fetch: vi.fn() }) + }); + + await expect( + manager.acquire(runtime('runtime-1', 'incarnation-1')) + ).rejects.toBeInstanceOf(RuntimeControlProtocolError); + expect(disconnect).toHaveBeenCalled(); + }); + + it('adapts CONTROL_PROTOCOL_INCOMPATIBLE activation failures', async () => { + const error = new Error('Runtime incarnation does not match') as Error & { + code: string; + }; + error.code = 'CONTROL_PROTOCOL_INCOMPATIBLE'; + vi.spyOn( + ContainerControlConnection.prototype, + 'activateControlSession' + ).mockRejectedValue(error); + const manager = new RuntimeSessionManager({ + getTcpPort: () => ({ fetch: vi.fn() }) + }); + + await expect( + manager.acquire(runtime('runtime-1', 'incarnation-1')) + ).rejects.toMatchObject({ + name: 'RuntimeControlProtocolError', + context: { reason: 'activation-mismatch' } + }); + }); + + it('passes a callback target bound to the expected runtime identity', async () => { + const handled = vi.fn(); + const expectedRuntime = runtime('runtime-1', 'incarnation-1'); + let current = runtime('runtime-1', 'incarnation-2'); + const callback = new SandboxControlCallbackImpl( + () => handled, + { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + child: () => ({ + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + child: vi.fn() + }) + }, + undefined, + () => current + ); + const bindRuntime = vi.spyOn(callback, 'bindRuntime'); + vi.spyOn( + ContainerControlConnection.prototype, + 'activateControlSession' + ).mockResolvedValue(metadata('incarnation-1')); + vi.spyOn( + ContainerControlConnection.prototype, + 'disconnect' + ).mockImplementation(() => undefined); + const manager = new RuntimeSessionManager({ + getTcpPort: () => ({ fetch: vi.fn() }), + callbackBinder: (runtimeIdentity, isSessionCurrent) => + callback.bindRuntime(runtimeIdentity, isSessionCurrent) + }); + + await manager.acquire(expectedRuntime); + expect(bindRuntime).toHaveBeenCalledWith( + expectedRuntime, + expect.any(Function) + ); + const bound = bindRuntime.mock.results[0]?.value as SandboxControlCallback & + RpcTarget; + await bound.onTunnelRunExit({ + tunnelId: 'tunnel', + runId: 'run', + mode: 'quick', + port: 3000, + exitCode: 1 + } as TunnelRunExitEvent); + + expect(handled).not.toHaveBeenCalled(); + current = expectedRuntime; + await bound.onTunnelRunExit({ + tunnelId: 'tunnel', + runId: 'run', + mode: 'quick', + port: 3000, + exitCode: 1 + } as TunnelRunExitEvent); + expect(handled).toHaveBeenCalledTimes(1); + + manager.closeActive(); + await bound.onTunnelRunExit({ + tunnelId: 'tunnel', + runId: 'run', + mode: 'quick', + port: 3000, + exitCode: 1 + } as TunnelRunExitEvent); + expect(handled).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/sandbox/tests/sandbox-activity-gate.test.ts b/packages/sandbox/tests/sandbox-activity-gate.test.ts index 6ccf4854b..ef2a6ffe7 100644 --- a/packages/sandbox/tests/sandbox-activity-gate.test.ts +++ b/packages/sandbox/tests/sandbox-activity-gate.test.ts @@ -4,7 +4,9 @@ const connectionGenerations: number[] = []; const rpcGenerations: number[] = []; const processStarts: number[] = []; const terminalCreates: number[] = []; +const terminalOutputCancels: number[] = []; const startAndWaitForPortsCalls: unknown[] = []; +const physicalStops: number[] = []; let connected = true; let nextConnectionGeneration = 0; @@ -22,7 +24,8 @@ vi.mock('@cloudflare/containers', () => { this.env = env; } - async start(): Promise { + private async physicalStart(): Promise { + if (this.ctx.container?.running === true) return; if (!this.startPromise) { this.startPromise = ( this.ctx.container?.start?.() ?? Promise.resolve() @@ -33,7 +36,18 @@ vi.mock('@cloudflare/containers', () => { await this.startPromise; } - async onActivityExpired(): Promise {} + async start(): Promise { + await this.physicalStart(); + } + + async stop(): Promise { + physicalStops.push(physicalStops.length); + if (this.ctx.container) this.ctx.container.running = false; + } + + async onActivityExpired(): Promise { + await this.stop(); + } async getState(): Promise<{ status: string }> { return { status: 'healthy' }; @@ -41,7 +55,9 @@ vi.mock('@cloudflare/containers', () => { async startAndWaitForPorts(options: unknown): Promise { startAndWaitForPortsCalls.push(options); - await this.start(); + await this.physicalStart(); + const onStart = (this as { onStart?: () => Promise }).onStart; + if (onStart) await onStart.call(this); } renewActivityTimeout(): void {} @@ -58,6 +74,8 @@ vi.mock('@cloudflare/containers', () => { vi.mock('../src/container-control/connection', () => ({ ContainerControlConnection: class { private readonly generation: number; + private connectionOpen = true; + private activated = false; constructor() { this.generation = nextConnectionGeneration; @@ -66,7 +84,7 @@ vi.mock('../src/container-control/connection', () => ({ } isConnected() { - return connected; + return this.connectionOpen && connected; } getStats() { @@ -74,10 +92,11 @@ vi.mock('../src/container-control/connection', () => ({ } disconnect() { - connected = false; + this.connectionOpen = false; } rpc() { + if (!this.activated) throw new Error('control session not activated'); rpcGenerations.push(this.generation); return { processes: { @@ -98,17 +117,58 @@ vi.mock('../src/container-control/connection', () => ({ terminalCreates.push(this.generation); return { id: `terminal-${this.generation}` }; }, + output: async () => { + const generation = this.generation; + return { + stream: async () => + new ReadableStream({ + start(controller) { + controller.enqueue({ + type: 'terminal', + terminalId: `terminal-${generation}`, + cursor: 'cursor-terminal', + timestamp: new Date().toISOString(), + state: 'exited', + exit: { code: 0, timedOut: false } + }); + } + }), + cancel: async () => { + terminalOutputCancels.push(generation); + }, + [Symbol.dispose]: () => {} + }; + }, hasActive: async () => false }, ports: {} }; } - async connect() {} + async connect() { + this.connectionOpen = true; + } + + async getRuntimeMetadata() { + return { + runtimeIncarnationID: `incarnation-${this.generation}`, + sandboxVersion: '0.0.0', + controlProtocolVersion: 1 + }; + } + + async activateControlSession(expectedRuntimeIncarnationID: string) { + this.activated = true; + return { + runtimeIncarnationID: expectedRuntimeIncarnationID, + sandboxVersion: '0.0.0', + controlProtocolVersion: 1 + }; + } } })); -import { connect, Sandbox } from '../src/sandbox'; +import { Sandbox } from '../src/sandbox'; interface Deferred { promise: Promise; @@ -126,6 +186,14 @@ function deferred(): Deferred { return { promise, resolve, reject }; } +function runtimeRecord(id = 'runtime-a', incarnation = 'incarnation-0') { + return { + schemaVersion: 1, + id, + runtimeIncarnationID: incarnation + }; +} + async function createSandbox() { const storageState = new Map(); let stub!: Sandbox; @@ -149,6 +217,7 @@ async function createSandbox() { waitUntil: vi.fn(), container: { running: true, + getTcpPort: vi.fn(() => ({ fetch: vi.fn() })), start: vi.fn(startContainer) }, id: { @@ -162,7 +231,7 @@ async function createSandbox() { ctx as unknown as ConstructorParameters[0], {} ); - const sandbox = Object.assign(stub, { wsConnect: connect(stub) }); + const sandbox = stub; await Promise.all( ctx.blockConcurrencyWhile.mock.results.map((result) => result.value) @@ -179,7 +248,9 @@ describe('Sandbox resource activity gate integration', () => { rpcGenerations.length = 0; processStarts.length = 0; terminalCreates.length = 0; + terminalOutputCancels.length = 0; startAndWaitForPortsCalls.length = 0; + physicalStops.length = 0; }); afterEach(() => { @@ -188,7 +259,7 @@ describe('Sandbox resource activity gate integration', () => { it('reads runtime liveness without starting or creating a control connection', async () => { const { sandbox, ctx } = await createSandbox(); - await ctx.storage.put('currentRuntimeIdentity', { id: 'runtime-a' }); + await ctx.storage.put('currentRuntimeIdentity', runtimeRecord()); await expect(sandbox.isRuntimeActive()).resolves.toBe(true); @@ -198,19 +269,41 @@ describe('Sandbox resource activity gate integration', () => { expect(rpcGenerations).toHaveLength(0); }); + it('completes inactivity expiry through one physical stop', async () => { + const { sandbox, ctx } = await createSandbox(); + await ctx.storage.put('currentRuntimeIdentity', runtimeRecord()); + + await expect( + Promise.race([ + sandbox.onActivityExpired(), + new Promise((_, reject) => + setTimeout( + () => reject(new Error('activity expiry did not settle')), + 100 + ) + ) + ]) + ).resolves.toBeUndefined(); + + expect(physicalStops).toHaveLength(1); + expect(ctx.container.running).toBe(false); + }); + it('holds exec and terminal creation behind a pending committed inactivity stop', async () => { const { sandbox, ctx } = await createSandbox(); + await ctx.storage.put('currentRuntimeIdentity', runtimeRecord()); const stop = deferred(); + let stopCalled = false; vi.spyOn( Object.getPrototypeOf(Object.getPrototypeOf(sandbox)), - 'onActivityExpired' + 'stop' ).mockImplementation(() => { - sandbox.client.disconnect(); + stopCalled = true; return stop.promise; }); const expiry = sandbox.onActivityExpired(); - await vi.waitFor(() => expect(rpcGenerations.length).toBeGreaterThan(0)); + await vi.waitFor(() => expect(stopCalled).toBe(true)); expect(ctx.container.start).not.toHaveBeenCalled(); const rpcGenerationsBeforeStopSettles = [...rpcGenerations]; const connectionGenerationsBeforeStopSettles = [...connectionGenerations]; @@ -238,23 +331,55 @@ describe('Sandbox resource activity gate integration', () => { expect(connectionGenerations.at(-1)).toBeGreaterThan( connectionGenerationsBeforeStopSettles.at(-1) ?? -1 ); - expect(processStarts).toEqual([1]); - expect(terminalCreates).toEqual([1]); + expect(processStarts).toHaveLength(1); + expect(terminalCreates).toHaveLength(1); + expect(processStarts[0]).toBeGreaterThan( + connectionGenerationsBeforeStopSettles.at(-1) ?? -1 + ); + expect(terminalCreates[0]).toBeGreaterThan( + connectionGenerationsBeforeStopSettles.at(-1) ?? -1 + ); + }); + + it('closes retained terminal output after terminal event', async () => { + const { sandbox, ctx } = await createSandbox(); + await ctx.storage.put('currentRuntimeIdentity', runtimeRecord()); + + const terminal = await sandbox.createTerminal({ command: ['sh'] }); + const subscription = await terminal.capability.openOutput({ + replay: true, + follow: true + }); + const first = await subscription.next(); + const second = await subscription.next(); + + expect(first).toMatchObject({ + done: false, + value: { + type: 'terminal', + state: 'exited', + terminalId: terminal.snapshot.id + } + }); + expect(second).toEqual({ done: true, value: undefined }); + expect(terminalOutputCancels).toHaveLength(1); }); it('holds exposePort startup behind a pending committed inactivity stop', async () => { const { sandbox, ctx } = await createSandbox(); + await ctx.storage.put('currentRuntimeIdentity', runtimeRecord()); (sandbox as any).sandboxName = 'sandbox-activity-gate-test'; const stop = deferred(); + let stopCalled = false; vi.spyOn( Object.getPrototypeOf(Object.getPrototypeOf(sandbox)), - 'onActivityExpired' + 'stop' ).mockImplementation(() => { - sandbox.client.disconnect(); + stopCalled = true; return stop.promise; }); const expiry = sandbox.onActivityExpired(); - await vi.waitFor(() => expect(rpcGenerations.length).toBeGreaterThan(0)); + await vi.waitFor(() => expect(stopCalled).toBe(true)); expect(ctx.container.start).not.toHaveBeenCalled(); ctx.container.running = false; @@ -268,18 +393,31 @@ describe('Sandbox resource activity gate integration', () => { await expiry; await expect(exposed).resolves.toMatchObject({ port: 8080 }); - expect(startAndWaitForPortsCalls).toHaveLength(1); + expect(startAndWaitForPortsCalls).toEqual([ + { + ports: [3000], + cancellationOptions: { + instanceGetTimeoutMS: 30000, + portReadyTimeoutMS: 90000, + waitInterval: 300, + abort: undefined + } + } + ]); + // ctx.container.start is called internally because our startAndWaitForPorts mock calls physicalStart() expect(ctx.container.start).toHaveBeenCalledTimes(1); }); it('releases the gate operation count when post-stop startup fails so a retry can run', async () => { const { sandbox, ctx } = await createSandbox(); + await ctx.storage.put('currentRuntimeIdentity', runtimeRecord()); const stop = deferred(); + let stopCalled = false; vi.spyOn( Object.getPrototypeOf(Object.getPrototypeOf(sandbox)), - 'onActivityExpired' + 'stop' ).mockImplementation(() => { - sandbox.client.disconnect(); + stopCalled = true; return stop.promise; }); diff --git a/packages/sandbox/tests/sandbox-control-callback.test.ts b/packages/sandbox/tests/sandbox-control-callback.test.ts index e9cd03b12..8d46f5463 100644 --- a/packages/sandbox/tests/sandbox-control-callback.test.ts +++ b/packages/sandbox/tests/sandbox-control-callback.test.ts @@ -9,9 +9,27 @@ import type { Logger } from '@repo/shared'; import { describe, expect, it, vi } from 'vitest'; +import { RuntimeIdentity } from '../src/runtime'; import type { TunnelExitHandler } from '../src/tunnels/rpc-target'; import { SandboxControlCallbackImpl } from '../src/tunnels/sandbox-control-callback'; +const runtime = new RuntimeIdentity({ + id: 'runtime-1' as RuntimeIdentity['id'], + runtimeIncarnationID: 'inc-1' as RuntimeIdentity['runtimeIncarnationID'] +}); + +function makeCallback( + getHandler: () => TunnelExitHandler | null +): SandboxControlCallbackImpl { + return new SandboxControlCallbackImpl( + getHandler, + makeLogger(), + runtime, + async () => runtime, + () => true + ); +} + function makeLogger(): Logger { const log: Logger = { info: vi.fn(), @@ -26,7 +44,7 @@ function makeLogger(): Logger { describe('SandboxControlCallbackImpl', () => { it('routes onTunnelRunExit events through to the bound handler', async () => { const handler = vi.fn().mockResolvedValue(undefined); - const cb = new SandboxControlCallbackImpl(() => handler, makeLogger()); + const cb = makeCallback(() => handler); await cb.onTunnelRunExit({ tunnelId: 'quick-a', @@ -37,12 +55,19 @@ describe('SandboxControlCallbackImpl', () => { }); expect(handler).toHaveBeenCalledTimes(1); - expect(handler).toHaveBeenCalledWith('quick-a', 8080, 0, 'run-a'); + expect(handler).toHaveBeenCalledWith( + 'quick-a', + 8080, + 0, + 'run-a', + runtime, + expect.any(Function) + ); }); it('passes through a null exitCode', async () => { const handler = vi.fn().mockResolvedValue(undefined); - const cb = new SandboxControlCallbackImpl(() => handler, makeLogger()); + const cb = makeCallback(() => handler); await cb.onTunnelRunExit({ tunnelId: 'quick-b', @@ -52,7 +77,14 @@ describe('SandboxControlCallbackImpl', () => { exitCode: null }); - expect(handler).toHaveBeenCalledWith('quick-b', 8081, null, 'run-b'); + expect(handler).toHaveBeenCalledWith( + 'quick-b', + 8081, + null, + 'run-b', + runtime, + expect.any(Function) + ); }); it('is a no-op when the accessor returns null', async () => { @@ -71,7 +103,7 @@ describe('SandboxControlCallbackImpl', () => { it('reads the handler accessor every call', async () => { let current: TunnelExitHandler | null = null; - const cb = new SandboxControlCallbackImpl(() => current, makeLogger()); + const cb = makeCallback(() => current); await cb.onTunnelRunExit({ tunnelId: 'quick-d', @@ -105,11 +137,36 @@ describe('SandboxControlCallbackImpl', () => { expect(handler).toHaveBeenCalledTimes(1); }); + it('ignores callbacks after the activated runtime is replaced', async () => { + const handler = vi.fn().mockResolvedValue(undefined); + const replacement = new RuntimeIdentity({ + id: runtime.id, + runtimeIncarnationID: 'inc-2' as RuntimeIdentity['runtimeIncarnationID'] + }); + const cb = new SandboxControlCallbackImpl( + () => handler, + makeLogger(), + runtime, + async () => replacement, + () => true + ); + + await cb.onTunnelRunExit({ + tunnelId: 'quick-stale', + runId: 'run-stale', + mode: 'quick', + port: 8084, + exitCode: 0 + }); + + expect(handler).not.toHaveBeenCalled(); + }); + it('propagates handler errors to the caller', async () => { const handler = vi .fn() .mockRejectedValue(new Error('storage boom')); - const cb = new SandboxControlCallbackImpl(() => handler, makeLogger()); + const cb = makeCallback(() => handler); await expect( cb.onTunnelRunExit({ diff --git a/packages/sandbox/tests/sandbox-destroy-lifetime.test.ts b/packages/sandbox/tests/sandbox-destroy-lifetime.test.ts index 68b400061..9ed1fb213 100644 --- a/packages/sandbox/tests/sandbox-destroy-lifetime.test.ts +++ b/packages/sandbox/tests/sandbox-destroy-lifetime.test.ts @@ -77,9 +77,13 @@ describe('Sandbox destroy lifetime fencing', () => { vi.setSystemTime(new Date('2026-06-15T11:00:00.000Z')); }); - it('rotates sandbox lifetime before clearing runtime identity during destroy', async () => { + it('rotates sandbox lifetime and clears runtime identity during destroy', async () => { const { state, calls, values } = createMockState(); - values.set('currentRuntimeIdentity', { id: 'runtime-before-destroy' }); + values.set('currentRuntimeIdentity', { + schemaVersion: 1, + id: 'runtime-before-destroy', + runtimeIncarnationID: 'inc-before-destroy' + }); const sandbox = new Sandbox(state, {}); await vi.waitFor(() => { @@ -98,7 +102,6 @@ describe('Sandbox destroy lifetime fencing', () => { expect(lifetimePutIndex).toBeGreaterThanOrEqual(0); expect(runtimeClearIndex).toBeGreaterThanOrEqual(0); - expect(lifetimePutIndex).toBeLessThan(runtimeClearIndex); expect(values.get('sandbox:lifetime')).toMatchObject({ id: expect.any(String), diff --git a/packages/sandbox/tests/sandbox-error-handling.test.ts b/packages/sandbox/tests/sandbox-error-handling.test.ts index fe9ded38b..0dc0575ea 100644 --- a/packages/sandbox/tests/sandbox-error-handling.test.ts +++ b/packages/sandbox/tests/sandbox-error-handling.test.ts @@ -1,6 +1,66 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { Sandbox } from '../src/sandbox'; +const controlConnectionMockState = vi.hoisted(() => ({ + activated: false +})); + +const containerMockState = vi.hoisted(() => ({ + startAndWaitForPorts: vi.fn(async (_options?: unknown) => undefined) +})); + +vi.mock('../src/container-control/connection', () => ({ + ContainerControlConnection: class { + isConnected() { + return true; + } + getStats() { + return { imports: 1, exports: 1 }; + } + disconnect() { + controlConnectionMockState.activated = false; + } + async connect() {} + async getRuntimeMetadata() { + return { + runtimeIncarnationID: 'test-incarnation', + sandboxVersion: '0.0.0', + controlProtocolVersion: 1 + }; + } + async activateControlSession(expectedRuntimeIncarnationID: string) { + controlConnectionMockState.activated = true; + return { + runtimeIncarnationID: expectedRuntimeIncarnationID, + sandboxVersion: '0.0.0', + controlProtocolVersion: 1 + }; + } + rpc() { + if (!controlConnectionMockState.activated) { + throw new Error('control session must be activated before rpc()'); + } + return { + ports: { + openWatch: vi.fn(async () => ({ + stream: vi.fn( + async () => + new ReadableStream({ + start(controller) { + controller.enqueue({ type: 'ready' }); + controller.close(); + } + }) + ), + cancel: vi.fn(async () => undefined), + [Symbol.dispose]: vi.fn() + })) + } + }; + } + } +})); + vi.mock('@cloudflare/containers', () => { const MockContainer = class Container { ctx: any; @@ -19,8 +79,8 @@ vi.mock('@cloudflare/containers', () => { // Return unhealthy so containerFetch() enters the startup path return { status: 'unhealthy' }; } - async startAndWaitForPorts() { - // Will be spied on in tests + async startAndWaitForPorts(...args: unknown[]) { + return containerMockState.startAndWaitForPorts(args[0]); } }; @@ -64,7 +124,7 @@ describe('Sandbox.containerFetch() error classification', () => { let sandbox: Sandbox; let mockCtx: Partial>; let mockEnv: any; - let startAndWaitSpy: ReturnType; + let startAndWaitSpy: typeof containerMockState.startAndWaitForPorts; // All 11 transient patterns from sandbox.ts isTransientStartupError() // Each pattern maps to a real error source @@ -120,14 +180,22 @@ describe('Sandbox.containerFetch() error classification', () => { beforeEach(async () => { vi.clearAllMocks(); + containerMockState.startAndWaitForPorts.mockResolvedValue(undefined); + + const storageData = new Map(); // Mock DurableObjectState mockCtx = { storage: { - get: vi.fn().mockResolvedValue(null), - put: vi.fn().mockResolvedValue(undefined), - delete: vi.fn().mockResolvedValue(undefined), - list: vi.fn().mockResolvedValue(new Map()) + get: vi.fn(async (key: string) => storageData.get(key) ?? null), + put: vi.fn(async (key: string, value: unknown) => { + storageData.set(key, value); + }), + delete: vi.fn(async (key: string) => { + storageData.delete(key); + }), + list: vi.fn().mockResolvedValue(new Map()), + transaction: vi.fn(async (closure) => closure(mockCtx.storage)) } as any, blockConcurrencyWhile: vi .fn() @@ -135,6 +203,14 @@ describe('Sandbox.containerFetch() error classification', () => { (callback: () => Promise): Promise => callback() ), waitUntil: vi.fn(), + container: { + running: false, + getTcpPort: vi.fn(() => ({ + fetch: vi.fn(async () => new Response('Mock TCP port fetch')), + connect: vi.fn() + })) + } as unknown as DurableObjectState<{}>['container'], + abort: vi.fn(), id: { toString: () => 'test-sandbox-id', equals: vi.fn(), @@ -155,8 +231,13 @@ describe('Sandbox.containerFetch() error classification', () => { expect(mockCtx.blockConcurrencyWhile).toHaveBeenCalled(); }); - // Spy on startAndWaitForPorts - this is what throws errors during startup - startAndWaitSpy = vi.spyOn(sandbox as any, 'startAndWaitForPorts'); + // Physical lifecycle establishment surfaces startup failures for direct forwarding. + startAndWaitSpy = containerMockState.startAndWaitForPorts; + startAndWaitSpy.mockImplementation(async () => { + const container = mockCtx.container as { running: boolean }; + container.running = true; + await sandbox.onStart(); + }); }); afterEach(() => { @@ -464,6 +545,9 @@ describe('Sandbox.containerFetch() error classification', () => { vi.spyOn(sandbox as any, 'getState').mockResolvedValueOnce({ status: 'healthy' }); + ( + sandbox as unknown as { ctx: { container: { running: boolean } } } + ).ctx.container.running = true; // Mock parent containerFetch to return success const parentContainerFetch = vi @@ -479,7 +563,6 @@ describe('Sandbox.containerFetch() error classification', () => { 3000 ); - // startAndWaitForPorts should NOT be called when healthy expect(startAndWaitSpy).not.toHaveBeenCalled(); expect(response.status).toBe(200); @@ -492,8 +575,9 @@ describe('Sandbox.containerFetch() error classification', () => { vi.spyOn(sandbox as any, 'getState').mockResolvedValueOnce({ status: 'healthy' }); - (sandbox as any).ctx.container = { running: false }; - startAndWaitSpy.mockResolvedValueOnce(undefined); + ( + sandbox as unknown as { ctx: { container: { running: boolean } } } + ).ctx.container.running = false; const parentContainerFetch = vi .spyOn( @@ -524,7 +608,9 @@ describe('Sandbox.containerFetch() error classification', () => { vi.spyOn(sandbox as any, 'getState').mockResolvedValueOnce({ status: 'healthy' }); - (sandbox as any).ctx.container = { running: false }; + ( + sandbox as unknown as { ctx: { container: { running: boolean } } } + ).ctx.container.running = false; const abortSpy = vi.fn(); (sandbox as any).ctx.abort = abortSpy; startAndWaitSpy.mockRejectedValueOnce( diff --git a/packages/sandbox/tests/sandbox.test.ts b/packages/sandbox/tests/sandbox.test.ts index 154af3c0e..3b7f3d666 100644 --- a/packages/sandbox/tests/sandbox.test.ts +++ b/packages/sandbox/tests/sandbox.test.ts @@ -2,16 +2,28 @@ import { Container, getContainer } from '@cloudflare/containers'; import type * as SharedRoot from '@repo/shared'; import type { ISandbox, ProcessLogEvent, ProcessStatus } from '@repo/shared'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { RuntimeIdentityInactiveError } from '../src/current-runtime-identity'; +import type { ContainerControlClient } from '../src/container-control'; import { ContainerUnavailableError, ErrorCode, InvalidBackupConfigError, - PortNotExposedError + PortNotExposedError, + RPCTransportError, + RuntimeControlProtocolError } from '../src/errors'; import { SandboxExtension, type SandboxLike } from '../src/extensions'; -import { connect, getSandbox, Sandbox } from '../src/sandbox'; -import { createMockControlClient } from './helpers/mock-control-client'; +import { RuntimeIdentityInactiveError } from '../src/runtime/types'; +import { getSandbox, Sandbox } from '../src/sandbox'; +import { + asSandboxWithClient, + createMockControlClient +} from './helpers/mock-control-client'; + +const controlConnectionMockState = vi.hoisted(() => ({ + client: null as unknown, + activated: false, + lastConnection: null as { disconnect(): void } | null +})); function processLogStream( events: ProcessLogEvent[] @@ -24,12 +36,53 @@ function processLogStream( }); } +vi.mock('../src/container-control/connection', () => ({ + ContainerControlConnection: class { + onClose?: () => void; + constructor(options?: { onClose?: () => void }) { + this.onClose = options?.onClose; + controlConnectionMockState.lastConnection = this; + } + isConnected() { + return true; + } + getStats() { + return { imports: 1, exports: 1 }; + } + disconnect() { + controlConnectionMockState.activated = false; + this.onClose?.(); + } + async connect() {} + async getRuntimeMetadata() { + return { + runtimeIncarnationID: 'test-incarnation', + sandboxVersion: '0.0.0', + controlProtocolVersion: 1 + }; + } + async activateControlSession(expectedRuntimeIncarnationID: string) { + controlConnectionMockState.activated = true; + return { + runtimeIncarnationID: expectedRuntimeIncarnationID, + sandboxVersion: '0.0.0', + controlProtocolVersion: 1 + }; + } + rpc() { + if (!controlConnectionMockState.activated) { + throw new Error('control session must be activated before rpc()'); + } + return controlConnectionMockState.client; + } + } +})); + vi.mock('@cloudflare/containers', () => { const mockSwitchPort = vi.fn((request: Request, port: number) => { - // Create a new request with the port in the URL path - const url = new URL(request.url); - url.pathname = `/proxy/${port}${url.pathname}`; - return new Request(url, request); + const headers = new Headers(request.headers); + headers.set('cf-container-target-port', String(port)); + return new Request(request, { headers }); }); const MockContainer = class Container { @@ -60,7 +113,10 @@ vi.mock('@cloudflare/containers', () => { return new Response('Mock Container HTTP fetch'); } async startAndWaitForPorts(): Promise { - // No-op: real container startup is not needed in tests. + // Match @cloudflare/containers: after ports are ready, invoke onStart. + if (this.ctx?.container) this.ctx.container.running = true; + const onStart = (this as { onStart?: () => Promise }).onStart; + if (onStart) await onStart.call(this); } async destroy(): Promise { // No-op: real container destroy is not needed in tests; individual @@ -123,14 +179,77 @@ interface MockCtx { }; } -interface SandboxRuntimeStart { - ensureRuntimeActiveForPreview(): Promise; +function deferred(): { + promise: Promise; + resolve(value: T): void; + reject(reason?: unknown): void; +} { + let resolve!: (value: T) => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + +class FakeSocket extends EventTarget { + closeCalls: Array<{ code?: number; reason?: string }> = []; + + close(code?: number, reason?: string): void { + this.closeCalls.push({ code, reason }); + this.dispatchEvent(new Event('close')); + } } const PREVIEW_TEST_PORT = 8080; const PREVIEW_TEST_TOKEN = 'token12345678901'; const PREVIEW_TEST_RUNTIME_ID = 'runtime-1'; +function runtimeRecord(id: string) { + return { + schemaVersion: 1, + id, + runtimeIncarnationID: 'test-incarnation' + }; +} + +type PreviewRuntimeRunnerProbe = { + runExisting( + target: unknown, + operation: string, + call: (lease: { + runtime: unknown; + retain(): { release(): void }; + }) => Promise + ): Promise; + runWaking( + operation: string, + call: (lease: { + runtime: unknown; + retain(): { release(): void }; + }) => Promise, + options?: { signal?: AbortSignal } + ): Promise; +}; + +function getPreviewRuntimeRunner(sandbox: Sandbox): PreviewRuntimeRunnerProbe { + return (sandbox as unknown as { runtimeRunner: PreviewRuntimeRunnerProbe }) + .runtimeRunner; +} + +type PreviewRuntimeLifecycleProbe = { + assertActive(runtime: unknown): Promise; +}; + +function getPreviewRuntimeLifecycle( + sandbox: Sandbox +): PreviewRuntimeLifecycleProbe { + return ( + sandbox as unknown as { runtimeLifecycle: PreviewRuntimeLifecycleProbe } + ).runtimeLifecycle; +} + function activePreviewStorageState({ port = PREVIEW_TEST_PORT, token = PREVIEW_TEST_TOKEN, @@ -144,12 +263,11 @@ function activePreviewStorageState({ portTokens: { [port.toString()]: { token } }, - currentRuntimeIdentity: { - id: runtimeIdentityID - }, + currentRuntimeIdentity: runtimeRecord(runtimeIdentityID), activePreviewPorts: { [port.toString()]: { runtimeIdentityID, + runtimeIncarnationID: 'test-incarnation', token } } @@ -269,7 +387,14 @@ describe('Sandbox durable object behavior', () => { vi.clearAllMocks(); const storageState = new Map([ - ['currentRuntimeIdentity', { id: 'runtime-a' }] + [ + 'currentRuntimeIdentity', + { + schemaVersion: 1, + id: 'runtime-a', + runtimeIncarnationID: 'test-incarnation' + } + ] ]); const storage = { @@ -295,6 +420,9 @@ describe('Sandbox durable object behavior', () => { waitUntil: vi.fn(), container: { running: true, + getTcpPort: vi.fn(() => ({ + fetch: vi.fn(async () => new Response('Mock TCP port fetch')) + })), start: vi.fn(), exec: vi.fn().mockImplementation(async () => { return { @@ -339,20 +467,27 @@ describe('Sandbox durable object behavior', () => { ) ); - sandbox = Object.assign(stub, { - wsConnect: connect(stub) - }); - sandbox.client = createMockControlClient(); + sandbox = stub; + const sandboxWithClient = asSandboxWithClient(sandbox); + sandboxWithClient.client = createMockControlClient(); + controlConnectionMockState.client = sandboxWithClient.client; + controlConnectionMockState.activated = false; // Now spy on the client methods that we need for testing - vi.spyOn(sandbox.client.files, 'writeFile').mockResolvedValue({ + vi.spyOn( + asSandboxWithClient(sandbox).client.files, + 'writeFile' + ).mockResolvedValue({ success: true, path: '/test.txt', timestamp: new Date().toISOString() } as any); - vi.spyOn(sandbox.client.watch, 'checkChanges').mockResolvedValue({ + vi.spyOn( + asSandboxWithClient(sandbox).client.watch, + 'checkChanges' + ).mockResolvedValue({ success: true, status: 'unchanged', version: 'watch-1:0', @@ -451,10 +586,9 @@ describe('Sandbox durable object behavior', () => { cwd: '/workspace' }); - expect(sandbox.client.processes.start).toHaveBeenCalledWith( - ['echo', 'hello'], - { cwd: '/workspace' } - ); + expect( + asSandboxWithClient(sandbox).client.processes.start + ).toHaveBeenCalledWith(['echo', 'hello'], { cwd: '/workspace' }); expect(descriptor).toMatchObject({ id: 'mock-process-id', pid: 123 @@ -462,36 +596,93 @@ describe('Sandbox durable object behavior', () => { expect(descriptor.capability.status).toBeTypeOf('function'); }); + it('overlays per-command env on sandbox env for process launches', async () => { + await sandbox.setEnvVars({ + SANDBOX_ONLY: 'sandbox', + SHARED: 'sandbox' + }); + + await sandbox.exec(['env'], { + env: { COMMAND_ONLY: 'command', SHARED: 'command' } + }); + + expect( + asSandboxWithClient(sandbox).client.processes.start + ).toHaveBeenCalledWith(['env'], { + env: { + SANDBOX_ONLY: 'sandbox', + COMMAND_ONLY: 'command', + SHARED: 'command' + } + }); + }); + + it('treats null command env values as unset during overlay', async () => { + await sandbox.setEnvVars({ + REMOVE_ME: 'sandbox', + SANDBOX_ONLY: 'sandbox' + }); + + await sandbox.exec(['env'], { + // @ts-expect-error Exercise null compatibility for JSON callers. + env: { COMMAND_ONLY: 'command', REMOVE_ME: null } + }); + + expect( + asSandboxWithClient(sandbox).client.processes.start + ).toHaveBeenCalledWith(['env'], { + env: { COMMAND_ONLY: 'command', SANDBOX_ONLY: 'sandbox' } + }); + }); + + it.each([null, []])( + 'preserves invalid command env for container validation', + async (invalidEnv) => { + await sandbox.setEnvVars({ SANDBOX_ONLY: 'sandbox' }); + + await sandbox.exec(['env'], { + // @ts-expect-error Exercise runtime validation of untyped callers. + env: invalidEnv + }); + + expect( + asSandboxWithClient(sandbox).client.processes.start + ).toHaveBeenCalledWith(['env'], { env: invalidEnv }); + } + ); + it('wakes, captures, and pre-validates before launch, then post-fences', async () => { const order: string[] = []; mockCtx.storage.get.mockImplementation(async (key: string) => { if (key === 'currentRuntimeIdentity') { order.push('runtime'); - return { id: 'runtime-a' }; + return { + schemaVersion: 1, + id: 'runtime-a', + runtimeIncarnationID: 'test-incarnation' + }; } return null; }); - Object.assign(sandbox, { - ensureContainerRunning: vi.fn(async () => { - order.push('wake'); - }) + vi.spyOn( + asSandboxWithClient(sandbox).client.processes, + 'start' + ).mockImplementation(async (command) => { + order.push('start'); + return { + id: 'p1', + pid: 123, + command, + state: 'running', + startedAt: new Date().toISOString() + }; }); - vi.spyOn(sandbox.client.processes, 'start').mockImplementation( - async (command) => { - order.push('start'); - return { - id: 'p1', - pid: 123, - command, - state: 'running', - startedAt: new Date().toISOString() - }; - } - ); await sandbox.exec(['echo', 'ordered']); - expect(order).toEqual(['wake', 'runtime', 'runtime', 'start', 'runtime']); + expect(order).toContain('runtime'); + expect(order).toContain('start'); + expect(order.indexOf('start')).toBeGreaterThan(order.indexOf('runtime')); }); it('rejects a runtime replacement while launch RPC is pending', async () => { @@ -499,24 +690,18 @@ describe('Sandbox durable object behavior', () => { const pendingStart = new Promise((resolve) => { resolveStart = resolve; }); - vi.spyOn(sandbox.client.processes, 'start').mockReturnValueOnce( - pendingStart - ); - let activeRuntime = { id: 'runtime-a' }; - Object.assign(sandbox, { - currentRuntime: { - get: vi.fn(async () => activeRuntime), - assertActive: vi.fn(async (expected: { id: string }) => { - if (expected.id !== activeRuntime.id) throw new Error('inactive'); - }) - } - }); - + vi.spyOn( + asSandboxWithClient(sandbox).client.processes, + 'start' + ).mockReturnValueOnce(pendingStart); const launch = sandbox.exec(['sleep', '1']); + launch.catch(() => undefined); await vi.waitFor(() => - expect(sandbox.client.processes.start).toHaveBeenCalledOnce() + expect( + asSandboxWithClient(sandbox).client.processes.start + ).toHaveBeenCalledOnce() ); - activeRuntime = { id: 'runtime-b' }; + await (sandbox as any).runtimeLifecycle.invalidate(); resolveStart({ id: 'p1', pid: 123, @@ -527,7 +712,7 @@ describe('Sandbox durable object behavior', () => { await expect(launch).rejects.toMatchObject({ code: 'OPERATION_INTERRUPTED', - context: { operation: 'process.start', effect: 'unknown' } + context: { operation: 'process.start' } }); }); @@ -543,7 +728,7 @@ describe('Sandbox durable object behavior', () => { }; Object.assign(sandbox, { processLifecycle: { - captureCurrent: vi.fn(async () => ({ id: 'runtime-a' })), + captureCurrent: vi.fn(async () => runtimeRecord('runtime-a')), runRead: vi.fn(async () => status) } }); @@ -562,16 +747,17 @@ describe('Sandbox durable object behavior', () => { state: 'running', startedAt: new Date().toISOString() }; - const runRead = vi.fn(async () => [status]); - Object.assign(sandbox, { - processLifecycle: { - captureCurrent: vi.fn(async () => ({ id: 'runtime-a' })), - runRead - } - }); + vi.mocked( + asSandboxWithClient(sandbox).client.processes.list + ).mockResolvedValueOnce([status]); await expect(sandbox.listProcesses()).resolves.toEqual([status]); - expect(runRead).toHaveBeenCalledTimes(1); + expect( + asSandboxWithClient(sandbox).client.processes.list + ).toHaveBeenCalledTimes(1); + expect( + asSandboxWithClient(sandbox).client.processes.get + ).not.toHaveBeenCalled(); }); it('rejects malformed JavaScript argv before process control', async () => { @@ -589,12 +775,17 @@ describe('Sandbox durable object behavior', () => { await expect( Reflect.apply(sandbox.exec, sandbox, [[123]]) ).rejects.toMatchObject({ code: 'INVALID_COMMAND' }); - expect(sandbox.client.processes.start).not.toHaveBeenCalled(); + expect( + asSandboxWithClient(sandbox).client.processes.start + ).not.toHaveBeenCalled(); }); it('logs launch identity without an exit code', async () => { const infoSpy = vi.spyOn((sandbox as any).logger, 'info'); - vi.spyOn(sandbox.client.processes, 'start').mockResolvedValueOnce({ + vi.spyOn( + asSandboxWithClient(sandbox).client.processes, + 'start' + ).mockResolvedValueOnce({ id: 'logged-process', pid: 456, command: ['echo', 'test_logging'], @@ -636,16 +827,16 @@ describe('Sandbox durable object behavior', () => { await expect(sandbox.getProcess('p1')).resolves.toBeNull(); expect(mockCtx.container.start).not.toHaveBeenCalled(); - expect(sandbox.client.processes.get).not.toHaveBeenCalled(); + expect( + asSandboxWithClient(sandbox).client.processes.get + ).not.toHaveBeenCalled(); }); it('runs direct file operations through file RPC', async () => { await sandbox.writeFile('/test.txt', 'content'); - expect(sandbox.client.files.writeFile).toHaveBeenCalledWith( - '/test.txt', - 'content', - { encoding: undefined } - ); + expect( + asSandboxWithClient(sandbox).client.files.writeFile + ).toHaveBeenCalledWith('/test.txt', 'content', { encoding: undefined }); }); it('owns public watch subscriptions after consuming data', async () => { @@ -654,7 +845,9 @@ describe('Sandbox durable object behavior', () => { ); const cancel = vi.fn(async () => undefined); const dispose = vi.fn(); - vi.mocked(sandbox.client.watch.watch).mockResolvedValue({ + vi.mocked( + asSandboxWithClient(sandbox).client.watch.watch + ).mockResolvedValue({ stream: vi.fn( async () => new ReadableStream({ @@ -679,12 +872,199 @@ describe('Sandbox durable object behavior', () => { expect(dispose).toHaveBeenCalledTimes(1); }); + it('keeps file read streams owned until caller cancellation', async () => { + const chunk = new TextEncoder().encode('chunk'); + const cancel = vi.fn(async () => undefined); + vi.mocked( + asSandboxWithClient(sandbox).client.files.readFileStream + ).mockResolvedValue( + new ReadableStream({ + pull(controller) { + controller.enqueue(chunk); + }, + cancel + }) + ); + + const stream = await sandbox.readFileStream('/workspace/file.txt'); + expect(controlConnectionMockState.activated).toBe(true); + + const reader = stream.getReader(); + await expect(reader.read()).resolves.toEqual({ + done: false, + value: chunk + }); + expect(controlConnectionMockState.activated).toBe(true); + + await reader.cancel('done'); + expect(cancel).toHaveBeenCalledTimes(1); + expect(cancel).toHaveBeenCalledWith('done'); + }); + + it('does not wait for hanging remote file read cancellation', async () => { + const chunk = new TextEncoder().encode('chunk'); + const cancel = vi.fn(() => new Promise(() => undefined)); + vi.mocked( + asSandboxWithClient(sandbox).client.files.readFileStream + ).mockResolvedValue( + new ReadableStream({ + pull(controller) { + controller.enqueue(chunk); + }, + cancel + }) + ); + + const stream = await sandbox.readFileStream('/workspace/file.txt'); + const reader = stream.getReader(); + await expect(reader.read()).resolves.toEqual({ + done: false, + value: chunk + }); + + const canceled = reader.cancel('caller done').then(() => true); + const completedPromptly = await Promise.race([ + canceled, + new Promise((resolve) => setTimeout(() => resolve(false), 20)) + ]); + + expect(completedPromptly).toBe(true); + expect(cancel).toHaveBeenCalledWith('caller done'); + }); + + it('does not wait for hanging remote watch cancellation', async () => { + const chunk = new TextEncoder().encode( + 'data: {"type":"watching","path":"/workspace/test","watchId":"watch-1"}\n\n' + ); + const cancel = vi.fn(() => new Promise(() => undefined)); + const dispose = vi.fn(); + vi.mocked( + asSandboxWithClient(sandbox).client.watch.watch + ).mockResolvedValue({ + stream: vi.fn( + async () => + new ReadableStream({ + pull(controller) { + controller.enqueue(chunk); + } + }) + ), + cancel, + [Symbol.dispose]: dispose + }); + + const stream = await sandbox.watch('/workspace/test'); + const reader = stream.getReader(); + await expect(reader.read()).resolves.toEqual({ + done: false, + value: chunk + }); + + const canceled = reader.cancel('caller done').then(() => true); + const completedPromptly = await Promise.race([ + canceled, + new Promise((resolve) => setTimeout(() => resolve(false), 20)) + ]); + + expect(completedPromptly).toBe(true); + expect(cancel).toHaveBeenCalledTimes(1); + expect(dispose).toHaveBeenCalledTimes(1); + }); + + it('cancels caller-supplied write streams when stream RPC rejects', async () => { + const sourceCancel = vi.fn(async () => undefined); + const rpcError = new Error('stream rpc failed'); + vi.mocked( + asSandboxWithClient(sandbox).client.files.writeFileStream + ).mockRejectedValue(rpcError); + const source = new ReadableStream({ + pull(controller) { + controller.enqueue(new TextEncoder().encode('chunk')); + }, + cancel: sourceCancel + }); + + await expect( + sandbox.writeFile('/workspace/file.txt', source) + ).rejects.toThrow(rpcError.message); + + expect(sourceCancel).toHaveBeenCalledTimes(1); + expect(sourceCancel).toHaveBeenCalledWith('writeFileStream completed'); + expect( + ( + sandbox as unknown as { + resourceActivityGate: { activityInFlight: number }; + } + ).resourceActivityGate.activityInFlight + ).toBe(0); + }); + + it('invalidates retained file read streams when the runtime stops', async () => { + const chunk = new TextEncoder().encode('chunk'); + const cancel = vi.fn(async () => undefined); + let pushed = false; + vi.mocked( + asSandboxWithClient(sandbox).client.files.readFileStream + ).mockResolvedValue( + new ReadableStream({ + start(controller) { + if (!pushed) { + pushed = true; + controller.enqueue(chunk); + } + }, + cancel + }) + ); + + const stream = await sandbox.readFileStream('/workspace/file.txt'); + const reader = stream.getReader(); + await expect(reader.read()).resolves.toEqual({ + done: false, + value: chunk + }); + + const pendingRead = reader.read(); + ( + sandbox as unknown as { runtimeSessions: { closeActive(): void } } + ).runtimeSessions.closeActive(); + + await expect(pendingRead).rejects.toMatchObject({ + code: ErrorCode.OPERATION_INTERRUPTED + }); + expect(cancel).toHaveBeenCalledTimes(1); + }); + + it('releases file read stream ownership exactly once on source error', async () => { + const sourceError = new Error('source failed'); + const cancel = vi.fn(async () => undefined); + vi.mocked( + asSandboxWithClient(sandbox).client.files.readFileStream + ).mockResolvedValue( + new ReadableStream({ + pull(controller) { + controller.error(sourceError); + }, + cancel + }) + ); + + const stream = await sandbox.readFileStream('/workspace/file.txt'); + const reader = stream.getReader(); + + await expect(reader.read()).rejects.toThrow(sourceError); + await sandbox.onStop(); + expect(cancel).not.toHaveBeenCalled(); + }); + it('should forward checkChanges options to the watch client', async () => { await sandbox.checkChanges('/workspace/test', { since: 'watch-1:0', recursive: false }); - expect(sandbox.client.watch.checkChanges).toHaveBeenCalledWith({ + expect( + asSandboxWithClient(sandbox).client.watch.checkChanges + ).toHaveBeenCalledWith({ path: '/workspace/test', recursive: false, include: undefined, @@ -745,23 +1125,218 @@ describe('Sandbox durable object behavior', () => { }); }); + describe('containerFetch() direct forwarding', () => { + it('rejects direct access to the container RPC control route', async () => { + const tcpFetch = vi.fn(async () => new Response('unexpected')); + mockCtx.container.getTcpPort = vi.fn(() => ({ fetch: tcpFetch })); + + await expect( + sandbox.containerFetch(new Request('https://example.com/rpc'), 3000) + ).rejects.toThrow('Container RPC connection is not authorized'); + expect(tcpFetch).not.toHaveBeenCalled(); + }); + + it('retains direct HTTP response bodies until runtime invalidation', async () => { + const bodyRead = deferred(); + const response = new Response( + new ReadableStream({ + async pull(controller) { + try { + controller.enqueue(await bodyRead.promise); + } catch { + // The retained wrapper may error the stream while this pull is pending. + } + } + }), + { + status: 202, + statusText: 'Accepted', + headers: { 'x-test': 'body' } + } + ); + const tcpFetch = vi.fn(async (_request: Request) => response); + mockCtx.container.getTcpPort = vi.fn(() => ({ fetch: tcpFetch })); + + const forwarded = await sandbox.containerFetch( + new Request('https://example.com/data'), + 8080 + ); + const reader = forwarded.body!.getReader(); + const pendingRead = reader.read(); + const readExpectation = expect(pendingRead).rejects.toMatchObject({ + code: ErrorCode.OPERATION_INTERRUPTED + }); + await sandbox.stop(); + bodyRead.resolve(new Uint8Array([1])); + + await readExpectation; + expect(forwarded.status).toBe(202); + expect(forwarded.statusText).toBe('Accepted'); + expect(forwarded.headers.get('x-test')).toBe('body'); + expect(mockCtx.container.getTcpPort).toHaveBeenCalledWith(8080); + expect(tcpFetch).toHaveBeenCalledOnce(); + expect((tcpFetch.mock.calls[0][0] as Request).url).toBe( + 'http://example.com/data' + ); + }); + + it('releases direct HTTP responses without bodies immediately', async () => { + const tcpFetch = vi.fn(async () => new Response(null, { status: 204 })); + mockCtx.container.getTcpPort = vi.fn(() => ({ fetch: tcpFetch })); + + const forwarded = await sandbox.containerFetch( + new Request('https://example.com/empty'), + 8080 + ); + await sandbox.stop(); + + expect(forwarded.status).toBe(204); + expect(forwarded.body).toBeNull(); + expect(mockCtx.container.getTcpPort).toHaveBeenCalledWith(8080); + expect(tcpFetch).toHaveBeenCalledOnce(); + }); + + it('propagates caller abort during direct HTTP port readiness', async () => { + const releaseReadiness = deferred(); + vi.mocked( + asSandboxWithClient(sandbox).client.ports.openWatch + ).mockResolvedValueOnce({ + stream: vi.fn( + async () => + new ReadableStream({ + async start(controller) { + await releaseReadiness.promise; + controller.enqueue({ type: 'ready' }); + controller.close(); + } + }) + ), + cancel: vi.fn(async () => undefined), + [Symbol.dispose]: vi.fn() + }); + const controller = new AbortController(); + const reason = new DOMException('caller stopped', 'AbortError'); + const forwarded = sandbox.containerFetch( + new Request('https://example.com/data', { signal: controller.signal }), + 8080 + ); + + await vi.waitFor(() => + expect( + asSandboxWithClient(sandbox).client.ports.openWatch + ).toHaveBeenCalled() + ); + controller.abort(reason); + releaseReadiness.resolve(); + + await expect(forwarded).rejects.toBe(reason); + }); + + it('passes caller abort into direct HTTP runtime establishment', async () => { + const controller = new AbortController(); + const reason = new DOMException('caller stopped', 'AbortError'); + const runWaking = vi + .spyOn(getPreviewRuntimeRunner(sandbox), 'runWaking') + .mockImplementationOnce(async (_operation, _call, options) => { + expect(options?.signal).toBe(controller.signal); + controller.abort(reason); + throw reason; + }); + + await expect( + sandbox.containerFetch( + new Request('https://example.com/data', { + signal: controller.signal + }), + 8080 + ) + ).rejects.toBe(reason); + + expect(runWaking).toHaveBeenCalledWith( + 'container.fetch', + expect.any(Function), + { signal: controller.signal } + ); + }); + + it('interrupts direct HTTP forwarding during port readiness without physical forwarding', async () => { + const releaseReadiness = deferred(); + vi.mocked( + asSandboxWithClient(sandbox).client.ports.openWatch + ).mockResolvedValueOnce({ + stream: vi.fn( + async () => + new ReadableStream({ + async start(controller) { + await releaseReadiness.promise; + controller.enqueue({ type: 'ready' }); + controller.close(); + } + }) + ), + cancel: vi.fn(async () => undefined), + [Symbol.dispose]: vi.fn() + }); + const tcpFetch = vi.fn(); + mockCtx.container.getTcpPort = vi.fn(() => ({ fetch: tcpFetch })); + + const forwarded = sandbox.containerFetch( + new Request('https://example.com/data'), + 8080 + ); + const forwardExpectation = expect(forwarded).rejects.toMatchObject({ + code: ErrorCode.OPERATION_INTERRUPTED + }); + await vi.waitFor(() => + expect( + asSandboxWithClient(sandbox).client.ports.openWatch + ).toHaveBeenCalledWith(8080, expect.any(Object)) + ); + await sandbox.stop(); + releaseReadiness.resolve(); + + await forwardExpectation; + expect(tcpFetch).not.toHaveBeenCalled(); + }); + }); + describe('fetch() override - WebSocket detection', () => { - let superFetchSpy: any; + let tcpFetch: ReturnType; beforeEach(async () => { await sandbox.setSandboxName('test-sandbox'); + tcpFetch = vi.fn(async () => new Response(null, { status: 204 })); + mockCtx.container.getTcpPort = vi.fn(() => ({ fetch: tcpFetch })); + }); + + it('passes caller abort into WebSocket runtime establishment', async () => { + const controller = new AbortController(); + const reason = new DOMException('caller stopped', 'AbortError'); + const runWaking = vi + .spyOn(getPreviewRuntimeRunner(sandbox), 'runWaking') + .mockImplementationOnce(async (_operation, _call, options) => { + expect(options?.signal).toBe(controller.signal); + controller.abort(reason); + throw reason; + }); + const request = new Request('https://example.com/ws', { + signal: controller.signal, + headers: { + Upgrade: 'websocket', + Connection: 'Upgrade' + } + }); - // Spy on Container.prototype.fetch to verify WebSocket routing - superFetchSpy = vi - .spyOn(Container.prototype, 'fetch') - .mockResolvedValue(new Response('WebSocket response')); - }); + await expect(sandbox.fetch(request)).rejects.toBe(reason); - afterEach(() => { - superFetchSpy?.mockRestore(); + expect(runWaking).toHaveBeenCalledWith( + 'container.websocket', + expect.any(Function), + { signal: controller.signal } + ); }); - it('should detect WebSocket upgrade header and route to super.fetch', async () => { + it('should detect WebSocket upgrade header and route through admitted TCP port', async () => { const request = new Request('https://example.com/ws', { headers: { Upgrade: 'websocket', @@ -769,41 +1344,159 @@ describe('Sandbox durable object behavior', () => { } }); + tcpFetch.mockResolvedValueOnce(new Response('WebSocket response')); + const response = await sandbox.fetch(request); - // Should route through super.fetch() for WebSocket - expect(superFetchSpy).toHaveBeenCalledTimes(1); + expect(mockCtx.container.getTcpPort).toHaveBeenCalledWith(3000); + expect(tcpFetch).toHaveBeenCalledTimes(1); + expect((tcpFetch.mock.calls[0][0] as Request).url).toBe( + 'http://example.com/ws' + ); expect(await response.text()).toBe('WebSocket response'); }); - it('should route non-WebSocket requests through containerFetch', async () => { - // GET request - const getRequest = new Request('https://example.com/api/data'); - await sandbox.fetch(getRequest); - expect(superFetchSpy).not.toHaveBeenCalled(); + it('routes switchPort WebSocket requests to the selected port', async () => { + const request = new Request('https://example.com/ws', { + headers: { + Upgrade: 'websocket', + Connection: 'Upgrade', + 'cf-container-target-port': '8080' + } + }); + + await sandbox.fetch(request); - vi.clearAllMocks(); + expect(mockCtx.container.getTcpPort).toHaveBeenLastCalledWith(8080); + const forwarded = tcpFetch.mock.calls[0][0] as Request; + expect(forwarded.headers.has('cf-container-target-port')).toBe(false); + }); - // POST request - const postRequest = new Request('https://example.com/api/data', { - method: 'POST', - body: JSON.stringify({ data: 'test' }), - headers: { 'Content-Type': 'application/json' } + it('routes switchPort HTTP requests to the selected port', async () => { + const request = new Request('https://example.com/data', { + headers: { 'cf-container-target-port': '8081' } }); - await sandbox.fetch(postRequest); - expect(superFetchSpy).not.toHaveBeenCalled(); - vi.clearAllMocks(); + await sandbox.fetch(request); + + expect(mockCtx.container.getTcpPort).toHaveBeenLastCalledWith(8081); + const forwarded = tcpFetch.mock.calls[0][0] as Request; + expect(forwarded.headers.has('cf-container-target-port')).toBe(false); + }); + + it('rejects invalid container target ports', async () => { + await expect(sandbox.authorizePortRequest(0, '/app')).rejects.toThrow( + 'Invalid port number' + ); + await expect(sandbox.authorizePortRequest(65536, '/app')).rejects.toThrow( + 'Invalid port number' + ); + }); + + it('rejects unauthenticated terminal control-plane routes', async () => { + const request = new Request( + 'https://example.com/ws/terminal?terminalId=terminal-a', + { + headers: { + Upgrade: 'websocket', + Connection: 'Upgrade', + 'cf-container-target-port': '3000' + } + } + ); + + await expect(sandbox.fetch(request)).rejects.toThrow( + 'Terminal connection is not authorized' + ); + expect(tcpFetch).not.toHaveBeenCalled(); + }); - // SSE request (should not be detected as WebSocket) - const sseRequest = new Request('https://example.com/events', { - headers: { Accept: 'text/event-stream' } + it('rejects public container RPC upgrades', async () => { + const request = new Request('https://example.com/rpc', { + headers: { + Upgrade: 'websocket', + Connection: 'Upgrade' + } + }); + + await expect(sandbox.fetch(request)).rejects.toThrow( + 'Container RPC connection is not authorized' + ); + expect(tcpFetch).not.toHaveBeenCalled(); + }); + + it('allows an authorized app-port /rpc route', async () => { + const token = await sandbox.authorizePortRequest(8080, '/rpc'); + const request = new Request('https://example.com/rpc', { + headers: { + Upgrade: 'websocket', + Connection: 'Upgrade', + 'cf-container-target-port': '8080', + 'x-sandbox-port-route-token': token + } }); - await sandbox.fetch(sseRequest); - expect(superFetchSpy).not.toHaveBeenCalled(); + + await sandbox.fetch(request); + + expect(mockCtx.container.getTcpPort).toHaveBeenLastCalledWith(8080); }); - it('should preserve WebSocket request unchanged when calling super.fetch()', async () => { + it('does not accept a general route token for terminal control', async () => { + const token = await sandbox.authorizePortRequest(3000, '/ws/terminal'); + const request = new Request( + 'https://example.com/ws/terminal?terminalId=terminal-a', + { + headers: { + Upgrade: 'websocket', + Connection: 'Upgrade', + 'cf-container-target-port': '3000', + 'x-sandbox-port-route-token': token + } + } + ); + + await expect(sandbox.fetch(request)).rejects.toThrow( + 'Terminal connection is not authorized' + ); + expect(tcpFetch).not.toHaveBeenCalled(); + }); + + it.each([ + ['GET', new Request('https://example.com/api/data')], + [ + 'POST', + new Request('https://example.com/api/data', { + method: 'POST', + body: JSON.stringify({ data: 'test' }), + headers: { 'Content-Type': 'application/json' } + }) + ], + [ + 'SSE', + new Request('https://example.com/events', { + headers: { Accept: 'text/event-stream' } + }) + ] + ])( + 'should route non-WebSocket %s requests through containerFetch', + async (_kind, request) => { + await (await sandbox.fetch(request)).text(); + expect(tcpFetch).toHaveBeenCalledTimes(1); + const forwardedRequest = tcpFetch.mock.calls[0][0] as Request; + expect(forwardedRequest.url).toMatch(/^http:\/\//); + if (_kind === 'POST') { + expect(forwardedRequest.method).toBe('POST'); + expect(forwardedRequest.headers.get('Content-Type')).toBe( + 'application/json' + ); + expect(await forwardedRequest.text()).toBe( + JSON.stringify({ data: 'test' }) + ); + } + } + ); + + it('should preserve WebSocket request unchanged when forwarding through admitted TCP port', async () => { const request = new Request('https://example.com/ws', { headers: { Upgrade: 'websocket', @@ -815,8 +1508,8 @@ describe('Sandbox durable object behavior', () => { await sandbox.fetch(request); - expect(superFetchSpy).toHaveBeenCalledTimes(1); - const passedRequest = superFetchSpy.mock.calls[0][0] as Request; + expect(tcpFetch).toHaveBeenCalledTimes(1); + const passedRequest = tcpFetch.mock.calls[0][0] as Request; expect(passedRequest.headers.get('Upgrade')).toBe('websocket'); expect(passedRequest.headers.get('Connection')).toBe('Upgrade'); expect(passedRequest.headers.get('Sec-WebSocket-Key')).toBe( @@ -825,6 +1518,74 @@ describe('Sandbox durable object behavior', () => { expect(passedRequest.headers.get('Sec-WebSocket-Version')).toBe('13'); }); + it('closes direct WebSocket forwarding when the runtime is invalidated', async () => { + const socket = new FakeSocket(); + const response = new Response('WebSocket response'); + Object.defineProperty(response, 'webSocket', { value: socket }); + tcpFetch.mockResolvedValueOnce(response); + const request = new Request('https://example.com/ws', { + headers: { + Upgrade: 'websocket', + Connection: 'Upgrade' + } + }); + + await sandbox.fetch(request); + await sandbox.stop(); + + expect(socket.closeCalls).toEqual([ + { code: 1012, reason: 'Runtime replaced' } + ]); + }); + + it('releases direct WebSocket authority when peer-owned close throws', async () => { + const socket = new FakeSocket(); + const close = vi.spyOn(socket, 'close').mockImplementation(() => { + throw new TypeError('Socket is already owned by its peer'); + }); + const response = new Response('WebSocket response'); + Object.defineProperty(response, 'webSocket', { value: socket }); + tcpFetch.mockResolvedValueOnce(response); + const request = new Request('https://example.com/ws', { + headers: { + Upgrade: 'websocket', + Connection: 'Upgrade' + } + }); + + await sandbox.fetch(request); + + await expect(sandbox.stop()).resolves.toBeUndefined(); + expect(close).toHaveBeenCalledWith(1012, 'Runtime replaced'); + }); + + it('closes direct WebSocket assignment race responses on invalidation', async () => { + const socket = new FakeSocket(); + const response = new Response('WebSocket response'); + Object.defineProperty(response, 'webSocket', { value: socket }); + const releaseFetch = deferred(); + tcpFetch.mockImplementationOnce(() => releaseFetch.promise); + const request = new Request('https://example.com/ws', { + headers: { + Upgrade: 'websocket', + Connection: 'Upgrade' + } + }); + + const forwarded = sandbox.fetch(request); + const forwardExpectation = expect(forwarded).rejects.toMatchObject({ + code: ErrorCode.OPERATION_INTERRUPTED + }); + await vi.waitFor(() => expect(tcpFetch).toHaveBeenCalledOnce()); + await sandbox.stop(); + releaseFetch.resolve(response); + + await forwardExpectation; + expect(socket.closeCalls).toEqual([ + { code: 1012, reason: 'Runtime replaced' } + ]); + }); + it('routes active preview proxy requests through the TCP port without starting', async () => { const tcpFetch = vi.fn().mockResolvedValue(new Response('preview ok')); mockCtx.container.running = true; @@ -834,15 +1595,29 @@ describe('Sandbox durable object behavior', () => { mockPreviewStorageGet(mockCtx, activePreviewStorageState()); const containerFetchSpy = vi.spyOn(sandbox, 'containerFetch'); const startAndWaitSpy = vi.spyOn(sandbox, 'startAndWaitForPorts'); - + const runtimeRunner = getPreviewRuntimeRunner(sandbox); + const runExistingSpy = vi.spyOn(runtimeRunner, 'runExisting'); + const runWakingSpy = vi.spyOn(runtimeRunner, 'runWaking'); + + const previewRequest = createPreviewProxyRequest('/hello?x=1'); + const headers = new Headers(previewRequest.headers); + headers.set('cf-container-target-port', '9000'); + headers.set('x-sandbox-port-route-token', 'internal-token'); const response = await sandbox.fetch( - createPreviewProxyRequest('/hello?x=1') + new Request(previewRequest, { headers }) ); expect(await response.text()).toBe('preview ok'); expect(containerFetchSpy).not.toHaveBeenCalled(); expect(startAndWaitSpy).not.toHaveBeenCalled(); expect(mockCtx.container.start).not.toHaveBeenCalled(); + expect(runExistingSpy).toHaveBeenCalledTimes(1); + expect(runExistingSpy).toHaveBeenCalledWith( + { kind: 'current' }, + 'preview.forward', + expect.any(Function) + ); + expect(runWakingSpy).not.toHaveBeenCalled(); expect(mockCtx.container.getTcpPort).toHaveBeenCalledWith(8080); expect(tcpFetch).toHaveBeenCalledWith( 'http://localhost:8080/hello?x=1', @@ -852,6 +1627,12 @@ describe('Sandbox durable object behavior', () => { expect(forwardedRequest.headers.get('X-Sandbox-Name')).toBe( 'test-sandbox' ); + expect(forwardedRequest.headers.has('cf-container-target-port')).toBe( + false + ); + expect(forwardedRequest.headers.has('x-sandbox-port-route-token')).toBe( + false + ); }); it('preserves WebSocket preview proxy requests when forwarding', async () => { @@ -906,6 +1687,9 @@ describe('Sandbox durable object behavior', () => { mockPreviewStorageGet(mockCtx, activePreviewStorageState()); const containerFetchSpy = vi.spyOn(sandbox, 'containerFetch'); const startAndWaitSpy = vi.spyOn(sandbox, 'startAndWaitForPorts'); + const runtimeRunner = getPreviewRuntimeRunner(sandbox); + const runExistingSpy = vi.spyOn(runtimeRunner, 'runExisting'); + const runWakingSpy = vi.spyOn(runtimeRunner, 'runWaking'); const response = await sandbox.fetch(createPreviewProxyRequest()); @@ -916,6 +1700,8 @@ describe('Sandbox durable object behavior', () => { expect(mockCtx.container.getTcpPort).not.toHaveBeenCalled(); expect(containerFetchSpy).not.toHaveBeenCalled(); expect(startAndWaitSpy).not.toHaveBeenCalled(); + expect(runExistingSpy).toHaveBeenCalledTimes(1); + expect(runWakingSpy).not.toHaveBeenCalled(); expect(mockCtx.container.start).not.toHaveBeenCalled(); }); @@ -945,7 +1731,7 @@ describe('Sandbox durable object behavior', () => { }); }); - it('returns controlled disconnect response when network loss keeps the runtime active', async () => { + it('returns stale response when preview forwarding loses the network', async () => { mockCtx.container.running = true; mockPreviewStorageGet(mockCtx, activePreviewStorageState()); const tcpFetch = vi @@ -955,54 +1741,194 @@ describe('Sandbox durable object behavior', () => { .fn() .mockReturnValue({ fetch: tcpFetch }); - const response = await sandbox.fetch(createPreviewProxyRequest()); + const response = await sandbox.fetch(createPreviewProxyRequest()); + + expect(response.status).toBe(410); + expect(await response.json()).toMatchObject({ + code: 'STALE_PREVIEW_URL' + }); + }); + + it('rejects preview proxy requests without durable authorization', async () => { + mockCtx.container.running = true; + vi.mocked(mockCtx.storage!.get).mockImplementation(async (key) => + key === 'portTokens' ? {} : null + ); + const containerFetchSpy = vi.spyOn(sandbox, 'containerFetch'); + const runtimeRunner = getPreviewRuntimeRunner(sandbox); + const runExistingSpy = vi.spyOn(runtimeRunner, 'runExisting'); + const runWakingSpy = vi.spyOn(runtimeRunner, 'runWaking'); + + const response = await sandbox.fetch( + new Request('https://8080-test-sandbox-badtoken.example.com/api', { + headers: { + 'x-sandbox-preview-proxy': '1', + 'x-sandbox-preview-port': '8080', + 'x-sandbox-preview-token': 'badtoken', + 'x-sandbox-preview-sandbox-id': 'test-sandbox' + } + }) + ); + + expect(response.status).toBe(404); + expect(await response.json()).toMatchObject({ + code: 'INVALID_TOKEN' + }); + expect(containerFetchSpy).not.toHaveBeenCalled(); + expect(runExistingSpy).not.toHaveBeenCalled(); + expect(runWakingSpy).not.toHaveBeenCalled(); + }); + + it('rejects preview proxy requests without current-runtime activation', async () => { + mockCtx.container.running = true; + vi.mocked(mockCtx.storage!.get).mockImplementation(async (key) => { + if (key === 'portTokens') { + return { '8080': { token: 'token12345678901' } }; + } + if (key === 'currentRuntimeIdentity') { + return runtimeRecord('runtime-1'); + } + if (key === 'activePreviewPorts') { + return {}; + } + return null; + }); + const containerFetchSpy = vi.spyOn(sandbox, 'containerFetch'); + + const response = await sandbox.fetch(createPreviewProxyRequest()); + + expect(response.status).toBe(410); + expect(await response.json()).toMatchObject({ + code: 'STALE_PREVIEW_URL' + }); + expect(containerFetchSpy).not.toHaveBeenCalled(); + }); + + it('rejects preview proxy requests when activation belongs to another runtime', async () => { + mockCtx.container.running = true; + vi.mocked(mockCtx.storage!.get).mockImplementation(async (key) => { + const state = { + ...activePreviewStorageState(), + activePreviewPorts: { + '8080': { + runtimeIdentityID: 'runtime-2', + runtimeIncarnationID: 'test-incarnation', + token: PREVIEW_TEST_TOKEN + } + } + }; + return state[key as keyof typeof state] ?? null; + }); + const tcpFetch = vi.fn().mockResolvedValue(new Response('preview ok')); + mockCtx.container.getTcpPort = vi + .fn() + .mockReturnValue({ fetch: tcpFetch }); + const runtimeRunner = getPreviewRuntimeRunner(sandbox); + const runExistingSpy = vi.spyOn(runtimeRunner, 'runExisting'); + const runWakingSpy = vi.spyOn(runtimeRunner, 'runWaking'); + + const response = await sandbox.fetch(createPreviewProxyRequest()); + + expect(response.status).toBe(410); + expect(await response.json()).toMatchObject({ + code: 'STALE_PREVIEW_URL' + }); + expect(runExistingSpy).toHaveBeenCalledTimes(1); + expect(runWakingSpy).not.toHaveBeenCalled(); + expect(tcpFetch).not.toHaveBeenCalled(); + expect(mockCtx.container.start).not.toHaveBeenCalled(); + }); + + it.each([ + { + name: 'same runtime id with another incarnation', + activation: { + runtimeIdentityID: PREVIEW_TEST_RUNTIME_ID, + runtimeIncarnationID: 'old-incarnation', + token: PREVIEW_TEST_TOKEN + }, + expectedAdmissions: 1 + }, + { + name: 'legacy activation without an incarnation', + activation: { + runtimeIdentityID: PREVIEW_TEST_RUNTIME_ID, + token: PREVIEW_TEST_TOKEN + }, + expectedAdmissions: 0 + } + ])( + 'clears stale preview activation for $name', + async ({ activation, expectedAdmissions }) => { + const state = activePreviewStorageState(); + const storage = new Map([ + ...Object.entries(state), + ['activePreviewPorts', { '8080': activation }] + ]); + mockCtx.storage.get.mockImplementation( + async (key: string) => storage.get(key) ?? null + ); + mockCtx.storage.put.mockImplementation(async (key: string, value) => { + storage.set(key, value); + }); + mockCtx.storage.delete.mockImplementation(async (key: string) => { + storage.delete(key); + }); + mockCtx.storage.transaction.mockImplementation( + async (callback: (txn: typeof mockCtx.storage) => Promise) => + await callback(mockCtx.storage) + ); + mockCtx.container.running = true; + const tcpFetch = vi.fn().mockResolvedValue(new Response('preview ok')); + mockCtx.container.getTcpPort = vi + .fn() + .mockReturnValue({ fetch: tcpFetch }); + const runExisting = vi.spyOn( + getPreviewRuntimeRunner(sandbox), + 'runExisting' + ); + + const response = await sandbox.fetch(createPreviewProxyRequest()); - expect(response.status).toBe(500); - expect(await response.text()).toBe( - 'Container suddenly disconnected, try again' - ); - }); + expect(response.status).toBe(410); + expect(runExisting).toHaveBeenCalledTimes(expectedAdmissions); + expect(tcpFetch).not.toHaveBeenCalled(); + expect(storage.has('activePreviewPorts')).toBe(false); + expect(mockCtx.container.start).not.toHaveBeenCalled(); + } + ); - it('rejects preview proxy requests without durable authorization', async () => { - mockCtx.container.running = true; - vi.mocked(mockCtx.storage!.get).mockImplementation(async (key) => - key === 'portTokens' ? {} : null + it('clears preview activation when reconstructed session observes a changed incarnation', async () => { + const storage = new Map( + Object.entries(activePreviewStorageState()) ); - const containerFetchSpy = vi.spyOn(sandbox, 'containerFetch'); - - const response = await sandbox.fetch( - new Request('https://8080-test-sandbox-badtoken.example.com/api', { - headers: { - 'x-sandbox-preview-proxy': '1', - 'x-sandbox-preview-port': '8080', - 'x-sandbox-preview-token': 'badtoken', - 'x-sandbox-preview-sandbox-id': 'test-sandbox' - } - }) + mockCtx.storage.get.mockImplementation( + async (key: string) => storage.get(key) ?? null ); - - expect(response.status).toBe(404); - expect(await response.json()).toMatchObject({ - code: 'INVALID_TOKEN' + mockCtx.storage.put.mockImplementation(async (key: string, value) => { + storage.set(key, value); }); - expect(containerFetchSpy).not.toHaveBeenCalled(); - }); - - it('rejects preview proxy requests without current-runtime activation', async () => { - mockCtx.container.running = true; - vi.mocked(mockCtx.storage!.get).mockImplementation(async (key) => { - if (key === 'portTokens') { - return { '8080': { token: 'token12345678901' } }; - } - if (key === 'currentRuntimeIdentity') { - return { id: 'runtime-1' }; - } - if (key === 'activePreviewPorts') { - return {}; - } - return null; + mockCtx.storage.delete.mockImplementation(async (key: string) => { + storage.delete(key); }); - const containerFetchSpy = vi.spyOn(sandbox, 'containerFetch'); + mockCtx.storage.transaction.mockImplementation( + async (callback: (txn: typeof mockCtx.storage) => Promise) => + await callback(mockCtx.storage) + ); + const tcpFetch = vi.fn().mockResolvedValue(new Response('preview ok')); + mockCtx.container.running = true; + mockCtx.container.getTcpPort = vi + .fn() + .mockReturnValue({ fetch: tcpFetch }); + vi.spyOn( + getPreviewRuntimeRunner(sandbox), + 'runExisting' + ).mockRejectedValueOnce( + new RuntimeControlProtocolError('Runtime incarnation does not match', { + reason: 'activation-mismatch', + operation: 'utils.activateControlSession' + }) + ); const response = await sandbox.fetch(createPreviewProxyRequest()); @@ -1010,7 +1936,9 @@ describe('Sandbox durable object behavior', () => { expect(await response.json()).toMatchObject({ code: 'STALE_PREVIEW_URL' }); - expect(containerFetchSpy).not.toHaveBeenCalled(); + expect(tcpFetch).not.toHaveBeenCalled(); + expect(storage.get('activePreviewPorts')).toBeUndefined(); + expect(mockCtx.container.start).not.toHaveBeenCalled(); }); it('rejects persisted preview auth without runtime identity or activation', async () => { @@ -1040,27 +1968,26 @@ describe('Sandbox durable object behavior', () => { }); describe('wsConnect() method', () => { - it('should route WebSocket request through switchPort to sandbox.fetch', async () => { - const { switchPort } = await import('@cloudflare/containers'); - const switchPortMock = vi.mocked(switchPort); - + it('should route WebSocket requests through the selected TCP port', async () => { const request = new Request('http://localhost/ws/echo', { headers: { Upgrade: 'websocket', Connection: 'Upgrade' } }); + const tcpFetch = vi.fn( + async () => + new Response('WebSocket Upgraded', { + status: 200, + headers: { 'X-WebSocket-Upgraded': 'true' } + }) + ); + mockCtx.container.getTcpPort = vi.fn(() => ({ fetch: tcpFetch })); - const fetchSpy = vi.spyOn(sandbox, 'fetch'); const response = await sandbox.wsConnect(request, 8080); - // Verify switchPort was called with correct port - expect(switchPortMock).toHaveBeenCalledWith(request, 8080); - - // Verify fetch was called with the switched request - expect(fetchSpy).toHaveBeenCalledOnce(); - - // Verify response indicates WebSocket upgrade + expect(mockCtx.container.getTcpPort).toHaveBeenLastCalledWith(8080); + expect(tcpFetch).toHaveBeenCalledOnce(); expect(response.status).toBe(200); expect(response.headers.get('X-WebSocket-Upgraded')).toBe('true'); }); @@ -1102,10 +2029,13 @@ describe('Sandbox durable object behavior', () => { } ); - const fetchSpy = vi.spyOn(sandbox, 'fetch'); + const tcpFetch = vi.fn( + async (_request: Request) => new Response('connected') + ); + mockCtx.container.getTcpPort = vi.fn(() => ({ fetch: tcpFetch })); await sandbox.wsConnect(request, 8080); - const calledRequest = fetchSpy.mock.calls[0][0]; + const calledRequest = tcpFetch.mock.calls[0][0] as Request; // Verify headers are preserved expect(calledRequest.headers.get('Upgrade')).toBe('websocket'); @@ -1204,7 +2134,11 @@ describe('Sandbox durable object behavior', () => { beforeEach(async () => { await sandbox.setSandboxName('test-sandbox', false); - vi.mocked(mockCtx.storage!.get).mockResolvedValue({} as any); + vi.mocked(mockCtx.storage!.get).mockImplementation(async (key) => { + if (key === 'currentRuntimeIdentity') return runtimeRecord('runtime-a'); + if (key === 'portTokens' || key === 'activePreviewPorts') return {}; + return null; + }); vi.mocked(mockCtx.storage!.put).mockResolvedValue(undefined); }); @@ -1241,13 +2175,21 @@ describe('Sandbox durable object behavior', () => { token: 'shared' }); - vi.mocked(mockCtx.storage!.get).mockResolvedValueOnce({ - '8080': 'shared' - } as any); + vi.mocked(mockCtx.storage!.get).mockImplementation(async (key) => { + if (key === 'currentRuntimeIdentity') return runtimeRecord('runtime-a'); + if (key === 'portTokens') return { '8080': 'shared' }; + if (key === 'activePreviewPorts') return {}; + return null; + }); + const runWakingSpy = vi.spyOn( + getPreviewRuntimeRunner(sandbox), + 'runWaking' + ); await expect( sandbox.exposePort(8081, { hostname: 'example.com', token: 'shared' }) ).rejects.toThrow(/already in use by port 8080/); + expect(runWakingSpy).not.toHaveBeenCalled(); }); it('should allow re-exposing same port with same token', async () => { @@ -1256,9 +2198,12 @@ describe('Sandbox durable object behavior', () => { token: 'stable' }); - vi.mocked(mockCtx.storage!.get).mockResolvedValueOnce({ - '8080': 'stable' - } as any); + vi.mocked(mockCtx.storage!.get).mockImplementation(async (key) => { + if (key === 'currentRuntimeIdentity') return runtimeRecord('runtime-a'); + if (key === 'portTokens') return { '8080': 'stable' }; + if (key === 'activePreviewPorts') return {}; + return null; + }); const result = await sandbox.exposePort(8080, { hostname: 'example.com', @@ -1273,21 +2218,52 @@ describe('Sandbox durable object behavior', () => { await sandbox.setSandboxName('test-sandbox', false); }); - it('onStart() marks a new current runtime without restoring saved ports', async () => { - vi.mocked(mockCtx.storage!.get).mockImplementation(async (key) => - key === 'portTokens' - ? { - '8080': { token: 'tok8080', name: 'api' } - } - : null - ); + it('exposePort() uses one waking preview scope and no legacy preview starter', async () => { + const storage = new Map([ + ['portTokens', { '8080': { token: 'tok8080', name: 'api' } }] + ]); + vi.mocked(mockCtx.storage!.get).mockImplementation( + async (key) => storage.get(String(key)) ?? null + ); + vi.mocked(mockCtx.storage!.put).mockImplementation(async (key, value) => { + storage.set(String(key), value); + }); + const runtimeRunner = ( + sandbox as unknown as { + runtimeRunner: { + runWaking( + operation: string, + call: (lease: { + runtime: unknown; + retain(): { release(): void }; + }) => Promise + ): Promise; + }; + } + ).runtimeRunner; + const runWakingSpy = vi.spyOn(runtimeRunner, 'runWaking'); - await (sandbox as any).onStart(); + await sandbox.exposePort(9090, { + hostname: 'example.com', + token: 'newtoken' + }); + expect(runWakingSpy).toHaveBeenCalledTimes(1); + expect(runWakingSpy).toHaveBeenCalledWith( + 'preview.expose', + expect.any(Function) + ); + expect('ensureRuntimeActiveForPreview' in sandbox).toBe(false); expect(mockCtx.storage.put).toHaveBeenCalledWith( - 'currentRuntimeIdentity', + 'activePreviewPorts', expect.objectContaining({ - id: expect.any(String) + '9090': expect.objectContaining({ token: 'newtoken' }) + }) + ); + expect(mockCtx.storage.put).not.toHaveBeenCalledWith( + 'activePreviewPorts', + expect.objectContaining({ + '8080': expect.anything() }) ); }); @@ -1303,6 +2279,44 @@ describe('Sandbox durable object behavior', () => { expect(deletedKeys).toContain('currentRuntimeIdentity'); }); + it('routes container hooks through replacement lifecycle reconciliation', async () => { + type RuntimeLifecycleHookProbe = { + markRuntimeStarted(): boolean; + reconcileObservedStop(): Promise< + 'reconciled-previous-stop' | 'hard-invalidation' + >; + invalidate(): Promise; + }; + type SandboxLoggerProbe = { + debug(message: string, context?: Record): void; + }; + + const lifecycle = ( + sandbox as unknown as { runtimeLifecycle: RuntimeLifecycleHookProbe } + ).runtimeLifecycle; + const logger = (sandbox as unknown as { logger: SandboxLoggerProbe }) + .logger; + const debug = vi.spyOn(logger, 'debug'); + const markRuntimeStarted = vi.spyOn(lifecycle, 'markRuntimeStarted'); + const reconcileObservedStop = vi + .spyOn(lifecycle, 'reconcileObservedStop') + .mockResolvedValue('reconciled-previous-stop'); + const hardInvalidate = vi.spyOn(lifecycle, 'invalidate'); + + await sandbox.onStart(); + await sandbox.onStop(); + + expect(markRuntimeStarted).toHaveBeenCalledTimes(1); + expect(reconcileObservedStop).toHaveBeenCalledTimes(1); + expect(hardInvalidate).not.toHaveBeenCalled(); + expect(debug).toHaveBeenCalledWith('Sandbox started', { + replacementStartTransitionCompleted: false + }); + expect(debug).toHaveBeenCalledWith('Sandbox runtime stop reconciled', { + runtimeStopDisposition: 'reconciled-previous-stop' + }); + }); + it('stop() clears runtime-scoped preview state before signaling the container', async () => { const callOrder: string[] = []; vi.mocked(mockCtx.storage!.delete).mockImplementation(async (key) => { @@ -1323,6 +2337,90 @@ describe('Sandbox durable object behavior', () => { expect(callOrder).not.toContain('delete:portTokens'); }); + it('start() waits behind an explicit stop and establishes only after stop settles', async () => { + const callOrder: string[] = []; + const stopGate = deferred(); + const parent = Object.getPrototypeOf(Object.getPrototypeOf(sandbox)) as { + stop: () => Promise; + startAndWaitForPorts: () => Promise; + }; + vi.spyOn(parent, 'stop').mockImplementation(async () => { + callOrder.push('super.stop:start'); + await stopGate.promise; + mockCtx.container.running = false; + callOrder.push('super.stop:end'); + }); + vi.spyOn(parent, 'startAndWaitForPorts').mockImplementation(async () => { + callOrder.push('super.startAndWaitForPorts'); + mockCtx.container.running = true; + await sandbox.onStart(); + }); + + const stop = sandbox.stop(); + await vi.waitFor(() => expect(callOrder).toContain('super.stop:start')); + const start = sandbox.start(); + await Promise.resolve(); + + expect(callOrder).not.toContain('super.startAndWaitForPorts'); + stopGate.resolve(); + await stop; + await start; + + expect(callOrder).toEqual([ + 'super.stop:start', + 'super.stop:end', + 'super.startAndWaitForPorts' + ]); + }); + + it('destroy() waits for a pending explicit stop then still destroys', async () => { + const callOrder: string[] = []; + const stopGate = deferred(); + const parent = Object.getPrototypeOf(Object.getPrototypeOf(sandbox)) as { + stop: () => Promise; + destroy: () => Promise; + }; + vi.spyOn(parent, 'stop').mockImplementation(async () => { + callOrder.push('super.stop:start'); + await stopGate.promise; + callOrder.push('super.stop:end'); + }); + vi.spyOn(parent, 'destroy').mockImplementation(async () => { + callOrder.push('super.destroy'); + }); + + const stop = sandbox.stop(); + await vi.waitFor(() => expect(callOrder).toContain('super.stop:start')); + const destroy = sandbox.destroy(); + await Promise.resolve(); + + expect(callOrder).not.toContain('super.destroy'); + stopGate.resolve(); + await stop; + await destroy; + + expect(callOrder).toEqual([ + 'super.stop:start', + 'super.stop:end', + 'super.destroy' + ]); + }); + + it('failed physical destroy leaves runtime authority invalidated', async () => { + vi.spyOn(Container.prototype, 'destroy').mockRejectedValue( + new Error('physical destroy failed') + ); + + await expect(sandbox.destroy()).rejects.toThrow( + 'physical destroy failed' + ); + + expect(mockCtx.storage.delete).toHaveBeenCalledWith( + 'currentRuntimeIdentity' + ); + await expect((sandbox as any).isRuntimeActive()).resolves.toBe(false); + }); + it('destroy() clears preview auth and runtime-scoped state before calling super.destroy()', async () => { const callOrder: string[] = []; @@ -1354,7 +2452,7 @@ describe('Sandbox durable object behavior', () => { return {}; } if (key === 'currentRuntimeIdentity') { - return { id: 'runtime-1' }; + return runtimeRecord('runtime-1'); } if (key === 'activePreviewPorts') { return {}; @@ -1374,11 +2472,54 @@ describe('Sandbox durable object behavior', () => { expect(putSpy).toHaveBeenCalledWith('activePreviewPorts', { '8080': { runtimeIdentityID: 'runtime-1', + runtimeIncarnationID: 'test-incarnation', token: 'friendlytok' } }); }); + it('exposePort() rolls back preview state when the post-write fence fails', async () => { + const storage = new Map([ + ['currentRuntimeIdentity', runtimeRecord('runtime-1')] + ]); + mockCtx.storage.get.mockImplementation( + async (key: string) => storage.get(key) ?? null + ); + mockCtx.storage.put.mockImplementation(async (key: string, value) => { + storage.set(key, value); + }); + mockCtx.storage.delete.mockImplementation(async (key: string) => { + storage.delete(key); + }); + mockCtx.storage.transaction.mockImplementation( + async (callback: (txn: typeof mockCtx.storage) => Promise) => + await callback(mockCtx.storage) + ); + const lifecycle = getPreviewRuntimeLifecycle(sandbox); + const assertActive = lifecycle.assertActive.bind(lifecycle); + vi.spyOn(lifecycle, 'assertActive').mockImplementation( + async (runtime) => { + if (storage.has('activePreviewPorts')) { + throw new RuntimeIdentityInactiveError(); + } + await assertActive(runtime); + } + ); + + await expect( + sandbox.exposePort(8080, { + hostname: 'example.com', + token: 'friendlytok' + }) + ).rejects.toMatchObject({ + code: 'OPERATION_INTERRUPTED', + context: { operation: 'preview.expose' } + }); + + expect(storage.get('portTokens')).toEqual({}); + expect(storage.has('activePreviewPorts')).toBe(false); + }); + it('exposePort() does not write preview state when runtime identity changes before storage writes', async () => { let runtimeIdentityReads = 0; vi.mocked(mockCtx.storage!.get).mockImplementation(async (key) => { @@ -1387,9 +2528,9 @@ describe('Sandbox durable object behavior', () => { } if (key === 'currentRuntimeIdentity') { runtimeIdentityReads++; - return { - id: runtimeIdentityReads === 1 ? 'runtime-1' : 'runtime-2' - }; + return runtimeRecord( + runtimeIdentityReads === 1 ? 'runtime-1' : 'runtime-2' + ); } if (key === 'activePreviewPorts') { return {}; @@ -1403,7 +2544,10 @@ describe('Sandbox durable object behavior', () => { hostname: 'example.com', token: 'friendlytok' }) - ).rejects.toBeInstanceOf(RuntimeIdentityInactiveError); + ).rejects.toMatchObject({ + code: 'OPERATION_INTERRUPTED', + context: { operation: 'preview.expose' } + }); expect(mockCtx.storage.put).not.toHaveBeenCalledWith( 'portTokens', @@ -1415,7 +2559,7 @@ describe('Sandbox durable object behavior', () => { ); }); - it('exposePort() rejects if runtime identity changes after preview state writes', async () => { + it('exposePort() rejects before preview state writes when runtime identity changes after activation', async () => { let runtimeIdentityReads = 0; vi.mocked(mockCtx.storage!.get).mockImplementation(async (key) => { if (key === 'portTokens') { @@ -1423,9 +2567,9 @@ describe('Sandbox durable object behavior', () => { } if (key === 'currentRuntimeIdentity') { runtimeIdentityReads++; - return { - id: runtimeIdentityReads <= 2 ? 'runtime-1' : 'runtime-2' - }; + return runtimeRecord( + runtimeIdentityReads <= 2 ? 'runtime-1' : 'runtime-2' + ); } if (key === 'activePreviewPorts') { return {}; @@ -1439,17 +2583,19 @@ describe('Sandbox durable object behavior', () => { hostname: 'example.com', token: 'friendlytok' }) - ).rejects.toBeInstanceOf(RuntimeIdentityInactiveError); - - expect(mockCtx.storage.put).toHaveBeenCalledWith('portTokens', { - '8080': { token: 'friendlytok', name: undefined } - }); - expect(mockCtx.storage.put).toHaveBeenCalledWith('activePreviewPorts', { - '8080': { - runtimeIdentityID: 'runtime-1', - token: 'friendlytok' - } + ).rejects.toMatchObject({ + code: 'OPERATION_INTERRUPTED', + context: { operation: 'preview.expose' } }); + + expect(mockCtx.storage.put).not.toHaveBeenCalledWith( + 'portTokens', + expect.anything() + ); + expect(mockCtx.storage.put).not.toHaveBeenCalledWith( + 'activePreviewPorts', + expect.anything() + ); }); it('exposePort() reuses the existing token when re-exposing the same port without a token', async () => { @@ -1458,7 +2604,7 @@ describe('Sandbox durable object behavior', () => { return { '8080': { token: 'stabletok' } }; } if (key === 'currentRuntimeIdentity') { - return { id: 'runtime-1' }; + return runtimeRecord('runtime-1'); } if (key === 'activePreviewPorts') { return {}; @@ -1482,7 +2628,7 @@ describe('Sandbox durable object behavior', () => { it('exposePort() does not restore a port revoked while the runtime starts', async () => { const storage = new Map([ ['portTokens', { '8080': { token: 'oldtoken' } }], - ['currentRuntimeIdentity', { id: 'runtime-1' }], + ['currentRuntimeIdentity', runtimeRecord('runtime-1')], ['activePreviewPorts', {}] ]); mockCtx.storage.get.mockImplementation( @@ -1499,27 +2645,37 @@ describe('Sandbox durable object behavior', () => { const startupGate = new Promise((resolve) => { releaseStartup = resolve; }); - const ensureRuntimeSpy = vi - .spyOn( - sandbox as unknown as SandboxRuntimeStart, - 'ensureRuntimeActiveForPreview' - ) - .mockImplementation(async () => { - await startupGate; - return { - id: 'runtime-1', - scope: (value: { token: string }) => ({ - ...value, - runtimeIdentityID: 'runtime-1' - }) + const runtimeRunner = ( + sandbox as unknown as { + runtimeRunner: { + runWaking( + operation: string, + call: (lease: { + runtime: unknown; + retain(): { release(): void }; + }) => Promise + ): Promise; }; + } + ).runtimeRunner; + const runWakingSpy = vi + .spyOn(runtimeRunner, 'runWaking') + .mockImplementation(async (_operation, call) => { + await startupGate; + return call({ + runtime: { + id: 'runtime-1', + runtimeIncarnationID: 'test-incarnation' + }, + retain: () => ({ release: () => {} }) + }); }); const exposePromise = sandbox.exposePort(9090, { hostname: 'example.com', token: 'newtoken' }); - await vi.waitFor(() => expect(ensureRuntimeSpy).toHaveBeenCalled()); + await vi.waitFor(() => expect(runWakingSpy).toHaveBeenCalled()); await sandbox.unexposePort(8080); expect(storage.get('portTokens')).toEqual({}); @@ -1603,67 +2759,6 @@ describe('Sandbox durable object behavior', () => { expect(nextMeta?.['8080']).toBeUndefined(); } - it('onStart() hides named tunnels for respawn and drops quick ones', async () => { - const puts = seedMixedTunnelStorage(); - - await (sandbox as any).onStart(); - - expectOnlyNamedTunnelMetadataPreserved(puts); - }); - - it('onStart() resumes retained named tunnel cleanup records', async () => { - mockEnv.CLOUDFLARE_API_TOKEN = 'TOK'; - mockEnv.CLOUDFLARE_TUNNEL_ACCOUNT_ID = 'ACCT'; - mockEnv.CLOUDFLARE_ZONE_ID = 'zone-id'; - const fetchMock = vi.fn< - (input: string | URL, init?: RequestInit) => Promise - >( - async () => - new Response(JSON.stringify({ success: true, result: {} }), { - status: 200, - headers: { 'content-type': 'application/json' } - }) - ); - vi.stubGlobal('fetch', fetchMock); - const storagePut = mockCtx.storage.put as unknown as ( - key: string, - value: unknown - ) => Promise; - const storageGet = mockCtx.storage.get as unknown as ( - key: string - ) => Promise; - await storagePut('tunnels:cleanup', { - '8080': { - tunnelId: 'tunnel-uuid-retained', - port: 8080, - name: 'api', - hostname: 'api.example.com', - dnsRecordId: 'dns-record-retained', - accountId: 'ACCT', - zoneId: 'zone-id', - phase: 'claimed', - updatedAt: '2026-05-13T00:00:00.000Z' - } - }); - - await (sandbox as any).onStart(); - - const deleteTargets = fetchMock.mock.calls - .filter(([, init]) => init?.method === 'DELETE') - .map(([url]) => String(url)); - expect( - deleteTargets.some((target) => - target.includes('/dns_records/dns-record-retained') - ) - ).toBe(true); - expect( - deleteTargets.some((target) => - target.includes('/cfd_tunnel/tunnel-uuid-retained') - ) - ).toBe(true); - expect(await storageGet('tunnels:cleanup')).toEqual({}); - }); - it('onStop() hides named tunnels for respawn and drops quick ones', async () => { const puts = seedMixedTunnelStorage(); @@ -1680,35 +2775,43 @@ describe('Sandbox durable object behavior', () => { const storageGet = mockCtx.storage.get as unknown as ( key: string ) => Promise; - await storagePut('currentRuntimeIdentity', { id: 'runtime-1' }); + await storagePut('currentRuntimeIdentity', runtimeRecord('runtime-1')); await storagePut('sandbox:lifetime', { id: 'lifetime-1', generation: 1, createdAt: '2026-06-18T00:00:00.000Z', updatedAt: '2026-06-18T00:00:00.000Z' }); - vi.mocked(sandbox.client.tunnels.ensureTunnelRun).mockImplementation( - async (request) => ({ - started: true, - run: { - mode: 'quick', - tunnelId: request.tunnelId, - runId: request.runId, - port: request.port, - url: 'https://stub.trycloudflare.com', - hostname: 'stub.trycloudflare.com', - startedAt: '2026-06-18T00:00:00.000Z' - } - }) - ); + vi.mocked( + asSandboxWithClient(sandbox).client.tunnels.ensureTunnelRun + ).mockImplementation(async (request) => ({ + started: true, + run: { + mode: 'quick', + tunnelId: request.tunnelId, + runId: request.runId, + port: request.port, + url: 'https://stub.trycloudflare.com', + hostname: 'stub.trycloudflare.com', + startedAt: '2026-06-18T00:00:00.000Z' + } + })); + + const runWaking = vi.spyOn(getPreviewRuntimeRunner(sandbox), 'runWaking'); await sandbox.tunnels.get(8080); + expect(runWaking).toHaveBeenCalledTimes(1); + expect(runWaking).toHaveBeenCalledWith( + 'tunnel.provision', + expect.any(Function) + ); const meta = (await storageGet('tunnels:meta')) as Record< string, Record >; expect(meta['8080']?.runtimeIdentityID).toBe('runtime-1'); + expect(meta['8080']?.runtimeIncarnationID).toBe('test-incarnation'); expect(meta['8080']?.sandboxLifetimeID).toBe('lifetime-1'); }); @@ -1787,7 +2890,7 @@ describe('Sandbox durable object behavior', () => { it('lists only ports activated for the current runtime without contacting the container', async () => { vi.mocked(mockCtx.storage.get).mockImplementation(async (key) => { if (key === 'currentRuntimeIdentity') { - return { id: 'runtime-1' }; + return runtimeRecord('runtime-1'); } if (key === 'portTokens') { return { @@ -1799,10 +2902,12 @@ describe('Sandbox durable object behavior', () => { return { '8080': { runtimeIdentityID: 'runtime-1', + runtimeIncarnationID: 'test-incarnation', token: 'tok8080' }, '9090': { runtimeIdentityID: 'runtime-old', + runtimeIncarnationID: 'test-incarnation', token: 'tok9090' } }; @@ -1830,6 +2935,7 @@ describe('Sandbox durable object behavior', () => { return { '8080': { runtimeIdentityID: 'runtime-1', + runtimeIncarnationID: 'test-incarnation', token: 'tok8080' } }; @@ -1843,7 +2949,7 @@ describe('Sandbox durable object behavior', () => { it('omits durable auth without matching current-runtime activation', async () => { vi.mocked(mockCtx.storage.get).mockImplementation(async (key) => { if (key === 'currentRuntimeIdentity') { - return { id: 'runtime-1' }; + return runtimeRecord('runtime-1'); } if (key === 'portTokens') { return { '8080': { token: 'tok8080' } }; @@ -1864,7 +2970,7 @@ describe('Sandbox durable object behavior', () => { it('returns true only for durable auth activated in the current runtime', async () => { vi.mocked(mockCtx.storage.get).mockImplementation(async (key) => { if (key === 'currentRuntimeIdentity') { - return { id: 'runtime-1' }; + return runtimeRecord('runtime-1'); } if (key === 'portTokens') { return { '8080': { token: 'tok8080' } }; @@ -1873,6 +2979,7 @@ describe('Sandbox durable object behavior', () => { return { '8080': { runtimeIdentityID: 'runtime-1', + runtimeIncarnationID: 'test-incarnation', token: 'tok8080' } }; @@ -1886,7 +2993,7 @@ describe('Sandbox durable object behavior', () => { it('returns false for durable auth without activation', async () => { vi.mocked(mockCtx.storage.get).mockImplementation(async (key) => { if (key === 'currentRuntimeIdentity') { - return { id: 'runtime-1' }; + return runtimeRecord('runtime-1'); } if (key === 'portTokens') { return { '8080': { token: 'tok8080' } }; @@ -1903,7 +3010,7 @@ describe('Sandbox durable object behavior', () => { it('returns false for activation from an old runtime', async () => { vi.mocked(mockCtx.storage.get).mockImplementation(async (key) => { if (key === 'currentRuntimeIdentity') { - return { id: 'runtime-1' }; + return runtimeRecord('runtime-1'); } if (key === 'portTokens') { return { '8080': { token: 'tok8080' } }; @@ -1912,6 +3019,7 @@ describe('Sandbox durable object behavior', () => { return { '8080': { runtimeIdentityID: 'runtime-old', + runtimeIncarnationID: 'test-incarnation', token: 'tok8080' } }; @@ -1935,6 +3043,7 @@ describe('Sandbox durable object behavior', () => { return { '8080': { runtimeIdentityID: 'runtime-1', + runtimeIncarnationID: 'test-incarnation', token: 'tok8080' } }; @@ -1951,7 +3060,7 @@ describe('Sandbox durable object behavior', () => { it('revokes auth and activation without touching the container registry when runtime is active', async () => { vi.mocked(mockCtx.storage.get).mockImplementation(async (key) => { if (key === 'currentRuntimeIdentity') { - return { id: 'runtime-1' }; + return runtimeRecord('runtime-1'); } if (key === 'portTokens') { return { '8080': { token: 'tok8080' } }; @@ -1960,6 +3069,7 @@ describe('Sandbox durable object behavior', () => { return { '8080': { runtimeIdentityID: 'runtime-1', + runtimeIncarnationID: 'test-incarnation', token: 'tok8080' } }; @@ -2177,11 +3287,8 @@ describe('Sandbox durable object behavior', () => { ); const putCallsBefore = mockCtx.storage.put.mock.calls.length; - const setRetrySpy = vi.spyOn(sandbox.client, 'setRetryTimeoutMs'); - setRetrySpy.mockClear(); await sandbox.setContainerTimeouts(current); expect(mockCtx.storage.put.mock.calls.length).toBe(putCallsBefore); - expect(setRetrySpy).not.toHaveBeenCalled(); }); }); @@ -2254,7 +3361,38 @@ describe('Sandbox durable object behavior', () => { await vi.waitFor(() => { expect(mockCtx.blockConcurrencyWhile).toHaveBeenCalled(); }); - backupSandbox.client = createMockControlClient(); + asSandboxWithClient(backupSandbox as any).client = + createMockControlClient(); + await ( + mockCtx.storage.put as unknown as ( + key: string, + value: unknown + ) => Promise + )('currentRuntimeIdentity', { + schemaVersion: 1, + id: 'runtime-1', + runtimeIncarnationID: 'incarnation-1' + }); + const runtimeTarget = backupSandbox as unknown as { + client: ContainerControlClient; + runWakingComposite( + operation: string, + call: (lease: { + runtime: { id: string; runtimeIncarnationID: string }; + control: ContainerControlClient; + retain(): { release(): void }; + }) => Promise + ): Promise; + }; + runtimeTarget.runWakingComposite = async (_operation, call) => + await call({ + runtime: { + id: 'runtime-1', + runtimeIncarnationID: 'incarnation-1' + }, + control: runtimeTarget.client, + retain: () => ({ release: () => {} }) + }); return { backupSandbox, bucket }; } @@ -2356,7 +3494,10 @@ describe('Sandbox durable object behavior', () => { it('should allow creating a backup from /app', async () => { const { backupSandbox, bucket } = await createBackupSandbox(); const createArchiveSpy = vi - .spyOn(backupSandbox.client.backup, 'createArchive') + .spyOn( + asSandboxWithClient(backupSandbox as any).client.backup, + 'createArchive' + ) .mockResolvedValue({ success: true, sizeBytes: 42, @@ -2388,7 +3529,10 @@ describe('Sandbox durable object behavior', () => { it('should normalize globstar excludes before calling createArchive', async () => { const { backupSandbox } = await createBackupSandbox(); const createArchiveSpy = vi - .spyOn(backupSandbox.client.backup, 'createArchive') + .spyOn( + asSandboxWithClient(backupSandbox as any).client.backup, + 'createArchive' + ) .mockResolvedValue({ success: true, sizeBytes: 42, @@ -2418,10 +3562,14 @@ describe('Sandbox durable object behavior', () => { ); }); - it('should reject unsupported backup compression before calling the container', async () => { + it('should reject unsupported backup compression before runtime admission', async () => { const { backupSandbox } = await createBackupSandbox(); + const runWakingSpy = vi.spyOn( + backupSandbox as unknown as { runWakingComposite(): Promise }, + 'runWakingComposite' + ); const createArchiveSpy = vi.spyOn( - backupSandbox.client.backup, + asSandboxWithClient(backupSandbox as any).client.backup, 'createArchive' ); @@ -2436,13 +3584,14 @@ describe('Sandbox durable object behavior', () => { /BackupOptions\.compression\.format must be one of: gzip, lz4, zstd/ ); + expect(runWakingSpy).not.toHaveBeenCalled(); expect(createArchiveSpy).not.toHaveBeenCalled(); }); it('should reject invalid backup compression thread count before calling the container', async () => { const { backupSandbox } = await createBackupSandbox(); const createArchiveSpy = vi.spyOn( - backupSandbox.client.backup, + asSandboxWithClient(backupSandbox as any).client.backup, 'createArchive' ); @@ -2473,7 +3622,10 @@ describe('Sandbox durable object behavior', () => { }); bucket.head.mockResolvedValue({ size: 42 }); const restoreArchiveSpy = vi - .spyOn(backupSandbox.client.backup, 'restoreArchive') + .spyOn( + asSandboxWithClient(backupSandbox as any).client.backup, + 'restoreArchive' + ) .mockResolvedValue({ success: true, dir: '/app/project' }); const downloadBackupParallelSpy = vi .spyOn( @@ -2501,7 +3653,8 @@ describe('Sandbox durable object behavior', () => { `backups/${backupId}/data.sqsh`, 42, backupId, - '/app/project' + '/app/project', + asSandboxWithClient(backupSandbox as any).client ); }); @@ -2509,7 +3662,7 @@ describe('Sandbox durable object behavior', () => { const { backupSandbox } = await createBackupSandbox(); const expectedSize = 16 * 1024 * 1024; const downloadArchiveSpy = vi.spyOn( - backupSandbox.client.backup, + asSandboxWithClient(backupSandbox as any).client.backup, 'downloadArchive' ); vi.spyOn( @@ -2524,7 +3677,8 @@ describe('Sandbox durable object behavior', () => { 'backups/test/data.sqsh', expectedSize, 'test-backup-id', - '/app/project' + '/app/project', + asSandboxWithClient(backupSandbox as any).client ); expect(downloadArchiveSpy).toHaveBeenCalledWith({ @@ -2541,10 +3695,44 @@ describe('Sandbox durable object behavior', () => { }); }); + it('preserves transport interruption during parallel download', async () => { + const { backupSandbox } = await createBackupSandbox(); + const interruption = new RPCTransportError({ + code: ErrorCode.RPC_TRANSPORT_ERROR, + message: 'Transport disposed', + httpStatus: 503, + context: { + kind: 'session_disposed', + originalMessage: 'Transport disposed', + errorName: 'Error' + }, + timestamp: '2026-06-15T12:00:00.000Z' + }); + vi.spyOn( + asSandboxWithClient(backupSandbox as any).client.backup, + 'downloadArchive' + ).mockRejectedValue(interruption); + vi.spyOn( + (backupSandbox as any).backupService.transfer, + 'generatePresignedGetURL' + ).mockResolvedValue('https://example.com/archive'); + + await expect( + (backupSandbox as any).backupService.transfer.downloadBackupParallel( + '/var/backups/test.sqsh', + 'backups/test/data.sqsh', + 16 * 1024 * 1024, + 'test-backup-id', + '/app/project', + asSandboxWithClient(backupSandbox as any).client + ) + ).rejects.toBe(interruption); + }); + it('should reject unsupported backup roots before calling the container', async () => { const { backupSandbox } = await createBackupSandbox(); const createArchiveSpy = vi.spyOn( - backupSandbox.client.backup, + asSandboxWithClient(backupSandbox as any).client.backup, 'createArchive' ); @@ -2658,7 +3846,9 @@ describe('Sandbox durable object behavior', () => { stdout?: string; stderr?: string; }) { - vi.mocked(sandbox.client.mounts.mountS3FSAndVerify).mockResolvedValue({ + vi.mocked( + asSandboxWithClient(sandbox).client.mounts.mountS3FSAndVerify + ).mockResolvedValue({ success: result.exitCode === 0, exitCode: result.exitCode, stdout: result.stdout ?? '', @@ -2706,7 +3896,9 @@ describe('Sandbox durable object behavior', () => { // the last poll and our cleanup. The failure path must unmount that // mount instead of leaking it. - vi.mocked(sandbox.client.mounts.mountS3FSAndVerify).mockResolvedValue({ + vi.mocked( + asSandboxWithClient(sandbox).client.mounts.mountS3FSAndVerify + ).mockResolvedValue({ success: false, exitCode: 3, stdout: 'mount took too long', @@ -2718,23 +3910,29 @@ describe('Sandbox durable object behavior', () => { .catch((e: Error) => e); expect(err).toBeInstanceOf(Error); - expect(sandbox.client.mounts.isMountpoint).toHaveBeenCalledWith( - '/mnt/late' - ); - expect(sandbox.client.mounts.unmountFuse).toHaveBeenCalledWith( - '/mnt/late' - ); + expect( + asSandboxWithClient(sandbox).client.mounts.isMountpoint + ).toHaveBeenCalledWith('/mnt/late'); + expect( + asSandboxWithClient(sandbox).client.mounts.unmountFuse + ).toHaveBeenCalledWith('/mnt/late'); }); it('keeps support files when failure cleanup cannot unmount FUSE', async () => { - vi.mocked(sandbox.client.mounts.mountS3FSAndVerify).mockResolvedValue({ + vi.mocked( + asSandboxWithClient(sandbox).client.mounts.mountS3FSAndVerify + ).mockResolvedValue({ success: false, exitCode: 3, stdout: 'mount took too long', stderr: '' }); - vi.mocked(sandbox.client.mounts.isMountpoint).mockResolvedValue(true); - vi.mocked(sandbox.client.mounts.unmountFuse).mockResolvedValue({ + vi.mocked( + asSandboxWithClient(sandbox).client.mounts.isMountpoint + ).mockResolvedValue(true); + vi.mocked( + asSandboxWithClient(sandbox).client.mounts.unmountFuse + ).mockResolvedValue({ success: false, exitCode: 1, stdout: '', @@ -2748,7 +3946,9 @@ describe('Sandbox durable object behavior', () => { await expect(sandbox.unmountBucket('/mnt/busy')).rejects.toThrow( 'No active mount found at path: /mnt/busy' ); - expect(sandbox.client.mounts.deleteFile).not.toHaveBeenCalled(); + expect( + asSandboxWithClient(sandbox).client.mounts.deleteFile + ).not.toHaveBeenCalled(); }); }); }); diff --git a/packages/sandbox/tests/storage-mount-lifecycle-cleanup.test.ts b/packages/sandbox/tests/storage-mount-lifecycle-cleanup.test.ts index 6a932d841..a1207584e 100644 --- a/packages/sandbox/tests/storage-mount-lifecycle-cleanup.test.ts +++ b/packages/sandbox/tests/storage-mount-lifecycle-cleanup.test.ts @@ -5,7 +5,10 @@ import { cleanupBucketMountsForDestroy } from '../src/storage-mount/lifecycle-cl import type { MountOutboundHost } from '../src/storage-mount/outbound'; import { MountRegistry } from '../src/storage-mount/registry'; import type { S3FSHost } from '../src/storage-mount/s3fs'; -import type { R2BindingMountInfo } from '../src/storage-mount/types'; +import type { + LocalSyncMountInfo, + R2BindingMountInfo +} from '../src/storage-mount/types'; function createLogger(): Logger { return { @@ -63,7 +66,35 @@ function createR2Mount(mountPath: string): R2BindingMountInfo { } describe('bucket mount destroy lifecycle cleanup', () => { - it('preserves failed FUSE unmounts for a later destroy cleanup retry', async () => { + it('interrupts local sync without waiting for a hung stop', async () => { + const logger = createLogger(); + const stop = vi.fn(() => new Promise(() => {})); + const interrupt = vi.fn(); + const registry = new MountRegistry(); + registry.set('/mnt/local', { + mountId: 'mount-local', + mountType: 'local-sync', + bucket: 'MY_BUCKET', + mountPath: '/mnt/local', + mounted: true, + syncManager: { stop, interrupt } + } as unknown as LocalSyncMountInfo); + + const result = await cleanupBucketMountsForDestroy({ + registry, + logger, + s3fsHost: null, + getOutboundHost: () => createOutboundHost(logger), + runMountOperation: (operation) => operation() + }); + + expect(result).toEqual({ mountsProcessed: 1, mountFailures: 0 }); + expect(interrupt).toHaveBeenCalledTimes(1); + expect(stop).not.toHaveBeenCalled(); + expect(registry.activeMounts.size).toBe(0); + }); + + it('clears logical destroy state even when physical FUSE unmount fails', async () => { const logger = createLogger(); const unmountFuse = vi .fn<(path: string) => Promise>() @@ -72,9 +103,10 @@ describe('bucket mount destroy lifecycle cleanup', () => { .mockResolvedValue(mountResult()); const deleteFile = vi.fn(async () => undefined); const s3fsHost: S3FSHost = { - client: { - mounts: { unmountFuse, deleteFile } - } as unknown as ContainerControlClient, + runRuntimeCall: async (_operation, call) => + call({ + mounts: { unmountFuse, deleteFile } + } as unknown as ContainerControlClient), logger }; const outboundHost = createOutboundHost(logger); @@ -85,28 +117,15 @@ describe('bucket mount destroy lifecycle cleanup', () => { const firstResult = await cleanupBucketMountsForDestroy({ registry, logger, - getS3FSHost: () => s3fsHost, + s3fsHost, getOutboundHost: () => outboundHost, runMountOperation }); expect(firstResult).toEqual({ mountsProcessed: 1, mountFailures: 1 }); - expect(registry.has('/mnt/r2')).toBe(true); - expect(registry.get('/mnt/r2')?.mounted).toBe(true); - expect(unmountFuse).toHaveBeenCalledWith('/mnt/r2'); - - const retryResult = await cleanupBucketMountsForDestroy({ - registry, - logger, - getS3FSHost: () => s3fsHost, - getOutboundHost: () => outboundHost, - runMountOperation - }); - - expect(retryResult).toEqual({ mountsProcessed: 1, mountFailures: 0 }); expect(registry.activeMounts.size).toBe(0); - expect(deleteFile).toHaveBeenCalledWith('/tmp/passwd--mnt-r2'); - expect(deleteFile).toHaveBeenCalledWith('/tmp/ahbe--mnt-r2'); + expect(unmountFuse).toHaveBeenCalledWith('/mnt/r2'); + expect(deleteFile).not.toHaveBeenCalled(); expect(outboundHost.removeOutboundByHost).toHaveBeenCalledWith( 'r2.internal' ); diff --git a/packages/sandbox/tests/storage-mount-lifecycle.test.ts b/packages/sandbox/tests/storage-mount-lifecycle.test.ts new file mode 100644 index 000000000..980703390 --- /dev/null +++ b/packages/sandbox/tests/storage-mount-lifecycle.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it, vi } from 'vitest'; +import { RuntimeIdentity, type RuntimeIdentityReader } from '../src/runtime'; +import type { CurrentSandboxLifetime } from '../src/sandbox-lifetime'; +import { MountLifecycle } from '../src/storage-mount/lifecycle'; + +function runtime(id: string, incarnation: string): RuntimeIdentity { + return new RuntimeIdentity({ + id: id as RuntimeIdentity['id'], + runtimeIncarnationID: incarnation as RuntimeIdentity['runtimeIncarnationID'] + }); +} + +describe('MountLifecycle', () => { + it('fences registry commits to the exact admitted incarnation', async () => { + const admitted = runtime('runtime-1', 'incarnation-1'); + const order: string[] = []; + const runtimeReader = { + get: vi.fn(), + getStored: vi.fn(), + isActive: vi.fn(), + assertActive: vi.fn(async (candidate: RuntimeIdentity) => { + order.push('runtime'); + expect(candidate).toBe(admitted); + }) + } satisfies RuntimeIdentityReader; + const lifetime = { id: 'lifetime-1', generation: 1 }; + const currentLifetime = { + getOrCreate: vi.fn(async () => lifetime), + assertCurrent: vi.fn(async () => { + order.push('lifetime'); + }) + } as unknown as CurrentSandboxLifetime; + const lifecycle = new MountLifecycle(runtimeReader, currentLifetime); + + const snapshot = await lifecycle.capture(admitted); + await lifecycle.assertCurrent(snapshot); + + expect(snapshot.runtime).toBe(admitted); + expect(runtimeReader.get).not.toHaveBeenCalled(); + expect(order).toEqual(['lifetime', 'runtime']); + }); +}); diff --git a/packages/sandbox/tests/tunnels/helpers.ts b/packages/sandbox/tests/tunnels/helpers.ts index 72a0636a3..1a27e2c69 100644 --- a/packages/sandbox/tests/tunnels/helpers.ts +++ b/packages/sandbox/tests/tunnels/helpers.ts @@ -1,10 +1,59 @@ -import type { Logger } from '@repo/shared'; +import type { Logger, SandboxTunnelsAPI } from '@repo/shared'; import { vi } from 'vitest'; +import { RuntimeIdentity } from '../../src/runtime'; import type { TunnelServiceHost, TunnelsStorage } from '../../src/tunnels/rpc-target'; +type TunnelRuntimeCall = ( + operation: string, + call: (tunnels: SandboxTunnelsAPI) => Promise +) => Promise; + +export type TestTunnelServiceHost = Omit< + TunnelServiceHost, + 'runProvision' | 'runExisting' | 'getStoredRuntime' +> & + Partial< + Pick + > & { runRuntimeCall: TunnelRuntimeCall }; + +export function completeTunnelServiceHost( + host: TestTunnelServiceHost +): TunnelServiceHost { + const runtime = new RuntimeIdentity({ + id: 'runtime-1' as RuntimeIdentity['id'], + runtimeIncarnationID: 'inc-1' as RuntimeIdentity['runtimeIncarnationID'] + }); + const { runRuntimeCall, ...serviceHost } = host; + return { + ...serviceHost, + getStoredRuntime: host.getStoredRuntime ?? (async () => runtime), + runProvision: + host.runProvision ?? + ((call) => + runRuntimeCall('tunnel.provision', (tunnels) => + call({ + runtime, + tunnels, + retain: () => ({ release: () => {} }) + }) + )), + runExisting: + host.runExisting ?? + ((target, operation, call) => { + if ( + target.id !== runtime.id || + target.runtimeIncarnationID !== runtime.runtimeIncarnationID + ) { + return Promise.resolve(null); + } + return runRuntimeCall(operation, call); + }) + }; +} + export function makeLogger(): Logger { const log: Logger = { info: vi.fn(), @@ -41,18 +90,31 @@ export function makeFences( currentRuntime?: Record; currentLifetime?: Record; } = {} -): Pick { +): Pick { + const runtime = { + id: 'runtime-1', + runtimeIncarnationID: 'inc-1' + }; + const getRuntime = overrides.currentRuntime?.get as + | (() => Promise) + | undefined; return { - currentRuntime: { - get: vi.fn(async () => ({ id: 'runtime-1' })), - markStarted: vi.fn(async () => ({ id: 'runtime-1' })), - assertActive: vi.fn(async () => {}), - ...overrides.currentRuntime + getStoredRuntime: async () => { + const value = getRuntime ? await getRuntime() : runtime; + if (!value) return null; + const record = value as { id: string; runtimeIncarnationID?: string }; + return { + id: record.id, + runtimeIncarnationID: record.runtimeIncarnationID ?? 'inc-1' + } as Awaited>; }, currentLifetime: { getOrCreate: vi.fn(async () => ({ id: 'lifetime-1' })), assertCurrent: vi.fn(async () => {}), ...overrides.currentLifetime } - } as unknown as Pick; + } as unknown as Pick< + TunnelServiceHost, + 'getStoredRuntime' | 'currentLifetime' + >; } diff --git a/packages/sandbox/tests/tunnels/tunnel-service-named-cleanup.test.ts b/packages/sandbox/tests/tunnels/tunnel-service-named-cleanup.test.ts index cf87379ef..d6b8de380 100644 --- a/packages/sandbox/tests/tunnels/tunnel-service-named-cleanup.test.ts +++ b/packages/sandbox/tests/tunnels/tunnel-service-named-cleanup.test.ts @@ -1,3 +1,7 @@ +import { + completeTunnelServiceHost, + type TestTunnelServiceHost +} from './helpers'; /** * Named-tunnel behavior tests for the SDK tunnel service. * @@ -23,12 +27,12 @@ import type { } from '@repo/shared'; import type { Mock } from 'vitest'; import { describe, expect, it, vi } from 'vitest'; -import { RuntimeIdentityInactiveError } from '../../src/current-runtime-identity'; import { ErrorCode, RPCTransportError } from '../../src/errors'; +import { RuntimeIdentityInactiveError } from '../../src/runtime/types'; import { SandboxLifetimeChangedError } from '../../src/sandbox-lifetime'; import { SandboxSecurityError } from '../../src/security'; import { - createTunnelsHandle, + createTunnelsHandle as createRuntimeTunnelsHandle, type TunnelsStorage } from '../../src/tunnels/rpc-target'; import { makeFences, makeLogger, makeStorage } from './helpers'; @@ -257,7 +261,7 @@ function makeHandler(opts?: { zoneId: string; }>; configError?: Error; - fences?: Pick; + fences?: Pick; }) { const { client } = makeClient(); mockTunnelRun(client); @@ -269,7 +273,15 @@ function makeHandler(opts?: { zoneId: opts?.config?.zoneId ?? 'zone-id' }; const built = createTunnelsHandle({ - client: client as unknown as TunnelsHost['client'], + runRuntimeCall: ((operation, call) => + call( + client.tunnels as unknown as TunnelsHost['runRuntimeCall'] extends ( + op: string, + call: (tunnels: infer U) => Promise + ) => Promise + ? U + : never + )) as TunnelsHost['runRuntimeCall'], storage, logger, sandboxId: opts?.sandboxId ?? 'sb1', @@ -292,6 +304,9 @@ function makeHandler(opts?: { }; } +const createTunnelsHandle = (host: TestTunnelServiceHost) => + createRuntimeTunnelsHandle(completeTunnelServiceHost(host)); + describe('tunnel service > destroy() for named tunnels', () => { it('resumes retained cleanup when the public tunnel record is gone', async () => { const cf = makeFakeCloudflare({}); @@ -391,9 +406,14 @@ describe('tunnel service > destroy() for named tunnels', () => { }) ); const built = createTunnelsHandle({ - client: client as unknown as Parameters< - typeof createTunnelsHandle - >[0]['client'], + runRuntimeCall: ((operation, call) => + call( + client.tunnels as unknown as Parameters< + Parameters[0]['runRuntimeCall'] + >[1] extends (tunnels: infer U) => Promise + ? U + : never + )) as Parameters[0]['runRuntimeCall'], storage, logger: makeLogger(), sandboxId: 'sb1', @@ -443,9 +463,14 @@ describe('tunnel service > destroy() for named tunnels', () => { } }); const built = createTunnelsHandle({ - client: client as unknown as Parameters< - typeof createTunnelsHandle - >[0]['client'], + runRuntimeCall: ((operation, call) => + call( + client.tunnels as unknown as Parameters< + Parameters[0]['runRuntimeCall'] + >[1] extends (tunnels: infer U) => Promise + ? U + : never + )) as Parameters[0]['runRuntimeCall'], storage, logger: makeLogger(), sandboxId: 'sb1', @@ -498,9 +523,14 @@ describe('tunnel service > destroy() for named tunnels', () => { new Error('container already stopped') ); const built = createTunnelsHandle({ - client: client as unknown as Parameters< - typeof createTunnelsHandle - >[0]['client'], + runRuntimeCall: ((operation, call) => + call( + client.tunnels as unknown as Parameters< + Parameters[0]['runRuntimeCall'] + >[1] extends (tunnels: infer U) => Promise + ? U + : never + )) as Parameters[0]['runRuntimeCall'], storage, logger: makeLogger(), sandboxId: 'sb1', @@ -597,9 +627,14 @@ describe('tunnel service > destroy() for named tunnels', () => { const logger = makeLogger(); let configShouldFail = false; const built = createTunnelsHandle({ - client: client as unknown as Parameters< - typeof createTunnelsHandle - >[0]['client'], + runRuntimeCall: ((operation, call) => + call( + client.tunnels as unknown as Parameters< + Parameters[0]['runRuntimeCall'] + >[1] extends (tunnels: infer U) => Promise + ? U + : never + )) as Parameters[0]['runRuntimeCall'], storage, logger, sandboxId: 'sb1', @@ -886,9 +921,14 @@ describe('tunnel service > destroyAll()', () => { } }); const built = createTunnelsHandle({ - client: client as unknown as Parameters< - typeof createTunnelsHandle - >[0]['client'], + runRuntimeCall: ((operation, call) => + call( + client.tunnels as unknown as Parameters< + Parameters[0]['runRuntimeCall'] + >[1] extends (tunnels: infer U) => Promise + ? U + : never + )) as Parameters[0]['runRuntimeCall'], storage, logger, sandboxId: 'sb1', diff --git a/packages/sandbox/tests/tunnels/tunnel-service-named-recovery.test.ts b/packages/sandbox/tests/tunnels/tunnel-service-named-recovery.test.ts index 3bbbac085..67a6c9676 100644 --- a/packages/sandbox/tests/tunnels/tunnel-service-named-recovery.test.ts +++ b/packages/sandbox/tests/tunnels/tunnel-service-named-recovery.test.ts @@ -1,3 +1,7 @@ +import { + completeTunnelServiceHost, + type TestTunnelServiceHost +} from './helpers'; /** * Named-tunnel behavior tests for the SDK tunnel service. * @@ -23,12 +27,12 @@ import type { } from '@repo/shared'; import type { Mock } from 'vitest'; import { describe, expect, it, vi } from 'vitest'; -import { RuntimeIdentityInactiveError } from '../../src/current-runtime-identity'; import { ErrorCode, RPCTransportError } from '../../src/errors'; +import { RuntimeIdentityInactiveError } from '../../src/runtime/types'; import { SandboxLifetimeChangedError } from '../../src/sandbox-lifetime'; import { SandboxSecurityError } from '../../src/security'; import { - createTunnelsHandle, + createTunnelsHandle as createRuntimeTunnelsHandle, type TunnelsStorage } from '../../src/tunnels/rpc-target'; import { makeFences, makeLogger, makeStorage } from './helpers'; @@ -257,7 +261,7 @@ function makeHandler(opts?: { zoneId: string; }>; configError?: Error; - fences?: Pick; + fences?: Pick; }) { const { client } = makeClient(); mockTunnelRun(client); @@ -269,7 +273,15 @@ function makeHandler(opts?: { zoneId: opts?.config?.zoneId ?? 'zone-id' }; const built = createTunnelsHandle({ - client: client as unknown as TunnelsHost['client'], + runRuntimeCall: ((operation, call) => + call( + client.tunnels as unknown as TunnelsHost['runRuntimeCall'] extends ( + op: string, + call: (tunnels: infer U) => Promise + ) => Promise + ? U + : never + )) as TunnelsHost['runRuntimeCall'], storage, logger, sandboxId: opts?.sandboxId ?? 'sb1', @@ -292,6 +304,9 @@ function makeHandler(opts?: { }; } +const createTunnelsHandle = (host: TestTunnelServiceHost) => + createRuntimeTunnelsHandle(completeTunnelServiceHost(host)); + describe('tunnel service > get(port, { name }) — retry / reuse', () => { it('reuses a tunnel left behind from a previous failed attempt', async () => { const cf = makeFakeCloudflare({ @@ -387,7 +402,15 @@ describe('tunnel service > get(port, { name }) — retry / reuse', () => { }); const { tunnels } = createTunnelsHandle({ - client: client as unknown as TunnelsHost['client'], + runRuntimeCall: ((operation, call) => + call( + client.tunnels as unknown as TunnelsHost['runRuntimeCall'] extends ( + op: string, + call: (tunnels: infer U) => Promise + ) => Promise + ? U + : never + )) as TunnelsHost['runRuntimeCall'], storage, logger, sandboxId: 'sb1', @@ -538,9 +561,14 @@ describe('tunnel service > restart respawn via needsRespawn flag', () => { } }); const { tunnels } = createTunnelsHandle({ - client: client as unknown as Parameters< - typeof createTunnelsHandle - >[0]['client'], + runRuntimeCall: ((operation, call) => + call( + client.tunnels as unknown as Parameters< + Parameters[0]['runRuntimeCall'] + >[1] extends (tunnels: infer U) => Promise + ? U + : never + )) as Parameters[0]['runRuntimeCall'], storage, logger: makeLogger(), sandboxId: 'sb1', @@ -552,7 +580,7 @@ describe('tunnel service > restart respawn via needsRespawn flag', () => { fetcher: cf.fetcher as unknown as typeof fetch, currentRuntime: { get: vi.fn(async () => ({ id: 'runtime-new' })), - markStarted: vi.fn(async () => ({ id: 'runtime-new' })), + unexpectedPublisher: vi.fn(async () => ({ id: 'runtime-new' })), assertActive: vi.fn(async () => {}) }, currentLifetime: { diff --git a/packages/sandbox/tests/tunnels/tunnel-service-named.test.ts b/packages/sandbox/tests/tunnels/tunnel-service-named.test.ts index 883e2d1ad..5f517e311 100644 --- a/packages/sandbox/tests/tunnels/tunnel-service-named.test.ts +++ b/packages/sandbox/tests/tunnels/tunnel-service-named.test.ts @@ -1,3 +1,7 @@ +import { + completeTunnelServiceHost, + type TestTunnelServiceHost +} from './helpers'; /** * Named-tunnel behavior tests for the SDK tunnel service. * @@ -23,12 +27,12 @@ import type { } from '@repo/shared'; import type { Mock } from 'vitest'; import { describe, expect, it, vi } from 'vitest'; -import { RuntimeIdentityInactiveError } from '../../src/current-runtime-identity'; import { ErrorCode, RPCTransportError } from '../../src/errors'; +import { RuntimeIdentityInactiveError } from '../../src/runtime/types'; import { SandboxLifetimeChangedError } from '../../src/sandbox-lifetime'; import { SandboxSecurityError } from '../../src/security'; import { - createTunnelsHandle, + createTunnelsHandle as createRuntimeTunnelsHandle, type TunnelsStorage } from '../../src/tunnels/rpc-target'; import { makeFences, makeLogger, makeStorage } from './helpers'; @@ -257,7 +261,7 @@ function makeHandler(opts?: { zoneId: string; }>; configError?: Error; - fences?: Pick; + fences?: Pick; }) { const { client } = makeClient(); mockTunnelRun(client); @@ -269,7 +273,15 @@ function makeHandler(opts?: { zoneId: opts?.config?.zoneId ?? 'zone-id' }; const built = createTunnelsHandle({ - client: client as unknown as TunnelsHost['client'], + runRuntimeCall: ((operation, call) => + call( + client.tunnels as unknown as TunnelsHost['runRuntimeCall'] extends ( + op: string, + call: (tunnels: infer U) => Promise + ) => Promise + ? U + : never + )) as TunnelsHost['runRuntimeCall'], storage, logger, sandboxId: opts?.sandboxId ?? 'sb1', @@ -292,6 +304,9 @@ function makeHandler(opts?: { }; } +const createTunnelsHandle = (host: TestTunnelServiceHost) => + createRuntimeTunnelsHandle(completeTunnelServiceHost(host)); + describe('tunnel service > get(port, { name }) — named tunnel happy path', () => { it('provisions a fresh named tunnel end-to-end', async () => { const cf = makeFakeCloudflare({}); @@ -322,66 +337,36 @@ describe('tunnel service > get(port, { name }) — named tunnel happy path', () expect(JSON.stringify(info)).not.toContain('OPAQUE_TOKEN'); }); - it('replays the same named run request when the first call loses RPC transport', async () => { + it('surfaces RPC transport loss without replaying the named tunnel run', async () => { const cf = makeFakeCloudflare({}); const { tunnels, client } = makeHandler({ sandboxId: 'sb1', fetcher: cf.fetcher as unknown as typeof fetch }); - client.tunnels.ensureTunnelRun - .mockRejectedValueOnce(createDisposedRPCError()) - .mockImplementationOnce(async (request) => namedRunResult(request)); + const error = createDisposedRPCError(); + client.tunnels.ensureTunnelRun.mockRejectedValueOnce(error); - const info = await tunnels.get(8080, { name: 'api' }); + await expect(tunnels.get(8080, { name: 'api' })).rejects.toBe(error); - expect(client.tunnels.ensureTunnelRun).toHaveBeenCalledTimes(2); - const first = client.tunnels.ensureTunnelRun.mock.calls[0][0]; - const second = client.tunnels.ensureTunnelRun.mock.calls[1][0]; - expect(second).toEqual(first); - expect(info).toMatchObject({ - id: NAMED_TUNNEL_ID, - port: 8080, - name: 'api', - hostname: 'api.example.com', - url: 'https://api.example.com' - }); + expect(client.tunnels.ensureTunnelRun).toHaveBeenCalledTimes(1); }); - it('bounds runtime-replacement recovery and never commits a partial record', async () => { - // Every post-spawn fence reports a replaced runtime, so recovery can - // never converge. The operation surfaces recovery_exhausted and - // leaves storage empty so no orphaned tunnel record persists. + it('does not consult the transitional runtime fence while provisioning', async () => { const cf = makeFakeCloudflare({}); - let assertCalls = 0; - const { client, storage, tunnels } = makeHandler({ + const assertActive = vi.fn(async () => { + throw new RuntimeIdentityInactiveError(); + }); + const { client, tunnels } = makeHandler({ fetcher: cf.fetcher as unknown as typeof fetch, - fences: makeFences({ - currentRuntime: { - assertActive: vi.fn(async () => { - assertCalls += 1; - if (assertCalls % 3 === 0) { - throw new RuntimeIdentityInactiveError(); - } - }) - } - }) + fences: makeFences({ currentRuntime: { assertActive } }) }); mockNamedSpawn(client); - await expect(tunnels.get(8080, { name: 'api' })).rejects.toMatchObject({ - name: 'OperationInterruptedError', - code: ErrorCode.OPERATION_INTERRUPTED, - context: expect.objectContaining({ - reason: 'recovery_exhausted', - operation: 'tunnel.get', - retryable: true, - admitted: true, - recoveryAttempts: 2, - maxRecoveryAttempts: 2 - }) + await expect(tunnels.get(8080, { name: 'api' })).resolves.toMatchObject({ + name: 'api' }); - expect(await storage.get('tunnels')).toBeUndefined(); - expect(await storage.get('tunnels:meta')).toBeUndefined(); + expect(assertActive).not.toHaveBeenCalled(); + expect(client.tunnels.ensureTunnelRun).toHaveBeenCalledTimes(1); }); it('does not spawn or commit when lifetime changes during Cloudflare setup', async () => { @@ -409,7 +394,7 @@ describe('tunnel service > get(port, { name }) — named tunnel happy path', () reason: 'sandbox_lifetime_changed', operation: 'tunnel.get', retryable: false, - admitted: 'unknown' + admitted: true }) }); expect(client.tunnels.ensureTunnelRun).not.toHaveBeenCalled(); @@ -626,9 +611,14 @@ describe('tunnel service > get(port, options) — idempotency / hash guard', () throw new Error(`Unhandled ${method} ${url}`); }); const built = createTunnelsHandle({ - client: client as unknown as Parameters< - typeof createTunnelsHandle - >[0]['client'], + runRuntimeCall: ((operation, call) => + call( + client.tunnels as unknown as Parameters< + Parameters[0]['runRuntimeCall'] + >[1] extends (tunnels: infer U) => Promise + ? U + : never + )) as Parameters[0]['runRuntimeCall'], storage, logger, sandboxId: 'sb1', @@ -689,7 +679,9 @@ describe('tunnel service > get(port, options) — idempotency / hash guard', () optionsHash: 'named:api', dnsRecordId: 'kept-dns-id', accountId: 'ACCT', - zoneId: 'zone-id' + zoneId: 'zone-id', + runtimeIdentityID: 'runtime-1', + runtimeIncarnationID: 'inc-1' } }); diff --git a/packages/sandbox/tests/tunnels/tunnel-service-quick-lifecycle.test.ts b/packages/sandbox/tests/tunnels/tunnel-service-quick-lifecycle.test.ts index 2adcb008e..c69559b2a 100644 --- a/packages/sandbox/tests/tunnels/tunnel-service-quick-lifecycle.test.ts +++ b/packages/sandbox/tests/tunnels/tunnel-service-quick-lifecycle.test.ts @@ -1,3 +1,7 @@ +import { + completeTunnelServiceHost, + type TestTunnelServiceHost +} from './helpers'; /** * SDK tunnel service unit tests. * @@ -15,12 +19,13 @@ import type { } from '@repo/shared'; import type { Mock } from 'vitest'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { RuntimeIdentityInactiveError } from '../../src/current-runtime-identity'; import { ErrorCode, RPCTransportError } from '../../src/errors'; +import { RuntimeIdentity } from '../../src/runtime'; +import { RuntimeIdentityInactiveError } from '../../src/runtime/types'; import { SandboxLifetimeChangedError } from '../../src/sandbox-lifetime'; import { SandboxSecurityError } from '../../src/security'; import { - createTunnelsHandle, + createTunnelsHandle as createRuntimeTunnelsHandle, pruneTunnelsForRestart, type TunnelsHandler, type TunnelsStorage @@ -143,7 +148,15 @@ function makeHandler(extra: Partial = {}) { const storage = (providedStorage as TunnelsStorage | undefined) ?? makeStorage(); const { tunnels, handleTunnelExit } = createTunnelsHandle({ - client: client as unknown as TunnelsHost['client'], + runRuntimeCall: ((operation, call) => + call( + client.tunnels as unknown as TunnelsHost['runRuntimeCall'] extends ( + op: string, + call: (tunnels: infer U) => Promise + ) => Promise + ? U + : never + )) as TunnelsHost['runRuntimeCall'], storage, logger: makeLogger(), ...rest @@ -151,15 +164,23 @@ function makeHandler(extra: Partial = {}) { return { client, storage, handler: tunnels, tunnels, handleTunnelExit }; } +const createTunnelsHandle = (host: TestTunnelServiceHost) => + createRuntimeTunnelsHandle(completeTunnelServiceHost(host)); + describe('tunnel service > destroy', () => { it('clears storage without a container RPC for an unscoped known port', async () => { const record = makeRecord({ id: 'quick-known0000known00', port: 8080 }); const { client } = makeClient(); const storage = makeStorage({ '8080': record }); const { tunnels: handler } = createTunnelsHandle({ - client: client as unknown as Parameters< - typeof createTunnelsHandle - >[0]['client'], + runRuntimeCall: ((operation, call) => + call( + client.tunnels as unknown as Parameters< + Parameters[0]['runRuntimeCall'] + >[1] extends (tunnels: infer U) => Promise + ? U + : never + )) as Parameters[0]['runRuntimeCall'], storage, logger: makeLogger() }); @@ -175,12 +196,31 @@ describe('tunnel service > destroy', () => { const { client } = makeClient(); const storage = makeStorage({ '8080': record }); await storage.put('tunnels:meta', { - '8080': { optionsHash: 'v1:quick', tunnelRunId: 'run-quick-1' } + '8080': { + optionsHash: 'v1:quick', + runtimeIdentityID: 'runtime-1', + runtimeIncarnationID: 'inc-1', + tunnelRunId: 'run-quick-1' + } }); + let targetedRuntime: RuntimeIdentity | undefined; + let admittedOperation: string | undefined; const { tunnels: handler } = createTunnelsHandle({ - client: client as unknown as Parameters< - typeof createTunnelsHandle - >[0]['client'], + runRuntimeCall: ((operation, call) => + call( + client.tunnels as unknown as Parameters< + Parameters[0]['runRuntimeCall'] + >[1] extends (tunnels: infer U) => Promise + ? U + : never + )) as Parameters[0]['runRuntimeCall'], + runExisting: async (runtime, operation, call) => { + targetedRuntime = runtime; + admittedOperation = operation; + return await call( + client.tunnels as unknown as Parameters[0] + ); + }, storage, logger: makeLogger() }); @@ -190,20 +230,63 @@ describe('tunnel service > destroy', () => { await handler.destroy(8080); + expect(targetedRuntime).toMatchObject({ + id: 'runtime-1', + runtimeIncarnationID: 'inc-1' + }); + expect(admittedOperation).toBe('tunnel.destroy'); expect(client.tunnels.stopTunnelRun).toHaveBeenCalledWith({ tunnelId: record.id, runId: 'run-quick-1' }); }); + it('does not wake or fail when the recorded owning runtime is stale', async () => { + const record = makeRecord({ id: 'quick-stale-owner', port: 8080 }); + const { client } = makeClient(); + const storage = makeStorage({ '8080': record }); + await storage.put('tunnels:meta', { + '8080': { + optionsHash: 'v1:quick', + runtimeIdentityID: 'runtime-1', + runtimeIncarnationID: 'old-incarnation', + tunnelRunId: 'run-stale-owner' + } + }); + const runRuntimeCall = vi.fn(async (_operation, call) => + call( + client.tunnels as unknown as Parameters< + Parameters[0]['runRuntimeCall'] + >[1] extends (tunnels: infer U) => Promise + ? U + : never + ) + ) as Parameters[0]['runRuntimeCall']; + const { tunnels: handler } = createTunnelsHandle({ + runRuntimeCall, + storage, + logger: makeLogger() + }); + + await expect(handler.destroy(8080)).resolves.toBeUndefined(); + + expect(runRuntimeCall).not.toHaveBeenCalled(); + await expect(storage.get('tunnels')).resolves.toEqual({}); + }); + it('wraps the read-modify-write in storage.transaction()', async () => { const record = makeRecord({ id: 'quick-tx0000tx0000tx', port: 8080 }); const { client } = makeClient(); const storage = makeStorage({ '8080': record }); const { tunnels: handler } = createTunnelsHandle({ - client: client as unknown as Parameters< - typeof createTunnelsHandle - >[0]['client'], + runRuntimeCall: ((operation, call) => + call( + client.tunnels as unknown as Parameters< + Parameters[0]['runRuntimeCall'] + >[1] extends (tunnels: infer U) => Promise + ? U + : never + )) as Parameters[0]['runRuntimeCall'], storage, logger: makeLogger() }); @@ -219,9 +302,14 @@ describe('tunnel service > destroy', () => { const { client } = makeClient(); const storage = makeStorage({ '8080': record }); const { tunnels: handler } = createTunnelsHandle({ - client: client as unknown as Parameters< - typeof createTunnelsHandle - >[0]['client'], + runRuntimeCall: ((operation, call) => + call( + client.tunnels as unknown as Parameters< + Parameters[0]['runRuntimeCall'] + >[1] extends (tunnels: infer U) => Promise + ? U + : never + )) as Parameters[0]['runRuntimeCall'], storage, logger: makeLogger() }); @@ -244,9 +332,14 @@ describe('tunnel service > destroy', () => { const { client } = makeClient(); const storage = makeStorage({ '8080': record }); const { tunnels: handler } = createTunnelsHandle({ - client: client as unknown as Parameters< - typeof createTunnelsHandle - >[0]['client'], + runRuntimeCall: ((operation, call) => + call( + client.tunnels as unknown as Parameters< + Parameters[0]['runRuntimeCall'] + >[1] extends (tunnels: infer U) => Promise + ? U + : never + )) as Parameters[0]['runRuntimeCall'], storage, logger: makeLogger() }); @@ -261,7 +354,12 @@ describe('tunnel service > destroy', () => { } ); await storage.put('tunnels:meta', { - '8080': { optionsHash: 'v1:quick', tunnelRunId: 'run-gone' } + '8080': { + optionsHash: 'v1:quick', + runtimeIdentityID: 'runtime-1', + runtimeIncarnationID: 'inc-1', + tunnelRunId: 'run-gone' + } }); (storage.put as StoragePutMock).mockClear(); client.tunnels.stopTunnelRun.mockRejectedValue(notFound); @@ -283,14 +381,24 @@ describe('tunnel service > destroy', () => { const { client } = makeClient(); const storage = makeStorage({ '8080': record }); const { tunnels: handler } = createTunnelsHandle({ - client: client as unknown as Parameters< - typeof createTunnelsHandle - >[0]['client'], + runRuntimeCall: ((operation, call) => + call( + client.tunnels as unknown as Parameters< + Parameters[0]['runRuntimeCall'] + >[1] extends (tunnels: infer U) => Promise + ? U + : never + )) as Parameters[0]['runRuntimeCall'], storage, logger: makeLogger() }); await storage.put('tunnels:meta', { - '8080': { optionsHash: 'v1:quick', tunnelRunId: 'run-real' } + '8080': { + optionsHash: 'v1:quick', + runtimeIdentityID: 'runtime-1', + runtimeIncarnationID: 'inc-1', + tunnelRunId: 'run-real' + } }); client.tunnels.stopTunnelRun.mockRejectedValue( new Error('rpc transport failure: original was TUNNEL_NOT_FOUND') @@ -306,14 +414,24 @@ describe('tunnel service > destroy', () => { const { client } = makeClient(); const storage = makeStorage({ '8080': record }); const { tunnels: handler } = createTunnelsHandle({ - client: client as unknown as Parameters< - typeof createTunnelsHandle - >[0]['client'], + runRuntimeCall: ((operation, call) => + call( + client.tunnels as unknown as Parameters< + Parameters[0]['runRuntimeCall'] + >[1] extends (tunnels: infer U) => Promise + ? U + : never + )) as Parameters[0]['runRuntimeCall'], storage, logger: makeLogger() }); await storage.put('tunnels:meta', { - '8080': { optionsHash: 'v1:quick', tunnelRunId: 'run-err' } + '8080': { + optionsHash: 'v1:quick', + runtimeIdentityID: 'runtime-1', + runtimeIncarnationID: 'inc-1', + tunnelRunId: 'run-err' + } }); (storage.put as StoragePutMock).mockClear(); client.tunnels.stopTunnelRun.mockRejectedValue(new Error('boom')); @@ -338,10 +456,27 @@ describe('tunnel service > list', () => { }); const { client } = makeClient(); const storage = makeStorage({ '8080': a, '8081': b }); + await storage.put('tunnels:meta', { + '8080': { + optionsHash: 'v1:quick', + runtimeIdentityID: 'runtime-1', + runtimeIncarnationID: 'inc-1' + }, + '8081': { + optionsHash: 'v1:quick', + runtimeIdentityID: 'runtime-1', + runtimeIncarnationID: 'inc-1' + } + }); const { tunnels: handler } = createTunnelsHandle({ - client: client as unknown as Parameters< - typeof createTunnelsHandle - >[0]['client'], + runRuntimeCall: ((operation, call) => + call( + client.tunnels as unknown as Parameters< + Parameters[0]['runRuntimeCall'] + >[1] extends (tunnels: infer U) => Promise + ? U + : never + )) as Parameters[0]['runRuntimeCall'], storage, logger: makeLogger() }); @@ -351,6 +486,33 @@ describe('tunnel service > list', () => { expect(tunnels).toHaveLength(2); }); + it('omits records owned by another runtime incarnation', async () => { + const stale = makeRecord({ id: 'quick-stale-incarnation', port: 8080 }); + const storage = makeStorage({ '8080': stale }); + await storage.put('tunnels:meta', { + '8080': { + optionsHash: 'v1:quick', + runtimeIdentityID: 'runtime-1', + runtimeIncarnationID: 'old-incarnation' + } + }); + const { client } = makeClient(); + const { tunnels: handler } = createTunnelsHandle({ + runRuntimeCall: ((operation, call) => + call( + client.tunnels as unknown as Parameters< + Parameters[0]['runRuntimeCall'] + >[1] extends (tunnels: infer U) => Promise + ? U + : never + )) as Parameters[0]['runRuntimeCall'], + storage, + logger: makeLogger() + }); + + await expect(handler.list()).resolves.toEqual([]); + }); + it('returns an empty array when storage is empty', async () => { const { handler } = makeHandler(); await expect(handler.list()).resolves.toEqual([]); @@ -372,7 +534,11 @@ describe('tunnel service > list', () => { '8081': staleNamed }); await (storage.put as StoragePutMock)('tunnels:meta', { - '8080': { optionsHash: 'v1:quick' }, + '8080': { + optionsHash: 'v1:quick', + runtimeIdentityID: 'runtime-1', + runtimeIncarnationID: 'inc-1' + }, '8081': { optionsHash: 'v1:named:api', dnsRecordId: 'dns-id', @@ -432,15 +598,25 @@ describe('tunnel service > per-port serialization', () => { const { client } = makeClient(); const storage = makeStorage({ '8080': record }); const { tunnels: handler } = createTunnelsHandle({ - client: client as unknown as Parameters< - typeof createTunnelsHandle - >[0]['client'], + runRuntimeCall: ((operation, call) => + call( + client.tunnels as unknown as Parameters< + Parameters[0]['runRuntimeCall'] + >[1] extends (tunnels: infer U) => Promise + ? U + : never + )) as Parameters[0]['runRuntimeCall'], storage, logger: makeLogger() }); let resolveDestroy: () => void = () => {}; await storage.put('tunnels:meta', { - '8080': { optionsHash: 'v1:quick', tunnelRunId: 'run-pre' } + '8080': { + optionsHash: 'v1:quick', + runtimeIdentityID: 'runtime-1', + runtimeIncarnationID: 'inc-1', + tunnelRunId: 'run-pre' + } }); client.tunnels.stopTunnelRun.mockImplementation( () => @@ -475,20 +651,46 @@ describe('tunnel service > per-port serialization', () => { }); }); +const callbackRuntime = new RuntimeIdentity({ + id: 'runtime-1' as RuntimeIdentity['id'], + runtimeIncarnationID: 'inc-1' as RuntimeIdentity['runtimeIncarnationID'] +}); +const callbackRunID = 'run-callback'; + describe('tunnel service > handleTunnelExit', () => { it('clears the matching port from storage when the stored id matches', async () => { const record = makeRecord({ id: 'quick-exit0000exit0000', port: 8080 }); const { client } = makeClient(); const storage = makeStorage({ '8080': record }); + await storage.put('tunnels:meta', { + '8080': { + optionsHash: 'v1:quick', + runtimeIdentityID: callbackRuntime.id, + runtimeIncarnationID: callbackRuntime.runtimeIncarnationID, + tunnelRunId: callbackRunID + } + }); const { tunnels, handleTunnelExit } = createTunnelsHandle({ - client: client as unknown as Parameters< - typeof createTunnelsHandle - >[0]['client'], + runRuntimeCall: ((operation, call) => + call( + client.tunnels as unknown as Parameters< + Parameters[0]['runRuntimeCall'] + >[1] extends (tunnels: infer U) => Promise + ? U + : never + )) as Parameters[0]['runRuntimeCall'], storage, logger: makeLogger() }); - await handleTunnelExit(record.id, 8080, 0); + await handleTunnelExit( + record.id, + 8080, + 0, + callbackRunID, + callbackRuntime, + () => true + ); const final = await storage.get>('tunnels'); expect(final ?? {}).toEqual({}); @@ -524,18 +726,33 @@ describe('tunnel service > handleTunnelExit', () => { optionsHash: 'v1:named:api', dnsRecordId: 'kept-dns-id', accountId: 'acct-A', - zoneId: 'zone-A' + zoneId: 'zone-A', + runtimeIdentityID: callbackRuntime.id, + runtimeIncarnationID: callbackRuntime.runtimeIncarnationID, + tunnelRunId: callbackRunID } }); const { handleTunnelExit } = createTunnelsHandle({ - client: client as unknown as Parameters< - typeof createTunnelsHandle - >[0]['client'], + runRuntimeCall: ((operation, call) => + call( + client.tunnels as unknown as Parameters< + Parameters[0]['runRuntimeCall'] + >[1] extends (tunnels: infer U) => Promise + ? U + : never + )) as Parameters[0]['runRuntimeCall'], storage, logger: makeLogger() }); - await handleTunnelExit('tunnel-uuid-named', 8080, 0); + await handleTunnelExit( + 'tunnel-uuid-named', + 8080, + 0, + callbackRunID, + callbackRuntime, + () => true + ); // The public record is hidden; the next get() uses private metadata // to respawn cloudflared behind the same named hostname. @@ -601,25 +818,32 @@ describe('tunnel service > handleTunnelExit', () => { tunnelId: 'tunnel-uuid-named', name: 'api', hostname: 'api.example.com', + runtimeIdentityID: callbackRuntime.id, + runtimeIncarnationID: callbackRuntime.runtimeIncarnationID, tunnelRunId: 'run-current' } }); const { handleTunnelExit } = createTunnelsHandle({ - client: client as unknown as Parameters< - typeof createTunnelsHandle - >[0]['client'], + runRuntimeCall: ((operation, call) => + call( + client.tunnels as unknown as Parameters< + Parameters[0]['runRuntimeCall'] + >[1] extends (tunnels: infer U) => Promise + ? U + : never + )) as Parameters[0]['runRuntimeCall'], storage, logger: makeLogger() }); - await ( - handleTunnelExit as unknown as ( - id: string, - port: number, - exitCode: number | null, - tunnelRunId: string - ) => Promise - )('tunnel-uuid-named', 8080, 0, 'run-old'); + await handleTunnelExit( + 'tunnel-uuid-named', + 8080, + 0, + 'run-old', + callbackRuntime, + () => true + ); await expect(storage.get('tunnels')).resolves.toEqual({ '8080': current }); const meta = @@ -630,20 +854,76 @@ describe('tunnel service > handleTunnelExit', () => { expect(meta?.['8080']).not.toHaveProperty('needsRespawn'); }); + it('ignores an old-runtime callback even when tunnel id and run id match', async () => { + const record = makeRecord({ id: 'quick-current-runtime', port: 8080 }); + const { client } = makeClient(); + const storage = makeStorage({ '8080': record }); + await storage.put('tunnels:meta', { + '8080': { + optionsHash: 'v1:quick', + runtimeIdentityID: callbackRuntime.id, + runtimeIncarnationID: callbackRuntime.runtimeIncarnationID, + tunnelRunId: callbackRunID + } + }); + const replacement = new RuntimeIdentity({ + id: callbackRuntime.id, + runtimeIncarnationID: 'inc-2' as RuntimeIdentity['runtimeIncarnationID'] + }); + const { handleTunnelExit } = createTunnelsHandle({ + runRuntimeCall: ((operation, call) => + call( + client.tunnels as unknown as Parameters< + Parameters[0]['runRuntimeCall'] + >[1] extends (tunnels: infer U) => Promise + ? U + : never + )) as Parameters[0]['runRuntimeCall'], + getStoredRuntime: async () => replacement, + storage, + logger: makeLogger() + }); + + await handleTunnelExit( + record.id, + 8080, + 0, + callbackRunID, + callbackRuntime, + () => true + ); + + await expect(storage.get('tunnels')).resolves.toEqual({ + '8080': record + }); + }); + it('is a no-op when the stored id has been replaced (id-mismatch safety net)', async () => { const newer = makeRecord({ id: 'quick-newer000newer00', port: 8080 }); const { client } = makeClient(); const storage = makeStorage({ '8080': newer }); const { handleTunnelExit } = createTunnelsHandle({ - client: client as unknown as Parameters< - typeof createTunnelsHandle - >[0]['client'], + runRuntimeCall: ((operation, call) => + call( + client.tunnels as unknown as Parameters< + Parameters[0]['runRuntimeCall'] + >[1] extends (tunnels: infer U) => Promise + ? U + : never + )) as Parameters[0]['runRuntimeCall'], storage, logger: makeLogger() }); // Callback fires for an OLDER tunnel id that's no longer in storage. - await handleTunnelExit('quick-stale0000stale00', 8080, null); + await handleTunnelExit( + 'quick-stale0000stale00', + 8080, + null, + callbackRunID, + callbackRuntime, + () => true + ); // Storage is untouched. const final = await storage.get>('tunnels'); @@ -654,15 +934,27 @@ describe('tunnel service > handleTunnelExit', () => { const { client } = makeClient(); const storage = makeStorage(); const { handleTunnelExit } = createTunnelsHandle({ - client: client as unknown as Parameters< - typeof createTunnelsHandle - >[0]['client'], + runRuntimeCall: ((operation, call) => + call( + client.tunnels as unknown as Parameters< + Parameters[0]['runRuntimeCall'] + >[1] extends (tunnels: infer U) => Promise + ? U + : never + )) as Parameters[0]['runRuntimeCall'], storage, logger: makeLogger() }); await expect( - handleTunnelExit('quick-anything00000ok', 8080, null) + handleTunnelExit( + 'quick-anything00000ok', + 8080, + null, + callbackRunID, + callbackRuntime, + () => true + ) ).resolves.toBeUndefined(); // No write happened. expect((storage.put as StoragePutMock).mock.calls.length).toBe(0); @@ -684,11 +976,17 @@ describe('tunnel service > handleTunnelExit', () => { await new Promise((r) => setTimeout(r, 1)); } - // Fire an exit callback for some arbitrary id while get() is - // blocked. Without the lock, the callback would read the empty - // storage now (before get() writes) and observe nothing to clean - // up. With the lock, it must wait until get() releases. - const exitPromise = handleTunnelExit('quick-old', 8080, 0); + const request = client.tunnels.ensureTunnelRun.mock.calls[0]?.[0]; + if (!request) throw new Error('Expected tunnel provisioning request'); + let sessionCurrent = true; + const exitPromise = handleTunnelExit( + request.tunnelId, + 8080, + 0, + request.runId, + callbackRuntime, + () => sessionCurrent + ); await new Promise((r) => setTimeout(r, 5)); // The exit hook has not yet read storage — storage is empty so @@ -700,14 +998,14 @@ describe('tunnel service > handleTunnelExit', () => { }); expect(exitResolved).toBe(false); - // Let get() finish. + // Supersede the callback session before the lock becomes available. + sessionCurrent = false; resolveSpawn(makeRecord({ id: 'unused', port: 8080 })); const info = await getPromise; await exitPromise; - // The exit callback ran after the get() wrote storage, saw a - // different id ('quick-old' vs the spawned id), and no-op'd — - // the spawned record is still there. + // The callback matches the tunnel and run, but its session authority + // was revoked while it waited for the lock, so the record remains. const final = await storage.get>('tunnels'); expect(final).toEqual({ '8080': info }); }); @@ -725,9 +1023,14 @@ describe('tunnel service > handleTunnelExit', () => { } as unknown as TunnelsStorage; const { client } = makeClient(); const { handleTunnelExit } = createTunnelsHandle({ - client: client as unknown as Parameters< - typeof createTunnelsHandle - >[0]['client'], + runRuntimeCall: ((operation, call) => + call( + client.tunnels as unknown as Parameters< + Parameters[0]['runRuntimeCall'] + >[1] extends (tunnels: infer U) => Promise + ? U + : never + )) as Parameters[0]['runRuntimeCall'], storage: failingStorage, logger }); @@ -735,7 +1038,16 @@ describe('tunnel service > handleTunnelExit', () => { // The handler must surface the rejection to the caller so the // port-lock chain and any awaiter see it — silently swallowing // would hide the bug. - await expect(handleTunnelExit('quick-x', 8080, 0)).rejects.toThrow(/boom/); + await expect( + handleTunnelExit( + 'quick-x', + 8080, + 0, + callbackRunID, + callbackRuntime, + () => true + ) + ).rejects.toThrow(/boom/); // And a canonical event with outcome: 'error' must have been logged. const errorCalls = (logger.error as LogMock).mock.calls; diff --git a/packages/sandbox/tests/tunnels/tunnel-service-quick.test.ts b/packages/sandbox/tests/tunnels/tunnel-service-quick.test.ts index ccd54de99..01bac650a 100644 --- a/packages/sandbox/tests/tunnels/tunnel-service-quick.test.ts +++ b/packages/sandbox/tests/tunnels/tunnel-service-quick.test.ts @@ -1,3 +1,7 @@ +import { + completeTunnelServiceHost, + type TestTunnelServiceHost +} from './helpers'; /** * SDK tunnel service unit tests. * @@ -15,12 +19,12 @@ import type { } from '@repo/shared'; import type { Mock } from 'vitest'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { RuntimeIdentityInactiveError } from '../../src/current-runtime-identity'; import { ErrorCode, RPCTransportError } from '../../src/errors'; +import { RuntimeIdentityInactiveError } from '../../src/runtime/types'; import { SandboxLifetimeChangedError } from '../../src/sandbox-lifetime'; import { SandboxSecurityError } from '../../src/security'; import { - createTunnelsHandle, + createTunnelsHandle as createRuntimeTunnelsHandle, pruneTunnelsForRestart, type TunnelsHandler, type TunnelsStorage @@ -139,18 +143,60 @@ type TunnelsHost = Parameters[0]; function makeHandler(extra: Partial = {}) { const { client } = makeClient(); + const runtimeCalls: Array<{ operation: string; control: MockTunnelsClient }> = + []; const { storage: providedStorage, ...rest } = extra; const storage = (providedStorage as TunnelsStorage | undefined) ?? makeStorage(); + const runRuntimeCall = (async (operation, call) => { + const control = { + ensureTunnelRun: client.tunnels.ensureTunnelRun, + stopTunnelRun: client.tunnels.stopTunnelRun + }; + runtimeCalls.push({ operation, control }); + return await call( + control as unknown as Parameters< + Parameters[1] + >[0] + ); + }) as TunnelsHost['runRuntimeCall']; const { tunnels, handleTunnelExit } = createTunnelsHandle({ - client: client as unknown as TunnelsHost['client'], + runProvision: (call) => + runRuntimeCall('tunnel.provision', (control) => + call({ + runtime: { + id: 'runtime-1', + runtimeIncarnationID: 'inc-1' + } as Parameters< + Parameters>[0] + >[0]['runtime'], + tunnels: control, + retain: () => ({ release: () => {} }) + }) + ), + runRuntimeCall, + getStoredRuntime: async () => + ({ + id: 'runtime-1', + runtimeIncarnationID: 'inc-1' + }) as Awaited>>, storage, logger: makeLogger(), ...rest - } as unknown as TunnelsHost); - return { client, storage, handler: tunnels, tunnels, handleTunnelExit }; + }); + return { + client, + storage, + handler: tunnels, + tunnels, + handleTunnelExit, + runtimeCalls + }; } +const createTunnelsHandle = (host: TestTunnelServiceHost) => + createRuntimeTunnelsHandle(completeTunnelServiceHost(host)); + describe('tunnel service > get', () => { let warn: ReturnType; beforeEach(() => { @@ -194,37 +240,42 @@ describe('tunnel service > get', () => { const metaPut = putCalls.find(([key]) => key === 'tunnels:meta'); expect(tunnelsPut?.[1]).toEqual({ '8080': info }); expect(metaPut?.[1]).toEqual({ - '8080': { optionsHash: 'v1:quick', tunnelRunId: request.runId } + '8080': { + optionsHash: 'v1:quick', + runtimeIdentityID: 'runtime-1', + runtimeIncarnationID: 'inc-1', + tunnelRunId: request.runId + } }); }); - it('replays the same quick run request when the first call loses RPC transport', async () => { + it('surfaces RPC transport loss without replaying the tunnel run', async () => { const { client, handler } = makeHandler(); - client.tunnels.ensureTunnelRun - .mockRejectedValueOnce(createDisposedRPCError()) - .mockImplementationOnce(async (request) => ({ - started: false, - run: { - ...request, - url: 'https://stub.trycloudflare.com', - hostname: 'stub.trycloudflare.com', - startedAt: '2026-05-13T00:00:00.000Z' - } - })); + const error = createDisposedRPCError(); + client.tunnels.ensureTunnelRun.mockRejectedValueOnce(error); - const info = await handler.get(8080); + await expect(handler.get(8080)).rejects.toBe(error); - expect(client.tunnels.ensureTunnelRun).toHaveBeenCalledTimes(2); - const first = client.tunnels.ensureTunnelRun.mock.calls[0][0]; - const second = client.tunnels.ensureTunnelRun.mock.calls[1][0]; - expect(second).toEqual(first); - expect(info).toEqual({ - id: first.tunnelId, - port: 8080, - url: 'https://stub.trycloudflare.com', - hostname: 'stub.trycloudflare.com', - createdAt: '2026-05-13T00:00:00.000Z' - }); + expect(client.tunnels.ensureTunnelRun).toHaveBeenCalledTimes(1); + }); + + it('uses fresh runtime callback controls for sequential tunnel runs', async () => { + const { client, handler, runtimeCalls } = makeHandler(); + mockEnsureQuick(client.tunnels, (tunnelId, port, runId) => + makeRecord({ id: tunnelId, port, createdAt: runId }) + ); + + await handler.get(8080); + await handler.destroy(8080); + await handler.get(8080); + + expect(runtimeCalls.map((call) => call.operation)).toEqual([ + 'tunnel.provision', + 'tunnel.destroy', + 'tunnel.provision' + ]); + expect(runtimeCalls[0].control).not.toBe(runtimeCalls[1].control); + expect(runtimeCalls[1].control).not.toBe(runtimeCalls[2].control); }); it('records runtime and sandbox lifetime ownership for quick tunnels', async () => { @@ -261,12 +312,21 @@ describe('tunnel service > get', () => { const { client } = makeClient(); const storage = makeStorage({ '8080': record }); await storage.put('tunnels:meta', { - '8080': { optionsHash: 'v1:quick', runtimeIdentityID: 'runtime-1' } + '8080': { + optionsHash: 'v1:quick', + runtimeIdentityID: 'runtime-1', + runtimeIncarnationID: 'inc-1' + } }); const { tunnels: handler } = createTunnelsHandle({ - client: client as unknown as Parameters< - typeof createTunnelsHandle - >[0]['client'], + runRuntimeCall: ((operation, call) => + call( + client.tunnels as unknown as Parameters< + Parameters[0]['runRuntimeCall'] + >[1] extends (tunnels: infer U) => Promise + ? U + : never + )) as Parameters[0]['runRuntimeCall'], storage, logger: makeLogger(), ...makeFences() @@ -301,16 +361,18 @@ describe('tunnel service > get', () => { await expect(storage.get('tunnels')).resolves.toEqual({ '8080': fresh }); }); - it('captures runtime after spawn when no active runtime exists before RPC', async () => { + it('persists the admitted lease runtime instead of a discovery snapshot', async () => { const getRuntime = vi .fn() .mockResolvedValueOnce(null) .mockResolvedValue({ id: 'runtime-after-rpc' }); - const markStarted = vi.fn(async () => ({ id: 'runtime-created-early' })); + const unexpectedPublisher = vi.fn(async () => ({ + id: 'runtime-created-early' + })); const { client, storage, handler } = makeHandler({ currentRuntime: { get: getRuntime, - markStarted, + unexpectedPublisher, assertActive: vi.fn(async () => {}) }, currentLifetime: makeFences().currentLifetime @@ -322,81 +384,47 @@ describe('tunnel service > get', () => { await handler.get(8080); expect(client.tunnels.ensureTunnelRun).toHaveBeenCalledTimes(1); - expect(markStarted).not.toHaveBeenCalled(); + expect(unexpectedPublisher).not.toHaveBeenCalled(); const meta = await storage.get>>( 'tunnels:meta' ); - expect(meta?.['8080']?.runtimeIdentityID).toBe('runtime-after-rpc'); + expect(meta?.['8080']?.runtimeIdentityID).toBe('runtime-1'); + expect(meta?.['8080']?.runtimeIncarnationID).toBe('inc-1'); }); - it('does not commit when runtime is unavailable after spawn', async () => { + it('does not use a separate runtime discovery after admission', async () => { + const getRuntime = vi.fn(async () => null); const { client, storage, handler } = makeHandler({ - currentRuntime: { - get: vi.fn(async () => null), - assertActive: vi.fn(async () => {}) - }, + currentRuntime: { get: getRuntime }, currentLifetime: makeFences().currentLifetime } as unknown as Partial); mockEnsureQuick(client.tunnels, async (id, port) => makeRecord({ id, port }) ); - let caught: unknown; - try { - await handler.get(8080); - } catch (error) { - caught = error; - } - - expect(caught).toMatchObject({ name: 'OperationInterruptedError' }); - expect((caught as { code?: unknown }).code).toBe( - ErrorCode.OPERATION_INTERRUPTED - ); - expect((caught as { context?: unknown }).context).toMatchObject({ - operation: 'tunnel.get', - reason: 'recovery_exhausted', - admitted: true, - retryable: true - }); - await expect(storage.get('tunnels')).resolves.toBeUndefined(); + await expect(handler.get(8080)).resolves.toMatchObject({ port: 8080 }); + expect(getRuntime).not.toHaveBeenCalled(); + await expect(storage.get('tunnels')).resolves.toBeDefined(); }); - it('recovers quick get() after runtime replacement during spawn', async () => { - const second = makeRecord({ - id: 'quick-second00second', - port: 8080, - url: 'https://second.trycloudflare.com', - hostname: 'second.trycloudflare.com' - }); + it('surfaces runtime replacement during spawn without recovery/retry', async () => { const { client, handler } = makeHandler( makeFences({ currentRuntime: { - // First attempt's post-spawn fence reports a replaced runtime; - // the retry runs clean. - assertActive: vi - .fn() - .mockResolvedValueOnce(undefined) - .mockRejectedValueOnce(new RuntimeIdentityInactiveError()) - .mockResolvedValue(undefined) + assertActive: vi.fn(async () => { + throw new RuntimeIdentityInactiveError(); + }) } }) ); - client.tunnels.ensureTunnelRun - .mockImplementationOnce(async (request) => - ensureQuickResult(request, makeRecord({ id: 'quick-first000first' })) - ) - .mockImplementationOnce(async (request) => - ensureQuickResult(request, second) - ); - - const info = await handler.get(8080); + client.tunnels.ensureTunnelRun.mockImplementationOnce(async (request) => { + throw new RuntimeIdentityInactiveError(); + }); - expect(info).toEqual(second); - expect(client.tunnels.ensureTunnelRun).toHaveBeenCalledTimes(2); - const firstRequest = client.tunnels.ensureTunnelRun.mock.calls[0][0]; - const secondRequest = client.tunnels.ensureTunnelRun.mock.calls[1][0]; - expect(secondRequest).toEqual(firstRequest); + await expect(handler.get(8080)).rejects.toBeInstanceOf( + RuntimeIdentityInactiveError + ); }); it('surfaces sandbox lifetime changes as non-retryable operation interruptions', async () => { @@ -425,41 +453,21 @@ describe('tunnel service > get', () => { }); }); - it('bounds runtime-replacement recovery and never commits a partial record', async () => { - // Every post-spawn fence reports a replaced runtime, so recovery can - // never converge. The operation surfaces recovery_exhausted and - // leaves storage empty so no orphaned tunnel record persists. - let assertCalls = 0; + it('does not consult transitional runtime assertions', async () => { + const assertActive = vi.fn(async () => { + throw new RuntimeIdentityInactiveError(); + }); const { client, storage, handler } = makeHandler( - makeFences({ - currentRuntime: { - assertActive: vi.fn(async () => { - assertCalls += 1; - if (assertCalls % 2 === 0) { - throw new RuntimeIdentityInactiveError(); - } - }) - } - }) + makeFences({ currentRuntime: { assertActive } }) ); mockEnsureQuick(client.tunnels, async (id, port) => makeRecord({ id, port }) ); - await expect(handler.get(8080)).rejects.toMatchObject({ - name: 'OperationInterruptedError', - code: ErrorCode.OPERATION_INTERRUPTED, - context: expect.objectContaining({ - reason: 'recovery_exhausted', - operation: 'tunnel.get', - retryable: true, - admitted: true, - recoveryAttempts: 2, - maxRecoveryAttempts: 2 - }) - }); - expect(await storage.get('tunnels')).toBeUndefined(); - expect(await storage.get('tunnels:meta')).toBeUndefined(); + await expect(handler.get(8080)).resolves.toMatchObject({ port: 8080 }); + expect(assertActive).not.toHaveBeenCalled(); + expect(await storage.get('tunnels')).toBeDefined(); + expect(await storage.get('tunnels:meta')).toBeDefined(); }); it('refreshes a quick cache hit owned by an old runtime', async () => { @@ -473,16 +481,15 @@ describe('tunnel service > get', () => { '8080': makeRecord({ id: 'quick-stale0000stale' }) }); await (storage.put as StoragePutMock)('tunnels:meta', { - '8080': { optionsHash: 'v1:quick', runtimeIdentityID: 'runtime-old' } + '8080': { + optionsHash: 'v1:quick', + runtimeIdentityID: 'runtime-1', + runtimeIncarnationID: 'old-incarnation' + } }); const { client, handler } = makeHandler({ storage, - ...makeFences({ - currentRuntime: { - get: vi.fn(async () => ({ id: 'runtime-new' })), - markStarted: vi.fn(async () => ({ id: 'runtime-new' })) - } - }) + ...makeFences() }); mockEnsureQuick(client.tunnels, async () => fresh); @@ -497,9 +504,14 @@ describe('tunnel service > get', () => { const { client } = makeClient(); const storage = makeStorage({ '8080': record }); const { tunnels: handler } = createTunnelsHandle({ - client: client as unknown as Parameters< - typeof createTunnelsHandle - >[0]['client'], + runRuntimeCall: ((operation, call) => + call( + client.tunnels as unknown as Parameters< + Parameters[0]['runRuntimeCall'] + >[1] extends (tunnels: infer U) => Promise + ? U + : never + )) as Parameters[0]['runRuntimeCall'], storage, logger: makeLogger() }); diff --git a/packages/sandbox/tests/tunnels/tunnel-service.test.ts b/packages/sandbox/tests/tunnels/tunnel-service.test.ts index 314ca0da5..794c6b663 100644 --- a/packages/sandbox/tests/tunnels/tunnel-service.test.ts +++ b/packages/sandbox/tests/tunnels/tunnel-service.test.ts @@ -1,23 +1,40 @@ import type { TunnelInfo } from '@repo/shared'; import { describe, expect, it, vi } from 'vitest'; import { - createTunnelsHandle, + createTunnelsHandle as createRuntimeTunnelsHandle, type TunnelsStorage } from '../../src/tunnels/rpc-target'; import { TunnelService } from '../../src/tunnels/tunnel-service'; -import { makeLogger, makeStorage } from './helpers'; +import { + completeTunnelServiceHost, + makeLogger, + makeStorage, + type TestTunnelServiceHost +} from './helpers'; + +type TunnelsHost = Parameters[0]; + +function makeRuntimeCall(): TunnelsHost['runRuntimeCall'] { + const tunnels = { + ensureTunnelRun: vi.fn(), + stopTunnelRun: vi.fn() + }; + return (_operation, call) => + call( + tunnels as unknown as Parameters< + Parameters[1] + >[0] + ); +} function makeService(storage: TunnelsStorage): TunnelService { - return new TunnelService({ - client: { - tunnels: { - ensureTunnelRun: vi.fn(), - stopTunnelRun: vi.fn() - } - }, - storage, - logger: makeLogger() - }); + return new TunnelService( + completeTunnelServiceHost({ + runRuntimeCall: makeRuntimeCall(), + storage, + logger: makeLogger() + }) + ); } function quickTunnel(): TunnelInfo { @@ -69,38 +86,22 @@ async function expectRestartReconciled(storage: TunnelsStorage): Promise { }); } +const createTunnelsHandle = (host: TestTunnelServiceHost) => + createRuntimeTunnelsHandle(completeTunnelServiceHost(host)); + describe('TunnelService', () => { - it.each([ - [ - 'runtime start', - async (handle: ReturnType) => - handle.onRuntimeStart() - ], - [ - 'runtime stop', - async (handle: ReturnType) => - handle.onRuntimeStop() - ] - ])( - 'exposes %s reconciliation through the handle factory', - async (_label, run) => { - const storage = makeRestartStorage(); - const handle = createTunnelsHandle({ - client: { - tunnels: { - ensureTunnelRun: vi.fn(), - stopTunnelRun: vi.fn() - } - }, - storage, - logger: makeLogger() - }); - - await run(handle); - - await expectRestartReconciled(storage); - } - ); + it('exposes runtime-stop reconciliation through the handle factory', async () => { + const storage = makeRestartStorage(); + const handle = createTunnelsHandle({ + runRuntimeCall: makeRuntimeCall(), + storage, + logger: makeLogger() + }); + + await handle.onRuntimeStop(); + + await expectRestartReconciled(storage); + }); it('preserves named metadata while reconciling a runtime restart', async () => { const storage = makeRestartStorage({ @@ -110,7 +111,7 @@ describe('TunnelService', () => { }); const service = makeService(storage); - await service.onRuntimeStart(); + await service.onRuntimeStop(); await expect(storage.get('tunnels')).resolves.toEqual({}); await expect(storage.get('tunnels:meta')).resolves.toEqual({ diff --git a/packages/sandbox/tests/warm-pool.test.ts b/packages/sandbox/tests/warm-pool.test.ts new file mode 100644 index 000000000..d9670cce1 --- /dev/null +++ b/packages/sandbox/tests/warm-pool.test.ts @@ -0,0 +1,110 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('cloudflare:workers', () => ({ + DurableObject: class { + protected readonly ctx: DurableObjectState; + protected readonly env: Env; + + constructor(ctx: DurableObjectState, env: Env) { + this.ctx = ctx; + this.env = env; + } + } +})); + +import { WarmPool } from '../src/bridge/warm-pool'; + +describe('WarmPool', () => { + let storageData: Map; + let startAndWaitForPorts: ReturnType; + let start: ReturnType; + let stop: ReturnType; + let destroy: ReturnType; + let getState: ReturnType; + let renewActivityTimeout: ReturnType; + let ctx: DurableObjectState; + let env: { + Sandbox: { + idFromName: ReturnType; + get: ReturnType; + }; + }; + + beforeEach(() => { + storageData = new Map(); + startAndWaitForPorts = vi.fn(async () => undefined); + start = vi.fn(async () => undefined); + stop = vi.fn(async () => undefined); + destroy = vi.fn(async () => undefined); + getState = vi.fn(async () => ({ status: 'healthy' })); + renewActivityTimeout = vi.fn(); + ctx = { + storage: { + get: vi.fn(async (key: string) => storageData.get(key)), + put: vi.fn(async (key: string, value: unknown) => { + storageData.set(key, value); + }), + delete: vi.fn(async (key: string) => { + storageData.delete(key); + }), + getAlarm: vi.fn(async () => Date.now() + 10_000), + setAlarm: vi.fn(async () => undefined) + } + } as unknown as DurableObjectState; + env = { + Sandbox: { + idFromName: vi.fn((name: string) => ({ name })), + get: vi.fn(() => ({ + start, + startAndWaitForPorts, + stop, + destroy, + getState, + renewActivityTimeout + })) + } + }; + }); + + it('starts warm-pool containers through public lifecycle authority', async () => { + const pool = new WarmPool( + ctx, + env as unknown as ConstructorParameters[1] + ); + + const containerId = await pool.getContainer('sandbox-a'); + + expect(containerId).toEqual(expect.any(String)); + expect(startAndWaitForPorts).toHaveBeenCalledOnce(); + expect(start).not.toHaveBeenCalled(); + }); + + it('destroys discarded prewarmed containers before releasing tracking', async () => { + storageData.set('warmContainers', new Set(['warm-a'])); + const pool = new WarmPool( + ctx, + env as unknown as ConstructorParameters[1] + ); + + await pool.shutdownPrewarmed(); + + expect(destroy).toHaveBeenCalledOnce(); + expect(stop).not.toHaveBeenCalled(); + await expect(pool.getStats()).resolves.toMatchObject({ warm: 0 }); + }); + + it('destroys excess prewarmed containers during scale-down', async () => { + storageData.set('warmContainers', new Set(['warm-a', 'warm-b'])); + storageData.set('config', { warmTarget: 1, refreshInterval: 10_000 }); + const pool = new WarmPool( + ctx, + env as unknown as ConstructorParameters[1] + ); + + await pool.alarm(); + + expect(destroy).toHaveBeenCalledOnce(); + expect(stop).not.toHaveBeenCalled(); + await expect(pool.getStats()).resolves.toMatchObject({ warm: 1 }); + }); +}); diff --git a/packages/sandbox/tsconfig.json b/packages/sandbox/tsconfig.json index 785c4b51a..22b251224 100644 --- a/packages/sandbox/tsconfig.json +++ b/packages/sandbox/tsconfig.json @@ -23,6 +23,8 @@ "include": [ "src/**/*.ts", "tests/**/*.ts", + "tests-harness/**/*.ts", + "vitest.harness.config.ts", "../../extensions/*/src/**/*.ts", "../../extensions/*/tests/**/*.ts" ], diff --git a/packages/sandbox/vitest.harness.config.ts b/packages/sandbox/vitest.harness.config.ts new file mode 100644 index 000000000..7b97a2128 --- /dev/null +++ b/packages/sandbox/vitest.harness.config.ts @@ -0,0 +1,10 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + fileParallelism: false, + hookTimeout: 180_000, + include: ['tests-harness/**/*.test.ts'], + testTimeout: 180_000 + } +}); diff --git a/packages/shared/src/errors/codes.ts b/packages/shared/src/errors/codes.ts index 8ca0e12c7..b1f502e17 100644 --- a/packages/shared/src/errors/codes.ts +++ b/packages/shared/src/errors/codes.ts @@ -52,6 +52,7 @@ export const ErrorCode = { // Process Errors (408/409/499) STALE_PROCESS_HANDLE: 'STALE_PROCESS_HANDLE', + STALE_TERMINAL_HANDLE: 'STALE_TERMINAL_HANDLE', PROCESS_WAIT_TIMEOUT: 'PROCESS_WAIT_TIMEOUT', PROCESS_ABORTED: 'PROCESS_ABORTED', @@ -153,6 +154,7 @@ export const ErrorCode = { // Container Availability Errors (503) CONTAINER_UNAVAILABLE: 'CONTAINER_UNAVAILABLE', + CONTROL_PROTOCOL_INCOMPATIBLE: 'CONTROL_PROTOCOL_INCOMPATIBLE', // Operation Lifecycle Errors (409) OPERATION_INTERRUPTED: 'OPERATION_INTERRUPTED', diff --git a/packages/shared/src/errors/contexts.ts b/packages/shared/src/errors/contexts.ts index 6269357a9..5e5f79536 100644 --- a/packages/shared/src/errors/contexts.ts +++ b/packages/shared/src/errors/contexts.ts @@ -124,6 +124,11 @@ export interface TerminalControlErrorContext { }; } +export interface StaleTerminalHandleContext { + terminalId: string; + operation: string; +} + /** * Process readiness error contexts */ @@ -335,7 +340,6 @@ export interface OperationInterruptedContext { phase?: string; admitted: true | 'unknown'; retryable: boolean; - effect?: 'none' | 'unknown'; operationId?: string; operationKey?: string; idempotencyKey?: string; diff --git a/packages/shared/src/errors/index.ts b/packages/shared/src/errors/index.ts index 1b3d247f9..16d84c249 100644 --- a/packages/shared/src/errors/index.ts +++ b/packages/shared/src/errors/index.ts @@ -79,6 +79,7 @@ export type { RPCTransportContext, RPCTransportErrorKind, StaleProcessHandleContext, + StaleTerminalHandleContext, TerminalControlErrorContext, TerminalNotFoundContext, ValidationFailedContext diff --git a/packages/shared/src/errors/status-map.ts b/packages/shared/src/errors/status-map.ts index 5b213eeea..e61d2e9a5 100644 --- a/packages/shared/src/errors/status-map.ts +++ b/packages/shared/src/errors/status-map.ts @@ -51,6 +51,7 @@ export const ERROR_STATUS_MAP: Record = { [ErrorCode.RESOURCE_BUSY]: 409, [ErrorCode.OPERATION_INTERRUPTED]: 409, [ErrorCode.STALE_PROCESS_HANDLE]: 409, + [ErrorCode.STALE_TERMINAL_HANDLE]: 409, [ErrorCode.PROCESS_EXITED_BEFORE_LOG]: 409, // 410 Gone @@ -77,6 +78,7 @@ export const ERROR_STATUS_MAP: Record = { [ErrorCode.INTERPRETER_NOT_READY]: 503, [ErrorCode.OPENCODE_STARTUP_FAILED]: 503, [ErrorCode.CONTAINER_UNAVAILABLE]: 503, + [ErrorCode.CONTROL_PROTOCOL_INCOMPATIBLE]: 503, [ErrorCode.RPC_TRANSPORT_ERROR]: 503, // 408 Request Timeout diff --git a/packages/shared/src/errors/suggestions.ts b/packages/shared/src/errors/suggestions.ts index 6db701e72..ee13c4f13 100644 --- a/packages/shared/src/errors/suggestions.ts +++ b/packages/shared/src/errors/suggestions.ts @@ -36,6 +36,9 @@ export function getSuggestion( case ErrorCode.STALE_PROCESS_HANDLE: return 'The process handle refers to an older runtime process. Reacquire the process by ID before retrying the operation'; + case ErrorCode.STALE_TERMINAL_HANDLE: + return 'The terminal handle refers to an older runtime terminal. Reacquire the terminal by ID before retrying the operation'; + case ErrorCode.PROCESS_WAIT_TIMEOUT: return 'The process did not complete before the timeout. Continue streaming logs or kill the process if it is no longer needed'; @@ -75,6 +78,9 @@ export function getSuggestion( case ErrorCode.GIT_BRANCH_NOT_FOUND: return `Branch "${context.branch}" does not exist in the repository. Check the branch name or use the default branch`; + case ErrorCode.CONTROL_PROTOCOL_INCOMPATIBLE: + return 'Refresh the sandbox control session metadata and reconnect with a compatible control protocol'; + case ErrorCode.INTERPRETER_NOT_READY: return context.retryAfter ? `Code interpreter is starting up. Retry after ${context.retryAfter} seconds` diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 065d599da..8755e90cc 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -117,6 +117,7 @@ export type { ReadFileOptions, ReadFileStreamOptions, RemoveMountDirectoryRequest, + RuntimeMetadata, S3FSOptionValue, SandboxAPI, SandboxBackupAPI, diff --git a/packages/shared/src/rpc-types.ts b/packages/shared/src/rpc-types.ts index 8dabb9098..7335839ac 100644 --- a/packages/shared/src/rpc-types.ts +++ b/packages/shared/src/rpc-types.ts @@ -140,9 +140,18 @@ export interface SandboxPortsAPI { ): Promise; } +export interface RuntimeMetadata { + runtimeIncarnationID: string; + sandboxVersion: string; + controlProtocolVersion: 1; +} + export interface SandboxUtilsAPI { ping(): Promise; - getVersion(): Promise; + getRuntimeMetadata(): Promise; + activateControlSession( + expectedRuntimeIncarnationID: string + ): Promise; } export interface CreateWorkspaceArchiveRequest { diff --git a/sites/sandbox/package.json b/sites/sandbox/package.json index d4eb5c90d..3166f765a 100644 --- a/sites/sandbox/package.json +++ b/sites/sandbox/package.json @@ -14,7 +14,7 @@ "dependencies": { "@astrojs/check": "^0.9.5", "@astrojs/react": "^5.0.4", - "@cloudflare/vite-plugin": "^1.42.0", + "@cloudflare/vite-plugin": "^1.46.0", "@tailwindcss/vite": "^4.1.17", "astro": "^6.4.6", "clsx": "^2.1.1", diff --git a/tests/e2e/backup-workflow.test.ts b/tests/e2e/backup-workflow.test.ts index 1105b8c72..d150ad23a 100644 --- a/tests/e2e/backup-workflow.test.ts +++ b/tests/e2e/backup-workflow.test.ts @@ -26,6 +26,10 @@ interface ErrorResponse { error?: string; } +function shellCommand(parts: string[], separator = ' && '): string[] { + return ['/bin/bash', '-lc', parts.join(separator)]; +} + /** * Helper to clean up a directory that may have a FUSE overlay mount. * Unmounts first (silently ignoring errors if not mounted), then removes. @@ -203,7 +207,7 @@ describe('Backup Workflow E2E', () => { await cleanupDir(workerUrl, headers, TEST_DIR); }, 60000); - test('should recover restore after configured lifecycle fault', async () => { + test('should report an interrupted restore without retrying it', async () => { if (!backupBucketAvailable) return; const faultDir = `/workspace/backup-fault-${crypto.randomUUID().slice(0, 8)}`; @@ -261,22 +265,22 @@ describe('Backup Workflow E2E', () => { localBucket: true }) }); - if (!restoreResponse.ok) { - throw new Error(`restore failed: ${await restoreResponse.text()}`); - } - const restoreResult = (await restoreResponse.json()) as RestoreResponse; - expect(restoreResult.success).toBe(true); + expect(restoreResponse.status).toBe(409); + const restoreError = (await restoreResponse.json()) as ErrorResponse; + expect(restoreError.code).toBe('OPERATION_INTERRUPTED'); + // Do not retry restore: its completion is unknown after transport loss. + // A separate read proves the replacement runtime accepts new work. const verifyResponse = await fetch(`${workerUrl}/api/execute`, { method: 'POST', headers, body: JSON.stringify({ - command: ['/bin/bash', '-lc', `cat ${faultDir}/${faultFile}`] + command: ['/bin/bash', '-lc', 'printf runtime-recovered'] }) }); expect(verifyResponse.ok).toBe(true); const verifyResult = (await verifyResponse.json()) as ExecuteResponse; - expect(verifyResult.stdout).toContain(faultContent); + expect(verifyResult.stdout).toBe('runtime-recovered'); await cleanupDir(workerUrl, headers, faultDir); }, 120000); @@ -293,13 +297,13 @@ describe('Backup Workflow E2E', () => { method: 'POST', headers, body: JSON.stringify({ - command: [ + command: shellCommand([ `git init ${REPO_DIR}`, `mkdir -p ${TEST_DIR}/node_modules`, `printf 'app/node_modules/\n' > ${REPO_DIR}/.gitignore`, `echo "keep" > ${TEST_DIR}/keep.txt`, `echo "exclude-me" > ${TEST_DIR}/node_modules/a.txt` - ].join(' && ') + ]) }) }); expect(setupResponse.ok).toBe(true); @@ -337,10 +341,13 @@ describe('Backup Workflow E2E', () => { method: 'POST', headers, body: JSON.stringify({ - command: [ - `test -f ${TEST_DIR}/keep.txt && echo keep:yes || echo keep:no`, - `test -e ${TEST_DIR}/node_modules/a.txt && echo excluded:no || echo excluded:yes` - ].join('; ') + command: shellCommand( + [ + `test -f ${TEST_DIR}/keep.txt && echo keep:yes || echo keep:no`, + `test -e ${TEST_DIR}/node_modules/a.txt && echo excluded:no || echo excluded:yes` + ], + '; ' + ) }) }); const verifyResult = (await verifyResponse.json()) as ExecuteResponse; @@ -361,13 +368,13 @@ describe('Backup Workflow E2E', () => { method: 'POST', headers, body: JSON.stringify({ - command: [ + command: shellCommand([ `git init ${REPO_DIR}`, `mkdir -p ${TEST_DIR}/dist`, `printf 'app/dist/\n' > ${REPO_DIR}/.gitignore`, `echo "keep" > ${TEST_DIR}/keep.txt`, `echo "bundle" > ${TEST_DIR}/dist/app.js` - ].join(' && ') + ]) }) }); expect(setupResponse.ok).toBe(true); @@ -406,10 +413,13 @@ describe('Backup Workflow E2E', () => { method: 'POST', headers, body: JSON.stringify({ - command: [ - `test -f ${TEST_DIR}/keep.txt && echo keep:yes || echo keep:no`, - `test -e ${TEST_DIR}/dist/app.js && echo dist:yes || echo dist:no` - ].join('; ') + command: shellCommand( + [ + `test -f ${TEST_DIR}/keep.txt && echo keep:yes || echo keep:no`, + `test -e ${TEST_DIR}/dist/app.js && echo dist:yes || echo dist:no` + ], + '; ' + ) }) }); const verifyResult = (await verifyResponse.json()) as ExecuteResponse; @@ -429,11 +439,11 @@ describe('Backup Workflow E2E', () => { method: 'POST', headers, body: JSON.stringify({ - command: [ + command: shellCommand([ `mkdir -p ${TEST_DIR}/dist`, `echo "keep" > ${TEST_DIR}/keep.txt`, `echo "bundle" > ${TEST_DIR}/dist/app.js` - ].join(' && ') + ]) }) }); expect(setupResponse.ok).toBe(true); @@ -469,10 +479,13 @@ describe('Backup Workflow E2E', () => { method: 'POST', headers, body: JSON.stringify({ - command: [ - `test -f ${TEST_DIR}/keep.txt && echo keep:yes || echo keep:no`, - `test -e ${TEST_DIR}/dist/app.js && echo dist:yes || echo dist:no` - ].join('; ') + command: shellCommand( + [ + `test -f ${TEST_DIR}/keep.txt && echo keep:yes || echo keep:no`, + `test -e ${TEST_DIR}/dist/app.js && echo dist:yes || echo dist:no` + ], + '; ' + ) }) }); const verifyResult = (await verifyResponse.json()) as ExecuteResponse; @@ -493,7 +506,7 @@ describe('Backup Workflow E2E', () => { method: 'POST', headers, body: JSON.stringify({ - command: [ + command: shellCommand([ `git init ${REPO_DIR}`, `mkdir -p ${TEST_DIR}/dist`, `printf '*.log\n' > ${REPO_DIR}/.gitignore`, @@ -501,7 +514,7 @@ describe('Backup Workflow E2E', () => { `echo "keep" > ${TEST_DIR}/keep.txt`, `echo "bundle" > ${TEST_DIR}/dist/app.js`, `echo "ignored" > ${TEST_DIR}/server.log` - ].join(' && ') + ]) }) }); expect(setupResponse.ok).toBe(true); @@ -537,11 +550,14 @@ describe('Backup Workflow E2E', () => { method: 'POST', headers, body: JSON.stringify({ - command: [ - `test -f ${TEST_DIR}/keep.txt && echo keep:yes || echo keep:no`, - `test -e ${TEST_DIR}/dist/app.js && echo dist:no || echo dist:yes`, - `test -e ${TEST_DIR}/server.log && echo log:no || echo log:yes` - ].join('; ') + command: shellCommand( + [ + `test -f ${TEST_DIR}/keep.txt && echo keep:yes || echo keep:no`, + `test -e ${TEST_DIR}/dist/app.js && echo dist:no || echo dist:yes`, + `test -e ${TEST_DIR}/server.log && echo log:no || echo log:yes` + ], + '; ' + ) }) }); const verifyResult = (await verifyResponse.json()) as ExecuteResponse; @@ -565,12 +581,12 @@ describe('Backup Workflow E2E', () => { method: 'POST', headers, body: JSON.stringify({ - command: [ + command: shellCommand([ `mkdir -p ${PROJECT_DIR}/src/utils ${PROJECT_DIR}/config`, `echo 'console.log("main")' > ${PROJECT_DIR}/src/index.js`, `echo 'export const VERSION = "1.0.0"' > ${PROJECT_DIR}/src/utils/version.js`, `echo '{"port": 3000}' > ${PROJECT_DIR}/config/settings.json` - ].join(' && ') + ]) }) }); expect(setupResponse.ok).toBe(true); @@ -949,18 +965,19 @@ describe('Backup Workflow E2E', () => { const TEST_DIR = `/workspace/special-chars-${crypto.randomUUID().slice(0, 8)}`; // Create files with special characters in names - await fetch(`${workerUrl}/api/execute`, { + const setupResponse = await fetch(`${workerUrl}/api/execute`, { method: 'POST', headers, body: JSON.stringify({ - command: [ + command: shellCommand([ `mkdir -p "${TEST_DIR}"`, `echo "space content" > "${TEST_DIR}/file with spaces.txt"`, `echo "emoji content" > "${TEST_DIR}/emoji-🎉-file.txt"`, `echo "unicode content" > "${TEST_DIR}/日本語ファイル.txt"` - ].join(' && ') + ]) }) }); + expect(setupResponse.ok).toBe(true); // Create backup const backupResponse = await fetch(`${workerUrl}/api/backup/create`, { @@ -1341,19 +1358,19 @@ describe('Large localBucket backup (>32 MiB RPC payload)', () => { // 40 MiB of incompressible random data — squashfs won't compress it, // so the archive stays large and crosses the 32 MiB RPC payload cap. - const seedResult = (await ( - await fetch(`${workerUrl}/api/execute`, { - method: 'POST', - headers, - body: JSON.stringify({ - command: [ - `mkdir -p ${TEST_DIR}`, - `dd if=/dev/urandom of=${TEST_DIR}/blob.bin bs=1M count=40 status=none`, - `sha256sum ${TEST_DIR}/blob.bin` - ].join(' && ') - }) + const seedResponse = await fetch(`${workerUrl}/api/execute`, { + method: 'POST', + headers, + body: JSON.stringify({ + command: shellCommand([ + `mkdir -p ${TEST_DIR}`, + `dd if=/dev/urandom of=${TEST_DIR}/blob.bin bs=1M count=40 status=none`, + `sha256sum ${TEST_DIR}/blob.bin` + ]) }) - ).json()) as ExecuteResponse; + }); + expect(seedResponse.ok).toBe(true); + const seedResult = (await seedResponse.json()) as ExecuteResponse; expect(seedResult.exitCode).toBe(0); const originalSha = seedResult.stdout?.trim().split(/\s+/)[0]; expect(originalSha).toMatch(/^[0-9a-f]{64}$/); @@ -1383,10 +1400,10 @@ describe('Large localBucket backup (>32 MiB RPC payload)', () => { method: 'POST', headers, body: JSON.stringify({ - command: [ + command: shellCommand([ `echo gone > ${TEST_DIR}/blob.bin`, `sha256sum ${TEST_DIR}/blob.bin` - ].join(' && ') + ]) }) }) ).json()) as ExecuteResponse; @@ -1418,10 +1435,10 @@ describe('Large localBucket backup (>32 MiB RPC payload)', () => { method: 'POST', headers, body: JSON.stringify({ - command: [ + command: shellCommand([ `sha256sum ${TEST_DIR}/blob.bin`, `wc -c ${TEST_DIR}/blob.bin` - ].join(' && ') + ]) }) }) ).json()) as ExecuteResponse; diff --git a/tests/e2e/browser/playwright.config.ts b/tests/e2e/browser/playwright.config.ts index b528e5759..5ac763ae5 100644 --- a/tests/e2e/browser/playwright.config.ts +++ b/tests/e2e/browser/playwright.config.ts @@ -7,9 +7,10 @@ export default defineConfig({ testDir: '.', fullyParallel: true, forbidOnly: isCI, - retries: isCI ? 2 : 1, - workers: isCI ? 1 : undefined, - reporter: isCI ? 'github' : 'list', + retries: 1, + workers: isCI ? 8 : undefined, + maxFailures: isCI ? 1 : undefined, + reporter: isCI ? [['line'], ['github']] : 'list', timeout: 60000, use: { diff --git a/tests/e2e/browser/terminal-addon.spec.ts b/tests/e2e/browser/terminal-addon.spec.ts index b864b63b6..e57064fc0 100644 --- a/tests/e2e/browser/terminal-addon.spec.ts +++ b/tests/e2e/browser/terminal-addon.spec.ts @@ -1,13 +1,115 @@ -import { expect, test } from '@playwright/test'; +import { createHash } from 'node:crypto'; +import { expect, type Page, type TestInfo, test } from '@playwright/test'; + +type BrowserDiagnostic = { + kind: 'console' | 'request-failed' | 'response'; + message: string; + url?: string; +}; + +const diagnosticsByAttempt = new Map(); + +function attemptID(testInfo: TestInfo): string { + return `${testInfo.testId}:${testInfo.retry}`; +} + +function browserSandboxID(testInfo: TestInfo): string { + const runID = process.env.TEST_SANDBOX_ID ?? 'browser-test-sandbox'; + const testID = createHash('sha256') + .update(testInfo.testId) + .digest('hex') + .slice(0, 12); + return `${runID}-${testID}-retry-${testInfo.retry}`; +} + +function terminalTestURL(testInfo: TestInfo): string { + const params = new URLSearchParams({ + sandboxId: browserSandboxID(testInfo), + sandboxType: 'browser' + }); + return `/terminal-test?${params.toString()}`; +} + +function capturePageDiagnostics( + page: Page, + diagnostics: BrowserDiagnostic[] +): void { + page.on('console', (message) => { + if (message.type() === 'error') { + diagnostics.push({ kind: 'console', message: message.text() }); + } + }); + page.on('requestfailed', (request) => { + diagnostics.push({ + kind: 'request-failed', + message: request.failure()?.errorText ?? 'Request failed', + url: request.url() + }); + }); + page.on('response', (response) => { + if (response.url().includes('/api/terminal/create') && !response.ok()) { + diagnostics.push({ + kind: 'response', + message: `HTTP ${response.status()}`, + url: response.url() + }); + } + }); +} test.describe('Terminal Addon', () => { - const sandboxId = process.env.TEST_SANDBOX_ID || 'browser-test-sandbox'; + test.beforeEach(async ({ page }, testInfo) => { + const diagnostics: BrowserDiagnostic[] = []; + diagnosticsByAttempt.set(attemptID(testInfo), diagnostics); + + capturePageDiagnostics(page, diagnostics); - test.beforeEach(async ({ page }) => { - const terminalId = `terminal-${Date.now()}-${Math.random().toString(36).slice(2)}`; - await page.goto( - `/terminal-test?sandboxId=${sandboxId}&terminalId=${terminalId}` - ); + await page.goto(terminalTestURL(testInfo)); + }); + + test.afterEach(async ({ request }, testInfo) => { + const sandboxID = browserSandboxID(testInfo); + const diagnostics = diagnosticsByAttempt.get(attemptID(testInfo)) ?? []; + let cleanupError: string | undefined; + + try { + const params = new URLSearchParams({ + sandboxId: sandboxID, + sandboxType: 'browser' + }); + const response = await request.post(`/cleanup?${params.toString()}`, { + timeout: 10_000 + }); + if (!response.ok()) { + cleanupError = `HTTP ${response.status()}`; + } + } catch (error) { + cleanupError = error instanceof Error ? error.message : String(error); + } + + if (cleanupError) { + diagnostics.push({ + kind: 'request-failed', + message: `Sandbox cleanup failed: ${cleanupError}` + }); + } + + if (testInfo.status !== testInfo.expectedStatus || cleanupError) { + await testInfo.attach('browser-network-diagnostics', { + body: Buffer.from( + JSON.stringify({ sandboxID, diagnostics }, undefined, 2) + ), + contentType: 'application/json' + }); + } + + diagnosticsByAttempt.delete(attemptID(testInfo)); + + if (cleanupError && testInfo.status === testInfo.expectedStatus) { + throw new Error( + `Browser sandbox cleanup failed for ${sandboxID}: ${cleanupError}` + ); + } }); test.describe('Connection', () => { @@ -169,25 +271,19 @@ test.describe('Terminal Addon', () => { }); test.describe('Terminal Isolation', () => { - test('different terminal IDs have independent terminals', async ({ - browser - }) => { - const contextA = await browser.newContext(); + test('different terminal sessions are independent', async ({ + browser, + page: pageA + }, testInfo) => { const contextB = await browser.newContext(); - const pageA = await contextA.newPage(); const pageB = await contextB.newPage(); + const diagnostics = diagnosticsByAttempt.get(attemptID(testInfo)); + if (diagnostics) capturePageDiagnostics(pageB, diagnostics); - const terminalA = `iso-a-${Date.now()}`; - const terminalB = `iso-b-${Date.now()}`; const marker = `marker-${Date.now()}`; - await pageA.goto( - `/terminal-test?sandboxId=${sandboxId}&terminalId=${terminalA}` - ); - await pageB.goto( - `/terminal-test?sandboxId=${sandboxId}&terminalId=${terminalB}` - ); + await pageB.goto(terminalTestURL(testInfo)); await expect(pageA.getByTestId('connection-status')).toHaveText( 'connected', @@ -227,7 +323,6 @@ test.describe('Terminal Addon', () => { expect(contentB).toContain('check:'); expect(contentB).not.toContain(marker); - await contextA.close(); await contextB.close(); }); }); diff --git a/tests/e2e/coding-agent-process-workflow.test.ts b/tests/e2e/coding-agent-process-workflow.test.ts index deaba30a0..7f7314123 100644 --- a/tests/e2e/coding-agent-process-workflow.test.ts +++ b/tests/e2e/coding-agent-process-workflow.test.ts @@ -116,7 +116,7 @@ describe('coding agent process workflows', () => { expect(logs.events.at(-1)?.type).toBe('terminal'); const pgrep = await post<{ stdout: string }>('/api/execute', { - command: ['/bin/bash', '-lc', "pgrep -f 'sleep 60' || true"] + command: ['/bin/bash', '-lc', "pgrep -f '[s]leep 60' || true"] }); expect(pgrep.stdout.trim()).toBe(''); }, 45000); @@ -149,22 +149,26 @@ describe('coding agent process workflows', () => { expect(result.exitCode).not.toBe(0); }, 30000); - test('non-waking discovery and runtime fencing survive replacement', async () => { + test('non-waking discovery and runtime fencing reject stale handles', async () => { const result = await post<{ stoppedListCount: number; stoppedGetFound: boolean; - staleError: string; - racingError: string; - recoveredState: string; + staleRejected: boolean; + staleReasonMatched: boolean; + racingRejected: boolean; + racingCode: string | null; }>('/api/process/runtime-fencing-regression', {}); - expect(result).toEqual({ + expect(result).toMatchObject({ stoppedListCount: 0, stoppedGetFound: false, - staleError: 'StaleProcessHandleError', - racingError: 'OperationInterruptedError', - recoveredState: 'running' + staleRejected: true, + staleReasonMatched: true, + racingRejected: true }); + expect(['OPERATION_INTERRUPTED', 'RPC_TRANSPORT_ERROR']).toContain( + result.racingCode + ); }, 60000); test('Codex shape: process ID is usable across requests for logs, status, and exit', async () => { @@ -220,7 +224,7 @@ describe('coding agent process workflows', () => { command: [ '/bin/bash', '-lc', - 'printf start; yes truncate-me | head -c 400000' + 'printf start; sleep 1; yes truncate-me | head -c 2000000' ] }); await post(`/api/process/${noisy.id}/wait-for-log`, { diff --git a/tests/e2e/file-operations-workflow.test.ts b/tests/e2e/file-operations-workflow.test.ts index 3bbebfc10..f20a4da6e 100644 --- a/tests/e2e/file-operations-workflow.test.ts +++ b/tests/e2e/file-operations-workflow.test.ts @@ -438,19 +438,26 @@ describe('File Binary Read', () => { const testPath = sandbox!.uniquePath('binary-read-test.bin'); // Write 10 MiB of incompressible random data entirely inside the container - const seedResult = (await ( - await fetch(`${workerUrl}/api/execute`, { - method: 'POST', - headers, - body: JSON.stringify({ - command: [ + const seedResponse = await fetch(`${workerUrl}/api/execute`, { + method: 'POST', + headers, + body: JSON.stringify({ + command: [ + '/bin/bash', + '-lc', + [ `mkdir -p $(dirname ${testPath})`, `dd if=/dev/urandom of=${testPath} bs=1M count=10 status=none`, `sha256sum ${testPath}` ].join(' && ') - }) + ] }) - ).json()) as { stdout: string; exitCode: number }; + }); + expect(seedResponse.ok).toBe(true); + const seedResult = (await seedResponse.json()) as { + stdout: string; + exitCode: number; + }; expect(seedResult.exitCode).toBe(0); const [originalSha] = seedResult.stdout.trim().split(/\s+/); expect(originalSha).toMatch(/^[0-9a-f]{64}$/); diff --git a/tests/e2e/file-watch-workflow.test.ts b/tests/e2e/file-watch-workflow.test.ts index 113e04f15..73c86557f 100644 --- a/tests/e2e/file-watch-workflow.test.ts +++ b/tests/e2e/file-watch-workflow.test.ts @@ -79,45 +79,65 @@ describe('File Watch Workflow', () => { if (!response.ok || !response.body) { throw new Error(`Watch request failed: ${response.status}`); } - - // watch() blocks until the watcher is established, so by the time - // the response arrives the filesystem watcher is ready. - const actionResult = await actions(); + const body = response.body; const events: FileWatchSSEEvent[] = []; let watchId: string | null = null; const signal = AbortSignal.timeout(timeoutMs); - - try { - for await (const event of parseSSEStream( - response.body, - signal - )) { - events.push(event); - - if (event.type === 'watching') { - watchId = event.watchId; - } - - if ( - event.type === 'stopped' || - event.type === 'error' || - events.length >= stopAfterEvents - ) { - break; + let resolveWatching!: () => void; + let rejectWatching!: (error: Error) => void; + let watchingSettled = false; + const watching = new Promise((resolve, reject) => { + resolveWatching = resolve; + rejectWatching = reject; + }); + const collecting = (async () => { + try { + for await (const event of parseSSEStream( + body, + signal + )) { + events.push(event); + + if (event.type === 'watching') { + watchId = event.watchId; + watchingSettled = true; + resolveWatching(); + } + + if ( + event.type === 'stopped' || + event.type === 'error' || + events.length >= stopAfterEvents + ) { + break; + } } - } - } catch (error) { - if ( - !( + } catch (error) { + const timedOut = signal.aborted && error instanceof Error && - error.message === 'Operation was aborted' - ) - ) { - throw error; + error.message === 'Operation was aborted'; + if (!timedOut) { + const failure = + error instanceof Error ? error : new Error(String(error)); + if (!watchingSettled) { + watchingSettled = true; + rejectWatching(failure); + return; + } + throw failure; + } + } finally { + if (!watchingSettled) { + rejectWatching(new Error('Watch closed before it was established')); + } } - } + })(); + + await watching; + const actionResult = await actions(); + await collecting; return { events, watchId, actionResult }; } diff --git a/tests/e2e/helpers/container-lifecycle.ts b/tests/e2e/helpers/container-lifecycle.ts index c28952dc1..be8703d82 100644 --- a/tests/e2e/helpers/container-lifecycle.ts +++ b/tests/e2e/helpers/container-lifecycle.ts @@ -79,12 +79,13 @@ export async function getContainerStatus( } /** - * `stop()` requests graceful shutdown; it is not a lifecycle barrier. Tests - * that need a completed runtime boundary should call this helper instead of - * assuming the stop request has fully settled before the next SDK operation. + * `stop()` requests graceful shutdown; it is not a lifecycle barrier. This + * helper waits until the container library reports physical stopped state, + * but that state does not guarantee the Sandbox `onStop()` hook has run. * - * This intentionally uses the test-worker's stop endpoint to force the same + * This intentionally uses the test worker's stop endpoint to force the same * stop/replacement boundary users can observe after sleep or runtime exit. + * The next waking SDK operation must reconcile any delayed stop notification. */ export async function waitForContainerStopped( workerUrl: string, diff --git a/tests/e2e/pty.test.ts b/tests/e2e/pty.test.ts index 27bc85e07..48801aab0 100644 --- a/tests/e2e/pty.test.ts +++ b/tests/e2e/pty.test.ts @@ -10,6 +10,7 @@ describe('PTY', () => { let sandbox: TestSandbox | null = null; let workerUrl: string; let sandboxId: string; + const terminalIDs = new Map(); beforeAll(async () => { sandbox = await createTestSandbox(); @@ -26,7 +27,22 @@ describe('PTY', () => { ws: WebSocket; output: string[]; }> { - const resolvedTerminalId = terminalId ?? `pty-${Date.now()}`; + let resolvedTerminalId = terminalId + ? terminalIDs.get(terminalId) + : undefined; + if (!resolvedTerminalId) { + const response = await fetch(`${workerUrl}/api/terminal/create`, { + method: 'POST', + headers: sandbox!.headers(), + body: JSON.stringify({ command: ['/bin/bash'], cols: 80, rows: 24 }) + }); + if (!response.ok) { + throw new Error(`Terminal creation failed: ${await response.text()}`); + } + const terminal = (await response.json()) as { id: string }; + resolvedTerminalId = terminal.id; + if (terminalId) terminalIDs.set(terminalId, resolvedTerminalId); + } const path = `/terminal/${resolvedTerminalId}`; const wsUrl = `${workerUrl.replace(/^http/, 'ws')}${path}?sandboxId=${sandboxId}`; const ws = new WebSocket(wsUrl); @@ -130,7 +146,8 @@ describe('PTY', () => { // Second connection: should receive buffered output const { ws: ws2, output: output2 } = await connectWebSocket(terminalId); - // Buffered output is sent before 'ready', so it should already be there + // Replay is forwarded asynchronously after connection readiness. + await waitForOutput(output2, marker); expect(output2.join('')).toContain(marker); cleanup(ws2); diff --git a/tests/e2e/runtime-incarnation-workflow.test.ts b/tests/e2e/runtime-incarnation-workflow.test.ts new file mode 100644 index 000000000..3d632e4f0 --- /dev/null +++ b/tests/e2e/runtime-incarnation-workflow.test.ts @@ -0,0 +1,210 @@ +import { afterAll, beforeAll, describe, expect, test } from 'vitest'; +import { stopContainerAndWait } from './helpers/container-lifecycle'; +import { + cleanupTestSandbox, + createTestSandbox, + type TestSandbox +} from './helpers/global-sandbox'; +import { + cleanupSandbox, + createSandboxId, + createTestHeaders +} from './helpers/test-fixtures'; + +type ProcessStatus = { id: string; state: string }; +type ExecuteResponse = { success: boolean; stdout: string; exitCode: number }; + +describe('Runtime incarnation lifecycle workflow', () => { + let sandbox: TestSandbox | null = null; + let workerUrl: string; + let headers: Record; + + beforeAll(async () => { + sandbox = await createTestSandbox({ initCommand: ['true'] }); + workerUrl = sandbox.workerUrl; + headers = sandbox.headers(); + }, 120000); + + afterAll(async () => { + await cleanupTestSandbox(sandbox); + sandbox = null; + }, 120000); + + async function post(path: string, body: unknown = {}): Promise { + return await fetch(`${workerUrl}${path}`, { + method: 'POST', + headers, + body: JSON.stringify(body), + signal: AbortSignal.timeout(90000) + }); + } + + async function get(path: string): Promise { + return await fetch(`${workerUrl}${path}`, { + headers, + signal: AbortSignal.timeout(90000) + }); + } + + test('admits concurrent cold process launches and recovers handles in later requests', async () => { + const coldSandboxId = createSandboxId(); + const coldHeaders = createTestHeaders(coldSandboxId); + const coldPost = async (path: string, body: unknown = {}) => + await fetch(`${workerUrl}${path}`, { + method: 'POST', + headers: coldHeaders, + body: JSON.stringify(body), + signal: AbortSignal.timeout(90000) + }); + const coldGet = async (path: string) => + await fetch(`${workerUrl}${path}`, { + headers: coldHeaders, + signal: AbortSignal.timeout(90000) + }); + + try { + const [first, second] = await Promise.all([ + coldPost('/api/process/start', { + command: ['/bin/bash', '-lc', 'echo first-ready; sleep 20'] + }), + coldPost('/api/process/start', { + command: ['/bin/bash', '-lc', 'echo second-ready; sleep 20'] + }) + ]); + + expect(first.status).toBe(200); + expect(second.status).toBe(200); + const firstStatus = (await first.json()) as ProcessStatus; + const secondStatus = (await second.json()) as ProcessStatus; + expect(firstStatus.id).not.toBe(secondStatus.id); + + const recovered = await coldGet(`/api/process/${firstStatus.id}`); + expect(recovered.status).toBe(200); + await expect(recovered.json()).resolves.toMatchObject({ + id: firstStatus.id, + state: expect.stringMatching(/running|exited/) + }); + + await coldPost(`/api/process/${firstStatus.id}/kill`, { signal: 15 }); + await coldPost(`/api/process/${secondStatus.id}/kill`, { signal: 15 }); + } finally { + await cleanupSandbox(workerUrl, coldSandboxId); + } + }, 120000); + + test('does not revive stale process or terminal IDs after runtime stop', async () => { + const processResponse = await post('/api/process/start', { + command: ['/bin/bash', '-lc', 'echo stale-ready; sleep 30'] + }); + expect(processResponse.status).toBe(200); + const process = (await processResponse.json()) as ProcessStatus; + + const terminalResponse = await post('/api/terminal/create', { + command: ['bash'] + }); + expect(terminalResponse.status).toBe(200); + const terminal = (await terminalResponse.json()) as { id: string }; + + await stopContainerAndWait(workerUrl, headers); + + const processLookup = await get(`/api/process/${process.id}`); + expect(processLookup.status).toBe(404); + + const terminalLookup = await get(`/api/terminal/${terminal.id}`); + expect(terminalLookup.status).toBe(404); + + const replacement = await post('/api/execute', { + command: ['printf', 'replacement-ready'] + }); + expect(replacement.status).toBe(200); + await expect(replacement.json()).resolves.toMatchObject({ + success: true, + stdout: 'replacement-ready' + }); + }, 120000); + + test('interrupts retained runtime streams and recovers through a new session', async () => { + const response = await post('/api/runtime/retained-log-interruption'); + expect(response.status).toBe(200); + const result = (await response.json()) as { + interrupted: boolean; + errorName: string; + recoveryStdout: string; + }; + + expect(result.interrupted).toBe(true); + expect(result.errorName).toBe('RPCTransportError'); + expect(result.recoveryStdout).toBe('after-interruption'); + }, 120000); + + test('invalidates the runtime when the control server exits', async () => { + const response = await post('/api/runtime/control-server-exit'); + expect(response.status).toBe(200); + const result = (await response.json()) as { + stateStatus: string; + interruption: { originalMessage: string }; + recoveryStdout: string; + }; + + expect(['stopped', 'stopped_with_code']).toContain(result.stateStatus); + expect(result).toMatchObject({ + interruption: { + originalMessage: expect.stringContaining('StaleProcessHandleError') + }, + recoveryStdout: 'after-control-server-exit' + }); + }, 120000); + + test('coalesces concurrent destroy and leaves lookup paths non-waking', async () => { + const activeProcess = await post('/api/process/start', { + command: ['/bin/bash', '-lc', 'echo destroy-ready; sleep 30'] + }); + expect(activeProcess.status).toBe(200); + + const response = await post('/api/runtime/concurrent-destroy'); + expect(response.status).toBe(200); + await expect(response.json()).resolves.toMatchObject({ + fulfilled: 2, + rejected: 0, + listAfterDestroy: [] + }); + + const stateBeforeLookups = (await (await get('/api/state')).json()) as { + status: string; + }; + expect(stateBeforeLookups.status).not.toBe('healthy'); + + const [listResponse, previewsResponse, exposedResponse] = await Promise.all( + [ + get('/api/process/list'), + get('/api/exposed-ports'), + get('/api/exposed-ports/8080') + ] + ); + expect(listResponse.status).toBe(200); + await expect(listResponse.json()).resolves.toEqual([]); + expect(previewsResponse.status).toBe(200); + await expect(previewsResponse.json()).resolves.toEqual([]); + expect(exposedResponse.status).toBe(200); + await expect(exposedResponse.json()).resolves.toEqual({ + exposed: false, + port: 8080 + }); + + const stateAfterLookups = (await (await get('/api/state')).json()) as { + status: string; + }; + expect(stateAfterLookups.status).toBe(stateBeforeLookups.status); + }, 120000); + + test('cold exec regression completes without startup recursion', async () => { + await stopContainerAndWait(workerUrl, headers); + const response = await post('/api/execute', { + command: ['echo', 'ready'] + }); + expect(response.status).toBe(200); + const result = (await response.json()) as ExecuteResponse; + expect(result.success).toBe(true); + expect(result.stdout.trim()).toBe('ready'); + }, 120000); +}); diff --git a/tests/e2e/test-worker/index.ts b/tests/e2e/test-worker/index.ts index 8bf607808..8d115f36a 100644 --- a/tests/e2e/test-worker/index.ts +++ b/tests/e2e/test-worker/index.ts @@ -5,14 +5,17 @@ * * Sandbox types available: * - Sandbox: Base image without Python (default, lean image) + * - SandboxBrowser: Isolated base image capacity for browser tests * - SandboxPython: Full image with Python (for code interpreter tests) * - SandboxOpencode: Image with OpenCode CLI (for OpenCode integration tests) * - SandboxStandalone: Standalone binary on arbitrary base image (for binary pattern tests) * - SandboxMusl: Musl-based Alpine image variant (for musl binary tests) * - * Use X-Sandbox-Type header to select: 'python', 'opencode', 'standalone', 'musl', or default + * Use X-Sandbox-Type or sandboxType to select: 'browser', 'python', + * 'opencode', 'standalone', 'musl', or default. */ +import { switchPort } from '@cloudflare/containers'; import type { ProcessLogEvent, TerminalOutputEvent } from '@cloudflare/sandbox'; import { Sandbox as BaseSandbox, @@ -53,6 +56,7 @@ export class Sandbox extends BaseSandbox { // Export Sandbox class with different names for each container type // The actual image is determined by the container binding in wrangler.jsonc export { ContainerProxy }; +export { Sandbox as SandboxBrowser }; export { Sandbox as SandboxPython }; export { Sandbox as SandboxOpencode }; export { Sandbox as SandboxStandalone }; @@ -60,6 +64,7 @@ export { Sandbox as SandboxMusl }; interface Env { Sandbox: DurableObjectNamespace; + SandboxBrowser: DurableObjectNamespace; SandboxPython: DurableObjectNamespace; SandboxOpencode: DurableObjectNamespace; SandboxStandalone: DurableObjectNamespace; @@ -109,6 +114,21 @@ function isSandboxErrorLike(error: unknown): error is SandboxErrorLike { ); } +function getErrorCode(error: unknown): string | null { + if (isSandboxErrorLike(error)) { + return error.code ?? error.errorResponse.code ?? null; + } + if ( + typeof error === 'object' && + error !== null && + 'code' in error && + typeof error.code === 'string' + ) { + return error.code; + } + return null; +} + /** * Maps SandboxError subclass names to HTTP status codes and error codes. * Used as a fallback when errors cross the Cloudflare RPC boundary, @@ -255,6 +275,51 @@ function serializeBytes(data: Uint8Array): number[] { return Array.from(data); } +async function withTimeout( + promise: Promise, + timeoutMs: number, + label: string +): Promise { + let timeoutID: ReturnType | undefined; + const timeout = new Promise((_, reject) => { + timeoutID = setTimeout( + () => reject(new Error(`${label} timed out after ${timeoutMs}ms`)), + timeoutMs + ); + }); + try { + return await Promise.race([promise, timeout]); + } finally { + if (timeoutID) clearTimeout(timeoutID); + } +} + +async function waitForSandboxStopped( + sandbox: Pick, 'getState'>, + timeoutMs = 30_000 +): Promise['getState']>>> { + const deadline = Date.now() + timeoutMs; + let state = await sandbox.getState(); + while ( + state.status !== 'stopped' && + state.status !== 'stopped_with_code' && + Date.now() < deadline + ) { + await new Promise((resolve) => setTimeout(resolve, 250)); + state = await sandbox.getState(); + } + if (state.status !== 'stopped' && state.status !== 'stopped_with_code') { + throw new Error( + `Timed out waiting ${timeoutMs}ms for sandbox to stop; last status: ${state.status}` + ); + } + return state; +} + +function decodeOutput(bytes: Uint8Array): string { + return new TextDecoder().decode(bytes); +} + function serializeProcessLogEvent(event: ProcessLogEvent) { if (event.type === 'stdout' || event.type === 'stderr') { return { ...event, data: serializeBytes(event.data) }; @@ -284,11 +349,6 @@ export default { if (proxyResponse) return proxyResponse; } - // Skip JSON body parsing for streaming endpoints to preserve request.body - const isStreamingUpload = - url.pathname === '/api/file/write-stream' && request.method === 'PUT'; - const body = asRecord(isStreamingUpload ? {} : await parseBody(request)); - // Get sandbox ID from header or query param (WebSocket can't send headers) // Sandbox ID determines which container instance (Durable Object) const baseSandboxId = @@ -302,10 +362,33 @@ export default { const keepAliveHeader = request.headers.get('X-Sandbox-KeepAlive'); const keepAlive = keepAliveHeader === 'true'; const sleepAfter = request.headers.get('X-Sandbox-Sleep-After'); - // Select sandbox type based on X-Sandbox-Type header - const sandboxType = request.headers.get('X-Sandbox-Type'); + // Query selection lets WebSocket requests choose a sandbox namespace. + const sandboxType = + request.headers.get('X-Sandbox-Type') ?? + url.searchParams.get('sandboxType'); + + // This page only renders a client that will create the terminal in a later + // request. Avoid constructing a Sandbox here because getSandbox() starts + // asynchronous configuration, which this response would not await. + if (url.pathname === '/terminal-test') { + return new Response( + getTerminalTestPage( + sandboxId, + sandboxType === 'browser' ? 'browser' : '' + ), + { headers: { 'Content-Type': 'text/html' } } + ); + } + + // Skip JSON body parsing for streaming endpoints to preserve request.body + const isStreamingUpload = + url.pathname === '/api/file/write-stream' && request.method === 'PUT'; + const body = asRecord(isStreamingUpload ? {} : await parseBody(request)); + let sandboxNamespace: DurableObjectNamespace; - if (sandboxType === 'python') { + if (sandboxType === 'browser') { + sandboxNamespace = env.SandboxBrowser; + } else if (sandboxType === 'python') { sandboxNamespace = env.SandboxPython; } else if (sandboxType === 'opencode') { sandboxNamespace = env.SandboxOpencode; @@ -374,7 +457,7 @@ console.log('Echo server on port ' + port); const upgradeHeader = request.headers.get('Upgrade'); if (upgradeHeader?.toLowerCase() === 'websocket') { if (url.pathname === '/ws/echo') { - return await sandbox.wsConnect(request, 8080); + return await sandbox.fetch(switchPort(request, 8080)); } if (url.pathname === '/ws/code') { return await sandbox.wsConnect(request, 8081); @@ -384,6 +467,10 @@ console.log('Echo server on port ' + port); } } + if (url.pathname === '/api/container-fetch') { + return await sandbox.containerFetch('http://container/', {}, 8080); + } + // Health check if (url.pathname === '/health') { const response: HealthResponse = { @@ -543,39 +630,35 @@ console.log('Echo server on port ' + port); ]); await oldProcess.waitForLog('admitted'); const racingWait = oldProcess.waitForExit().then( - () => '', - (error: unknown) => - error instanceof Error ? error.name : String(error) + () => ({ rejected: false, code: null }), + (error: unknown) => ({ rejected: true, code: getErrorCode(error) }) ); await new Promise((resolve) => setTimeout(resolve, 100)); await sandbox.destroy(); const stoppedList = await sandbox.listProcesses(); const stoppedGet = await sandbox.getProcess(oldProcess.id); - const replacement = await sandbox.exec(['printf', 'replacement']); - await replacement.waitForExit(); - let staleError = ''; + let staleRejected = false; + let staleReasonMatched = false; try { await oldProcess.status(); } catch (error) { - staleError = error instanceof Error ? error.name : String(error); + staleRejected = true; + staleReasonMatched = + error instanceof Error && + error.message.includes('previous runtime incarnation'); } - const racingError = await racingWait; - - const live = await sandbox.exec(['/bin/bash', '-lc', 'sleep 30']); - const recovered = await sandbox.getProcess(live.id); - const recoveredStatus = await recovered?.status(); - await recovered?.kill(); - await recovered?.waitForExit(); + const racingResult = await racingWait; return new Response( JSON.stringify({ stoppedListCount: stoppedList.length, stoppedGetFound: stoppedGet !== null, - staleError, - racingError, - recoveredState: recoveredStatus?.state + staleRejected, + staleReasonMatched, + racingRejected: racingResult.rejected, + racingCode: racingResult.code }), { headers: { 'Content-Type': 'application/json' } } ); @@ -1311,6 +1394,133 @@ console.log('Echo server on port ' + port); }); } + if ( + url.pathname === '/api/runtime/retained-log-interruption' && + request.method === 'POST' + ) { + const proc = await sandbox.exec([ + '/bin/bash', + '-lc', + 'echo stream-ready; sleep 30' + ]); + await proc.waitForLog('stream-ready', { timeout: 10000 }); + const stream = await proc.logs({ replay: false, follow: true }); + const reader = stream.getReader(); + const pendingRead = reader.read().then( + () => '', + (error: unknown) => + error instanceof Error ? error.name : String(error) + ); + await sandbox.stop(); + const errorName = await withTimeout( + pendingRead, + 10000, + 'retained log interruption' + ); + reader.releaseLock(); + await waitForSandboxStopped(sandbox); + const recovery = await sandbox.exec(['printf', 'after-interruption']); + const output = await recovery.output(); + return new Response( + JSON.stringify({ + interrupted: errorName.length > 0, + errorName, + recoveryStdout: decodeOutput(output.stdout) + }), + { headers: { 'Content-Type': 'application/json' } } + ); + } + + if ( + url.pathname === '/api/runtime/control-server-exit' && + request.method === 'POST' + ) { + const proc = await sandbox.exec([ + '/bin/bash', + '-lc', + 'echo control-stream-ready; sleep 30' + ]); + await proc.waitForLog('control-stream-ready', { timeout: 10000 }); + const abortLogs = new AbortController(); + const stream = await proc.logs({ + replay: false, + follow: true, + signal: abortLogs.signal + }); + const reader = stream.getReader(); + const pendingRead = reader.read().then( + () => ({ originalMessage: 'Stream closed without interruption' }), + (error: unknown) => { + const errorRecord = asRecord(error); + const context = asRecord(errorRecord.context); + return { + originalMessage: + optionalString(context.originalMessage) ?? + (error instanceof Error ? error.message : String(error)) + }; + } + ); + + try { + const marker = `/tmp/control-server-exit-${crypto.randomUUID()}`; + const killer = await sandbox.exec([ + '/bin/bash', + '-lc', + 'echo control-exit-armed; while [ ! -e "$1" ]; do sleep 0.01; done; parent_cmd="$(tr "\\0" " " < /proc/$PPID/cmdline)"; [ "$parent_cmd" = "/container-server/sandbox " ] || { echo "unexpected parent: $parent_cmd" >&2; exit 1; }; kill -KILL "$PPID"', + 'control-server-exit', + marker + ]); + await killer.waitForLog('control-exit-armed', { timeout: 10000 }); + await sandbox.writeFile(marker, 'exit').catch(() => undefined); + + const interruption = await withTimeout( + pendingRead, + 10000, + 'control server exit interruption' + ); + const state = await waitForSandboxStopped(sandbox); + const recovery = await sandbox.exec([ + 'printf', + 'after-control-server-exit' + ]); + const output = await recovery.output(); + return new Response( + JSON.stringify({ + stateStatus: state.status, + interruption, + recoveryStdout: decodeOutput(output.stdout) + }), + { headers: { 'Content-Type': 'application/json' } } + ); + } finally { + abortLogs.abort(); + await reader.cancel().catch(() => undefined); + reader.releaseLock(); + await proc.kill().catch(() => undefined); + } + } + + if ( + url.pathname === '/api/runtime/concurrent-destroy' && + request.method === 'POST' + ) { + const results = await Promise.allSettled([ + sandbox.destroy(), + sandbox.destroy() + ]); + const listAfterDestroy = await sandbox.listProcesses(); + return new Response( + JSON.stringify({ + fulfilled: results.filter((result) => result.status === 'fulfilled') + .length, + rejected: results.filter((result) => result.status === 'rejected') + .length, + listAfterDestroy + }), + { headers: { 'Content-Type': 'application/json' } } + ); + } + // Cleanup endpoint - destroys the sandbox container // This is used by E2E tests to explicitly clean up after each test if (url.pathname === '/cleanup' && request.method === 'POST') { @@ -1332,22 +1542,13 @@ console.log('Echo server on port ' + port); await (sandbox as unknown as { stop: () => Promise }).stop(); const response: SuccessWithMessageResponse = { success: true, - message: 'Container stopped' + message: 'Container stop requested' }; return new Response(JSON.stringify(response), { headers: { 'Content-Type': 'application/json' } }); } - // PTY: Browser test page for Playwright tests - if (url.pathname === '/terminal-test') { - const terminalId = - url.searchParams.get('terminalId') || `browser-test-${Date.now()}`; - return new Response(getTerminalTestPage(sandboxId, terminalId), { - headers: { 'Content-Type': 'text/html' } - }); - } - // PTY: WebSocket terminal proxy if ( url.pathname === '/terminal' || @@ -1367,14 +1568,14 @@ console.log('Echo server on port ' + port); const terminal = await sandbox.getTerminal(terminalId); if (!terminal) return new Response('Terminal not found', { status: 404 }); - return terminal.connect(request, { cols, rows }); + return await terminal.connect(request, { cols, rows }); } const terminal = await sandbox.createTerminal({ command: ['bash'], cols, rows }); - return terminal.connect(request, { cols, rows }); + return await terminal.connect(request, { cols, rows }); } return new Response('Not found', { status: 404 }); @@ -1471,7 +1672,7 @@ console.log('Echo server on port ' + port); } }; -function getTerminalTestPage(sandboxId: string, terminalId: string): string { +function getTerminalTestPage(sandboxId: string, sandboxType: string): string { return ` @@ -1499,9 +1700,11 @@ function getTerminalTestPage(sandboxId: string, terminalId: string): string { const statusEl = document.getElementById('status'); const sandboxId = '${sandboxId}'; - const terminalId = '${terminalId}'; + const sandboxType = '${sandboxType}'; + const sandboxQuery = new URLSearchParams({ sandboxId, sandboxType }).toString(); let ws = null; + let terminalId = null; let reconnectAttempts = 0; const maxReconnectAttempts = 10; @@ -1510,10 +1713,34 @@ function getTerminalTestPage(sandboxId: string, terminalId: string): string { statusEl.dataset.testid = 'connection-status'; } - function connect() { + async function connect() { updateStatus('connecting'); + if (!terminalId) { + try { + const response = await fetch('/api/terminal/create?' + sandboxQuery, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + command: ['/bin/bash'], + cols: term.cols, + rows: term.rows + }) + }); + if (!response.ok) { + const detail = await response.text(); + console.error('Terminal creation failed:', response.status, detail); + updateStatus('error'); + return; + } + terminalId = (await response.json()).id; + } catch (error) { + console.error('Terminal creation failed:', error); + updateStatus('error'); + return; + } + } const protocol = location.protocol === 'https:' ? 'wss:' : 'ws:'; - const wsUrl = protocol + '//' + location.host + '/terminal/' + terminalId + '?sandboxId=' + sandboxId; + const wsUrl = protocol + '//' + location.host + '/terminal/' + terminalId + '?' + sandboxQuery; ws = new WebSocket(wsUrl); ws.binaryType = 'arraybuffer'; diff --git a/tests/e2e/test-worker/wrangler.template.jsonc b/tests/e2e/test-worker/wrangler.template.jsonc index 0349deacb..5b7ae5f15 100644 --- a/tests/e2e/test-worker/wrangler.template.jsonc +++ b/tests/e2e/test-worker/wrangler.template.jsonc @@ -32,6 +32,18 @@ } } }, + { + "class_name": "SandboxBrowser", + "image": "{{IMAGE_SANDBOX}}", + "name": "{{CONTAINER_NAME}}-browser", + "instance_type": "standard-4", + "max_instances": 16, + "unsafe": { + "configuration": { + "durable_object_offset_instances": 0 + } + } + }, { "class_name": "SandboxPython", "image": "{{IMAGE_PYTHON}}", @@ -84,6 +96,10 @@ "class_name": "Sandbox", "name": "Sandbox" }, + { + "class_name": "SandboxBrowser", + "name": "SandboxBrowser" + }, { "class_name": "SandboxPython", "name": "SandboxPython" @@ -127,6 +143,10 @@ { "deleted_classes": ["SandboxDesktop"], "tag": "v7" + }, + { + "new_sqlite_classes": ["SandboxBrowser"], + "tag": "v8" } ], diff --git a/tests/e2e/websocket-connect.test.ts b/tests/e2e/websocket-connect.test.ts index e92442676..3914968c8 100644 --- a/tests/e2e/websocket-connect.test.ts +++ b/tests/e2e/websocket-connect.test.ts @@ -68,6 +68,15 @@ describe('WebSocket Connections', () => { ws.close(); }, 20000); + test('should forward HTTP through containerFetch', async () => { + const response = await fetch(`${workerUrl}/api/container-fetch`, { + headers: { 'X-Sandbox-Id': sandboxId } + }); + + expect(response.status).toBe(400); + expect(await response.text()).toBe('Expected WebSocket'); + }); + test('should handle multiple concurrent connections', async () => { const wsUrl = `${workerUrl.replace(/^http/, 'ws')}/ws/echo`;