diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 685e559..7da90b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,16 +14,6 @@ jobs: timeout-minutes: 240 steps: - uses: actions/checkout@v7.0.1 - - name: Reserve native build space - run: | - # These preinstalled SDKs are unused by this job. All Rust build tools - # come from the pinned container, and Node/Terraform are installed below. - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc - available=$(df -Pk . | awk 'NR == 2 {print $4}') - if [ "$available" -lt 20971520 ]; then - echo "Native builds need 20 GiB free. Set RAT_NATIVE_RUNNER to an ARM64 runner with more disk." - exit 1 - fi - uses: actions/setup-node@v7.0.0 with: node-version: 22 @@ -36,12 +26,10 @@ jobs: id: codex-runtime with: path: .runtime/codex/linux-arm64 - key: codex-linux-arm64-${{ hashFiles('runtime/codex/**', '!runtime/codex/model-capabilities.json', '!runtime/codex/README.md', 'scripts/build-codex-runtime.mjs', 'scripts/codex-runtime-targets.mjs') }} - - name: Build pinned Linux ARM64 runtime + key: official-codex-linux-arm64-${{ hashFiles('runtime/codex/source.json', 'runtime/codex/companion-packages.json', 'package-lock.json', 'scripts/build-codex-runtime.mjs', 'scripts/codex-runtime-artifact.mjs', 'scripts/codex-runtime-targets.mjs') }} + - name: Package official Linux ARM64 runtime if: steps.codex-runtime.outputs.cache-hit != 'true' - run: docker buildx build --platform linux/arm64 --file runtime/codex/Dockerfile --output type=local,dest=.runtime/codex/linux-arm64 . - # Cache a completed compilation even if a later acceptance check fails. - # Restoring this cache never skips package, checksum or native image tests. + run: node scripts/build-codex-runtime.mjs .runtime/codex/linux-arm64 - uses: actions/cache/save@v4 if: steps.codex-runtime.outputs.cache-hit != 'true' with: @@ -85,31 +73,35 @@ jobs: - run: npm ci - name: Verify x86 sandbox enforcement with the pinned upstream runtime run: npx vitest run tests/pi/managed-effects.test.ts tests/runner/managed-sandbox.test.ts tests/simulation/linux-kvm-helper.test.ts tests/simulation/native-vm-prototype.test.ts --no-file-parallelism - - name: Reserve native build space - run: | - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc - available=$(df -Pk . | awk 'NR == 2 {print $4}') - test "$available" -ge 20971520 - uses: actions/cache/restore@v4 id: codex-runtime with: path: .runtime/codex/linux-x64 - key: codex-linux-x64-${{ hashFiles('runtime/codex/**', '!runtime/codex/model-capabilities.json', '!runtime/codex/README.md', 'scripts/build-codex-runtime.mjs', 'scripts/codex-runtime-targets.mjs') }} - - name: Build pinned Linux x64 runtime + key: official-codex-linux-x64-${{ hashFiles('runtime/codex/source.json', 'runtime/codex/companion-packages.json', 'package-lock.json', 'scripts/build-codex-runtime.mjs', 'scripts/codex-runtime-artifact.mjs', 'scripts/codex-runtime-targets.mjs') }} + - name: Package official Linux x64 runtime if: steps.codex-runtime.outputs.cache-hit != 'true' - run: docker buildx build --platform linux/amd64 --file runtime/codex/Dockerfile --output type=local,dest=.runtime/codex/linux-x64 . + run: node scripts/build-codex-runtime.mjs .runtime/codex/linux-x64 - uses: actions/cache/save@v4 if: steps.codex-runtime.outputs.cache-hit != 'true' with: path: .runtime/codex/linux-x64 key: ${{ steps.codex-runtime.outputs.cache-primary-key }} - name: Validate x64 artifact provenance - run: node --input-type=module -e "import { validateCodexRuntimeArtifact } from './scripts/codex-runtime-artifact.mjs'; await validateCodexRuntimeArtifact('.runtime/codex/linux-x64', { expectedTarget:'x86_64-unknown-linux-gnu' });" - - name: Verify patched x86 sandbox and local Pi transport + run: node --input-type=module -e "import { validateCodexRuntimeArtifact } from './scripts/codex-runtime-artifact.mjs'; await validateCodexRuntimeArtifact('.runtime/codex/linux-x64', { expectedTarget:'x86_64-unknown-linux-musl' });" + - name: Verify x64 native contracts, checkpoints and Pi sandbox env: CODEX_CONFORMANCE_BINARY: ${{ github.workspace }}/.runtime/codex/linux-x64/runtime/bin/codex CODEX_REQUIRE_PARITY: 'true' - run: npx vitest run tests/pi/managed-effects.test.ts tests/pi/managed-subprocess.test.ts tests/runner/managed-sandbox.test.ts tests/agents/local-pi-transport.test.ts tests/agents/local-pi.test.ts --no-file-parallelism + run: | + npm run test:agents:parity:host + npx vitest run tests/runner/managed-sandbox.test.ts --no-file-parallelism + - name: Verify x64 isolated cold recovery + env: + CODEX_RUNTIME_ARTIFACT: ${{ github.workspace }}/.runtime/codex/linux-x64 + run: | + npm run runtime:local:build + docker buildx build --platform linux/amd64 --load --file testing/native-parity.Dockerfile --build-arg WORKER_IMAGE=rat-things-local:1 --tag rat-things-native-parity:x64 . + docker run --rm --platform linux/amd64 --privileged --init --user 0 --env CODEX_COLD_NATIVE_E2E=true --entrypoint npx rat-things-native-parity:x64 --no-install vitest run tests/runner/codex-cold-verification.test.ts --no-file-parallelism - name: Archive verified Linux x64 runtime run: tar -C .runtime/codex/linux-x64 -czf .runtime/codex/linux-x64.tar.gz artifact.json runtime - name: Export verified Linux x64 runtime diff --git a/AGENTS.md b/AGENTS.md index 17daaff..6ab5c59 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -40,10 +40,10 @@ npm run test:e2e:localstack Terraform requires packaged Lambda artifacts: -Packaging also requires the patched Linux ARM64 Codex artifact. Build it with +Packaging also requires the official, unmodified Linux ARM64 Codex artifact. Package it with `runtime/codex/Dockerfile` or select an exported artifact with `CODEX_RUNTIME_ARTIFACT`; see `testing/README.md`. `npm run check` includes packaging -and validates this artifact. CI caches it by source, patch and builder identity. +and validates this artifact. CI caches it by official package and packager identity. ```bash npm run package diff --git a/README.md b/README.md index d79932d..ebe9a3d 100644 --- a/README.md +++ b/README.md @@ -195,7 +195,7 @@ npm run console:serve ``` For the full repository checks, run `npm run check`. It includes Lambda packaging -and requires a patched Linux ARM64 Codex artifact, Docker and Terraform; see +and requires an official Linux ARM64 Codex artifact, Docker and Terraform; see [development prerequisites](testing/README.md). To deploy a backend, start with the [AWS quickstart](docs/quickstart.md). diff --git a/desktop/tests/local_pi_e2e.rs b/desktop/tests/local_pi_e2e.rs index 489881a..317bdd6 100644 --- a/desktop/tests/local_pi_e2e.rs +++ b/desktop/tests/local_pi_e2e.rs @@ -412,7 +412,7 @@ fn saved_pi_agent_runs_managed_tools_and_restores_in_rust_console() { &mut app, "Task", &format!( - "Perform this exact managed workspace verification using all four tools: (1) write_file path {file} with content {first}; (2) read_file that path and confirm its exact content; (3) list_files /workspace and locate that file; (4) exec_command with command array [\"node\",\"-e\",\"console.log('RUST_UI_WORKER '+process.getuid()+' '+process.platform)\"] and confirm it prints RUST_UI_WORKER 10001 linux. Then respond with exactly {first}." + "Perform this exact managed workspace verification using all four tools: (1) write_file path {file} with content {first}; (2) read_file that path and confirm its exact content; (3) list_files /workspace and locate that file; (4) exec_command with command array [\"node\",\"--disable-warning=UNDICI-EHPA\",\"-e\",\"console.log('RUST_UI_WORKER '+process.getuid()+' '+process.platform)\"] and confirm it prints RUST_UI_WORKER 10001 linux. Then respond with exactly {first}." ), ); capture(&mut app, &evidence, "managed-session-form.png"); @@ -471,7 +471,7 @@ fn saved_pi_agent_runs_managed_tools_and_restores_in_rust_console() { }); let guest_command = require_operation( &first_commands, - &json!({"command":["node","-e","console.log('RUST_UI_WORKER '+process.getuid()+' '+process.platform)"]}), + &json!({"command":["node","--disable-warning=UNDICI-EHPA","-e","console.log('RUST_UI_WORKER '+process.getuid()+' '+process.platform)"]}), |text| text.trim() == "RUST_UI_WORKER 10001 linux", ); show_command(&mut app, guest_command); @@ -499,7 +499,7 @@ fn saved_pi_agent_runs_managed_tools_and_restores_in_rust_console() { &mut app, "Message", &format!( - "Continue this same Session. Use read_file on {file} to confirm the existing content is {first}. Then use write_file to replace its content with exactly {second}, use read_file again to verify it, and use exec_command with command array [\"node\",\"-e\",\"console.log('RUST_UI_CONTINUED '+process.getuid())\"]. Respond with exactly {second}." + "Continue this same Session. Use read_file on {file} to confirm the existing content is {first}. Then use write_file to replace its content with exactly {second}, use read_file again to verify it, and use exec_command with command array [\"node\",\"--disable-warning=UNDICI-EHPA\",\"-e\",\"console.log('RUST_UI_CONTINUED '+process.getuid())\"]. Respond with exactly {second}." ), ); click(&mut app, "Send"); @@ -541,7 +541,7 @@ fn saved_pi_agent_runs_managed_tools_and_restores_in_rust_console() { }); let continued_command = require_operation( &continued_commands, - &json!({"command":["node","-e","console.log('RUST_UI_CONTINUED '+process.getuid())"]}), + &json!({"command":["node","--disable-warning=UNDICI-EHPA","-e","console.log('RUST_UI_CONTINUED '+process.getuid())"]}), |text| text.trim() == "RUST_UI_CONTINUED 10001", ); show_command(&mut app, continued_command); diff --git a/docs/agents-api.md b/docs/agents-api.md index efa0d61..8588e37 100644 --- a/docs/agents-api.md +++ b/docs/agents-api.md @@ -298,6 +298,11 @@ for await (const event of await client.beta.agents.sessions.events.stream(sessio SSE is a live stream. Recover missed work from saved Items and Turns; reconnecting does not replay an event log. Disconnecting a stream does not cancel execution. + +Command-output deltas arrive once the command finishes, immediately before its +completed Item. This keeps their text in the order reported by the native +executor's completed aggregate, including its output limit and any omission +marker. Assistant text and reasoning summaries continue to stream as they arrive. Function tool results must identify the pending call and Turn. Successful results provide `output`; failed results provide `error`. Human approval requests are not part of the runtime: the deployment admits a fixed capability envelope before @@ -397,6 +402,24 @@ retains flushed workspace files and setup identity after confirmed termination o worker. Neither runtime restores running processes. Checkpointing is host-controlled; there is no public checkpoint endpoint or automatic idle timer. +Codex checkpoints require successful completed Turns. Before publication, the +host stops native writers, flushes the Session journal, synchronizes storage and +opens a disposable copy with the official Codex binary. Root and child thread +identities and complete public native history must survive a fresh-process +resume. Verification has no network or credentials and cannot run model work or +tools. Extra coordination details omitted by Codex's public history remain in +the host journal. This does not verify Codex's hidden mailbox state. + +Codex cold checkpoints currently support the built-in OpenAI provider and custom +Responses API providers. Bedrock, Ollama and LM Studio are refused before the +running process stops because their cold recovery has not been validated. This +restriction does not prevent ordinary Sessions or Pi Durable execution. + +Checkpointing refuses unsupported startup integrations, external native storage, +failed or interrupted native history, and unverifiable saved state. An ordinary +process exit alone cannot publish a checkpoint. These limits also apply when +forking requires a new Codex checkpoint. + ## Fork a Session Set `session_checkpoints_enabled = true` with S3 Files configured to enable forks; diff --git a/docs/development-and-deployment.md b/docs/development-and-deployment.md index e5bb82b..fce9801 100644 --- a/docs/development-and-deployment.md +++ b/docs/development-and-deployment.md @@ -11,8 +11,8 @@ public sharing. Return here when choosing a longer-lived installation shape. ## Prerequisites - Node.js 22.20+, npm, and Git. -- Docker with Compose for LocalStack and Buildx for the patched Linux ARM64 runtime and image canaries. -- A verified patched native artifact for packaging; build or import it using +- Docker with Compose for LocalStack and Buildx for the official Linux ARM64 runtime and image canaries. +- A verified official native artifact for packaging; build or import it using [`testing/README.md`](../testing/README.md#agents-api-native-compatibility-gate). - Terraform 1.16.6, matching `.terraform-version`, CI and the LocalStack container, and AWS credentials for infrastructure work. The module requires Terraform 1.9+ diff --git a/docs/local-workspace.md b/docs/local-workspace.md index e694930..1553512 100644 --- a/docs/local-workspace.md +++ b/docs/local-workspace.md @@ -60,7 +60,7 @@ active worker. Conversations with **No environment** do not require a VM. Windows and Linux with Docker Desktop, Intel Macs, and native Linux Docker with the gVisor `runsc` runtime are in preview. Set `RAT_THINGS_PREVIEW_PLATFORMS=1` to try them. Native Linux Docker without gVisor is refused, because the worker would share -the host kernel. A source checkout can build the patched Linux x86-64 worker. +the host kernel. A source checkout can package the official Codex release into the Linux x86-64 worker. The [KVM development setup](../testing/local-runtime/linux-kvm/README.md) uses a dedicated VM on Linux x86-64 when `/dev/kvm` and vsock are available. It requires an explicit asset path and is separate from the signed macOS installer. A gVisor diff --git a/docs/quickstart.md b/docs/quickstart.md index 5e690d5..164476d 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -10,7 +10,7 @@ Use the [deployment guide](development-and-deployment.md) to configure the Terra module, model credentials, ARM64 runtime image and Agents HTTP service. Package Lambda artifacts before Terraform validation or deployment. Deployment creates AWS resources; ordinary local checks do not provision MicroVMs. Packaging requires the -patched Linux ARM64 Codex artifact described in the [contributor guide](../testing/README.md#agents-api-native-compatibility-gate). Do not substitute the stock npm binary. +official Linux ARM64 Codex artifact described in the [contributor guide](../testing/README.md#agents-api-native-compatibility-gate). Keep its verified manifest and complete companion files together. ## Automated setup diff --git a/docs/security.md b/docs/security.md index 86defcf..16acea0 100644 --- a/docs/security.md +++ b/docs/security.md @@ -14,6 +14,13 @@ execution boundary. Trusted orchestration runs as root and launches Codex or Pi as UID/GID 10001. The worker sandbox defaults to workspace-only writes with agent networking off. The host-side local harness retains its separate read-only/no-network policy. +Managed execution verifies the official Codex process's effective permission +profile before exposing tools. Unexpected merged grants, unknown policy files or +incomplete policy inspection prevent startup. Agent networking off uses a network +namespace and a proxy with no allowed destinations. Local socket metadata and +anonymous socketpairs remain available for subprocess I/O; external connections +and named Unix socket access remain denied. + The EC2 worker supports connected Session lifetime beyond Lambda MicroVM's maximum lifetime. Both backends must preserve the same ownership, process and credential boundaries. Self-hosted executor connections also require an operator-controlled sandbox; transport encryption does @@ -91,7 +98,7 @@ from actor attribution. No caller-provided URL or credential may establish owner | Superseded worker write | Immutable execution generation, backend identity and conditional state changes | Lost process memory cannot be recreated from public Items alone | | Artifact disclosure | Private encrypted objects, owner checks, saved Turn/path identity and explicit publication grants | Publication URLs are bearer credentials; provider copies have independent retention | | Resource exhaustion | Bounded requests, pagination, queues and deployment concurrency controls | Configure tenant budgets, API throttles and provider-event abuse limits | -| Supply-chain compromise | Locked packages, pinned source/base image, recorded patches and artifact digests | Review build provenance and scan/sign release artifacts | +| Supply-chain compromise | Locked packages, pinned source/base image, official package integrity and complete artifact digests | Review build provenance and scan/sign release artifacts | ## Fixed capability-envelope security model @@ -201,6 +208,13 @@ the same owner/Session storage under its fenced execution identity. Native check harness history; public-item recovery preserves historical facts without replaying tool effects. Neither recovery path promises survival of a process whose VM has been destroyed. +Codex checkpoint publication requires a successful cold resume of copied native +history after all writers stop and storage is synchronized. A separate verifier +runs without credentials or network access; it cannot modify the saved candidate. +Evidence covers root and child identities, public native history, and the flushed +host journal. It does not inspect hidden Codex mailbox state or preserve running +commands. Unsupported startup integrations or unverifiable state prevent handoff. + The EC2 supervisor requires the outer dedicated VM for its privileged mount/network setup. The agent does not receive the Docker socket or host control configuration. Metadata and the local control listener are denied to the guest while trusted root orchestration remains able to operate. diff --git a/docs/support.md b/docs/support.md index c269176..bb486cc 100644 --- a/docs/support.md +++ b/docs/support.md @@ -14,8 +14,9 @@ published release cadence. Operators apply updates through the existing [local runtime setup](local-workspace.md#give-your-session-a-linux-workspace). An update should identify the reviewed repository revision and the matching -complete runtime artifacts for the target platform. Codex source, ordered patches, -companion packages and file inventories are pinned together. Keep the artifact +complete runtime artifacts for the target platform. The official Codex release, +companion packages and file inventories are pinned together; Rat Things does not +patch the Codex executable. Keep the artifact manifest with its runtime directory; replacing only the Codex executable with an unrelated upstream release is not a supported upgrade. Artifact verification checks identity and file integrity; it does not establish a signature or prove diff --git a/microvm/runtime-server.mjs b/microvm/runtime-server.mjs index e197b42..541a3cc 100644 --- a/microvm/runtime-server.mjs +++ b/microvm/runtime-server.mjs @@ -302,9 +302,9 @@ function startRun(run) { storagePreparationDurationMs = Date.now() - preparationStartedAt; run.environment.SESSION_STATE_ROOT = stateRoot; run.environment.CODEX_HOME = join(stateRoot, 'codex-home'); - // SQLite WAL/connection locks stay on this worker's local filesystem. - // Native rollouts remain durable in CODEX_HOME and rebuild the indexes - // when a replacement worker resumes the saved thread. + // Ordinary runs bind this directory to worker-local storage. Checkpoint + // mode retains the databases and WAL beside the native home; paginated + // history cannot be reconstructed from legacy rollout files alone. run.environment.CODEX_SQLITE_HOME = join(stateRoot, 'codex-home', 'sqlite'); run.environment.BROWSER_PROFILE_ROOT = join(stateRoot, 'codex-home', 'browser-profile'); run.environment.WORKSPACE_ROOT = stateRoot; diff --git a/package-lock.json b/package-lock.json index 9b49055..9fbd30c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -44,7 +44,7 @@ "devDependencies": { "@aws-sdk/client-dynamodb-streams": "3.1088.0", "@aws-sdk/client-ecs": "3.1088.0", - "@openai/codex": "0.154.0", + "@openai/codex": "0.164.0", "@playwright/mcp": "0.0.83", "@playwright/test": "1.62.1", "@types/archiver": "7.0.0", @@ -1628,9 +1628,9 @@ } }, "node_modules/@openai/codex": { - "version": "0.154.0", - "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.154.0.tgz", - "integrity": "sha512-FV/x1OHXYv/ifjf3mXj9ThTTAWcUZN6cGIRQRhRxkKNOPuImu1WW0c8ev1vUkE9XGH90dEnYG1tBjIkxRikg0w==", + "version": "0.164.0", + "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.164.0.tgz", + "integrity": "sha512-TWj7UGyPRe3Pj/A3ddNAxs1te3a/W/xAd+J0LHrrFTmsSfahGMOLF/vn88D3QCIgAT3B1rm1y/Z3NYE0rx6cZQ==", "dev": true, "license": "Apache-2.0", "bin": { @@ -1640,19 +1640,19 @@ "node": ">=16" }, "optionalDependencies": { - "@openai/codex-darwin-arm64": "npm:@openai/codex@0.154.0-darwin-arm64", - "@openai/codex-darwin-x64": "npm:@openai/codex@0.154.0-darwin-x64", - "@openai/codex-linux-arm64": "npm:@openai/codex@0.154.0-linux-arm64", - "@openai/codex-linux-x64": "npm:@openai/codex@0.154.0-linux-x64", - "@openai/codex-win32-arm64": "npm:@openai/codex@0.154.0-win32-arm64", - "@openai/codex-win32-x64": "npm:@openai/codex@0.154.0-win32-x64" + "@openai/codex-darwin-arm64": "npm:@openai/codex@0.164.0-darwin-arm64", + "@openai/codex-darwin-x64": "npm:@openai/codex@0.164.0-darwin-x64", + "@openai/codex-linux-arm64": "npm:@openai/codex@0.164.0-linux-arm64", + "@openai/codex-linux-x64": "npm:@openai/codex@0.164.0-linux-x64", + "@openai/codex-win32-arm64": "npm:@openai/codex@0.164.0-win32-arm64", + "@openai/codex-win32-x64": "npm:@openai/codex@0.164.0-win32-x64" } }, "node_modules/@openai/codex-darwin-arm64": { "name": "@openai/codex", - "version": "0.154.0-darwin-arm64", - "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.154.0-darwin-arm64.tgz", - "integrity": "sha512-HP/vJCH/t2hB9Kg6hotN9UglClJ6/z584fal5lEP14C9gNAgAQS4/kTQC7l5V+BA3TqwDPwINSjul28cX8AYXg==", + "version": "0.164.0-darwin-arm64", + "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.164.0-darwin-arm64.tgz", + "integrity": "sha512-pDizgOUAtq3/k3AbUFHbaU19WpYKCja+oFpNSTLKrDQj4lTUNSDuTPJip4uQVn2/1EipQdpQrGMOQBVuI1znAA==", "cpu": [ "arm64" ], @@ -1668,9 +1668,9 @@ }, "node_modules/@openai/codex-darwin-x64": { "name": "@openai/codex", - "version": "0.154.0-darwin-x64", - "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.154.0-darwin-x64.tgz", - "integrity": "sha512-2aqz+72Hop8PF2RYglQ4JnGjm3OlRIrTykJIT0hyLeUgM6NCFy09RgTmqRCoWliKQZjEn9jjZqUEp7QujAj77g==", + "version": "0.164.0-darwin-x64", + "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.164.0-darwin-x64.tgz", + "integrity": "sha512-lIe+udnguPGzBezkgbkj2Eaz0YqOhNXT7jja136Fj0sXfI+UKNS2icmaK14GcoRAgeVEpDTtC1ftamWBPmD1zg==", "cpu": [ "x64" ], @@ -1686,9 +1686,9 @@ }, "node_modules/@openai/codex-linux-arm64": { "name": "@openai/codex", - "version": "0.154.0-linux-arm64", - "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.154.0-linux-arm64.tgz", - "integrity": "sha512-KmTCB6ST484zeYlPpKP/K5P/gRaYmt6TihVD+zotoe6O9q0JSBP+FYvCz4A/zZXR7xDOHURTSjHp0sD8wWS0YQ==", + "version": "0.164.0-linux-arm64", + "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.164.0-linux-arm64.tgz", + "integrity": "sha512-LL5iTQjyDL4YRX44EWqhIfEQ5uGnmgxkQBJSxjZSeJ8L1Og83pdlzUJW/PRJ76oaVwOVgh6HXwuPEiBNEHU4lg==", "cpu": [ "arm64" ], @@ -1704,9 +1704,9 @@ }, "node_modules/@openai/codex-linux-x64": { "name": "@openai/codex", - "version": "0.154.0-linux-x64", - "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.154.0-linux-x64.tgz", - "integrity": "sha512-a4FI3A8sGtwGrOqltrPbrS2hajrHQG591EwmRfiRoLMb10VxdBtUGW4gu6IJVYENiYGA7k3P4jlRHEoCZU/s9Q==", + "version": "0.164.0-linux-x64", + "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.164.0-linux-x64.tgz", + "integrity": "sha512-XLAoaWsGOT2khzl+P4FYDPTkzAr9W0VFXFaVkqlFdkM0soa1gH/4S0wSD57Ek3HHwkirAEGz2/Mwehseh+l5Mg==", "cpu": [ "x64" ], @@ -1722,9 +1722,9 @@ }, "node_modules/@openai/codex-win32-arm64": { "name": "@openai/codex", - "version": "0.154.0-win32-arm64", - "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.154.0-win32-arm64.tgz", - "integrity": "sha512-CRUmZnE0Y/a8aLMrrA681EytOGaPaF659wJAiI4I3hsbQjaeYBSPV7PkCjy4Qn5LR/fmwIUORVH+6JaBNQL+tw==", + "version": "0.164.0-win32-arm64", + "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.164.0-win32-arm64.tgz", + "integrity": "sha512-Zkzt8HNmB/vOnlDo5D7IOlrt5rHaxYQB7iGbNP7bHcXZtrk6gaD7UriwaieGYJpcf8DqAT/0tqzY4xbs8ZwbmQ==", "cpu": [ "arm64" ], @@ -1740,9 +1740,9 @@ }, "node_modules/@openai/codex-win32-x64": { "name": "@openai/codex", - "version": "0.154.0-win32-x64", - "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.154.0-win32-x64.tgz", - "integrity": "sha512-Stg2KEJPIKVqPPR1wCverGOR4ey3RR3cvakR07w7FNKQUMzmHaOZomRsP2bR1qOT/67yHsks9rB+MCMfIWXcRA==", + "version": "0.164.0-win32-x64", + "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.164.0-win32-x64.tgz", + "integrity": "sha512-urWPhwh9YavuoAPIkmVYKBTtd5OtGVL4Ki0Fb28dhK/Mz6Pt4LSy+gbRrZFz5+RXrw9ubvXuXDCcd8beYcWhDg==", "cpu": [ "x64" ], diff --git a/package.json b/package.json index 2f3e0e3..3ace87c 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "test:watch": "vitest", "codex:build": "node scripts/build-codex-runtime.mjs", "test:agents:parity": "CODEX_REQUIRE_PARITY=true vitest run tests/agents tests/pi --no-file-parallelism", + "test:agents:parity:host": "npm run test:agents:parity -- --exclude tests/pi/packaged-child.test.ts", "localstack:up": "./testing/bootstrap.sh", "localstack:status": "./testing/bootstrap.sh --status", "localstack:down": "./testing/bootstrap.sh --down", @@ -112,7 +113,7 @@ "devDependencies": { "@aws-sdk/client-dynamodb-streams": "3.1088.0", "@aws-sdk/client-ecs": "3.1088.0", - "@openai/codex": "0.154.0", + "@openai/codex": "0.164.0", "@playwright/mcp": "0.0.83", "@playwright/test": "1.62.1", "@types/archiver": "7.0.0", diff --git a/relay/Dockerfile b/relay/Dockerfile index b70a22b..7f95087 100644 --- a/relay/Dockerfile +++ b/relay/Dockerfile @@ -3,7 +3,7 @@ FROM node:22-bookworm-slim@sha256:83f487e0a63425e5b4d146fb5e5be574bcbe1b7b843d3e # the pinned image's roots so both HTTPS clients verify the same authorities. RUN mkdir -p /etc/ssl/certs && node -e "require('node:fs').writeFileSync('/etc/ssl/certs/ca-certificates.crt', require('node:tls').rootCertificates.join('\n') + '\n')" ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt -RUN npm install --global @openai/codex@0.154.0 && npm cache clean --force +RUN npm install --global @openai/codex@0.164.0 && npm cache clean --force WORKDIR /app COPY dist/environment-relay.mjs /app/environment-relay.mjs COPY dist/agents-server.mjs /app/agents-server.mjs diff --git a/runtime/codex/Dockerfile b/runtime/codex/Dockerfile index 052b0e6..5cf5e7d 100644 --- a/runtime/codex/Dockerfile +++ b/runtime/codex/Dockerfile @@ -2,27 +2,14 @@ ARG TARGETARCH FROM public.ecr.aws/lambda/microvms:al2023-minimal@sha256:05cb9b38d841e7ff1b693dc9e894909612f340bf99ec97d426e8000a5bbe96c3 AS base-arm64 FROM public.ecr.aws/amazonlinux/amazonlinux:2023@sha256:c07ab0773eddf9449c0994a87d8148300db98bdf8d8dad672a9dc26321c104fc AS base-amd64 -# Both builders retain the worker's Amazon Linux 2023 glibc baseline. -FROM base-${TARGETARCH} AS build -ARG TARGETARCH -RUN dnf install -y gcc gcc-c++ make cmake clang pkgconfig libcap-devel perl-core \ - python3 git tar gzip xz unzip nodejs22 nodejs22-npm binutils findutils \ - && dnf clean all && rm -rf /var/cache/dnf -RUN dnf install -y openssl-devel && dnf clean all && rm -rf /var/cache/dnf -RUN curl --fail --location --silent --show-error https://sh.rustup.rs --output /tmp/rustup.sh \ - && sh /tmp/rustup.sh -y --profile minimal --default-toolchain 1.95.0 --no-modify-path \ - && rm /tmp/rustup.sh -ENV PATH=/root/.cargo/bin:$PATH CARGO_BUILD_JOBS=1 +FROM base-${TARGETARCH} AS package +RUN dnf install -y nodejs22 nodejs22-npm && dnf clean all && rm -rf /var/cache/dnf WORKDIR /rat COPY runtime/codex /rat/runtime/codex -COPY scripts/build-codex-runtime.mjs /rat/scripts/build-codex-runtime.mjs -COPY scripts/codex-runtime-targets.mjs /rat/scripts/codex-runtime-targets.mjs -RUN npm install --omit=dev --no-audit --no-fund \ +COPY package-lock.json /rat/package-lock.json +COPY scripts/build-codex-runtime.mjs scripts/codex-runtime-targets.mjs scripts/codex-runtime-artifact.mjs /rat/scripts/ +RUN npm install --ignore-scripts --omit=dev --no-audit --no-fund --package-lock=false \ "@openai/codex@$(node -p 'JSON.parse(require("fs").readFileSync("runtime/codex/source.json")).upstreamVersion')" -RUN --mount=type=cache,target=/build,id=rat-codex-${TARGETARCH}-build \ - --mount=type=cache,target=/root/.cargo/registry,id=rat-codex-${TARGETARCH}-registry \ - --mount=type=cache,target=/root/.cargo/git,id=rat-codex-${TARGETARCH}-git \ - node scripts/build-codex-runtime.mjs /build \ - && mkdir /out && cp -a /build/runtime /out/runtime && cp /build/artifact.json /out/artifact.json +RUN node scripts/build-codex-runtime.mjs /out FROM scratch AS artifact -COPY --from=build /out/ / +COPY --from=package /out/ / diff --git a/runtime/codex/Dockerfile.dockerignore b/runtime/codex/Dockerfile.dockerignore index 692443f..31a238e 100644 --- a/runtime/codex/Dockerfile.dockerignore +++ b/runtime/codex/Dockerfile.dockerignore @@ -1,7 +1,9 @@ ** +!package-lock.json !runtime/ !runtime/codex/ !runtime/codex/** !scripts/ !scripts/build-codex-runtime.mjs !scripts/codex-runtime-targets.mjs +!scripts/codex-runtime-artifact.mjs diff --git a/runtime/codex/README.md b/runtime/codex/README.md index e77e7b5..9d31b52 100644 --- a/runtime/codex/README.md +++ b/runtime/codex/README.md @@ -1,105 +1,108 @@ # Maintain the Codex runtime -Codex is the default runtime; Pi Durable is optional. This directory defines the -patched Codex executable and matching companions used by the worker and local -runtime builds. Treat an upstream upgrade as a behavior change, even when every -patch applies cleanly. - -## What is pinned - -[source.json](source.json) selects the upstream repository, exact commit, release -version and ordered patches. [companion-packages.json](companion-packages.json) -locks each platform's npm package identity, tarball integrity, descriptor and file -hashes against [package-lock.json](../../package-lock.json). The compiled executable -travels with its matching code-mode host, `rg`, `zsh` and Linux `bwrap` resources. -Replacing only `bin/codex` from an unrelated release is not an upgrade. - -[artifact-policy.json](artifact-policy.json) binds the source and ordered patch -hashes to the reviewed patched Git tree. The [builder](../../scripts/build-codex-runtime.mjs) -checks the checkout before and after its locked release build. Its exported -`artifact.json` records that identity, target, build settings and complete file -inventory; keep it beside `runtime/`. The [validator](../../scripts/codex-runtime-artifact.mjs) -compares this evidence with repository policy and companion hashes, rather than -trusting an artifact's own claims. Artifact validation is not a signature or a -runtime behavior test. - -## Why each patch exists - -The order below is the application order in `source.json`. Rust test names refer -to tests in the patched upstream checkout; repository links identify separate -integration checks. Building the executable does not run those Rust tests. - -| Patch | Contract to preserve | Existing regression evidence | -| --- | --- | --- | -| [release-lock.patch](release-lock.patch) | Align workspace package versions in `Cargo.lock` with the pinned release so `cargo build --locked` succeeds without rewriting the lock. | The locked build and exact-tree audit; no separate runtime behavior change. | -| [interrupt-barrier.patch](interrupt-barrier.patch) | Await child task interruption before acknowledging it, so a follow-up on that child does not race the previous execution guard at the concurrency limit. | Strict interruption/follow-up cases in [native-session.test.ts](../../tests/agents/native-session.test.ts), plus nested limit checks in [native-admission.test.ts](../../tests/agents/native-admission.test.ts). | -| [api-model-settings.patch](api-model-settings.patch) | Preserve explicit `auto`, `default`, `flex`, `priority` and `fast` service tiers through config and Turn updates; the provider decides support without a silent catalog/UI downgrade. | Patched config, Session and protocol service-tier unit tests; [native-session-settings.test.ts](../../tests/agents/native-session-settings.test.ts) checks outgoing requests across Turns. | -| [api-turn-errors.patch](api-turn-errors.patch) | Preserve typed provider failures across HTTP and SSE, including credit exhaustion and timeouts; keep private diagnostics out of normalized errors and credit exhaustion non-retryable. | Patched API bridge/SSE unit tests; [native-turn-errors.test.ts](../../tests/agents/native-turn-errors.test.ts) checks public categories, private-message removal and request count. | -| [checkpoint-shutdown.patch](checkpoint-shutdown.patch) | Add private `thread/checkpoint`: require an idle durable thread and acknowledge successful persistence shutdown, with a bounded wait. The host must seal admission first. | [session-checkpoint.test.ts](../../tests/runner/session-checkpoint.test.ts) checks orchestration with an RPC fixture. The opt-in [AWS checkpoint proof](../../tests/aws/session-checkpoint.test.ts) exercises actual worker replacement and native history; the fixture alone does not prove a durable flush. | -| [step-settings-test-import.patch](step-settings-test-import.patch) | Restore the `ServiceTier` import needed by the upstream step-settings tests after the settings patch removes the production import. | Compile and run the affected `codex-core` unit tests; this is a test-build fix. | -| [unified-exec-output-subscription.patch](unified-exec-output-subscription.patch) | Subscribe when the process output channel is created, retaining early output until the streaming consumer takes its receiver. | Patched Rust test `exit_watcher_preserves_output_emitted_before_streaming_starts` checks both streamed bytes and completed command output. | -| [chatgpt-service-tier.patch](chatgpt-service-tier.patch) | Omit `auto`/`default` for ChatGPT authentication while preserving other explicit tiers and API-key/Bedrock request values across HTTP, WebSocket and compaction requests. | Patched `responses_service_tier_*` Rust tests cover these transports/auth modes, including repeated Turns and WebSocket continuation. They use local fixtures, not a live provider. | -| [local-subprocess-stdio.patch](local-subprocess-stdio.patch) | Permit the local socket metadata and write-half-close that Node subprocess pipes require under disabled networking. Keep connection operations, non-Unix socket creation, other socket options and other shutdown modes denied. | Patched `restricted_network_supports_node_subprocess_stdio` and `restricted_network_socket_metadata_does_not_enable_connections` tests; [managed-subprocess.test.ts](../../tests/pi/managed-subprocess.test.ts) checks the compiled artifact in strict Linux parity, and both KVM harnesses exercise the guest. | - -## Upgrade one candidate at a time - -1. Preserve the last verified artifact and its matching repository revision before - changing inputs. Review upstream changes to the patched contracts, sandbox, - app-server protocol and companion layout. Record an upstream issue/PR when a - patch is proposed there; do not assume an equivalent fix from its title alone. -2. Update the source pin and exact npm dependency/lockfile together. Rebase the - ordered patches, review every changed hunk, and refresh companion inventories - from the matching integrity-verified npm packages for all supported targets. - Refresh [model-defaults.json](model-defaults.json) from the pinned upstream - catalog and review [model-capabilities.json](model-capabilities.json) against - actual API behavior. The builder verifies the defaults' source hash and values. -3. Use a new build directory for the candidate. Run `npm run codex:build -- - /absolute/path/to/new-build --prepare-only`, inspect the resulting source diff, - and derive the expected patched tree from those reviewed inputs. Update - `artifact-policy.json` with that tree and the source identity hash; never take - the expected tree from an untrusted artifact merely to make validation pass. -4. Compile and run the affected upstream Rust test modules, including the tests - named above. Then build and validate each platform below. Use the exact candidate - executable for `CODEX_CONFORMANCE_BINARY`; keep strict parity enabled through - `npm run test:agents:parity`. A fixture pass establishes the exercised protocol - behavior, not live authentication or VM isolation. -5. Run `npm run check` with the matching ARM64 packaging artifact, the worker-image - gates and the relevant local acceptance harnesses. Record artifact identities, - commands, results and remaining gaps in the PR/evidence directory. Promote - source policy and artifacts together only after the required gates pass. - -Use the build commands and prerequisites in the -[native compatibility guide](../../testing/README.md#agents-api-native-compatibility-gate). -The [target map](../../scripts/codex-runtime-targets.mjs) is authoritative: - -| Lane | Build and acceptance | -| --- | --- | -| Linux ARM64, `aarch64-unknown-linux-gnu` | Build `runtime/codex/Dockerfile` with `--platform linux/arm64` on the worker's Amazon Linux 2023 baseline. Run packaging, `test:e2e:native-image` and `test:e2e:microvm-image`. Lambda packaging remains ARM64. | -| Linux x64, `x86_64-unknown-linux-gnu` | Build the same Dockerfile with `--platform linux/amd64`. Validate the x64 artifact, run strict native parity and sandbox tests with it, then exercise the [KVM managed Session harnesses](../../testing/local-runtime/linux-kvm/README.md). Building an image does not prove isolation. | -| Apple silicon macOS, `aarch64-apple-darwin` | Build natively with `codex:build` and run strict native parity on macOS. Pair it with the matching ARM64 Linux guest; follow the [guest build](../../testing/local-runtime/native-guest-build.md), [signature fixtures](../../testing/local-runtime/native-signatures/README.md) and [installed runtime acceptance](../../testing/local-runtime/README.md). Keep the signed bundle/verifier contract intact. | - -The [CI workflow](../../.github/workflows/ci.yml) builds and exports ARM64 and x64 -artifacts after their configured gates. Downloaded exports still require provenance -validation. Its macOS job checks the desktop, not a patched macOS runtime release; -do not count that job as macOS runtime acceptance. Caches save compilation work and -never replace artifact validation or semantic tests. For checkpoint changes, include the -explicitly opted-in AWS checkpoint proof before claiming cloud handoff acceptance; -ordinary checks must not provision AWS resources. - -## Upstream, drop or roll back - -Prefer small upstream fixes with the reproducer attached. Drop a patch only when -the pinned upstream source implements the required contract without it, or Rat -Things deliberately removes that dependency. Remove it from `source.json`, refresh -the tree policy, and rerun its semantic regression and affected platform gates. -Keep the regression after dropping the patch. Build/test-only patches can go once -the clean upstream release builds and its affected tests compile without them. - -If a candidate fails, retain the previous verified release. Roll back the source -definition, patches, lockfile, companion inventory and policy together, and select -their matching complete artifact and worker image or signed app release. Do not -relax validation or fall back to the stock npm binary. Check durable-state -compatibility before opening Sessions written by the newer candidate; preserve -their storage and withhold rollback admission if compatibility is unknown. Keep -the last verified artifact outside disposable build caches until its replacement -has passed acceptance. +Rat Things uses the official, unmodified Codex 0.164.0 release. It does not compile +Codex, apply source patches or rewrite its executable. Runtime-specific behavior +belongs in Rat Things adapters and uses the public Codex app-server protocol. + +## Release identity + +[source.json](source.json) records the upstream repository, release commit and +version. [companion-packages.json](companion-packages.json) records the official +npm package URL, integrity, descriptor and every runtime file hash for each +supported platform. [package-lock.json](../../package-lock.json) pins the same +packages. The entrypoint, code-mode and voice companions, shared libraries, +resources and bundled `bwrap` must all match that inventory. Bundling voice +resources does not enable voice. + +The [packager](../../scripts/build-codex-runtime.mjs) copies the installed official +package into a new directory and writes `artifact.json` beside `runtime/`. It +then runs the [validator](../../scripts/codex-runtime-artifact.mjs). Validation +requires the pinned distribution identity and exact bytes for every file, +including `bin/codex`; a self-consistent manifest cannot authorize a modified +binary. Missing or extra files, symlinks, unsupported targets and old custom-build +manifests are rejected. The original package descriptor is preserved. + +Linux packages use the upstream musl targets. Worker and VM helper target names +are separate from the Codex distribution target. Linux ARM64 remains required +for Lambda packaging; Linux x64 is used by the local KVM development backend. + +```sh +npm ci +npm run codex:build -- /absolute/path/to/new-runtime +``` + +The historical `codex:build` command packages the official release for the host +platform; it performs no Rust compilation. Use a new output directory. To prepare +a different Linux architecture, use the container packager: + +```sh +docker buildx build --platform linux/arm64 --file runtime/codex/Dockerfile \ + --output type=local,dest=/absolute/path/to/new-arm64-runtime . +``` + +Keep `artifact.json` and `runtime/` together. Select an export with +`CODEX_RUNTIME_ARTIFACT`. Installing a newer global `codex` does not update Rat +Things' pinned worker artifact. + +## Adapter contracts + +Codex settings follow the capabilities exposed by the pinned release. Rat Things +maps its public settings at the boundary and rejects unsupported explicit choices +before starting model work. Provider failures retain the precision available in +Codex's public typed events; private diagnostic text is not used to invent error +categories or returned as a public message. + +Sandbox enforcement remains a prerequisite for managed execution. On Linux, +network-disabled execution uses a validated proxy policy with no allowed +destinations so Node subprocess I/O can work without granting external network +access. The host owns the effective policy and refuses unexpected configuration +or approval requests. Tests must verify actual denied writes and connections, +not infer enforcement from a configuration flag. + +Native checkpoint publication requires completed-turn recovery verification by +the host. An ordinary Codex process exit is not a persistence acknowledgement. +The host must stop all writers, synchronize storage, and verify the saved native +root and child identities and complete public history in a disposable isolated +process before publishing a candidate. Verification must not perform model work +or tools, and its writes must not replace the saved candidate. Running commands, +pending interactions and arbitrary background work are not migrated. Failed +verification must fail the handoff; it must not silently reconstruct a prompt or +start a new native thread. The opt-in AWS checkpoint harness remains necessary +before claiming cloud worker replacement acceptance. + +The cold verifier supports the built-in OpenAI provider and custom Responses API +providers. It rejects Bedrock, Ollama and LM Studio before stopping the live +process until their recovery paths are validated; ordinary Sessions and Pi +Durable execution are unaffected. + +## Upgrade and rollback + +1. Preserve the previous artifact and matching repository revision. Select an + official stable release and inspect its public protocol, sandbox behavior and + companion layout. +2. Update the source pin and npm lock together. Verify each official package's + tarball integrity before refreshing the complete inventory. Update model + defaults from that release's catalog; absent models keep a null default. +3. Package into a new directory and verify its complete inventory. Update Rat + Things adapters for upstream changes. Do not patch Codex to preserve an old + assertion or substitute an unrelated executable. +4. Run the strict native contracts, sandbox and checkpoint regressions against + those exact official bytes. Then run `npm run check`, worker-image checks and + relevant managed acceptance harnesses described in [testing](../../testing/README.md). +5. Record the release, artifact identities, validation and limitations in the PR. + Promote the release only after the required gates pass. + +CI packages official ARM64 and x64 releases and exports them after their required +gates. Cached artifacts still undergo validation and behavior checks. The macOS +CI job validates the desktop; it does not establish signed runtime acceptance. +The native macOS bundle assembler preserves every official Codex and companion +byte, including existing signatures. It signs only the Rat Things VM helper and +outer bundle, then checks the official payload hashes again. Native macOS runtime +distribution remains blocked: admission of the upstream signatures still requires +design and validation on a Mac. The existing installer signer checks remain in +force, and assembled candidates retain `distributionReady: false`. + +Rollback must move the source definition, package lock, inventories, adapters and +runtime artifact together. Preserve newer session storage and verify that the +older version can read it before admitting a rollback. Never bypass checksum, +sandbox or checkpoint verification to make an older artifact run. diff --git a/runtime/codex/api-model-settings.patch b/runtime/codex/api-model-settings.patch deleted file mode 100644 index 92bb157..0000000 --- a/runtime/codex/api-model-settings.patch +++ /dev/null @@ -1,315 +0,0 @@ -diff --git a/codex-rs/core/src/config/config_tests.rs b/codex-rs/core/src/config/config_tests.rs -index a093aaf..900c7d2 100644 ---- a/codex-rs/core/src/config/config_tests.rs -+++ b/codex-rs/core/src/config/config_tests.rs -@@ -9839,7 +9839,7 @@ async fn default_service_tier_override_uses_default_request_value() -> std::io:: - } - - #[tokio::test] --async fn legacy_fast_service_tier_override_uses_priority_request_value() -> std::io::Result<()> { -+async fn legacy_fast_service_tier_override_preserves_request_value() -> std::io::Result<()> { - let fixture = create_test_fixture()?; - - let config = Config::load_from_base_config_with_overrides( -@@ -9853,10 +9853,7 @@ async fn legacy_fast_service_tier_override_uses_priority_request_value() -> std: - ) - .await?; - -- assert_eq!( -- config.service_tier, -- Some(ServiceTier::Fast.request_value().to_string()) -- ); -+ assert_eq!(config.service_tier, Some("fast".to_string())); - Ok(()) - } - -@@ -9909,7 +9906,7 @@ async fn config_toml_service_tier_accepts_arbitrary_string() -> std::io::Result< - } - - #[tokio::test] --async fn config_toml_legacy_fast_service_tier_uses_priority_request_value() -> std::io::Result<()> { -+async fn config_toml_legacy_fast_service_tier_preserves_request_value() -> std::io::Result<()> { - let mut fixture = create_test_fixture()?; - fixture.cfg.service_tier = Some("fast".to_string()); - let cwd = fixture.cwd_path(); -@@ -9925,10 +9922,7 @@ async fn config_toml_legacy_fast_service_tier_uses_priority_request_value() -> s - ) - .await?; - -- assert_eq!( -- config.service_tier, -- Some(ServiceTier::Fast.request_value().to_string()) -- ); -+ assert_eq!(config.service_tier, Some("fast".to_string())); - Ok(()) - } - -diff --git a/codex-rs/core/src/config/mod.rs b/codex-rs/core/src/config/mod.rs -index ebda977..9ac707b 100644 ---- a/codex-rs/core/src/config/mod.rs -+++ b/codex-rs/core/src/config/mod.rs -@@ -102,7 +102,6 @@ use codex_protocol::config_types::Personality; - use codex_protocol::config_types::ReasoningSummary; - use codex_protocol::config_types::SERVICE_TIER_DEFAULT_REQUEST_VALUE; - use codex_protocol::config_types::SandboxMode; --use codex_protocol::config_types::ServiceTier; - use codex_protocol::config_types::ShellEnvironmentPolicy; - use codex_protocol::config_types::TrustLevel; - use codex_protocol::config_types::Verbosity; -@@ -3884,15 +3883,6 @@ impl Config { - Some(None) => Some(SERVICE_TIER_DEFAULT_REQUEST_VALUE.to_string()), - None => cfg.service_tier, - }; -- let service_tier = service_tier.and_then(|service_tier| { -- match ServiceTier::from_request_value(&service_tier) { -- Some(ServiceTier::Fast) => features -- .enabled(Feature::FastMode) -- .then(|| ServiceTier::Fast.request_value().to_string()), -- Some(ServiceTier::Flex) => Some(ServiceTier::Flex.request_value().to_string()), -- None => Some(service_tier), -- } -- }); - - let compact_prompt = compact_prompt.or(cfg.compact_prompt).and_then(|value| { - let trimmed = value.trim(); -diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs -index 8ffc144..8485d20 100644 ---- a/codex-rs/core/src/session/mod.rs -+++ b/codex-rs/core/src/session/mod.rs -@@ -986,6 +986,14 @@ fn get_service_tier( - fast_mode_enabled: bool, - model_info: &ModelInfo, - ) -> Option { -+ // The API provider, rather than the interactive model catalogue, decides -+ // whether an explicit routing choice is supported. Never silently downgrade it. -+ if configured_service_tier -+ .as_deref() -+ .is_some_and(|tier| codex_protocol::config_types::AGENTS_API_SERVICE_TIERS.contains(&tier)) -+ { -+ return configured_service_tier; -+ } - if !fast_mode_enabled { - return None; - } -@@ -1002,6 +1010,7 @@ fn unsupported_service_tier_warning( - ) -> Option { - let service_tier = configured_service_tier.filter(|service_tier| { - fast_mode_enabled -+ && !codex_protocol::config_types::AGENTS_API_SERVICE_TIERS.contains(service_tier) - && *service_tier != SERVICE_TIER_DEFAULT_REQUEST_VALUE - && !model_info.supports_service_tier(service_tier) - })?; -diff --git a/codex-rs/core/src/session/step_settings.rs b/codex-rs/core/src/session/step_settings.rs -index 49502e8..af1bee7 100644 ---- a/codex-rs/core/src/session/step_settings.rs -+++ b/codex-rs/core/src/session/step_settings.rs -@@ -12,7 +12,6 @@ use codex_protocol::config_types::CollaborationMode; - use codex_protocol::config_types::Personality; - use codex_protocol::config_types::ReasoningSummary; - use codex_protocol::config_types::SERVICE_TIER_DEFAULT_REQUEST_VALUE; --use codex_protocol::config_types::ServiceTier; - use codex_protocol::openai_models::ModelInfo; - use codex_protocol::openai_models::ReasoningEffort; - use codex_protocol::protocol::AskForApproval; -@@ -275,13 +274,8 @@ impl StepSettings { - next.reasoning_summary = Some(summary); - } - if let Some(service_tier) = update.service_tier.clone() { -- // TODO(aibrahim): Remove once v2 clients no longer send the legacy -- // "fast" service tier value. - next.service_tier = Some(match service_tier { -- Some(service_tier) => ServiceTier::from_request_value(&service_tier) -- .map_or(service_tier, |service_tier| { -- service_tier.request_value().to_string() -- }), -+ Some(service_tier) => service_tier, - None => SERVICE_TIER_DEFAULT_REQUEST_VALUE.to_string(), - }); - } -diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs -index 116e450..2c1d33a 100644 ---- a/codex-rs/core/src/session/tests.rs -+++ b/codex-rs/core/src/session/tests.rs -@@ -5096,71 +5096,36 @@ fn get_service_tier_does_not_default_when_model_has_no_default() { - } - - #[test] --fn get_service_tier_drops_unsupported_configured_tier_when_fast_mode_enabled() { -- let model_info = model_with_default_service_tier(Some(ServiceTier::Fast.request_value())); -- -- assert_eq!( -- get_service_tier( -- Some("unsupported".to_string()), -- /*fast_mode_enabled*/ true, -- &model_info, -- ), -- None -- ); -- assert_eq!( -- get_service_tier( -- Some(ServiceTier::Flex.request_value().to_string()), -- /*fast_mode_enabled*/ true, -- &model_info, -- ), -- None -- ); -- assert_eq!( -- get_service_tier( -- Some(SERVICE_TIER_DEFAULT_REQUEST_VALUE.to_string()), -- /*fast_mode_enabled*/ true, -- &model_info, -- ), -- Some(SERVICE_TIER_DEFAULT_REQUEST_VALUE.to_string()) -- ); --} -- --#[test] --fn get_service_tier_ignores_configured_tier_when_fast_mode_disabled() { -- let model_info = model_with_default_service_tier(Some(ServiceTier::Fast.request_value())); -- -- assert_eq!( -- get_service_tier( -- Some(ServiceTier::Fast.request_value().to_string()), -- /*fast_mode_enabled*/ false, -- &model_info, -- ), -- None -- ); -- assert_eq!( -- get_service_tier( -- Some(SERVICE_TIER_DEFAULT_REQUEST_VALUE.to_string()), -- /*fast_mode_enabled*/ false, -- &model_info, -- ), -- None -- ); -- assert_eq!( -- get_service_tier( -- Some("unsupported".to_string()), -- /*fast_mode_enabled*/ false, -- &model_info, -- ), -- None -- ); -- assert_eq!( -- get_service_tier( -- /*configured_service_tier*/ None, -- /*fast_mode_enabled*/ false, -- &model_info, -- ), -- None -- ); -+fn get_service_tier_preserves_explicit_api_choices_without_catalog_or_ui_support() { -+ let model_info = model_with_default_service_tier(/*default_service_tier*/ None); -+ for fast_mode_enabled in [false, true] { -+ for tier in codex_protocol::config_types::AGENTS_API_SERVICE_TIERS { -+ assert_eq!( -+ get_service_tier(Some(tier.to_string()), fast_mode_enabled, &model_info), -+ Some(tier.to_string()) -+ ); -+ assert_eq!( -+ unsupported_service_tier_warning(Some(tier), fast_mode_enabled, &model_info), -+ None -+ ); -+ } -+ assert_eq!( -+ get_service_tier( -+ Some("unsupported".to_string()), -+ fast_mode_enabled, -+ &model_info -+ ), -+ None -+ ); -+ assert_eq!( -+ get_service_tier( -+ /*configured_service_tier*/ None, -+ fast_mode_enabled, -+ &model_info -+ ), -+ None -+ ); -+ } - } - - #[tokio::test] -@@ -5187,7 +5152,7 @@ async fn session_settings_null_service_tier_update_uses_default_service_tier() { - } - - #[tokio::test] --async fn session_settings_legacy_fast_service_tier_update_uses_priority_request_value() { -+async fn session_settings_fast_service_tier_update_preserves_request_value() { - let session_configuration = make_session_configuration_for_tests().await; - - let updated = session_configuration -@@ -5203,10 +5168,7 @@ async fn session_settings_legacy_fast_service_tier_update_uses_priority_request_ - ) - .expect("legacy fast service tier update should apply"); - -- assert_eq!( -- updated.step_settings.service_tier, -- Some(ServiceTier::Fast.request_value().to_string()) -- ); -+ assert_eq!(updated.step_settings.service_tier, Some("fast".to_string())); - } - - pub(crate) async fn make_session_configuration_for_tests() -> SessionConfiguration { -diff --git a/codex-rs/protocol/src/config_types.rs b/codex-rs/protocol/src/config_types.rs -index b5ac448..4bdaa7d 100644 ---- a/codex-rs/protocol/src/config_types.rs -+++ b/codex-rs/protocol/src/config_types.rs -@@ -535,6 +535,9 @@ pub enum ServiceTier { - /// selected no service tier, so model catalog defaults should not apply. - pub const SERVICE_TIER_DEFAULT_REQUEST_VALUE: &str = "default"; - -+/// Explicit Agents API routing choices are validated by the model provider. -+pub const AGENTS_API_SERVICE_TIERS: [&str; 5] = ["auto", "default", "flex", "priority", "fast"]; -+ - impl ServiceTier { - pub const fn request_value(self) -> &'static str { - match self { -diff --git a/codex-rs/protocol/src/openai_models.rs b/codex-rs/protocol/src/openai_models.rs -index 8c021b1..ef0632d 100644 ---- a/codex-rs/protocol/src/openai_models.rs -+++ b/codex-rs/protocol/src/openai_models.rs -@@ -923,8 +923,8 @@ impl ModelInfo { - - pub fn service_tier_for_request(&self, service_tier: Option) -> Option { - service_tier.filter(|service_tier| { -- service_tier != SERVICE_TIER_DEFAULT_REQUEST_VALUE -- && self.supports_service_tier(service_tier) -+ crate::config_types::AGENTS_API_SERVICE_TIERS.contains(&service_tier.as_str()) -+ || self.supports_service_tier(service_tier) - }) - } - } -@@ -2050,7 +2050,7 @@ mod tests { - } - - #[test] -- fn service_tier_for_request_omits_explicit_default_tier() { -+ fn service_tier_for_request_preserves_explicit_default_tier() { - let model = ModelInfo { - default_service_tier: Some(ServiceTier::Fast.request_value().to_string()), - service_tiers: vec![ModelServiceTier { -@@ -2063,7 +2063,7 @@ mod tests { - - assert_eq!( - model.service_tier_for_request(Some(SERVICE_TIER_DEFAULT_REQUEST_VALUE.to_string())), -- None -+ Some(SERVICE_TIER_DEFAULT_REQUEST_VALUE.to_string()) - ); - } - -@@ -2087,6 +2087,12 @@ mod tests { - model.service_tier_for_request(Some("unsupported".to_string())), - None - ); -+ for tier in crate::config_types::AGENTS_API_SERVICE_TIERS { -+ assert_eq!( -+ model.service_tier_for_request(Some(tier.to_string())), -+ Some(tier.to_string()) -+ ); -+ } - assert_eq!(model.service_tier_for_request(/*service_tier*/ None), None); - } - } diff --git a/runtime/codex/api-turn-errors.patch b/runtime/codex/api-turn-errors.patch deleted file mode 100644 index 3236630..0000000 --- a/runtime/codex/api-turn-errors.patch +++ /dev/null @@ -1,347 +0,0 @@ -diff --git a/codex-rs/app-server-protocol/src/protocol/v2/shared.rs b/codex-rs/app-server-protocol/src/protocol/v2/shared.rs -index b5a68a0..e6d174b 100644 ---- a/codex-rs/app-server-protocol/src/protocol/v2/shared.rs -+++ b/codex-rs/app-server-protocol/src/protocol/v2/shared.rs -@@ -78,6 +78,8 @@ pub enum CodexErrorInfo { - ContextWindowExceeded, - SessionBudgetExceeded, - UsageLimitExceeded, -+ CreditBalanceExhausted, -+ RequestTimeout, - RateLimitExceeded, - ServerOverloaded, - CyberPolicy, -@@ -126,6 +128,8 @@ impl From for CodexErrorInfo { - CoreCodexErrorInfo::ContextWindowExceeded => CodexErrorInfo::ContextWindowExceeded, - CoreCodexErrorInfo::SessionBudgetExceeded => CodexErrorInfo::SessionBudgetExceeded, - CoreCodexErrorInfo::UsageLimitExceeded => CodexErrorInfo::UsageLimitExceeded, -+ CoreCodexErrorInfo::CreditBalanceExhausted => CodexErrorInfo::CreditBalanceExhausted, -+ CoreCodexErrorInfo::RequestTimeout => CodexErrorInfo::RequestTimeout, - CoreCodexErrorInfo::RateLimitExceeded => CodexErrorInfo::RateLimitExceeded, - CoreCodexErrorInfo::ServerOverloaded => CodexErrorInfo::ServerOverloaded, - CoreCodexErrorInfo::CyberPolicy => CodexErrorInfo::CyberPolicy, -diff --git a/codex-rs/cli/src/doctor.rs b/codex-rs/cli/src/doctor.rs -index e42fa9b..5f0a722 100644 ---- a/codex-rs/cli/src/doctor.rs -+++ b/codex-rs/cli/src/doctor.rs -@@ -2364,6 +2364,7 @@ fn websocket_error_detail(err: &ApiError) -> String { - ApiError::Stream(message) => format!("handshake stream error: {message}"), - ApiError::ContextWindowExceeded - | ApiError::QuotaExceeded -+ | ApiError::CreditBalanceExhausted - | ApiError::UsageNotIncluded - | ApiError::Retryable { .. } - | ApiError::RateLimitExceeded { .. } -diff --git a/codex-rs/codex-api/src/api_bridge.rs b/codex-rs/codex-api/src/api_bridge.rs -index e8d7c47..1b33d11 100644 ---- a/codex-rs/codex-api/src/api_bridge.rs -+++ b/codex-rs/codex-api/src/api_bridge.rs -@@ -22,6 +22,7 @@ pub fn map_api_error(err: ApiError) -> CodexErr { - match err { - ApiError::ContextWindowExceeded => CodexErr::ContextWindowExceeded, - ApiError::QuotaExceeded => CodexErr::QuotaExceeded, -+ ApiError::CreditBalanceExhausted => CodexErr::CreditBalanceExhausted, - ApiError::UsageNotIncluded => CodexErr::UsageNotIncluded, - ApiError::Retryable { message, delay } => { - let error = CodexErr::Stream(message); -@@ -40,6 +41,9 @@ pub fn map_api_error(err: ApiError) -> CodexErr { - ApiError::Stream(msg) => CodexErr::Stream(msg), - ApiError::ServerOverloaded => CodexErr::ServerOverloaded, - ApiError::Api { status, message } => { -+ if let Some(error) = typed_error_body(&message) { -+ return error; -+ } - let user_message = api_error_user_message(status, &message); - CodexErr::UnexpectedStatus(UnexpectedResponseError { - status, -@@ -71,6 +75,9 @@ pub fn map_api_error(err: ApiError) -> CodexErr { - body, - } => { - let body_text = body.unwrap_or_default(); -+ if let Some(error) = typed_error_body(&body_text) { -+ return error; -+ } - - if status == http::StatusCode::SERVICE_UNAVAILABLE - && let Ok(value) = serde_json::from_str::(&body_text) -@@ -196,6 +203,24 @@ pub fn map_api_error(err: ApiError) -> CodexErr { - } - } - -+// Typed failures take precedence over a transport status without retaining the -+// provider body in the resulting error. -+fn typed_error_body(body: &str) -> Option { -+ let value = serde_json::from_str::(body).ok()?; -+ match value.get("error")?.get("code")?.as_str()? { -+ "credit_balance_exhausted" => Some(CodexErr::CreditBalanceExhausted), -+ "context_length_exceeded" => Some(CodexErr::ContextWindowExceeded), -+ "session_budget_exceeded" => Some(CodexErr::SessionBudgetExceeded), -+ "insufficient_quota" | "usage_limit_exceeded" => Some(CodexErr::QuotaExceeded), -+ "server_error" => Some(CodexErr::InternalServerError), -+ "server_overloaded" | "server_is_overloaded" | "slow_down" => { -+ Some(CodexErr::ServerOverloaded) -+ } -+ "request_timeout" => Some(CodexErr::RequestTimeout), -+ _ => None, -+ } -+} -+ - const ACTIVE_LIMIT_HEADER: &str = "x-codex-active-limit"; - const REQUEST_ID_HEADER: &str = "x-request-id"; - const OAI_REQUEST_ID_HEADER: &str = "x-oai-request-id"; -diff --git a/codex-rs/codex-api/src/api_bridge_tests.rs b/codex-rs/codex-api/src/api_bridge_tests.rs -index 666dea8..a2df4c3 100644 ---- a/codex-rs/codex-api/src/api_bridge_tests.rs -+++ b/codex-rs/codex-api/src/api_bridge_tests.rs -@@ -521,3 +521,67 @@ fn map_api_error_extracts_identity_auth_details_from_headers() { - ); - assert_eq!(err.identity_error_code.as_deref(), Some("token_expired")); - } -+ -+#[test] -+fn credit_balance_errors_preserve_their_category_and_do_not_retry() { -+ let body = -+ r#"{"error":{"code":"credit_balance_exhausted","message":"private provider diagnostic"}}"#; -+ for error in [ -+ ApiError::CreditBalanceExhausted, -+ ApiError::Api { -+ status: http::StatusCode::PAYMENT_REQUIRED, -+ message: body.to_string(), -+ }, -+ ApiError::Transport(TransportError::Http { -+ status: http::StatusCode::TOO_MANY_REQUESTS, -+ url: None, -+ headers: None, -+ body: Some(body.to_string()), -+ }), -+ ] { -+ let mapped = map_api_error(error); -+ assert_eq!( -+ mapped.to_codex_protocol_error(), -+ CodexErrorInfo::CreditBalanceExhausted -+ ); -+ assert!(!mapped.is_retryable()); -+ assert_eq!(mapped.to_string(), "API credit balance exhausted."); -+ } -+} -+ -+#[test] -+fn typed_provider_errors_are_independent_of_http_transport_status() { -+ for (code, expected) in [ -+ ( -+ "context_length_exceeded", -+ CodexErrorInfo::ContextWindowExceeded, -+ ), -+ ("insufficient_quota", CodexErrorInfo::UsageLimitExceeded), -+ ( -+ "session_budget_exceeded", -+ CodexErrorInfo::SessionBudgetExceeded, -+ ), -+ ("server_error", CodexErrorInfo::InternalServerError), -+ ("server_overloaded", CodexErrorInfo::ServerOverloaded), -+ ("request_timeout", CodexErrorInfo::RequestTimeout), -+ ] { -+ let body = serde_json::json!({"error": {"code": code, "message": "private diagnostic"}}) -+ .to_string(); -+ for error in [ -+ ApiError::Api { -+ status: http::StatusCode::BAD_REQUEST, -+ message: body.clone(), -+ }, -+ ApiError::Transport(TransportError::Http { -+ status: http::StatusCode::BAD_REQUEST, -+ url: None, -+ headers: None, -+ body: Some(body), -+ }), -+ ] { -+ let mapped = map_api_error(error); -+ assert_eq!(mapped.to_codex_protocol_error(), expected); -+ assert!(!mapped.to_string().contains("private diagnostic")); -+ } -+ } -+} -diff --git a/codex-rs/codex-api/src/error.rs b/codex-rs/codex-api/src/error.rs -index d1d83cf..1030d81 100644 ---- a/codex-rs/codex-api/src/error.rs -+++ b/codex-rs/codex-api/src/error.rs -@@ -17,6 +17,8 @@ pub enum ApiError { - ContextWindowExceeded, - #[error("quota exceeded")] - QuotaExceeded, -+ #[error("API credit balance exhausted")] -+ CreditBalanceExhausted, - #[error("usage not included")] - UsageNotIncluded, - #[error("retryable error: {message}")] -diff --git a/codex-rs/codex-api/src/sse/responses.rs b/codex-rs/codex-api/src/sse/responses.rs -index e4f5f04..126005d 100644 ---- a/codex-rs/codex-api/src/sse/responses.rs -+++ b/codex-rs/codex-api/src/sse/responses.rs -@@ -420,7 +420,23 @@ pub fn process_responses_event( - if let Some(error) = resp_val.get("error") - && let Ok(error) = serde_json::from_value::(error.clone()) - { -- if is_context_window_error(&error) { -+ if error.code.as_deref() == Some("credit_balance_exhausted") { -+ response_error = ApiError::CreditBalanceExhausted; -+ } else if matches!( -+ error.code.as_deref(), -+ Some( -+ "server_error" -+ | "request_timeout" -+ | "session_budget_exceeded" -+ | "usage_limit_exceeded" -+ | "server_overloaded" -+ ) -+ ) { -+ response_error = ApiError::Api { -+ status: http::StatusCode::INTERNAL_SERVER_ERROR, -+ message: serde_json::json!({"error": {"code": error.code}}).to_string(), -+ }; -+ } else if is_context_window_error(&error) { - response_error = ApiError::ContextWindowExceeded; - } else if is_quota_exceeded_error(&error) { - response_error = ApiError::QuotaExceeded; -@@ -1181,6 +1197,15 @@ mod tests { - assert_matches!(events[0], Err(ApiError::ContextWindowExceeded)); - } - -+ #[tokio::test] -+ async fn credit_balance_exhausted_error_is_fatal() { -+ let raw = r#"{"type":"response.failed","response":{"id":"credit_failure","error":{"code":"credit_balance_exhausted","message":"No API credits remain."}}}"#; -+ let sse = format!("data: {raw}\n\n"); -+ let events = collect_events(&[sse.as_bytes()]).await; -+ assert_eq!(events.len(), 1); -+ assert_matches!(events[0], Err(ApiError::CreditBalanceExhausted)); -+ } -+ - #[tokio::test] - async fn quota_exceeded_error_is_fatal() { - let raw_error = r#"{"type":"response.failed","sequence_number":3,"response":{"id":"resp_fatal_quota","object":"response","created_at":1759771626,"status":"failed","background":false,"error":{"code":"insufficient_quota","message":"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."},"incomplete_details":null}}"#; -diff --git a/codex-rs/ext/goal/src/extension.rs b/codex-rs/ext/goal/src/extension.rs -index 8984899..633b99d 100644 ---- a/codex-rs/ext/goal/src/extension.rs -+++ b/codex-rs/ext/goal/src/extension.rs -@@ -365,7 +365,9 @@ where - }; - - let reason = match input.error { -- CodexErrorInfo::UsageLimitExceeded => ActiveGoalStopReason::UsageLimit, -+ CodexErrorInfo::UsageLimitExceeded | CodexErrorInfo::CreditBalanceExhausted => { -+ ActiveGoalStopReason::UsageLimit -+ } - // The turn has ended because the error was non-retryable or its - // retries were exhausted. Block the goal to prevent automatic - // continuation from looping and consuming tokens, as can happen -diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/metrics.rs b/codex-rs/ext/guardian-v2/src/async_scorer/metrics.rs -index 9b60a47..4058215 100644 ---- a/codex-rs/ext/guardian-v2/src/async_scorer/metrics.rs -+++ b/codex-rs/ext/guardian-v2/src/async_scorer/metrics.rs -@@ -43,6 +43,7 @@ pub(super) fn sampler_failure_reason(error: &LunaSamplerError) -> &'static str { - ApiError::Stream(_) => "stream_error", - ApiError::ContextWindowExceeded => "context_window_exceeded", - ApiError::QuotaExceeded => "quota_exceeded", -+ ApiError::CreditBalanceExhausted => "credit_balance_exhausted", - ApiError::UsageNotIncluded => "usage_not_included", - ApiError::Retryable { .. } => "retryable_api_error", - ApiError::RateLimitExceeded { .. } | ApiError::RateLimit(_) => "rate_limit", -diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/sampler.rs b/codex-rs/ext/guardian-v2/src/async_scorer/sampler.rs -index 993e2d4..1f68eb6 100644 ---- a/codex-rs/ext/guardian-v2/src/async_scorer/sampler.rs -+++ b/codex-rs/ext/guardian-v2/src/async_scorer/sampler.rs -@@ -240,6 +240,7 @@ impl LunaSampler { - ApiError::Transport(TransportError::Build(_)) - | ApiError::ContextWindowExceeded - | ApiError::QuotaExceeded -+ | ApiError::CreditBalanceExhausted - | ApiError::UsageNotIncluded - | ApiError::RateLimit(_) - | ApiError::InvalidRequest { .. } -diff --git a/codex-rs/protocol/src/error.rs b/codex-rs/protocol/src/error.rs -index be0192f..f0f8ea9 100644 ---- a/codex-rs/protocol/src/error.rs -+++ b/codex-rs/protocol/src/error.rs -@@ -147,6 +147,8 @@ pub enum CodexErrorDetails { - ConnectionFailed(ConnectionFailedError), - #[error("Quota exceeded. Check your plan and billing details.")] - QuotaExceeded, -+ #[error("API credit balance exhausted.")] -+ CreditBalanceExhausted, - #[error( - "To use Codex with your ChatGPT plan, upgrade to Plus: https://chatgpt.com/explore/plus." - )] -@@ -321,6 +323,7 @@ impl CodexErr { - Interrupted, - ServerOverloaded, - QuotaExceeded, -+ CreditBalanceExhausted, - UsageNotIncluded, - InternalServerError, - InternalAgentDied, -@@ -377,6 +380,7 @@ impl CodexErr { - | CodexErrorDetails::Fatal(_) - | CodexErrorDetails::UsageNotIncluded - | CodexErrorDetails::QuotaExceeded -+ | CodexErrorDetails::CreditBalanceExhausted - | CodexErrorDetails::InvalidImageRequest() - | CodexErrorDetails::InvalidRequest(_) - | CodexErrorDetails::ToolCollision(_) -@@ -431,6 +435,7 @@ impl CodexErr { - pub fn to_codex_protocol_error(&self) -> CodexErrorInfo { - match &self.details { - CodexErrorDetails::ContextWindowExceeded => CodexErrorInfo::ContextWindowExceeded, -+ CodexErrorDetails::CreditBalanceExhausted => CodexErrorInfo::CreditBalanceExhausted, - CodexErrorDetails::SessionBudgetExceeded => CodexErrorInfo::SessionBudgetExceeded, - CodexErrorDetails::RateLimitExceeded(_) => CodexErrorInfo::RateLimitExceeded, - CodexErrorDetails::UsageLimitReached(_) -@@ -452,6 +457,15 @@ impl CodexErr { - http_status_code: self.http_status_code_value(), - } - } -+ CodexErrorDetails::UnexpectedStatus(error) => CodexErrorInfo::HttpConnectionFailed { -+ http_status_code: Some(error.status.as_u16()), -+ }, -+ CodexErrorDetails::InvalidRequest(_) | CodexErrorDetails::InvalidImageRequest() => { -+ CodexErrorInfo::BadRequest -+ } -+ CodexErrorDetails::RequestTimeout | CodexErrorDetails::Timeout => { -+ CodexErrorInfo::RequestTimeout -+ } - CodexErrorDetails::RefreshTokenFailed(_) => CodexErrorInfo::Unauthorized, - CodexErrorDetails::SessionConfiguredNotFirstEvent - | CodexErrorDetails::InternalServerError -diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs -index 66e4315..1eb4740 100644 ---- a/codex-rs/protocol/src/protocol.rs -+++ b/codex-rs/protocol/src/protocol.rs -@@ -1852,6 +1852,8 @@ pub enum CodexErrorInfo { - ContextWindowExceeded, - SessionBudgetExceeded, - UsageLimitExceeded, -+ CreditBalanceExhausted, -+ RequestTimeout, - RateLimitExceeded, - ServerOverloaded, - CyberPolicy, -@@ -1892,6 +1894,8 @@ impl CodexErrorInfo { - Self::ContextWindowExceeded - | Self::SessionBudgetExceeded - | Self::UsageLimitExceeded -+ | Self::CreditBalanceExhausted -+ | Self::RequestTimeout - | Self::RateLimitExceeded - | Self::ServerOverloaded - | Self::CyberPolicy -diff --git a/codex-rs/response-debug-context/src/lib.rs b/codex-rs/response-debug-context/src/lib.rs -index 42825a7..9fb0553 100644 ---- a/codex-rs/response-debug-context/src/lib.rs -+++ b/codex-rs/response-debug-context/src/lib.rs -@@ -78,6 +78,7 @@ pub fn telemetry_api_error_message(error: &ApiError) -> String { - ApiError::Stream(err) => err.to_string(), - ApiError::ContextWindowExceeded => "context window exceeded".to_string(), - ApiError::QuotaExceeded => "quota exceeded".to_string(), -+ ApiError::CreditBalanceExhausted => "credit balance exhausted".to_string(), - ApiError::UsageNotIncluded => "usage not included".to_string(), - ApiError::Retryable { .. } => "retryable error".to_string(), - ApiError::RateLimitExceeded { .. } => "rate limit exceeded".to_string(), diff --git a/runtime/codex/artifact-policy.json b/runtime/codex/artifact-policy.json deleted file mode 100644 index f8768ba..0000000 --- a/runtime/codex/artifact-policy.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "sourceSha256": "19690d78c38c17fdc79f5d80b3fd04b7743e7c0763a58c4583b71d2662c1bb31", - "patchedTree": "a17024192873c259bd70268551db7bd581dc35cc" -} diff --git a/runtime/codex/chatgpt-service-tier.patch b/runtime/codex/chatgpt-service-tier.patch deleted file mode 100644 index 9ae01b6..0000000 --- a/runtime/codex/chatgpt-service-tier.patch +++ /dev/null @@ -1,385 +0,0 @@ -diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs -index a813022..aa95792 100644 ---- a/codex-rs/core/src/client.rs -+++ b/codex-rs/core/src/client.rs -@@ -183,6 +183,13 @@ pub(crate) struct CompactConversationRequestSettings { - pub(crate) service_tier: Option, - } - -+fn service_tier_for_auth(service_tier: Option, auth: Option<&CodexAuth>) -> Option { -+ service_tier.filter(|tier| { -+ !auth.is_some_and(CodexAuth::is_chatgpt_auth) -+ || !matches!(tier.as_str(), "auto" | "default") -+ }) -+} -+ - fn session_telemetry_for_request( - session_telemetry: &SessionTelemetry, - request: &ResponsesApiRequest, -@@ -610,6 +617,7 @@ impl ModelClient { - settings.effort, - settings.summary, - settings.service_tier, -+ client_setup.auth.as_ref(), - responses_metadata, - )?; - let ResponsesApiRequest { -@@ -888,6 +896,7 @@ impl ModelClient { - } - } - -+ #[allow(clippy::too_many_arguments)] - fn build_responses_request( - &self, - prompt: &Prompt, -@@ -895,6 +904,7 @@ impl ModelClient { - effort: Option, - summary: ReasoningSummaryConfig, - service_tier: Option, -+ auth: Option<&CodexAuth>, - responses_metadata: &CodexResponsesMetadata, - ) -> Result { - let mut input = prompt.get_formatted_input_for_request(model_info.use_responses_lite); -@@ -974,7 +984,8 @@ impl ModelClient { - prompt.output_schema_strict, - ); - let prompt_cache_key = Some(self.prompt_cache_key(responses_metadata)); -- let service_tier = model_info.service_tier_for_request(service_tier); -+ let service_tier = -+ service_tier_for_auth(model_info.service_tier_for_request(service_tier), auth); - let request = ResponsesApiRequest { - model: model_info.slug.clone(), - instructions, -@@ -1612,6 +1623,7 @@ impl ModelClientSession { - effort.clone(), - summary, - service_tier.clone(), -+ client_setup.auth.as_ref(), - responses_metadata, - )?; - self.client.set_guardian_metadata( -@@ -1761,6 +1773,7 @@ impl ModelClientSession { - effort.clone(), - summary, - service_tier.clone(), -+ client_setup.auth.as_ref(), - responses_metadata, - )?; - if endpoint == ResponsesEndpoint::Guardian { -diff --git a/codex-rs/core/src/client_tests.rs b/codex-rs/core/src/client_tests.rs -index 348d169..97d4781 100644 ---- a/codex-rs/core/src/client_tests.rs -+++ b/codex-rs/core/src/client_tests.rs -@@ -63,6 +63,11 @@ use codex_rollout_trace::RawTraceEventPayload; - use codex_rollout_trace::RolloutTrace; - use codex_rollout_trace::TraceWriter; - use codex_rollout_trace::replay_bundle; -+use core_test_support::responses::ev_completed; -+use core_test_support::responses::ev_response_created; -+use core_test_support::responses::mount_sse_sequence; -+use core_test_support::responses::sse; -+use core_test_support::responses::start_websocket_server; - use futures::StreamExt; - use pretty_assertions::assert_eq; - use serde_json::json; -@@ -306,6 +311,7 @@ fn responses_lite_prefix_ids_track_thread_and_payload() -> anyhow::Result<()> { - /*effort*/ None, - codex_protocol::config_types::ReasoningSummary::None, - /*service_tier*/ None, -+ /*auth*/ None, - &test_responses_metadata_for_client( - client, - /*turn_id*/ None, -@@ -366,6 +372,284 @@ fn test_session_telemetry() -> SessionTelemetry { - ) - } - -+const REQUEST_SERVICE_TIERS: [Option<&str>; 8] = [ -+ None, -+ Some("priority"), -+ Some("auto"), -+ Some("auto"), -+ Some("flex"), -+ Some("default"), -+ Some("fast"), -+ None, -+]; -+const CHATGPT_SERVICE_TIERS: [Option<&str>; 8] = [ -+ None, -+ Some("priority"), -+ None, -+ None, -+ Some("flex"), -+ None, -+ Some("fast"), -+ None, -+]; -+ -+fn service_tier_test_client( -+ base_url: String, -+ auth: Option, -+ supports_websockets: bool, -+) -> ModelClient { -+ let mut provider = ModelProviderInfo::create_openai_provider(Some(base_url)); -+ provider.requires_openai_auth = auth.is_some(); -+ provider.supports_websockets = supports_websockets; -+ provider.request_max_retries = Some(0); -+ provider.stream_max_retries = Some(0); -+ provider.stream_idle_timeout_ms = Some(5_000); -+ ModelClient::new( -+ auth.map(AuthManager::from_auth_for_testing), -+ AgentIdentityAuthPolicy::JwtOnly, -+ ThreadId::new(), -+ provider, -+ SessionSource::Cli, -+ "test_originator".to_string(), -+ /*model_verbosity*/ None, -+ /*content_item_kinds_enabled*/ true, -+ /*enable_request_compression*/ false, -+ /*include_timing_metrics*/ false, -+ /*beta_features_header*/ None, -+ /*concurrent_reasoning_summaries_enabled*/ false, -+ /*attestation_provider*/ None, -+ HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault), -+ ) -+} -+ -+async fn stream_service_tier_sequence(client: &ModelClient) -> anyhow::Result<()> { -+ let mut session = client.new_session(); -+ let mut prompt = Prompt::default(); -+ for (index, service_tier) in REQUEST_SERVICE_TIERS.into_iter().enumerate() { -+ prompt.input.push(ResponseItem::Message { -+ id: None, -+ role: "user".to_string(), -+ content: vec![ContentItem::InputText { -+ text: format!("turn {index}"), -+ }], -+ phase: None, -+ internal_chat_message_metadata_passthrough: None, -+ }); -+ let metadata = test_responses_metadata_for_client( -+ client, -+ Some(&format!("turn-{index}")), -+ format!("{}:0", client.state.thread_id), -+ /*parent_thread_id*/ None, -+ TestCodexResponsesRequestKind::Turn, -+ ); -+ let mut stream = session -+ .stream( -+ &prompt, -+ &test_model_info(), -+ &test_session_telemetry(), -+ /*effort*/ None, -+ codex_protocol::config_types::ReasoningSummary::None, -+ service_tier.map(str::to_string), -+ &metadata, -+ &InferenceTraceContext::disabled(), -+ ) -+ .await?; -+ let mut completed = false; -+ while let Some(event) = stream.next().await { -+ if let ResponseEvent::Completed { response_id, .. } = event? { -+ assert_eq!(response_id, format!("resp-{index}")); -+ completed = true; -+ } -+ } -+ assert!(completed, "response stream should complete"); -+ } -+ Ok(()) -+} -+ -+#[test_case::test_case(Some(CodexAuth::create_dummy_chatgpt_auth_for_testing()), CHATGPT_SERVICE_TIERS; "chatgpt")] -+#[test_case::test_case(Some(CodexAuth::from_external_chatgpt_tokens(TEST_CHATGPT_ID_TOKEN, "account-123", Some("pro")).expect("test auth")), CHATGPT_SERVICE_TIERS; "external_chatgpt")] -+#[test_case::test_case(Some(CodexAuth::from_api_key("test-api-key")), REQUEST_SERVICE_TIERS; "api_key")] -+#[test_case::test_case(None, REQUEST_SERVICE_TIERS; "provider_auth")] -+#[tokio::test] -+async fn responses_service_tier_http_uses_auth_contract( -+ auth: Option, -+ expected_tiers: [Option<&str>; 8], -+) -> anyhow::Result<()> { -+ let server = MockServer::start().await; -+ let responses = REQUEST_SERVICE_TIERS -+ .iter() -+ .enumerate() -+ .map(|(index, _)| { -+ sse(vec![ -+ ev_response_created(&format!("resp-{index}")), -+ ev_completed(&format!("resp-{index}")), -+ ]) -+ }) -+ .collect(); -+ let response_mock = mount_sse_sequence(&server, responses).await; -+ let client = service_tier_test_client( -+ format!("{}/v1", server.uri()), -+ auth, -+ /*supports_websockets*/ false, -+ ); -+ -+ stream_service_tier_sequence(&client).await?; -+ -+ let requests = response_mock.requests(); -+ assert_eq!(requests.len(), expected_tiers.len()); -+ for (request, expected_tier) in requests.iter().zip(expected_tiers) { -+ assert_eq!( -+ request.body_json().get("service_tier"), -+ expected_tier.map(serde_json::Value::from).as_ref(), -+ ); -+ } -+ Ok(()) -+} -+ -+#[test_case::test_case(Some(CodexAuth::create_dummy_chatgpt_auth_for_testing()), CHATGPT_SERVICE_TIERS; "chatgpt")] -+#[test_case::test_case(Some(CodexAuth::from_external_chatgpt_tokens(TEST_CHATGPT_ID_TOKEN, "account-123", Some("pro")).expect("test auth")), CHATGPT_SERVICE_TIERS; "external_chatgpt")] -+#[test_case::test_case(Some(CodexAuth::from_api_key("test-api-key")), REQUEST_SERVICE_TIERS; "api_key")] -+#[test_case::test_case(None, REQUEST_SERVICE_TIERS; "provider_auth")] -+#[tokio::test] -+async fn responses_service_tier_websocket_uses_auth_contract( -+ auth: Option, -+ expected_tiers: [Option<&str>; 8], -+) -> anyhow::Result<()> { -+ let responses = REQUEST_SERVICE_TIERS -+ .iter() -+ .enumerate() -+ .map(|(index, _)| { -+ vec![ -+ ev_response_created(&format!("resp-{index}")), -+ ev_completed(&format!("resp-{index}")), -+ ] -+ }) -+ .collect(); -+ let server = start_websocket_server(vec![responses]).await; -+ let client = service_tier_test_client( -+ format!("{}/v1", server.uri()), -+ auth, -+ /*supports_websockets*/ true, -+ ); -+ -+ stream_service_tier_sequence(&client).await?; -+ -+ let requests = server.single_connection(); -+ assert_eq!(requests.len(), expected_tiers.len()); -+ for (request, expected_tier) in requests.iter().zip(expected_tiers) { -+ assert_eq!( -+ request.body_json().get("service_tier"), -+ expected_tier.map(serde_json::Value::from).as_ref(), -+ ); -+ } -+ assert!( -+ requests[2] -+ .body_json() -+ .get("previous_response_id") -+ .is_none() -+ ); -+ assert_eq!(requests[3].body_json()["previous_response_id"], "resp-2"); -+ server.shutdown().await; -+ Ok(()) -+} -+ -+#[test_case::test_case(Some(CodexAuth::create_dummy_chatgpt_auth_for_testing()), CHATGPT_SERVICE_TIERS; "chatgpt")] -+#[test_case::test_case(Some(CodexAuth::from_api_key("test-api-key")), REQUEST_SERVICE_TIERS; "api_key")] -+#[tokio::test] -+async fn responses_service_tier_compact_uses_auth_contract( -+ auth: Option, -+ expected_tiers: [Option<&str>; 8], -+) -> anyhow::Result<()> { -+ let server = MockServer::start().await; -+ Mock::given(method("POST")) -+ .and(path("/v1/responses/compact")) -+ .respond_with(ResponseTemplate::new(/*status*/ 200).set_body_json(json!({ "output": [] }))) -+ .expect(/*requests*/ 8) -+ .mount(&server) -+ .await; -+ let client = service_tier_test_client( -+ format!("{}/v1", server.uri()), -+ auth, -+ /*supports_websockets*/ false, -+ ); -+ let prompt = Prompt { -+ input: vec![ResponseItem::Message { -+ id: None, -+ role: "user".to_string(), -+ content: vec![ContentItem::InputText { -+ text: "compact this".to_string(), -+ }], -+ phase: None, -+ internal_chat_message_metadata_passthrough: None, -+ }], -+ ..Default::default() -+ }; -+ for service_tier in REQUEST_SERVICE_TIERS { -+ client -+ .compact_conversation_history( -+ &prompt, -+ &test_model_info(), -+ /*turn_state*/ None, -+ CompactConversationRequestSettings { -+ effort: None, -+ summary: codex_protocol::config_types::ReasoningSummary::None, -+ service_tier: service_tier.map(str::to_string), -+ }, -+ &test_session_telemetry(), -+ &CompactionTraceContext::disabled(), -+ &test_responses_metadata_for_client( -+ &client, -+ /*turn_id*/ None, -+ format!("{}:0", client.state.thread_id), -+ /*parent_thread_id*/ None, -+ TestCodexResponsesRequestKind::Turn, -+ ), -+ ) -+ .await?; -+ } -+ let requests = server.received_requests().await.expect("compact requests"); -+ assert_eq!(requests.len(), expected_tiers.len()); -+ for (request, expected_tier) in requests.iter().zip(expected_tiers) { -+ let body: serde_json::Value = request.body_json()?; -+ assert_eq!( -+ body.get("service_tier"), -+ expected_tier.map(serde_json::Value::from).as_ref(), -+ ); -+ } -+ Ok(()) -+} -+ -+#[test] -+fn responses_service_tier_preserves_bedrock_request_values() -> anyhow::Result<()> { -+ let client = test_model_client(SessionSource::Cli); -+ let auth = CodexAuth::BedrockApiKey(codex_login::auth::BedrockApiKeyAuth { -+ api_key: "test-bedrock-key".to_string(), -+ region: "us-west-2".to_string(), -+ }); -+ for service_tier in REQUEST_SERVICE_TIERS { -+ let request = client.build_responses_request( -+ &Prompt::default(), -+ &test_model_info(), -+ /*effort*/ None, -+ codex_protocol::config_types::ReasoningSummary::None, -+ service_tier.map(str::to_string), -+ Some(&auth), -+ &test_responses_metadata_for_client( -+ &client, -+ /*turn_id*/ None, -+ format!("{}:0", client.state.thread_id), -+ /*parent_thread_id*/ None, -+ TestCodexResponsesRequestKind::Turn, -+ ), -+ )?; -+ assert_eq!( -+ serde_json::to_value(request)?.get("service_tier"), -+ service_tier.map(serde_json::Value::from).as_ref(), -+ ); -+ } -+ Ok(()) -+} -+ - fn spawned_session_source() -> SessionSource { - SessionSource::SubAgent(SubAgentSource::ThreadSpawn { - parent_thread_id: ThreadId::new(), -@@ -389,6 +673,7 @@ fn reasoning_effort_in_request( - Some(effort), - codex_protocol::config_types::ReasoningSummary::None, - /*service_tier*/ None, -+ /*auth*/ None, - &test_responses_metadata_for_client( - &client, - /*turn_id*/ None, diff --git a/runtime/codex/checkpoint-shutdown.patch b/runtime/codex/checkpoint-shutdown.patch deleted file mode 100644 index 4c220c7..0000000 --- a/runtime/codex/checkpoint-shutdown.patch +++ /dev/null @@ -1,161 +0,0 @@ -diff --git a/codex-rs/app-server-protocol/src/protocol/common.rs b/codex-rs/app-server-protocol/src/protocol/common.rs ---- a/codex-rs/app-server-protocol/src/protocol/common.rs -+++ b/codex-rs/app-server-protocol/src/protocol/common.rs -@@ -820,6 +820,11 @@ - params: v2::ThreadLoadedListParams, - serialization: None, - response: v2::ThreadLoadedListResponse, -+ }, -+ ThreadCheckpoint => "thread/checkpoint" { -+ params: v2::ThreadCheckpointParams, -+ serialization: thread_id(params.thread_id), -+ response: v2::ThreadCheckpointResponse, - }, - ThreadRead => "thread/read" { - params: v2::ThreadReadParams, -diff --git a/codex-rs/app-server-protocol/src/protocol/v2/thread.rs b/codex-rs/app-server-protocol/src/protocol/v2/thread.rs ---- a/codex-rs/app-server-protocol/src/protocol/v2/thread.rs -+++ b/codex-rs/app-server-protocol/src/protocol/v2/thread.rs -@@ -1664,6 +1664,21 @@ - #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] - #[serde(rename_all = "camelCase")] - #[ts(export_to = "v2/")] -+pub struct ThreadCheckpointParams { -+ pub thread_id: String, -+} -+ -+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] -+#[serde(rename_all = "camelCase")] -+#[ts(export_to = "v2/")] -+pub struct ThreadCheckpointResponse { -+ pub thread_id: String, -+ pub flushed: bool, -+} -+ -+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] -+#[serde(rename_all = "camelCase")] -+#[ts(export_to = "v2/")] - pub struct ThreadReadParams { - pub thread_id: String, - /// When true, include turns and their items from rollout history. -diff --git a/codex-rs/app-server/src/message_processor.rs b/codex-rs/app-server/src/message_processor.rs ---- a/codex-rs/app-server/src/message_processor.rs -+++ b/codex-rs/app-server/src/message_processor.rs -@@ -1359,6 +1359,9 @@ - ClientRequest::ThreadLoadedList { params, .. } => { - self.thread_processor.thread_loaded_list(params).await - } -+ ClientRequest::ThreadCheckpoint { params, .. } => { -+ self.thread_processor.thread_checkpoint(params).await -+ } - ClientRequest::ThreadRead { params, .. } => { - self.thread_processor.thread_read(&request_id, params).await - } -diff --git a/codex-rs/app-server/src/request_processors/thread_processor.rs b/codex-rs/app-server/src/request_processors/thread_processor.rs ---- a/codex-rs/app-server/src/request_processors/thread_processor.rs -+++ b/codex-rs/app-server/src/request_processors/thread_processor.rs -@@ -529,6 +529,22 @@ - ) - .await - .map(|()| None) -+ } -+ -+ /// Private host handoff primitive; callers must seal admission on every thread. -+ pub(crate) async fn thread_checkpoint( -+ &self, -+ params: codex_app_server_protocol::ThreadCheckpointParams, -+ ) -> Result, JSONRPCErrorError> { -+ let (_, thread) = self.load_thread(¶ms.thread_id).await?; -+ tokio::time::timeout(Duration::from_secs(30), thread.checkpoint_shutdown()) -+ .await -+ .map_err(|_| internal_error("checkpoint shutdown timed out"))? -+ .map_err(|_| internal_error("checkpoint shutdown was not durably acknowledged"))?; -+ Ok(Some(codex_app_server_protocol::ThreadCheckpointResponse { -+ thread_id: params.thread_id, -+ flushed: true, -+ }.into())) - } - - pub(crate) async fn thread_unsubscribe( -diff --git a/codex-rs/core/src/session/session.rs b/codex-rs/core/src/session/session.rs ---- a/codex-rs/core/src/session/session.rs -+++ b/codex-rs/core/src/session/session.rs -@@ -41,6 +41,8 @@ - /// - /// A session has at most 1 running task at a time, and can be interrupted by user input. - pub(crate) struct Session { -+ /// Set only by successful persistence shutdown, before loop termination. -+ pub(crate) checkpoint_shutdown_succeeded: std::sync::atomic::AtomicBool, - pub(crate) thread_id: ThreadId, - pub(crate) installation_id: String, - pub(super) tx_event: Sender, -@@ -1501,6 +1503,7 @@ - }; - let (mcp_prewarm_tx, mcp_prewarm_rx) = async_channel::bounded(1); - let sess = Arc::new(Session { -+ checkpoint_shutdown_succeeded: std::sync::atomic::AtomicBool::new(false), - thread_id, - installation_id, - tx_event: tx_event.clone(), -diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs ---- a/codex-rs/core/src/session/tests.rs -+++ b/codex-rs/core/src/session/tests.rs -@@ -6575,6 +6575,7 @@ - }; - - let session = Session { -+ checkpoint_shutdown_succeeded: std::sync::atomic::AtomicBool::new(false), - thread_id, - installation_id: "11111111-1111-4111-8111-111111111111".to_string(), - tx_event, -@@ -8882,6 +8883,7 @@ - }; - - let session = Arc::new(Session { -+ checkpoint_shutdown_succeeded: std::sync::atomic::AtomicBool::new(false), - thread_id, - installation_id: "11111111-1111-4111-8111-111111111111".to_string(), - tx_event, -diff --git a/codex-rs/core/src/session/handlers.rs b/codex-rs/core/src/session/handlers.rs ---- a/codex-rs/core/src/session/handlers.rs -+++ b/codex-rs/core/src/session/handlers.rs -@@ -460,9 +460,14 @@ - - // Gracefully flush and shutdown thread persistence on session end so tests - // that inspect durable state do not race with the background writer. -- if let Some(live_thread) = sess.live_thread() -- && let Err(e) = live_thread.shutdown().await -- { -+ let checkpoint_shutdown_result = match sess.live_thread() { -+ Some(live_thread) => live_thread.shutdown().await, -+ None => Ok(()), -+ }; -+ sess.checkpoint_shutdown_succeeded.store( -+ sess.live_thread().is_some() && checkpoint_shutdown_result.is_ok(), std::sync::atomic::Ordering::Release, -+ ); -+ if let Err(e) = checkpoint_shutdown_result { - warn!("failed to shutdown thread persistence: {e}"); - let event = Event { - id: sub_id.clone(), -diff --git a/codex-rs/core/src/codex_thread.rs b/codex-rs/core/src/codex_thread.rs ---- a/codex-rs/core/src/codex_thread.rs -+++ b/codex-rs/core/src/codex_thread.rs -@@ -233,6 +233,18 @@ - &self.session.services.thread_extension_data - } - -+ /// A normal ShutdownComplete notification does not imply persistence succeeded. -+ pub async fn checkpoint_shutdown(&self) -> CodexResult<()> { -+ if self.session.active_turn.lock().await.is_some() || self.session.live_thread().is_none() { -+ return Err(CodexErr::UnsupportedOperation("checkpoint requires an idle durable thread".to_string())); -+ } -+ self.shutdown_and_wait().await?; -+ if !self.session.checkpoint_shutdown_succeeded.load(std::sync::atomic::Ordering::Acquire) { -+ return Err(CodexErr::Fatal("checkpoint persistence shutdown failed".to_string())); -+ } -+ Ok(()) -+ } -+ - pub async fn shutdown_and_wait(&self) -> CodexResult<()> { - self.io.shutdown_and_wait().await - } diff --git a/runtime/codex/companion-packages.json b/runtime/codex/companion-packages.json index 3a6c774..991e9ed 100644 --- a/runtime/codex/companion-packages.json +++ b/runtime/codex/companion-packages.json @@ -1,12 +1,12 @@ { "@openai/codex-darwin-arm64": { "name": "@openai/codex", - "version": "0.154.0-darwin-arm64", - "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.154.0-darwin-arm64.tgz", - "integrity": "sha512-HP/vJCH/t2hB9Kg6hotN9UglClJ6/z584fal5lEP14C9gNAgAQS4/kTQC7l5V+BA3TqwDPwINSjul28cX8AYXg==", + "version": "0.164.0-darwin-arm64", + "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.164.0-darwin-arm64.tgz", + "integrity": "sha512-pDizgOUAtq3/k3AbUFHbaU19WpYKCja+oFpNSTLKrDQj4lTUNSDuTPJip4uQVn2/1EipQdpQrGMOQBVuI1znAA==", "descriptor": { "layoutVersion": 1, - "version": "0.154.0", + "version": "0.164.0", "target": "aarch64-apple-darwin", "variant": "codex", "entrypoint": "bin/codex", @@ -16,39 +16,219 @@ "files": [ { "path": "bin/codex", - "sha256": "4f85982624b3898c8991cb80c0981b2aa71070e3537046c9a95950318a95afcc", + "sha256": "d8990e3ee7acfbc26f91265b12a1a83f0fee40d0e0f8f1f40a37d4ec21f499cf", "executable": true }, { "path": "bin/codex-code-mode-host", - "sha256": "426d73aaeb2aeef45e98b5add99e8ef9594a31673d281489bb2cb06b38c27423", + "sha256": "416721e33af8822041dc650793512b3e58d43e0be4f3a5c6187eb4dfaad3148a", "executable": true }, { "path": "codex-package.json", - "sha256": "6bace96c38debc466a94cd71d48464df1b1dc27a4ac75d8cebbcb2b6a3a6540c", + "sha256": "c77eb3abe21da837655153670c53ccbb89c26995ba3ccad2b72204b4670e1ad8", "executable": false }, { "path": "codex-path/rg", - "sha256": "6bbb665c5a40785e74c7ccf1c2e606efa7e70b09540c1e00787c3f88827e668d", + "sha256": "fd819df955fce19b2ee89c89c94e19a16127481de1ab0e5099add82cdfdc96a0", "executable": true }, { - "path": "codex-resources/zsh/bin/zsh", - "sha256": "25649b16e12a7b1b26686a9553d8a141866ee445319fddb5263d57784f0fd10b", + "path": "codex-resources/voice/bin/codex-voice-host", + "sha256": "ecfcf05fe021c5ad2e1c1e3975cf884eecbb90ad1a32e25fa7f216d6f178e437", "executable": true + }, + { + "path": "codex-resources/voice/lib/libffi.8.dylib", + "sha256": "76a199d09fc77a42b59317185f1e42b1ec0f5f288e8ff367d68a893318d5f036", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libgio-2.0.0.dylib", + "sha256": "e48f50ccb00d78ec1f8b422e0fff69ec1751ed34a147662a70bd638ed617fb1a", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libglib-2.0.0.dylib", + "sha256": "47d1519d17e11f220d79d29a4926008433286a98ef92c6503902917d370f3bbd", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libgmodule-2.0.0.dylib", + "sha256": "7b63fe175daa2aaafe131cae60851c5a8c71b209617ced0d2d4f273b73c8860b", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libgobject-2.0.0.dylib", + "sha256": "444d41be63bb6593914ad2e73a127cc4dc8ad4907bd5c1c5e6e9657a02b4ed38", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libgstapp-1.0.0.dylib", + "sha256": "1fb42be568a30d07d4f54cbfdb9f64b3b906b68f5e7e536edd159ad7a1ed2541", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libgstaudio-1.0.0.dylib", + "sha256": "d4a48c604ac793f6dbc3afdc68c61d1c1144b9c35720f440669cc7be39758792", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libgstbase-1.0.0.dylib", + "sha256": "50715fac0c1ee1168a58b8191867977001831a8cd1b2ffc4d40064e92d7ec357", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libgstnet-1.0.0.dylib", + "sha256": "efb57393d18a236202b3e2a4f4326f0761c4e9da678d1fb2cefb62a953efd5db", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libgstpbutils-1.0.0.dylib", + "sha256": "adb620bdc2190a157e2e5f765a703640a73c498fb081d83629acc76f840d2011", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libgstreamer-1.0.0.dylib", + "sha256": "a1b30c536ced0265b3a8da01c6e79366c4f81f84367497ee03fd09a9efb711ad", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libgstrtp-1.0.0.dylib", + "sha256": "04e3e4ae32e9f31112de4e0f0be4a7562aef7b9a8447e8c2ac0c40f2f4833d27", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libgsttag-1.0.0.dylib", + "sha256": "c451148da91f64d149d9993e98f7ca8a1a338ed0545fa80a559201aede82ca3d", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libgstvideo-1.0.0.dylib", + "sha256": "597ee1f7cd55013b2d42d8d049d09fdffe64d85ee5e0907596fe1752d0f08122", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libintl.8.dylib", + "sha256": "cdbc95ae3e8350047bc8b7510761b4a70285afe2a6924aa912109f93a9a12629", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libopus.0.dylib", + "sha256": "951c20cc9df5e242bd386e1145dbc755d61fc23619848166c63c1ec1e03a91b5", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libpcre2-8.0.dylib", + "sha256": "944fa068df87698d23452b6a590de741012897b54419cd785bcbd2c1bf3d5409", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libz.1.dylib", + "sha256": "73a57393eaea45753dcc60a130122dfdd10b981be65e98b216af1d0a7eb652a4", + "executable": true + }, + { + "path": "codex-resources/voice/licenses/LGPL-2.1.txt", + "sha256": "ad2eec519ebd4b5df86ea84dff24ae3bfa2edea846a703b58902dd221ae375db", + "executable": false + }, + { + "path": "codex-resources/voice/licenses/libffi.txt", + "sha256": "17b64dc60f3b6897a60f971e288b973f655c2edcdf08b25f3c3dd5549857881c", + "executable": false + }, + { + "path": "codex-resources/voice/licenses/Opus.txt", + "sha256": "01e1167d54a096d123cf6dfbbeb19587278845c6481d2d66d545669846079551", + "executable": false + }, + { + "path": "codex-resources/voice/licenses/PCRE2.md", + "sha256": "197d8a73ffee0d6b09adba2f9c677b5f5aede24edf89258a68e48248d010d811", + "executable": false + }, + { + "path": "codex-resources/voice/licenses/proxy-libintl.txt", + "sha256": "d245807f90032872d1438d741ed21e2490e1175dc8aa3afa5ddb6c8e529b58e5", + "executable": false + }, + { + "path": "codex-resources/voice/licenses/sljit.txt", + "sha256": "5f216505c0f6ea3273caec89e766eef93cdeb7bbb0c429f9360116d7c938feeb", + "executable": false + }, + { + "path": "codex-resources/voice/licenses/zlib.txt", + "sha256": "e32ff4e00d9d94930537635291da39e7e612703334bf6fde8c7f1686fe8a45a2", + "executable": false + }, + { + "path": "codex-resources/voice/manifest.json", + "sha256": "d6663e9c89b260f2936501e5822495081fb66c9eb629386a9fed0aef17b977fb", + "executable": false + }, + { + "path": "codex-resources/voice/NOTICE.md", + "sha256": "40097a1799441051aa0b6645cdc819cceb19be73cdd265eb602aa2b15e0b1f48", + "executable": false + }, + { + "path": "codex-resources/voice/plugins/libgstapp.dylib", + "sha256": "681e43dbe55976e63ee4931e532e45be48215ff7bcacf4b1f50cffae4c9954b4", + "executable": true + }, + { + "path": "codex-resources/voice/plugins/libgstaudioconvert.dylib", + "sha256": "ac1060be8fab2d9859fd06f960463ffdefc0b4a0e0e5a4ad288704d9fc447816", + "executable": true + }, + { + "path": "codex-resources/voice/plugins/libgstaudioresample.dylib", + "sha256": "af908e7efbb6bc41ccd0b1f79285321436a09ea856b63dcea361f577b8efaf5f", + "executable": true + }, + { + "path": "codex-resources/voice/plugins/libgstcoreelements.dylib", + "sha256": "9c95059f3fcc08ea94f11aa6dfdd9feaa839e1d7e324413b36f58e57c2689be0", + "executable": true + }, + { + "path": "codex-resources/voice/plugins/libgstopus.dylib", + "sha256": "0bbd4c12369394322edbfec44bcc6e6b3c6833699b98f40bbfa614dad11e66a7", + "executable": true + }, + { + "path": "codex-resources/voice/plugins/libgstrtp.dylib", + "sha256": "79a6612d349973ee87cd33e278db37d85ca7355d1dc8b8248c3c7fd591f0fbd7", + "executable": true + }, + { + "path": "codex-resources/voice/plugins/libgstrtpmanager.dylib", + "sha256": "828a6ab627e1cb4ea4a87ba1c9b8d2a5872d8d3990672df8523e4d946b69cf8a", + "executable": true + }, + { + "path": "codex-resources/voice/runtime.json", + "sha256": "8ed3dddbd77cc253897240234a756823d94e936921424eb098c9e1b549da9d95", + "executable": false + }, + { + "path": "codex-resources/voice/sources.json", + "sha256": "e2f6b124f6277e7c3c4f5c7cc8830c66190f671ad65e972bb2cc9507dbfcb937", + "executable": false } ] }, "@openai/codex-linux-arm64": { "name": "@openai/codex", - "version": "0.154.0-linux-arm64", - "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.154.0-linux-arm64.tgz", - "integrity": "sha512-KmTCB6ST484zeYlPpKP/K5P/gRaYmt6TihVD+zotoe6O9q0JSBP+FYvCz4A/zZXR7xDOHURTSjHp0sD8wWS0YQ==", + "version": "0.164.0-linux-arm64", + "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.164.0-linux-arm64.tgz", + "integrity": "sha512-LL5iTQjyDL4YRX44EWqhIfEQ5uGnmgxkQBJSxjZSeJ8L1Og83pdlzUJW/PRJ76oaVwOVgh6HXwuPEiBNEHU4lg==", "descriptor": { "layoutVersion": 1, - "version": "0.154.0", + "version": "0.164.0", "target": "aarch64-unknown-linux-musl", "variant": "codex", "entrypoint": "bin/codex", @@ -58,17 +238,17 @@ "files": [ { "path": "bin/codex", - "sha256": "9b7c1c7abdc26fc3c4f47c77656a8e9121def5483dbae830ef1ee561758448a9", + "sha256": "8813c8d8a4cea2eeba271b72ab22f49628b184b24ab18dbef06b4ce41e9a8d39", "executable": true }, { "path": "bin/codex-code-mode-host", - "sha256": "f31e1c5ffbbca7884aff2f0f8795d3da197f4aafb114033a399dfc17a5119031", + "sha256": "7d28b996b825b13ea058af67655b619a26a125d61d4fb2985694a9c4e77d2e21", "executable": true }, { "path": "codex-package.json", - "sha256": "abfae2ff248420c32f531f3ba0cb83ea27bb9ba9355872658b591e1939a60288", + "sha256": "28dd4b4e31da1d3caaca29fade8f7e5fe1da8d8cfc1c0bfe266bc9f341c57570", "executable": false }, { @@ -78,24 +258,209 @@ }, { "path": "codex-resources/bwrap", - "sha256": "58bd88f39d02a0b5ac553c2f334edfff1ec74afb9b8f4233dfc5b69225038f92", + "sha256": "0d784834f1674ff02aee25b42f2cbcfd27c5305863806cd6475d0eac8e33c378", + "executable": true + }, + { + "path": "codex-resources/voice/bin/codex-voice-host", + "sha256": "c70476f0eb59854c0b5b73d8ade1d27bd274dfae906102351167c21fd3d9d2cc", + "executable": true + }, + { + "path": "codex-resources/voice/lib/gstreamer-1.0/libgstapp.so", + "sha256": "77560791fa89d709d02fb8241b88834e826cf0fa74f00098867552abb931e703", + "executable": true + }, + { + "path": "codex-resources/voice/lib/gstreamer-1.0/libgstaudioconvert.so", + "sha256": "533c3ec676b04e6401658c628acf6e2ff80d44b3107e9befac4fc0e30b9331a1", + "executable": true + }, + { + "path": "codex-resources/voice/lib/gstreamer-1.0/libgstaudioresample.so", + "sha256": "08ef8eb4bd1eebffc54ec80a1419f89cb58f32728fa48cb455b5f2553fbdd16a", + "executable": true + }, + { + "path": "codex-resources/voice/lib/gstreamer-1.0/libgstcoreelements.so", + "sha256": "ebdc736002ff6b337c1bd1cd0daa6e8ba1c63bbd2425bda0c2601842ab54fff0", + "executable": true + }, + { + "path": "codex-resources/voice/lib/gstreamer-1.0/libgstopus.so", + "sha256": "a6097568f03071c28bed85bc027386fa6d84b2b85f732805b9c5b8bf266157d7", + "executable": true + }, + { + "path": "codex-resources/voice/lib/gstreamer-1.0/libgstrtp.so", + "sha256": "b817af549089771f622a9369aa7b360969be3d06c32e07d24842c162098782cf", + "executable": true + }, + { + "path": "codex-resources/voice/lib/gstreamer-1.0/libgstrtpmanager.so", + "sha256": "0c0ee9409a88161ee5d4c52570c08fdd68995e0fa1cbbd5e2c627ad9efa50838", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libffi.so.8", + "sha256": "aa8da05db63784a05495256b92092d5cd0ce2d1babcaa59f730a7a3042350cb9", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libgio-2.0.so.0", + "sha256": "f38f93d562656bc166cf8716d23dde05c40fe6068350f8d686e51052e470aab8", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libglib-2.0.so.0", + "sha256": "e47a6825d63dc4ce7078103830b61640922869562510c7c1a9db6af44a053ee9", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libgmodule-2.0.so.0", + "sha256": "1aae3a9a1471cbb51d6b13feda0c8e4de98afaf920c3535f4e74cc090b0176d2", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libgobject-2.0.so.0", + "sha256": "96b78d19cdf1737f03873ef73b5528e81e322b5dc02223ca1e5d38801ac8fcb5", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libgstallocators-1.0.so.0", + "sha256": "b5778d8c9d416151161785c2b82045bf08cc1d85d8cea88854ae2167fcf8aa54", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libgstapp-1.0.so.0", + "sha256": "67ce511205f0e0cad6e83fc328c64eaa35ea98e0d5f378395f8489eaf128b856", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libgstaudio-1.0.so.0", + "sha256": "df250bd1d07a2b023f365a270707486d5f00e851ffdd6347249ea0415e3aebe7", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libgstbase-1.0.so.0", + "sha256": "29b6dd83ee18d01cdf33e4cef589fa901aa612b16987a732bcbbb22a9fd86358", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libgstnet-1.0.so.0", + "sha256": "1e30dd69b436b10092cd5b2d854b557db73d886354b00750a37b1f6b9db2aa9f", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libgstpbutils-1.0.so.0", + "sha256": "d62afad3c9066c5a2d44ff907c07afae830bdd7d4e1d717f0d7575f7518f52dd", "executable": true }, { - "path": "codex-resources/zsh/bin/zsh", - "sha256": "7feeacd883e1dc749847936948c378653c80a69ec4a9542f0f126b411882c179", + "path": "codex-resources/voice/lib/libgstreamer-1.0.so.0", + "sha256": "18075362dfe989e31979deb6e1d54db740e6c78fca1a0b964ab9a3eb6f48a38d", "executable": true + }, + { + "path": "codex-resources/voice/lib/libgstrtp-1.0.so.0", + "sha256": "5202ffd864930bc0c5e87457e9c8d71c9269deed3877d4da52cf0048f2d78a49", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libgsttag-1.0.so.0", + "sha256": "73bb88bcc76a7c57a31066c16b3d0b1590105b278117242820da5399e5cbde02", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libgstvideo-1.0.so.0", + "sha256": "f4a7e532adcf5bf0527057408c71553ea3b4090f84b45bd896b0dc2c2f55ccca", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libintl.so.8", + "sha256": "d244097a6dd6d84cfdc76807bb8b084528dfc0b5bedca18f0ddb54904cd45329", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libopus.so.0", + "sha256": "7426975e16d5109831b4f88ea0681c251e332ed188e3b483d799fa285b538e25", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libpcre2-8.so.0", + "sha256": "ed12ebb1003748d284f98c4f90f060a80e25f085b0251c8d8ebcc7fa8ba58f35", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libz.so.1", + "sha256": "284a7e1475cccabdda7fb91d9917a95a1abfbe73688033acc7ac27241d97ceca", + "executable": true + }, + { + "path": "codex-resources/voice/licenses/LGPL-2.1.txt", + "sha256": "ad2eec519ebd4b5df86ea84dff24ae3bfa2edea846a703b58902dd221ae375db", + "executable": false + }, + { + "path": "codex-resources/voice/licenses/libffi.txt", + "sha256": "17b64dc60f3b6897a60f971e288b973f655c2edcdf08b25f3c3dd5549857881c", + "executable": false + }, + { + "path": "codex-resources/voice/licenses/Opus.txt", + "sha256": "01e1167d54a096d123cf6dfbbeb19587278845c6481d2d66d545669846079551", + "executable": false + }, + { + "path": "codex-resources/voice/licenses/PCRE2.md", + "sha256": "197d8a73ffee0d6b09adba2f9c677b5f5aede24edf89258a68e48248d010d811", + "executable": false + }, + { + "path": "codex-resources/voice/licenses/proxy-libintl.txt", + "sha256": "d245807f90032872d1438d741ed21e2490e1175dc8aa3afa5ddb6c8e529b58e5", + "executable": false + }, + { + "path": "codex-resources/voice/licenses/sljit.txt", + "sha256": "5f216505c0f6ea3273caec89e766eef93cdeb7bbb0c429f9360116d7c938feeb", + "executable": false + }, + { + "path": "codex-resources/voice/licenses/zlib.txt", + "sha256": "e32ff4e00d9d94930537635291da39e7e612703334bf6fde8c7f1686fe8a45a2", + "executable": false + }, + { + "path": "codex-resources/voice/manifest.json", + "sha256": "ee01011027ad58abae1bc88b47cefb91fed468ea4f8b042bef1b1ecaf4c65fef", + "executable": false + }, + { + "path": "codex-resources/voice/NOTICE.md", + "sha256": "40097a1799441051aa0b6645cdc819cceb19be73cdd265eb602aa2b15e0b1f48", + "executable": false + }, + { + "path": "codex-resources/voice/runtime.json", + "sha256": "6230eac46f615633854f4705efdb1ca1667a64c58bdf28a7cff8aea5fef87338", + "executable": false + }, + { + "path": "codex-resources/voice/sources.json", + "sha256": "e2f6b124f6277e7c3c4f5c7cc8830c66190f671ad65e972bb2cc9507dbfcb937", + "executable": false } ] }, "@openai/codex-linux-x64": { "name": "@openai/codex", - "version": "0.154.0-linux-x64", - "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.154.0-linux-x64.tgz", - "integrity": "sha512-a4FI3A8sGtwGrOqltrPbrS2hajrHQG591EwmRfiRoLMb10VxdBtUGW4gu6IJVYENiYGA7k3P4jlRHEoCZU/s9Q==", + "version": "0.164.0-linux-x64", + "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.164.0-linux-x64.tgz", + "integrity": "sha512-XLAoaWsGOT2khzl+P4FYDPTkzAr9W0VFXFaVkqlFdkM0soa1gH/4S0wSD57Ek3HHwkirAEGz2/Mwehseh+l5Mg==", "descriptor": { "layoutVersion": 1, - "version": "0.154.0", + "version": "0.164.0", "target": "x86_64-unknown-linux-musl", "variant": "codex", "entrypoint": "bin/codex", @@ -105,17 +470,17 @@ "files": [ { "path": "bin/codex", - "sha256": "3188814c35471432d4123203e0eb38e5bddc60226e3d7ddf0e59e649ea140022", + "sha256": "b93f6811d3f653f0fb83be4050d857fb9abbb111dfa2ce160563327daff7f306", "executable": true }, { "path": "bin/codex-code-mode-host", - "sha256": "0c57be435e73b70d9106c850d751cd259a7f04da958a453d7ef59090d82b70f1", + "sha256": "8ee4f472e92ac226a189e98998633b2d07f989f208515750764db660369f2926", "executable": true }, { "path": "codex-package.json", - "sha256": "b039964d28d57b2a7e929ee9986303582501c7cc9787f6c018b6d3c7b35c87d7", + "sha256": "328cc6b7d776fed066a3b475766ec01bd8b8f31d91e4955a5588f37ff9ddb51b", "executable": false }, { @@ -125,13 +490,198 @@ }, { "path": "codex-resources/bwrap", - "sha256": "01fb705f067bd5365b63d8ad2323a61c8d007733ca5e649437e086f3fb9935d8", + "sha256": "3a81c19ad10a70436529c58e77898e2a447abe80167f813fc20ec34bd9eafdeb", + "executable": true + }, + { + "path": "codex-resources/voice/bin/codex-voice-host", + "sha256": "a374997203c3aae9b6a8dd05f355256b4066f045ac271fb8dc8c52769bdbda8a", + "executable": true + }, + { + "path": "codex-resources/voice/lib/gstreamer-1.0/libgstapp.so", + "sha256": "0e43aee9255c9d938df41ff6af1cf65d119aacf78ca1427616cf69623df9ee0c", + "executable": true + }, + { + "path": "codex-resources/voice/lib/gstreamer-1.0/libgstaudioconvert.so", + "sha256": "482ec6d32a496f6322682cc0b119bd0af82ea31a0f3026dd07c9a9917346d2f6", "executable": true }, { - "path": "codex-resources/zsh/bin/zsh", - "sha256": "67faaaa89242c4a332e16e508a1977cffc24bf7fca31d4411cdfd101f3831ef3", + "path": "codex-resources/voice/lib/gstreamer-1.0/libgstaudioresample.so", + "sha256": "e9abcc301e817a81b6f01fba3d8701a8767a23090603b5f3baa0357837f2346f", "executable": true + }, + { + "path": "codex-resources/voice/lib/gstreamer-1.0/libgstcoreelements.so", + "sha256": "951d7579c56ebeef6b3db85914ef1001e6a466ed54b443b194eeec997d159c71", + "executable": true + }, + { + "path": "codex-resources/voice/lib/gstreamer-1.0/libgstopus.so", + "sha256": "fd01916d777e4c0b82e7baee5a9ee8493f04b25194d59a7084cab1bb554169a8", + "executable": true + }, + { + "path": "codex-resources/voice/lib/gstreamer-1.0/libgstrtp.so", + "sha256": "392684a4895d869399a255cba729ec0075f1849abe3b1fc9b1f56af5ba32a2cc", + "executable": true + }, + { + "path": "codex-resources/voice/lib/gstreamer-1.0/libgstrtpmanager.so", + "sha256": "07117be91f9fa2e449386fcedafe757f34436029b86843ddf92a72a088f8242f", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libffi.so.8", + "sha256": "bdec19c4e46b8fbf1c3e0b39f7b3fad933b77b43e2c37631157c97950fd2b174", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libgio-2.0.so.0", + "sha256": "21cf9737aa0b78e176723112f419115aa0433f97cb13465b6e2c0a41faf8853c", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libglib-2.0.so.0", + "sha256": "f45fc3cabf7b363fd4b77d81914921ff838fd023a9ee4058a5e91bfde46d5406", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libgmodule-2.0.so.0", + "sha256": "e30d348e2497900e44a43261976921afcb023bb0a326aaea9733c6a87512bd6b", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libgobject-2.0.so.0", + "sha256": "f6d4f8f6d95c0e10e0b07baf265e5b53f35fe5c2840b341efe4122662c38651f", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libgstallocators-1.0.so.0", + "sha256": "9ea81406081c9d069915da4e4b7b3fd65f9294e94ae9d291956258abe17f4e6d", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libgstapp-1.0.so.0", + "sha256": "681f3f137cb194bdffa6a8d314dbe72002dbd6c3f72d26bffa4d97013648d94e", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libgstaudio-1.0.so.0", + "sha256": "369a54082f9b68b532dc4ffa98e7fee49f9efb79c93a9fdd622dc6ad42a20bd9", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libgstbase-1.0.so.0", + "sha256": "54c4200b2164ad19badee88fcb999989d37980a57b9ea3fccc204549da675745", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libgstnet-1.0.so.0", + "sha256": "60690a11caa2b561bc1fd8deb4f495a28aa3b942b6bff1c68c01acb77b8432b5", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libgstpbutils-1.0.so.0", + "sha256": "061c78737ebb8575d5c06fca16c555b2a763eb8b6ee9aa6736d41d8412ab78ed", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libgstreamer-1.0.so.0", + "sha256": "7ca382c8ec951368384f5c85fdccb31fc08e090be402e811575809f6babedfd0", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libgstrtp-1.0.so.0", + "sha256": "1d5e6a4d84d6da9b19599d8be5ee7414453b4e5c8e98ac8dfd0cbff3820b3b44", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libgsttag-1.0.so.0", + "sha256": "26386441f8d324d05448399f7b25641cea4975b7224411588fcdde8748f54085", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libgstvideo-1.0.so.0", + "sha256": "1fdf86805535aabb7c175c13f73957af9fbe7527a99b48d6adf835ed943cec4b", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libintl.so.8", + "sha256": "3745518bbcd678e70f7560496596c8adc8b9215d23839c5eb2d0dea0bce3975a", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libopus.so.0", + "sha256": "7f1b4bd7b36f6e3d906102e93cf1c9f82053befe7ffd76584ce50222cb2ae3df", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libpcre2-8.so.0", + "sha256": "e39519b242f40686734ed8d0218f68cac24f0e956ec64c8c1b49b9da5cc9788b", + "executable": true + }, + { + "path": "codex-resources/voice/lib/libz.so.1", + "sha256": "6c5d765f42d9368cc4bdbd11f725c1bcf6b14b78322033d3c94f787fc6261f5e", + "executable": true + }, + { + "path": "codex-resources/voice/licenses/LGPL-2.1.txt", + "sha256": "ad2eec519ebd4b5df86ea84dff24ae3bfa2edea846a703b58902dd221ae375db", + "executable": false + }, + { + "path": "codex-resources/voice/licenses/libffi.txt", + "sha256": "17b64dc60f3b6897a60f971e288b973f655c2edcdf08b25f3c3dd5549857881c", + "executable": false + }, + { + "path": "codex-resources/voice/licenses/Opus.txt", + "sha256": "01e1167d54a096d123cf6dfbbeb19587278845c6481d2d66d545669846079551", + "executable": false + }, + { + "path": "codex-resources/voice/licenses/PCRE2.md", + "sha256": "197d8a73ffee0d6b09adba2f9c677b5f5aede24edf89258a68e48248d010d811", + "executable": false + }, + { + "path": "codex-resources/voice/licenses/proxy-libintl.txt", + "sha256": "d245807f90032872d1438d741ed21e2490e1175dc8aa3afa5ddb6c8e529b58e5", + "executable": false + }, + { + "path": "codex-resources/voice/licenses/sljit.txt", + "sha256": "5f216505c0f6ea3273caec89e766eef93cdeb7bbb0c429f9360116d7c938feeb", + "executable": false + }, + { + "path": "codex-resources/voice/licenses/zlib.txt", + "sha256": "e32ff4e00d9d94930537635291da39e7e612703334bf6fde8c7f1686fe8a45a2", + "executable": false + }, + { + "path": "codex-resources/voice/manifest.json", + "sha256": "b4c7fbc3b46647c1a382c7129f47b7a4e453e5a18d671c726bb04d12f610f58c", + "executable": false + }, + { + "path": "codex-resources/voice/NOTICE.md", + "sha256": "40097a1799441051aa0b6645cdc819cceb19be73cdd265eb602aa2b15e0b1f48", + "executable": false + }, + { + "path": "codex-resources/voice/runtime.json", + "sha256": "350690ea1502d50be347483b17ef60480a5f3a236b9cfa638eb12bbe51c2cd63", + "executable": false + }, + { + "path": "codex-resources/voice/sources.json", + "sha256": "e2f6b124f6277e7c3c4f5c7cc8830c66190f671ad65e972bb2cc9507dbfcb937", + "executable": false } ] } diff --git a/runtime/codex/interrupt-barrier.patch b/runtime/codex/interrupt-barrier.patch deleted file mode 100644 index ce4fc2b..0000000 --- a/runtime/codex/interrupt-barrier.patch +++ /dev/null @@ -1,29 +0,0 @@ ---- a/codex-rs/core/src/agent/control.rs -+++ b/codex-rs/core/src/agent/control.rs -@@ -359,19 +359,13 @@ - /// Interrupt the current task for an existing agent thread. - pub(crate) async fn interrupt_agent(&self, agent_id: ThreadId) -> CodexResult { - let state = self.upgrade()?; -- self.handle_thread_request_result( -- agent_id, -- &state, -- state -- .send_op( -- agent_id, -- Op::Interrupt, -- /*parent_turn_id*/ None, -- /*root_turn_id*/ None, -- ) -- .await, -- ) -- .await -+ let thread = state.get_thread(agent_id).await?; -+ // Completion of this tool is a barrier for subsequent follow-up work. -+ // Merely enqueueing Op::Interrupt lets a follow-up see no active Turn -+ // while cancellation still holds its execution guard and rejects the -+ // existing child at the concurrency limit. -+ thread.session.interrupt_task().await; -+ Ok(Uuid::now_v7().to_string()) - } - - async fn handle_thread_request_result( diff --git a/runtime/codex/local-subprocess-stdio.patch b/runtime/codex/local-subprocess-stdio.patch deleted file mode 100644 index 754ecec..0000000 --- a/runtime/codex/local-subprocess-stdio.patch +++ /dev/null @@ -1,141 +0,0 @@ -diff --git a/codex-rs/linux-sandbox/src/landlock.rs b/codex-rs/linux-sandbox/src/landlock.rs ---- a/codex-rs/linux-sandbox/src/landlock.rs -+++ b/codex-rs/linux-sandbox/src/landlock.rs -@@ -191,8 +191,6 @@ - deny_syscall(&mut rules, libc::SYS_bind); - deny_syscall(&mut rules, libc::SYS_listen); - deny_syscall(&mut rules, libc::SYS_getpeername); -- deny_syscall(&mut rules, libc::SYS_getsockname); -- deny_syscall(&mut rules, libc::SYS_shutdown); - deny_syscall(&mut rules, libc::SYS_sendto); - deny_syscall(&mut rules, libc::SYS_sendmmsg); - // NOTE: allowing recvfrom allows some tools like: `cargo clippy` -@@ -200,7 +198,39 @@ - // management. - // deny_syscall(&mut rules, libc::SYS_recvfrom); - deny_syscall(&mut rules, libc::SYS_recvmmsg); -- deny_syscall(&mut rules, libc::SYS_getsockopt); -+ // Local subprocess pipes use AF_UNIX socketpairs. Permit only the -+ // descriptor metadata and write-half-close required by libuv; none -+ // of these operations creates or connects a socket. Other socket -+ // options, mutations, connection operations and families stay denied. -+ // getsockname is read-only metadata and needs no seccomp rule. -+ rules.insert( -+ libc::SYS_shutdown, -+ vec![SeccompRule::new(vec![SeccompCondition::new( -+ 1, -+ SeccompCmpArgLen::Dword, -+ SeccompCmpOp::Ne, -+ libc::SHUT_WR as u64, -+ )?])?], -+ ); -+ // Separate rules are ORed: reject either a different level or a -+ // different option. Combining these conditions would allow too much. -+ rules.insert( -+ libc::SYS_getsockopt, -+ vec![ -+ SeccompRule::new(vec![SeccompCondition::new( -+ 1, -+ SeccompCmpArgLen::Dword, -+ SeccompCmpOp::Ne, -+ libc::SOL_SOCKET as u64, -+ )?])?, -+ SeccompRule::new(vec![SeccompCondition::new( -+ 2, -+ SeccompCmpArgLen::Dword, -+ SeccompCmpOp::Ne, -+ libc::SO_TYPE as u64, -+ )?])?, -+ ], -+ ); - deny_syscall(&mut rules, libc::SYS_setsockopt); - - // For `socket` we allow AF_UNIX (arg0 == AF_UNIX) and deny -diff --git a/codex-rs/linux-sandbox/tests/suite/landlock.rs b/codex-rs/linux-sandbox/tests/suite/landlock.rs ---- a/codex-rs/linux-sandbox/tests/suite/landlock.rs -+++ b/codex-rs/linux-sandbox/tests/suite/landlock.rs -@@ -1258,3 +1258,84 @@ - // all images ship bash, so we guard against 127 as well. - assert_network_blocked(&["bash", "-c", "echo hi > /dev/tcp/127.0.0.1/80"]).await; - } -+ -+// These compatibility regressions require Node and Python, and intentionally do -+// not accept a missing interpreter or unavailable sandbox as a passing result. -+#[tokio::test] -+async fn restricted_network_supports_node_subprocess_stdio() { -+ let script = r#" -+const assert = require('node:assert/strict'); -+const { spawnSync, spawn } = require('node:child_process'); -+const child = "let input='';process.stdin.setEncoding('utf8');process.stdin.on('data',chunk=>input+=chunk);process.stdin.on('end',()=>{process.stdout.write('out:'+input);process.stderr.write('err:'+input)});"; -+const sync = spawnSync(process.execPath, ['-e', child], { input: 'sync', encoding: 'utf8', timeout: 3000 }); -+assert.equal(sync.error, undefined); -+assert.equal(sync.status, 0); -+assert.equal(sync.stdout, 'out:sync'); -+assert.equal(sync.stderr, 'err:sync'); -+const failed = spawnSync(process.execPath, ['-e', "console.error('intentional-failure');process.exitCode=7"], { encoding: 'utf8', timeout: 3000 }); -+assert.equal(failed.error, undefined); -+assert.equal(failed.status, 7); -+assert.equal(failed.stderr, 'intentional-failure\n'); -+const asyncChild = spawn(process.execPath, ['-e', child]); -+let stdout = '', stderr = ''; -+asyncChild.stdout.on('data', chunk => stdout += chunk); -+asyncChild.stderr.on('data', chunk => stderr += chunk); -+asyncChild.on('error', error => { throw error; }); -+asyncChild.stdin.on('error', error => { throw error; }); -+asyncChild.on('close', code => { -+ assert.equal(code, 0); -+ assert.equal(stdout, 'out:async'); -+ assert.equal(stderr, 'err:async'); -+ console.log('local-subprocess-stdio-ok'); -+}); -+asyncChild.stdin.end('async'); -+"#; -+ let output = run_cmd_output(&["node", "-e", script], &[], LONG_TIMEOUT_MS).await; -+ assert_eq!(output.exit_code, 0, "{}", output.stderr.text); -+ assert_eq!(output.stdout.text, "local-subprocess-stdio-ok\n"); -+} -+ -+#[tokio::test] -+async fn restricted_network_socket_metadata_does_not_enable_connections() { -+ let script = r#" -+import errno, os, socket -+ -+def denied(operation): -+ try: -+ operation() -+ except PermissionError as error: -+ assert error.errno == errno.EPERM, error -+ else: -+ raise AssertionError('operation was not denied with EPERM') -+ -+left, right = socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM) -+assert left.getsockname() == '' -+assert left.getsockopt(socket.SOL_SOCKET, socket.SO_TYPE) == socket.SOCK_STREAM -+os.write(left.fileno(), b'local') -+assert os.read(right.fileno(), 5) == b'local' -+left.shutdown(socket.SHUT_WR) -+assert os.read(right.fileno(), 1) == b'' -+# Each mismatching field independently denies, including both mismatching. -+denied(lambda: right.getsockopt(socket.IPPROTO_TCP, socket.SO_TYPE)) -+denied(lambda: right.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)) -+denied(lambda: right.getsockopt(socket.IPPROTO_TCP, socket.SO_ERROR)) -+denied(lambda: right.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 65536)) -+denied(lambda: right.getpeername()) -+denied(lambda: right.shutdown(socket.SHUT_RD)) -+denied(lambda: right.shutdown(socket.SHUT_RDWR)) -+left.close() -+right.close() -+for family in (socket.AF_INET, socket.AF_INET6, socket.AF_VSOCK, socket.AF_NETLINK): -+ denied(lambda: socket.socket(family, socket.SOCK_STREAM)) -+ denied(lambda: socket.socketpair(family, socket.SOCK_STREAM)) -+with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as endpoint: -+ denied(lambda: endpoint.bind('/tmp/rat-network-denied')) -+ denied(lambda: endpoint.connect('/tmp/rat-network-denied')) -+ denied(lambda: endpoint.listen(1)) -+ denied(lambda: endpoint.accept()) -+print('local-socket-only-ok') -+"#; -+ let output = run_cmd_output(&["python3", "-c", script], &[], LONG_TIMEOUT_MS).await; -+ assert_eq!(output.exit_code, 0, "{}", output.stderr.text); -+ assert_eq!(output.stdout.text, "local-socket-only-ok\n"); -+} diff --git a/runtime/codex/model-capabilities.json b/runtime/codex/model-capabilities.json index 0a8e56e..d291757 100644 --- a/runtime/codex/model-capabilities.json +++ b/runtime/codex/model-capabilities.json @@ -1,6 +1,6 @@ { - "revision": "6b9826e3aa83b1a5947db50f4332cb9c65f1b340", - "sourceSha256": "f3b8104396daf6381bed9d7c4b154a8664f9b5089b44b01a9d54f421566ec9a7", + "revision": "6aaac694803004a63c6f393d153c96d22521a6b9", + "sourceSha256": "943ca7fe1d19ed019054158303f0dbbd80b3bef43b3aa425a71aa7cb3fb52c2b", "models": { "gpt-6-astra": { "reasoning": [ @@ -113,6 +113,35 @@ "high", "xhigh" ] + }, + "gpt-6.1-sol": { + "reasoning": [ + "low", + "medium", + "high", + "xhigh", + "max", + "ultra" + ] + }, + "gpt-6-sol": { + "reasoning": [ + "low", + "medium", + "high", + "xhigh", + "max", + "ultra" + ] + }, + "gpt-6-luna": { + "reasoning": [ + "low", + "medium", + "high", + "xhigh", + "max" + ] } }, "reasoningOverridesSource": "https://developers.openai.com/api/docs/guides/latest-model?model=gpt-5.4", diff --git a/runtime/codex/model-defaults.json b/runtime/codex/model-defaults.json index a27db14..4cccb73 100644 --- a/runtime/codex/model-defaults.json +++ b/runtime/codex/model-defaults.json @@ -1,18 +1,18 @@ { - "revision": "6b9826e3aa83b1a5947db50f4332cb9c65f1b340", + "revision": "6aaac694803004a63c6f393d153c96d22521a6b9", "sourcePath": "codex-rs/models-manager/models.json", - "sourceSha256": "f3b8104396daf6381bed9d7c4b154a8664f9b5089b44b01a9d54f421566ec9a7", + "sourceSha256": "943ca7fe1d19ed019054158303f0dbbd80b3bef43b3aa425a71aa7cb3fb52c2b", "defaults": { "gpt-6-astra": "low", + "gpt-6.1-sol": "low", + "gpt-6-sol": "medium", + "gpt-6-luna": "medium", "gpt-5.6-sol": "low", "gpt-5.6-terra": "medium", "gpt-5.6-luna": "medium", "gpt-daybreak-blue-latest": "low", "gpt-daybreak-red-latest": "medium", "gpt-5.5": "medium", - "gpt-5.4": "medium", - "gpt-5.4-mini": "medium", - "gpt-5.2": "medium", "codex-auto-review": "medium" } } diff --git a/runtime/codex/release-lock.patch b/runtime/codex/release-lock.patch deleted file mode 100644 index 2e74ec4..0000000 --- a/runtime/codex/release-lock.patch +++ /dev/null @@ -1,1334 +0,0 @@ -diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock -index 45fca1b..7ba1057 100644 ---- a/codex-rs/Cargo.lock -+++ b/codex-rs/Cargo.lock -@@ -416,7 +416,7 @@ checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" - - [[package]] - name = "app_test_support" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "base64 0.22.1", -@@ -2040,7 +2040,7 @@ dependencies = [ - - [[package]] - name = "codex-agent-extension" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "codex-core", -@@ -2052,7 +2052,7 @@ dependencies = [ - - [[package]] - name = "codex-agent-graph-store" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "codex-protocol", - "codex-state", -@@ -2067,7 +2067,7 @@ dependencies = [ - - [[package]] - name = "codex-agent-identity" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "base64 0.22.1", -@@ -2087,7 +2087,7 @@ dependencies = [ - - [[package]] - name = "codex-agent-roles" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "codex-config", - "codex-file-system", -@@ -2100,7 +2100,7 @@ dependencies = [ - - [[package]] - name = "codex-analytics" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "codex-app-server-protocol", - "codex-git-utils", -@@ -2121,7 +2121,7 @@ dependencies = [ - - [[package]] - name = "codex-ansi-escape" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "ansi-to-tui", - "ratatui", -@@ -2130,7 +2130,7 @@ dependencies = [ - - [[package]] - name = "codex-api" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "assert_matches", -@@ -2165,7 +2165,7 @@ dependencies = [ - - [[package]] - name = "codex-app-server" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "app_test_support", -@@ -2268,7 +2268,7 @@ dependencies = [ - - [[package]] - name = "codex-app-server-client" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "codex-app-server", - "codex-app-server-protocol", -@@ -2295,7 +2295,7 @@ dependencies = [ - - [[package]] - name = "codex-app-server-daemon" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "codex-app-server-protocol", -@@ -2318,7 +2318,7 @@ dependencies = [ - - [[package]] - name = "codex-app-server-protocol" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "codex-app-server-protocol-noop-macros", -@@ -2352,11 +2352,11 @@ dependencies = [ - - [[package]] - name = "codex-app-server-protocol-noop-macros" --version = "0.0.0" -+version = "0.154.0" - - [[package]] - name = "codex-app-server-test-client" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "clap", -@@ -2378,7 +2378,7 @@ dependencies = [ - - [[package]] - name = "codex-app-server-transport" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "axum", -@@ -2420,7 +2420,7 @@ dependencies = [ - - [[package]] - name = "codex-apply-patch" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "assert_cmd", -@@ -2440,7 +2440,7 @@ dependencies = [ - - [[package]] - name = "codex-arg0" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "codex-apply-patch", -@@ -2462,7 +2462,7 @@ dependencies = [ - - [[package]] - name = "codex-async-utils" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "pretty_assertions", - "tokio", -@@ -2471,7 +2471,7 @@ dependencies = [ - - [[package]] - name = "codex-attachment-store" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "base64 0.22.1", - "codex-utils-image", -@@ -2482,7 +2482,7 @@ dependencies = [ - - [[package]] - name = "codex-aws-auth" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "aws-config", - "aws-credential-types", -@@ -2497,7 +2497,7 @@ dependencies = [ - - [[package]] - name = "codex-backend-client" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "codex-api", -@@ -2517,7 +2517,7 @@ dependencies = [ - - [[package]] - name = "codex-backend-openapi-models" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "serde", - "serde_json", -@@ -2526,7 +2526,7 @@ dependencies = [ - - [[package]] - name = "codex-build-info" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "codex-install-context", - "pretty_assertions", -@@ -2539,7 +2539,7 @@ dependencies = [ - - [[package]] - name = "codex-bwrap" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "cc", - "libc", -@@ -2548,7 +2548,7 @@ dependencies = [ - - [[package]] - name = "codex-chatgpt" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "clap", -@@ -2570,7 +2570,7 @@ dependencies = [ - - [[package]] - name = "codex-cli" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "app_test_support", -@@ -2663,7 +2663,7 @@ dependencies = [ - - [[package]] - name = "codex-client" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "codex-http-client", - "eventsource-stream", -@@ -2676,7 +2676,7 @@ dependencies = [ - - [[package]] - name = "codex-cloud-config" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "base64 0.22.1", - "chrono", -@@ -2701,7 +2701,7 @@ dependencies = [ - - [[package]] - name = "codex-cloud-tasks" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "chrono", -@@ -2734,7 +2734,7 @@ dependencies = [ - - [[package]] - name = "codex-cloud-tasks-client" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "chrono", -@@ -2749,7 +2749,7 @@ dependencies = [ - - [[package]] - name = "codex-cloud-tasks-mock-client" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "chrono", - "codex-cloud-tasks-client", -@@ -2758,7 +2758,7 @@ dependencies = [ - - [[package]] - name = "codex-code-mode" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "codex-code-mode-protocol", - "codex-http-client", -@@ -2781,7 +2781,7 @@ dependencies = [ - - [[package]] - name = "codex-code-mode-host" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "axum", -@@ -2809,7 +2809,7 @@ dependencies = [ - - [[package]] - name = "codex-code-mode-protocol" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "codex-protocol", - "glob", -@@ -2827,7 +2827,7 @@ dependencies = [ - - [[package]] - name = "codex-code-mode-runtime" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "base64 0.22.1", - "codex-code-mode-protocol", -@@ -2845,11 +2845,11 @@ dependencies = [ - - [[package]] - name = "codex-collaboration-mode-templates" --version = "0.0.0" -+version = "0.154.0" - - [[package]] - name = "codex-config" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "base64 0.22.1", -@@ -2898,7 +2898,7 @@ dependencies = [ - - [[package]] - name = "codex-config-schema" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "clap", -@@ -2907,7 +2907,7 @@ dependencies = [ - - [[package]] - name = "codex-connectors" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "arc-swap", -@@ -2929,7 +2929,7 @@ dependencies = [ - - [[package]] - name = "codex-connectors-extension" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "codex-connectors", - "codex-core-plugins", -@@ -2943,7 +2943,7 @@ dependencies = [ - - [[package]] - name = "codex-context-fragments" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "codex-protocol", - "codex-utils-string", -@@ -2952,7 +2952,7 @@ dependencies = [ - - [[package]] - name = "codex-core" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "arc-swap", -@@ -3087,7 +3087,7 @@ dependencies = [ - - [[package]] - name = "codex-core-api" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "codex-analytics", - "codex-app-server-protocol", -@@ -3110,7 +3110,7 @@ dependencies = [ - - [[package]] - name = "codex-core-plugins" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "chrono", -@@ -3165,7 +3165,7 @@ dependencies = [ - - [[package]] - name = "codex-diagnostics" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "libc", - "pretty_assertions", -@@ -3173,7 +3173,7 @@ dependencies = [ - - [[package]] - name = "codex-exec" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "assert_cmd", -@@ -3224,7 +3224,7 @@ dependencies = [ - - [[package]] - name = "codex-exec-server" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "arc-swap", -@@ -3284,7 +3284,7 @@ dependencies = [ - - [[package]] - name = "codex-exec-server-protocol" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "base64 0.22.1", - "codex-file-system", -@@ -3299,7 +3299,7 @@ dependencies = [ - - [[package]] - name = "codex-exec-server-test-support" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "codex-exec-server", -@@ -3314,7 +3314,7 @@ dependencies = [ - - [[package]] - name = "codex-execpolicy" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "clap", -@@ -3332,7 +3332,7 @@ dependencies = [ - - [[package]] - name = "codex-experimental-api-macros" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "proc-macro2", - "quote", -@@ -3341,7 +3341,7 @@ dependencies = [ - - [[package]] - name = "codex-extension-api" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "codex-config", - "codex-context-fragments", -@@ -3359,7 +3359,7 @@ dependencies = [ - - [[package]] - name = "codex-extension-items" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "codex-utils-absolute-path", - "pretty_assertions", -@@ -3371,7 +3371,7 @@ dependencies = [ - - [[package]] - name = "codex-external-agent-migration" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "chrono", - "codex-analytics", -@@ -3401,7 +3401,7 @@ dependencies = [ - - [[package]] - name = "codex-features" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "codex-otel", - "codex-protocol", -@@ -3414,7 +3414,7 @@ dependencies = [ - - [[package]] - name = "codex-feedback" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "bytes", -@@ -3436,7 +3436,7 @@ dependencies = [ - - [[package]] - name = "codex-file-search" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "clap", -@@ -3452,7 +3452,7 @@ dependencies = [ - - [[package]] - name = "codex-file-system" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "bytes", - "codex-protocol", -@@ -3464,7 +3464,7 @@ dependencies = [ - - [[package]] - name = "codex-file-watcher" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "notify", - "pretty_assertions", -@@ -3475,7 +3475,7 @@ dependencies = [ - - [[package]] - name = "codex-git-attribution" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "codex-backend-client", - "codex-extension-api", -@@ -3488,7 +3488,7 @@ dependencies = [ - - [[package]] - name = "codex-git-utils" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "chrono", -@@ -3514,7 +3514,7 @@ dependencies = [ - - [[package]] - name = "codex-goal-extension" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "chrono", -@@ -3538,7 +3538,7 @@ dependencies = [ - - [[package]] - name = "codex-guardian-context" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "codex-history", - "codex-protocol", -@@ -3548,7 +3548,7 @@ dependencies = [ - - [[package]] - name = "codex-guardian-v2" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "codex-analytics", -@@ -3578,7 +3578,7 @@ dependencies = [ - - [[package]] - name = "codex-history" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "codex-protocol", -@@ -3590,7 +3590,7 @@ dependencies = [ - - [[package]] - name = "codex-history-notes-extension" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "codex-analytics", - "codex-api", -@@ -3615,7 +3615,7 @@ dependencies = [ - - [[package]] - name = "codex-home" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "codex-extension-api", - "codex-utils-absolute-path", -@@ -3626,7 +3626,7 @@ dependencies = [ - - [[package]] - name = "codex-hooks" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "async-channel", -@@ -3652,7 +3652,7 @@ dependencies = [ - - [[package]] - name = "codex-http-client" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "bytes", - "codex-utils-cargo-bin", -@@ -3684,7 +3684,7 @@ dependencies = [ - - [[package]] - name = "codex-image-generation-extension" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "base64 0.22.1", - "codex-api", -@@ -3711,7 +3711,7 @@ dependencies = [ - - [[package]] - name = "codex-install-context" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "codex-utils-absolute-path", - "codex-utils-home-dir", -@@ -3724,7 +3724,7 @@ dependencies = [ - - [[package]] - name = "codex-keyring-store" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "keyring", - "tracing", -@@ -3732,7 +3732,7 @@ dependencies = [ - - [[package]] - name = "codex-linux-sandbox" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "clap", - "codex-core", -@@ -3759,7 +3759,7 @@ dependencies = [ - - [[package]] - name = "codex-lmstudio" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "codex-core", - "codex-http-client", -@@ -3773,7 +3773,7 @@ dependencies = [ - - [[package]] - name = "codex-login" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "base64 0.22.1", -@@ -3816,7 +3816,7 @@ dependencies = [ - - [[package]] - name = "codex-mcp" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "arc-swap", -@@ -3856,7 +3856,7 @@ dependencies = [ - - [[package]] - name = "codex-mcp-extension" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "axum", -@@ -3887,7 +3887,7 @@ dependencies = [ - - [[package]] - name = "codex-memories-extension" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "codex-core", - "codex-extension-api", -@@ -3908,7 +3908,7 @@ dependencies = [ - - [[package]] - name = "codex-memories-read" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "codex-protocol", - "codex-shell-command", -@@ -3918,7 +3918,7 @@ dependencies = [ - - [[package]] - name = "codex-memories-write" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "chrono", -@@ -3955,7 +3955,7 @@ dependencies = [ - - [[package]] - name = "codex-message-history" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "codex-config", - "memchr", -@@ -3969,7 +3969,7 @@ dependencies = [ - - [[package]] - name = "codex-model-provider" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "codex-agent-identity", - "codex-api", -@@ -3993,7 +3993,7 @@ dependencies = [ - - [[package]] - name = "codex-model-provider-info" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "codex-api", - "codex-protocol", -@@ -4010,7 +4010,7 @@ dependencies = [ - - [[package]] - name = "codex-models-manager" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "chrono", - "codex-collaboration-mode-templates", -@@ -4029,7 +4029,7 @@ dependencies = [ - - [[package]] - name = "codex-mxc-sandbox" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "appcontainer_common", -@@ -4041,7 +4041,7 @@ dependencies = [ - - [[package]] - name = "codex-network-proxy" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "base64 0.22.1", -@@ -4080,7 +4080,7 @@ dependencies = [ - - [[package]] - name = "codex-ollama" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "assert_matches", - "async-stream", -@@ -4100,7 +4100,7 @@ dependencies = [ - - [[package]] - name = "codex-otel" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "chrono", - "codex-api", -@@ -4133,7 +4133,7 @@ dependencies = [ - - [[package]] - name = "codex-otel-trace-websocket" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "axum", -@@ -4144,7 +4144,7 @@ dependencies = [ - - [[package]] - name = "codex-plugin" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "codex-config", - "codex-protocol", -@@ -4158,7 +4158,7 @@ dependencies = [ - - [[package]] - name = "codex-process-hardening" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "libc", - "pretty_assertions", -@@ -4166,7 +4166,7 @@ dependencies = [ - - [[package]] - name = "codex-prompts" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "codex-context-fragments", -@@ -4180,7 +4180,7 @@ dependencies = [ - - [[package]] - name = "codex-protocol" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "chardetng", -@@ -4224,7 +4224,7 @@ dependencies = [ - - [[package]] - name = "codex-queue-extension" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "codex-core", -@@ -4245,7 +4245,7 @@ dependencies = [ - - [[package]] - name = "codex-realtime-webrtc" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "codex-build-info", -@@ -4262,7 +4262,7 @@ dependencies = [ - - [[package]] - name = "codex-response-debug-context" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "base64 0.22.1", - "codex-api", -@@ -4273,7 +4273,7 @@ dependencies = [ - - [[package]] - name = "codex-responses-api-proxy" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "clap", -@@ -4290,7 +4290,7 @@ dependencies = [ - - [[package]] - name = "codex-rmcp-client" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "axum", -@@ -4336,7 +4336,7 @@ dependencies = [ - - [[package]] - name = "codex-rollout" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "chrono", -@@ -4363,7 +4363,7 @@ dependencies = [ - - [[package]] - name = "codex-rollout-trace" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "codex-code-mode", -@@ -4379,7 +4379,7 @@ dependencies = [ - - [[package]] - name = "codex-sandboxing" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "codex-mxc-sandbox", -@@ -4405,7 +4405,7 @@ dependencies = [ - - [[package]] - name = "codex-secrets" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "age", - "anyhow", -@@ -4426,7 +4426,7 @@ dependencies = [ - - [[package]] - name = "codex-shell-command" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "base64 0.22.1", -@@ -4449,7 +4449,7 @@ dependencies = [ - - [[package]] - name = "codex-shell-escalation" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "clap", -@@ -4469,7 +4469,7 @@ dependencies = [ - - [[package]] - name = "codex-skills" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "codex-protocol", - "codex-shell-command", -@@ -4486,7 +4486,7 @@ dependencies = [ - - [[package]] - name = "codex-skills-extension" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "assert_matches", - "codex-analytics", -@@ -4523,7 +4523,7 @@ dependencies = [ - - [[package]] - name = "codex-state" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "chrono", -@@ -4547,7 +4547,7 @@ dependencies = [ - - [[package]] - name = "codex-stdio-to-uds" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "codex-uds", -@@ -4559,7 +4559,7 @@ dependencies = [ - - [[package]] - name = "codex-terminal-detection" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "pretty_assertions", - "tracing", -@@ -4567,7 +4567,7 @@ dependencies = [ - - [[package]] - name = "codex-test-binary-support" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "codex-arg0", - "tempfile", -@@ -4575,7 +4575,7 @@ dependencies = [ - - [[package]] - name = "codex-thread-manager-sample" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "clap", -@@ -4586,7 +4586,7 @@ dependencies = [ - - [[package]] - name = "codex-thread-store" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "chrono", - "codex-app-server-protocol", -@@ -4616,7 +4616,7 @@ dependencies = [ - - [[package]] - name = "codex-tools" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "bitflags 2.13.1", - "codex-code-mode", -@@ -4641,7 +4641,7 @@ dependencies = [ - - [[package]] - name = "codex-tui" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "app_test_support", -@@ -4758,7 +4758,7 @@ dependencies = [ - - [[package]] - name = "codex-uds" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "async-io", - "pretty_assertions", -@@ -4771,7 +4771,7 @@ dependencies = [ - - [[package]] - name = "codex-utils-absolute-path" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "dirs", - "dunce", -@@ -4785,14 +4785,14 @@ dependencies = [ - - [[package]] - name = "codex-utils-approval-presets" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "codex-protocol", - ] - - [[package]] - name = "codex-utils-audio" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "base64 0.22.1", - "codex-protocol", -@@ -4806,7 +4806,7 @@ dependencies = [ - - [[package]] - name = "codex-utils-cache" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "lru", - "sha1 0.10.6", -@@ -4815,7 +4815,7 @@ dependencies = [ - - [[package]] - name = "codex-utils-cargo-bin" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "assert_cmd", - "runfiles", -@@ -4824,7 +4824,7 @@ dependencies = [ - - [[package]] - name = "codex-utils-cli" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "clap", - "codex-protocol", -@@ -4836,15 +4836,15 @@ dependencies = [ - - [[package]] - name = "codex-utils-elapsed" --version = "0.0.0" -+version = "0.154.0" - - [[package]] - name = "codex-utils-fuzzy-match" --version = "0.0.0" -+version = "0.154.0" - - [[package]] - name = "codex-utils-git-discovery" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "codex-git-utils", - "codex-utils-absolute-path", -@@ -4857,7 +4857,7 @@ dependencies = [ - - [[package]] - name = "codex-utils-home-dir" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "codex-utils-absolute-path", - "dirs", -@@ -4867,7 +4867,7 @@ dependencies = [ - - [[package]] - name = "codex-utils-image" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "base64 0.22.1", - "codex-utils-cache", -@@ -4880,7 +4880,7 @@ dependencies = [ - - [[package]] - name = "codex-utils-json-to-toml" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "pretty_assertions", - "serde_json", -@@ -4889,7 +4889,7 @@ dependencies = [ - - [[package]] - name = "codex-utils-oss" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "codex-core", - "codex-lmstudio", -@@ -4899,7 +4899,7 @@ dependencies = [ - - [[package]] - name = "codex-utils-output-truncation" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "codex-protocol", - "codex-utils-string", -@@ -4908,7 +4908,7 @@ dependencies = [ - - [[package]] - name = "codex-utils-path" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "codex-utils-absolute-path", - "dunce", -@@ -4919,7 +4919,7 @@ dependencies = [ - - [[package]] - name = "codex-utils-path-uri" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "base64 0.22.1", - "codex-utils-absolute-path", -@@ -4935,7 +4935,7 @@ dependencies = [ - - [[package]] - name = "codex-utils-plugins" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "codex-exec-server", - "codex-exec-server-protocol", -@@ -4949,7 +4949,7 @@ dependencies = [ - - [[package]] - name = "codex-utils-pty" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "filedescriptor", -@@ -4965,7 +4965,7 @@ dependencies = [ - - [[package]] - name = "codex-utils-readiness" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "assert_matches", - "thiserror 2.0.18", -@@ -4975,7 +4975,7 @@ dependencies = [ - - [[package]] - name = "codex-utils-redacted-string" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "schemars 0.8.22", - "serde", -@@ -4983,14 +4983,14 @@ dependencies = [ - - [[package]] - name = "codex-utils-rustls-provider" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "rustls", - ] - - [[package]] - name = "codex-utils-sandbox-summary" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "codex-protocol", - "codex-utils-absolute-path", -@@ -4999,7 +4999,7 @@ dependencies = [ - - [[package]] - name = "codex-utils-sleep-inhibitor" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "core-foundation 0.9.4", - "libc", -@@ -5009,14 +5009,14 @@ dependencies = [ - - [[package]] - name = "codex-utils-stream-parser" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "pretty_assertions", - ] - - [[package]] - name = "codex-utils-string" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "pretty_assertions", - "regex-lite", -@@ -5026,14 +5026,14 @@ dependencies = [ - - [[package]] - name = "codex-utils-template" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "pretty_assertions", - ] - - [[package]] - name = "codex-v8-poc" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "pretty_assertions", - "v8", -@@ -5041,7 +5041,7 @@ dependencies = [ - - [[package]] - name = "codex-voice-host" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "codex-install-context", -@@ -5068,7 +5068,7 @@ dependencies = [ - - [[package]] - name = "codex-web-search-extension" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "codex-api", - "codex-core", -@@ -5089,7 +5089,7 @@ dependencies = [ - - [[package]] - name = "codex-websocket-client" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "codex-http-client", - "codex-utils-rustls-provider", -@@ -5105,7 +5105,7 @@ dependencies = [ - - [[package]] - name = "codex-windows-sandbox" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "base64 0.22.1", -@@ -5131,7 +5131,7 @@ dependencies = [ - - [[package]] - name = "codex-windows-sandbox-service" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "codex-cloud-config", -@@ -5148,7 +5148,7 @@ dependencies = [ - - [[package]] - name = "codex-workload-identity" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "codex-http-client", - "pretty_assertions", -@@ -5163,7 +5163,7 @@ dependencies = [ - - [[package]] - name = "codex-worktree" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "codex-git-utils", -@@ -5403,7 +5403,7 @@ dependencies = [ - - [[package]] - name = "core_test_support" --version = "0.0.0" -+version = "0.154.0" - dependencies = [ - "anyhow", - "assert_cmd", diff --git a/runtime/codex/source.json b/runtime/codex/source.json index 316d33b..9210bc1 100644 --- a/runtime/codex/source.json +++ b/runtime/codex/source.json @@ -1,16 +1,5 @@ { "repository": "https://github.com/openai/codex.git", - "revision": "6b9826e3aa83b1a5947db50f4332cb9c65f1b340", - "upstreamVersion": "0.154.0", - "patches": [ - "release-lock.patch", - "interrupt-barrier.patch", - "api-model-settings.patch", - "api-turn-errors.patch", - "checkpoint-shutdown.patch", - "step-settings-test-import.patch", - "unified-exec-output-subscription.patch", - "chatgpt-service-tier.patch", - "local-subprocess-stdio.patch" - ] + "revision": "6aaac694803004a63c6f393d153c96d22521a6b9", + "upstreamVersion": "0.164.0" } diff --git a/runtime/codex/step-settings-test-import.patch b/runtime/codex/step-settings-test-import.patch deleted file mode 100644 index 7fac824..0000000 --- a/runtime/codex/step-settings-test-import.patch +++ /dev/null @@ -1,12 +0,0 @@ -diff --git a/codex-rs/core/src/session/step_settings_tests.rs b/codex-rs/core/src/session/step_settings_tests.rs -index ba04038..eeb4763 100644 ---- a/codex-rs/core/src/session/step_settings_tests.rs -+++ b/codex-rs/core/src/session/step_settings_tests.rs -@@ -11,6 +11,7 @@ use codex_features::Feature; - use codex_models_manager::manager::StaticModelsManager; - use codex_models_manager::model_info::model_info_from_slug; - use codex_protocol::config_types::ModeKind; -+use codex_protocol::config_types::ServiceTier; - use codex_protocol::config_types::Settings; - use codex_protocol::models::BaseInstructionsProvenance; - use codex_protocol::models::PermissionProfile; diff --git a/runtime/codex/unified-exec-output-subscription.patch b/runtime/codex/unified-exec-output-subscription.patch deleted file mode 100644 index 584e8ba..0000000 --- a/runtime/codex/unified-exec-output-subscription.patch +++ /dev/null @@ -1,160 +0,0 @@ -diff --git a/codex-rs/core/src/unified_exec/async_watcher_tests.rs b/codex-rs/core/src/unified_exec/async_watcher_tests.rs -index bffc5cf..29e3d8b 100644 ---- a/codex-rs/core/src/unified_exec/async_watcher_tests.rs -+++ b/codex-rs/core/src/unified_exec/async_watcher_tests.rs -@@ -33,6 +33,16 @@ struct StreamingOutputHarness { - } - - async fn streaming_output_harness() -> anyhow::Result { -+ let harness = unstarted_streaming_output_harness().await?; -+ start_streaming_output( -+ &harness.process, -+ &harness.context, -+ Arc::clone(&harness.transcript), -+ ); -+ Ok(harness) -+} -+ -+async fn unstarted_streaming_output_harness() -> anyhow::Result { - let (writer_tx, _writer_rx) = tokio::sync::mpsc::channel::>(1); - let (stdout_tx, stdout_rx) = tokio::sync::broadcast::channel::>(8); - let (exit_tx, exit_rx) = tokio::sync::oneshot::channel::(); -@@ -59,8 +69,6 @@ async fn streaming_output_harness() -> anyhow::Result { - "streaming-output-test".to_string(), - ); - let transcript = Arc::new(tokio::sync::Mutex::new(HeadTailBuffer::default())); -- start_streaming_output(&process, &context, Arc::clone(&transcript)); -- - Ok(StreamingOutputHarness { - process, - stdout_tx, -@@ -71,6 +79,89 @@ async fn streaming_output_harness() -> anyhow::Result { - }) - } - -+#[tokio::test] -+async fn exit_watcher_preserves_output_emitted_before_streaming_starts() -> anyhow::Result<()> { -+ let StreamingOutputHarness { -+ process, -+ stdout_tx, -+ exit_tx, -+ transcript, -+ context, -+ rx_event, -+ } = unstarted_streaming_output_harness().await?; -+ let output_handles = process.output_handles(); -+ let early_output_buffered = output_handles.output_notify.notified(); -+ tokio::pin!(early_output_buffered); -+ early_output_buffered.as_mut().enable(); -+ -+ stdout_tx -+ .send(b"EARLY-OUTPUT-MARKER\n".to_vec()) -+ .expect("send early output"); -+ early_output_buffered.await; -+ let tool_output = std::mem::take(&mut *output_handles.output_buffer.lock().await); -+ assert_eq!(tool_output.to_bytes(), b"EARLY-OUTPUT-MARKER\n"); -+ -+ start_streaming_output(&process, &context, Arc::clone(&transcript)); -+ let cwd = context -+ .step_context -+ .environments -+ .primary() -+ .expect("primary environment") -+ .cwd() -+ .clone(); -+ spawn_exit_watcher( -+ Arc::clone(&process), -+ Arc::clone(&context.session), -+ Arc::clone(&context.step_context.turn), -+ context.call_id.clone(), -+ vec!["proof".to_string()], -+ cwd, -+ /*process_id*/ 123, -+ /*plugin_attribution*/ None, -+ transcript, -+ Instant::now(), -+ /*network_denial_monitor*/ None, -+ /*plugin_metrics_sidecar*/ None, -+ ); -+ drop(context); -+ -+ stdout_tx -+ .send(b"LATE-OUTPUT-MARKER\n".to_vec()) -+ .expect("send late output"); -+ drop(stdout_tx); -+ exit_tx.send(0).expect("send exit"); -+ -+ let mut streamed_output = Vec::new(); -+ let mut completions = Vec::new(); -+ while let Ok(event) = rx_event.recv().await { -+ match event.msg { -+ EventMsg::ExecCommandOutputDelta(delta) => streamed_output.extend(delta.chunk), -+ EventMsg::ItemCompleted(completed) => { -+ let TurnItem::CommandExecution(item) = completed.item else { -+ panic!("expected CommandExecution"); -+ }; -+ completions.push((item.status, item.exit_code, item.aggregated_output)); -+ } -+ EventMsg::ExecCommandEnd(_) => {} -+ event => panic!("unexpected event: {event:?}"), -+ } -+ } -+ assert_eq!( -+ completions, -+ vec![( -+ CommandExecutionStatus::Completed, -+ Some(0), -+ Some("EARLY-OUTPUT-MARKER\nLATE-OUTPUT-MARKER\n".to_string()), -+ )] -+ ); -+ assert_eq!( -+ streamed_output, -+ b"EARLY-OUTPUT-MARKER\nLATE-OUTPUT-MARKER\n" -+ ); -+ -+ Ok(()) -+} -+ - #[tokio::test] - async fn streaming_output_preserves_multibyte_characters_across_chunks() -> anyhow::Result<()> { - let StreamingOutputHarness { -diff --git a/codex-rs/core/src/unified_exec/process.rs b/codex-rs/core/src/unified_exec/process.rs -index 81946e4..49dd1ec 100644 ---- a/codex-rs/core/src/unified_exec/process.rs -+++ b/codex-rs/core/src/unified_exec/process.rs -@@ -90,6 +90,7 @@ enum ProcessHandle { - pub(crate) struct UnifiedExecProcess { - process_handle: ProcessHandle, - output_tx: broadcast::Sender>, -+ output_rx: std::sync::Mutex>>>, - output: OutputHandles, - output_drained: Arc, - interaction_lock: Arc>, -@@ -125,12 +126,13 @@ impl UnifiedExecProcess { - cancellation_token: CancellationToken::new(), - }; - let output_drained = Arc::new(Notify::new()); -- let (output_tx, _) = broadcast::channel(64); -+ let (output_tx, output_rx) = broadcast::channel(64); - let (state_tx, state_rx) = watch::channel(ProcessState::default()); - - Self { - process_handle, - output_tx, -+ output_rx: std::sync::Mutex::new(Some(output_rx)), - output, - output_drained, - interaction_lock: Arc::new(Mutex::new(())), -@@ -173,7 +175,11 @@ impl UnifiedExecProcess { - } - - pub(super) fn output_receiver(&self) -> tokio::sync::broadcast::Receiver> { -- self.output_tx.subscribe() -+ self.output_rx -+ .lock() -+ .expect("unified exec output receiver lock poisoned") -+ .take() -+ .expect("unified exec output receiver already taken") - } - - pub(super) fn cancellation_token(&self) -> CancellationToken { diff --git a/runtime/local/verify-runtime.swift b/runtime/local/verify-runtime.swift index 5d9343d..8264ffc 100644 --- a/runtime/local/verify-runtime.swift +++ b/runtime/local/verify-runtime.swift @@ -47,11 +47,36 @@ struct ExecutablePolicy { } let executablePolicy = [ - ExecutablePolicy(path: "Contents/Helpers/rat-native-worker", identifier: "dev.rat-things.runtime.vm-helper"), ExecutablePolicy(path: "Contents/Resources/codex/bin/codex", identifier: "dev.rat-things.runtime.codex"), ExecutablePolicy(path: "Contents/Resources/codex/bin/codex-code-mode-host", identifier: "dev.rat-things.runtime.code-mode-host"), ExecutablePolicy(path: "Contents/Resources/codex/codex-path/rg", identifier: "dev.rat-things.runtime.ripgrep"), - ExecutablePolicy(path: "Contents/Resources/codex/codex-resources/zsh/bin/zsh", identifier: "dev.rat-things.runtime.zsh") + ExecutablePolicy(path: "Contents/Resources/codex/codex-resources/voice/bin/codex-voice-host", identifier: "dev.rat-things.runtime.voice-host"), + ExecutablePolicy(path: "Contents/Resources/codex/codex-resources/voice/lib/libffi.8.dylib", identifier: "dev.rat-things.runtime.voice.lib.libffi.8.dylib"), + ExecutablePolicy(path: "Contents/Resources/codex/codex-resources/voice/lib/libgio-2.0.0.dylib", identifier: "dev.rat-things.runtime.voice.lib.libgio-2.0.0.dylib"), + ExecutablePolicy(path: "Contents/Resources/codex/codex-resources/voice/lib/libglib-2.0.0.dylib", identifier: "dev.rat-things.runtime.voice.lib.libglib-2.0.0.dylib"), + ExecutablePolicy(path: "Contents/Resources/codex/codex-resources/voice/lib/libgmodule-2.0.0.dylib", identifier: "dev.rat-things.runtime.voice.lib.libgmodule-2.0.0.dylib"), + ExecutablePolicy(path: "Contents/Resources/codex/codex-resources/voice/lib/libgobject-2.0.0.dylib", identifier: "dev.rat-things.runtime.voice.lib.libgobject-2.0.0.dylib"), + ExecutablePolicy(path: "Contents/Resources/codex/codex-resources/voice/lib/libgstapp-1.0.0.dylib", identifier: "dev.rat-things.runtime.voice.lib.libgstapp-1.0.0.dylib"), + ExecutablePolicy(path: "Contents/Resources/codex/codex-resources/voice/lib/libgstaudio-1.0.0.dylib", identifier: "dev.rat-things.runtime.voice.lib.libgstaudio-1.0.0.dylib"), + ExecutablePolicy(path: "Contents/Resources/codex/codex-resources/voice/lib/libgstbase-1.0.0.dylib", identifier: "dev.rat-things.runtime.voice.lib.libgstbase-1.0.0.dylib"), + ExecutablePolicy(path: "Contents/Resources/codex/codex-resources/voice/lib/libgstnet-1.0.0.dylib", identifier: "dev.rat-things.runtime.voice.lib.libgstnet-1.0.0.dylib"), + ExecutablePolicy(path: "Contents/Resources/codex/codex-resources/voice/lib/libgstpbutils-1.0.0.dylib", identifier: "dev.rat-things.runtime.voice.lib.libgstpbutils-1.0.0.dylib"), + ExecutablePolicy(path: "Contents/Resources/codex/codex-resources/voice/lib/libgstreamer-1.0.0.dylib", identifier: "dev.rat-things.runtime.voice.lib.libgstreamer-1.0.0.dylib"), + ExecutablePolicy(path: "Contents/Resources/codex/codex-resources/voice/lib/libgstrtp-1.0.0.dylib", identifier: "dev.rat-things.runtime.voice.lib.libgstrtp-1.0.0.dylib"), + ExecutablePolicy(path: "Contents/Resources/codex/codex-resources/voice/lib/libgsttag-1.0.0.dylib", identifier: "dev.rat-things.runtime.voice.lib.libgsttag-1.0.0.dylib"), + ExecutablePolicy(path: "Contents/Resources/codex/codex-resources/voice/lib/libgstvideo-1.0.0.dylib", identifier: "dev.rat-things.runtime.voice.lib.libgstvideo-1.0.0.dylib"), + ExecutablePolicy(path: "Contents/Resources/codex/codex-resources/voice/lib/libintl.8.dylib", identifier: "dev.rat-things.runtime.voice.lib.libintl.8.dylib"), + ExecutablePolicy(path: "Contents/Resources/codex/codex-resources/voice/lib/libopus.0.dylib", identifier: "dev.rat-things.runtime.voice.lib.libopus.0.dylib"), + ExecutablePolicy(path: "Contents/Resources/codex/codex-resources/voice/lib/libpcre2-8.0.dylib", identifier: "dev.rat-things.runtime.voice.lib.libpcre2-8.0.dylib"), + ExecutablePolicy(path: "Contents/Resources/codex/codex-resources/voice/lib/libz.1.dylib", identifier: "dev.rat-things.runtime.voice.lib.libz.1.dylib"), + ExecutablePolicy(path: "Contents/Resources/codex/codex-resources/voice/plugins/libgstapp.dylib", identifier: "dev.rat-things.runtime.voice.plugins.libgstapp.dylib"), + ExecutablePolicy(path: "Contents/Resources/codex/codex-resources/voice/plugins/libgstaudioconvert.dylib", identifier: "dev.rat-things.runtime.voice.plugins.libgstaudioconvert.dylib"), + ExecutablePolicy(path: "Contents/Resources/codex/codex-resources/voice/plugins/libgstaudioresample.dylib", identifier: "dev.rat-things.runtime.voice.plugins.libgstaudioresample.dylib"), + ExecutablePolicy(path: "Contents/Resources/codex/codex-resources/voice/plugins/libgstcoreelements.dylib", identifier: "dev.rat-things.runtime.voice.plugins.libgstcoreelements.dylib"), + ExecutablePolicy(path: "Contents/Resources/codex/codex-resources/voice/plugins/libgstopus.dylib", identifier: "dev.rat-things.runtime.voice.plugins.libgstopus.dylib"), + ExecutablePolicy(path: "Contents/Resources/codex/codex-resources/voice/plugins/libgstrtp.dylib", identifier: "dev.rat-things.runtime.voice.plugins.libgstrtp.dylib"), + ExecutablePolicy(path: "Contents/Resources/codex/codex-resources/voice/plugins/libgstrtpmanager.dylib", identifier: "dev.rat-things.runtime.voice.plugins.libgstrtpmanager.dylib"), + ExecutablePolicy(path: "Contents/Helpers/rat-native-worker", identifier: "dev.rat-things.runtime.vm-helper") ] let executablePaths = Set(executablePolicy.map(\.path)) let signaturePaths: Set = [ diff --git a/runtime/native/host-code-policy.json b/runtime/native/host-code-policy.json new file mode 100644 index 0000000..6f364b9 --- /dev/null +++ b/runtime/native/host-code-policy.json @@ -0,0 +1,189 @@ +[ + { + "path": "Contents/Resources/codex/bin/codex", + "role": "host-codex", + "identifier": "dev.rat-things.runtime.codex", + "machOType": 2, + "entitlements": "jit" + }, + { + "path": "Contents/Resources/codex/bin/codex-code-mode-host", + "role": "host-companion", + "identifier": "dev.rat-things.runtime.code-mode-host", + "machOType": 2, + "entitlements": "jit" + }, + { + "path": "Contents/Resources/codex/codex-path/rg", + "role": "host-companion", + "identifier": "dev.rat-things.runtime.ripgrep", + "machOType": 2 + }, + { + "path": "Contents/Resources/codex/codex-resources/voice/bin/codex-voice-host", + "role": "host-companion", + "identifier": "dev.rat-things.runtime.voice-host", + "machOType": 2, + "rpaths": [ + "@loader_path/../../_solib_darwin_arm64/voice/native_link_macos_aarch64", + "@loader_path/../lib" + ] + }, + { + "path": "Contents/Resources/codex/codex-resources/voice/lib/libffi.8.dylib", + "role": "host-companion", + "identifier": "dev.rat-things.runtime.voice.lib.libffi.8.dylib", + "machOType": 6 + }, + { + "path": "Contents/Resources/codex/codex-resources/voice/lib/libgio-2.0.0.dylib", + "role": "host-companion", + "identifier": "dev.rat-things.runtime.voice.lib.libgio-2.0.0.dylib", + "machOType": 6 + }, + { + "path": "Contents/Resources/codex/codex-resources/voice/lib/libglib-2.0.0.dylib", + "role": "host-companion", + "identifier": "dev.rat-things.runtime.voice.lib.libglib-2.0.0.dylib", + "machOType": 6 + }, + { + "path": "Contents/Resources/codex/codex-resources/voice/lib/libgmodule-2.0.0.dylib", + "role": "host-companion", + "identifier": "dev.rat-things.runtime.voice.lib.libgmodule-2.0.0.dylib", + "machOType": 6 + }, + { + "path": "Contents/Resources/codex/codex-resources/voice/lib/libgobject-2.0.0.dylib", + "role": "host-companion", + "identifier": "dev.rat-things.runtime.voice.lib.libgobject-2.0.0.dylib", + "machOType": 6 + }, + { + "path": "Contents/Resources/codex/codex-resources/voice/lib/libgstapp-1.0.0.dylib", + "role": "host-companion", + "identifier": "dev.rat-things.runtime.voice.lib.libgstapp-1.0.0.dylib", + "machOType": 6 + }, + { + "path": "Contents/Resources/codex/codex-resources/voice/lib/libgstaudio-1.0.0.dylib", + "role": "host-companion", + "identifier": "dev.rat-things.runtime.voice.lib.libgstaudio-1.0.0.dylib", + "machOType": 6 + }, + { + "path": "Contents/Resources/codex/codex-resources/voice/lib/libgstbase-1.0.0.dylib", + "role": "host-companion", + "identifier": "dev.rat-things.runtime.voice.lib.libgstbase-1.0.0.dylib", + "machOType": 6 + }, + { + "path": "Contents/Resources/codex/codex-resources/voice/lib/libgstnet-1.0.0.dylib", + "role": "host-companion", + "identifier": "dev.rat-things.runtime.voice.lib.libgstnet-1.0.0.dylib", + "machOType": 6 + }, + { + "path": "Contents/Resources/codex/codex-resources/voice/lib/libgstpbutils-1.0.0.dylib", + "role": "host-companion", + "identifier": "dev.rat-things.runtime.voice.lib.libgstpbutils-1.0.0.dylib", + "machOType": 6 + }, + { + "path": "Contents/Resources/codex/codex-resources/voice/lib/libgstreamer-1.0.0.dylib", + "role": "host-companion", + "identifier": "dev.rat-things.runtime.voice.lib.libgstreamer-1.0.0.dylib", + "machOType": 6 + }, + { + "path": "Contents/Resources/codex/codex-resources/voice/lib/libgstrtp-1.0.0.dylib", + "role": "host-companion", + "identifier": "dev.rat-things.runtime.voice.lib.libgstrtp-1.0.0.dylib", + "machOType": 6 + }, + { + "path": "Contents/Resources/codex/codex-resources/voice/lib/libgsttag-1.0.0.dylib", + "role": "host-companion", + "identifier": "dev.rat-things.runtime.voice.lib.libgsttag-1.0.0.dylib", + "machOType": 6 + }, + { + "path": "Contents/Resources/codex/codex-resources/voice/lib/libgstvideo-1.0.0.dylib", + "role": "host-companion", + "identifier": "dev.rat-things.runtime.voice.lib.libgstvideo-1.0.0.dylib", + "machOType": 6 + }, + { + "path": "Contents/Resources/codex/codex-resources/voice/lib/libintl.8.dylib", + "role": "host-companion", + "identifier": "dev.rat-things.runtime.voice.lib.libintl.8.dylib", + "machOType": 6 + }, + { + "path": "Contents/Resources/codex/codex-resources/voice/lib/libopus.0.dylib", + "role": "host-companion", + "identifier": "dev.rat-things.runtime.voice.lib.libopus.0.dylib", + "machOType": 6 + }, + { + "path": "Contents/Resources/codex/codex-resources/voice/lib/libpcre2-8.0.dylib", + "role": "host-companion", + "identifier": "dev.rat-things.runtime.voice.lib.libpcre2-8.0.dylib", + "machOType": 6 + }, + { + "path": "Contents/Resources/codex/codex-resources/voice/lib/libz.1.dylib", + "role": "host-companion", + "identifier": "dev.rat-things.runtime.voice.lib.libz.1.dylib", + "machOType": 6 + }, + { + "path": "Contents/Resources/codex/codex-resources/voice/plugins/libgstapp.dylib", + "role": "host-companion", + "identifier": "dev.rat-things.runtime.voice.plugins.libgstapp.dylib", + "machOType": 6 + }, + { + "path": "Contents/Resources/codex/codex-resources/voice/plugins/libgstaudioconvert.dylib", + "role": "host-companion", + "identifier": "dev.rat-things.runtime.voice.plugins.libgstaudioconvert.dylib", + "machOType": 6 + }, + { + "path": "Contents/Resources/codex/codex-resources/voice/plugins/libgstaudioresample.dylib", + "role": "host-companion", + "identifier": "dev.rat-things.runtime.voice.plugins.libgstaudioresample.dylib", + "machOType": 6 + }, + { + "path": "Contents/Resources/codex/codex-resources/voice/plugins/libgstcoreelements.dylib", + "role": "host-companion", + "identifier": "dev.rat-things.runtime.voice.plugins.libgstcoreelements.dylib", + "machOType": 6 + }, + { + "path": "Contents/Resources/codex/codex-resources/voice/plugins/libgstopus.dylib", + "role": "host-companion", + "identifier": "dev.rat-things.runtime.voice.plugins.libgstopus.dylib", + "machOType": 6 + }, + { + "path": "Contents/Resources/codex/codex-resources/voice/plugins/libgstrtp.dylib", + "role": "host-companion", + "identifier": "dev.rat-things.runtime.voice.plugins.libgstrtp.dylib", + "machOType": 6 + }, + { + "path": "Contents/Resources/codex/codex-resources/voice/plugins/libgstrtpmanager.dylib", + "role": "host-companion", + "identifier": "dev.rat-things.runtime.voice.plugins.libgstrtpmanager.dylib", + "machOType": 6 + }, + { + "path": "Contents/Helpers/rat-native-worker", + "role": "vm-helper", + "identifier": "dev.rat-things.runtime.vm-helper", + "machOType": 2, + "entitlements": "virtualization" + } +] diff --git a/runtime/native/init.sh b/runtime/native/init.sh index 4012883..28d7fc4 100644 --- a/runtime/native/init.sh +++ b/runtime/native/init.sh @@ -9,6 +9,10 @@ if [ "${1:-}" != rat-root-mounted ]; then exec chroot . /init rat-root-mounted fi mount -t proc proc /proc +# Resolve the guest's own hostname locally before executor configuration discovery. +printf '%s\n' rat-things-worker > /proc/sys/kernel/hostname +printf '%s\n' rat-things-worker > /etc/hostname +printf '%s\n' '127.0.0.1 localhost rat-things-worker' '::1 localhost rat-things-worker' > /etc/hosts mount -t sysfs sysfs /sys mount -t devtmpfs devtmpfs /dev mkdir -p /dev/pts /dev/shm /run /tmp /workspace /mnt/rat-disk diff --git a/scripts/build-codex-runtime.mjs b/scripts/build-codex-runtime.mjs index 692892b..f9b919f 100644 --- a/scripts/build-codex-runtime.mjs +++ b/scripts/build-codex-runtime.mjs @@ -1,113 +1,32 @@ -import { execFile, spawn } from 'node:child_process'; -import { promisify } from 'node:util'; -import { readFile, writeFile, mkdir, readdir, statfs, cp, mkdtemp, rm } from 'node:fs/promises'; -import { dirname, join, resolve, relative } from 'node:path'; +import { cp, mkdir, readFile, writeFile } from 'node:fs/promises'; +import { dirname, join, resolve } from 'node:path'; import { createRequire } from 'node:module'; import { fileURLToPath } from 'node:url'; -import { createHash } from 'node:crypto'; import { codexRuntimeTargets } from './codex-runtime-targets.mjs'; +import { validateCodexRuntimeArtifact } from './codex-runtime-artifact.mjs'; const root = resolve(dirname(fileURLToPath(import.meta.url)), '..'); -const definition = join(root, 'runtime/codex'); -const source = JSON.parse(await readFile(join(definition, 'source.json'), 'utf8')); -const digest = (bytes) => createHash('sha256').update(bytes).digest('hex'); -const buildRoot = process.argv[2] && resolve(process.argv[2]); -const prepareOnly = process.argv[3] === '--prepare-only'; -if (!buildRoot || process.argv.length > 4 || process.argv[3] && !prepareOnly) throw new Error('Usage: node scripts/build-codex-runtime.mjs BUILD_DIRECTORY [--prepare-only]'); -// Resolve host compatibility before creating a build directory or fetching source. -const targetDefinition = codexRuntimeTargets[`${process.platform}-${process.arch}`]; -if (!prepareOnly && !targetDefinition) throw new Error('The worker runtime build supports ARM64 macOS and ARM64 or x64 Linux. Use the matching Linux Docker builder for worker artifacts.'); -await mkdir(buildRoot, { recursive: true }); -const marker = join(buildRoot, 'rat-codex-source.json'); -const identity = JSON.stringify({ ...source, patches: await Promise.all(source.patches.map(async (name) => ({ name, sha256: createHash('sha256').update(await readFile(join(definition, name))).digest('hex') }))) }); -const previous = await readFile(marker, 'utf8').catch((error) => { if (error.code === 'ENOENT') return undefined; throw error; }); -if (previous !== undefined && previous !== identity) throw new Error('Build directory belongs to a different source revision or patch. Use a new directory.'); -if (previous === undefined && (await readdir(buildRoot)).length) throw new Error('Use an empty build directory. Existing files will not be replaced.'); -const run = async (program, args, cwd = buildRoot, env = process.env) => { - const child = spawn(program, args, { cwd, env, stdio: 'inherit' }); - await new Promise((resolve, reject) => { child.once('error', reject); child.once('close', (code, signal) => code === 0 ? resolve() : reject(new Error(`${program} exited with ${signal ?? code}`))); }); +const output = process.argv[2] && resolve(process.argv[2]); +if (!output || process.argv.length !== 3) throw new Error('Usage: node scripts/build-codex-runtime.mjs NEW_OUTPUT_DIRECTORY'); +const target = codexRuntimeTargets[`${process.platform}-${process.arch}`]; +if (!target) throw new Error('Official runtime packaging supports ARM64 macOS and ARM64 or x64 Linux.'); +const source = JSON.parse(await readFile(join(root, 'runtime/codex/source.json'), 'utf8')); +if ('patches' in source) throw new Error('Only unmodified official Codex releases can be packaged.'); +const inventory = JSON.parse(await readFile(join(root, 'runtime/codex/companion-packages.json'), 'utf8'))[target.packageName]; +const packagePath = createRequire(import.meta.url).resolve(`${target.packageName}/package.json`); +const installed = JSON.parse(await readFile(packagePath, 'utf8')); +if (installed.version !== `${source.upstreamVersion}-${target.platform}-${target.arch}`) throw new Error('Install the exact pinned official Codex package first.'); +const vendor = join(dirname(packagePath), 'vendor', target.vendorTarget); +await mkdir(dirname(output), { recursive: true }); +await mkdir(output); +await cp(vendor, join(output, 'runtime'), { recursive: true, errorOnExist: true, force: false }); +const manifest = { + source, platform: target.platform, arch: target.arch, + distribution: { kind: 'official-npm', target: target.vendorTarget, integrity: inventory.integrity }, + upstreamPackage: { name: target.packageName, version: installed.version }, + sha256: inventory.files.find(file => file.path === 'bin/codex').sha256, + files: inventory.files.map(({ path, sha256 }) => ({ path, sha256 })), }; -const checkout = join(buildRoot, 'source'); -if (previous === undefined) { - await writeFile(marker, identity, { flag: 'wx' }); - await run('git', ['init', checkout]); - await run('git', ['remote', 'add', 'origin', source.repository], checkout); - await run('git', ['fetch', '--depth=1', 'origin', source.revision], checkout); - await run('git', ['checkout', '--detach', source.revision], checkout); - for (const patch of source.patches) { - await run('git', ['apply', '--check', join(definition, patch)], checkout); - await run('git', ['apply', join(definition, patch)], checkout); - } -} -const { stdout: revision } = await promisify(execFile)('git', ['rev-parse', 'HEAD'], { cwd: checkout }); -if (revision.trim() !== source.revision) throw new Error('Native source revision changed'); -for (const patch of source.patches) await run('git', ['apply', '--reverse', '--check', join(definition, patch)], checkout); -const patchedTree = await verifySourceTree(); -const modelDefaults = JSON.parse(await readFile(join(definition, 'model-defaults.json'), 'utf8')); -const modelCatalogue = await readFile(join(checkout, modelDefaults.sourcePath)); -const expectedDefaults = Object.fromEntries(JSON.parse(modelCatalogue).models.map((model) => [model.slug, model.default_reasoning_level])); -if (modelDefaults.revision !== source.revision || createHash('sha256').update(modelCatalogue).digest('hex') !== modelDefaults.sourceSha256 - || JSON.stringify(modelDefaults.defaults) !== JSON.stringify(expectedDefaults)) { - throw new Error('Resolved API model defaults do not match the pinned harness catalogue.'); -} -if (!prepareOnly) { - const space = await statfs(buildRoot); - if (space.bavail * space.bsize < 16 * 1024 ** 3) throw new Error('Native compilation requires at least 16 GiB of free working space. Source preparation is complete.'); - const target = join(buildRoot, 'target'); - const jobs = Number(process.env.CARGO_BUILD_JOBS ?? 1); - if (!Number.isInteger(jobs) || jobs < 1) throw new Error('CARGO_BUILD_JOBS must be a positive integer.'); - const build = { profile: 'release', debug: 'none', incremental: false, locked: true, jobs }; - // Keep the exact upstream package's companion host and resources. Patching the - // CLI alone silently disables code mode when its sibling executable is absent. - const packageName = targetDefinition.packageName; - const packagePath = createRequire(import.meta.url).resolve(`${packageName}/package.json`); - const upstreamPackage = JSON.parse(await readFile(packagePath, 'utf8')); - if (upstreamPackage.version !== `${source.upstreamVersion}-${process.platform}-${process.arch}`) throw new Error('Install the exact pinned Codex companion package before building.'); - const vendor = join(dirname(packagePath), 'vendor', targetDefinition.vendorTarget); - const vendorFiles = await files(vendor); - const bwrap = vendorFiles.find((path) => path.endsWith('/bwrap')); - if (process.platform === 'linux' && !bwrap) throw new Error('The pinned Linux runtime has no bundled bwrap.'); - const bwrapSha256 = bwrap ? digest(await readFile(bwrap)) : undefined; - await run('cargo', ['build', '--locked', '--release', '--bin', 'codex'], join(checkout, 'codex-rs'), { - ...process.env, CARGO_TARGET_DIR: target, CARGO_PROFILE_RELEASE_DEBUG: 'none', CARGO_INCREMENTAL: '0', CARGO_BUILD_JOBS: String(jobs), - ...(bwrapSha256 ? { CODEX_BWRAP_SHA256: bwrapSha256 } : {}), - }); - if (await verifySourceTree() !== patchedTree) throw new Error('Native source changed during compilation.'); - const { stdout: compilerVersion } = await promisify(execFile)('rustc', ['-vV'], { cwd: join(checkout, 'codex-rs') }); - const nativeTarget = compilerVersion.split('\n').find(line => line.startsWith('host: '))?.slice(6); - if (nativeTarget !== targetDefinition.nativeTarget) throw new Error('Native compiler target does not match the requested runtime.'); - const runtime = join(buildRoot, 'runtime'); - await cp(vendor, runtime, { recursive: true }); - const descriptorPath = join(runtime, 'codex-package.json'); - const descriptor = JSON.parse(await readFile(descriptorPath, 'utf8')); - await writeFile(descriptorPath, JSON.stringify({ ...descriptor, target: nativeTarget }, null, 2) + '\n'); - const binary = join(runtime, 'bin/codex'); - await cp(join(target, 'release', 'codex'), binary); - const sha256 = digest(await readFile(binary)); - const manifest = await Promise.all((await files(runtime)).map(async (path) => ({ path: relative(runtime, path), sha256: digest(await readFile(path)) }))); - await writeFile(join(buildRoot, 'artifact.json'), JSON.stringify({ source: JSON.parse(identity), build: { ...build, nativeTarget, patchedTree }, binary, sha256, - platform: process.platform, arch: process.arch, upstreamPackage: { name: packageName, version: upstreamPackage.version }, files: manifest }, null, 2) + '\n'); - await run(binary, ['--version']); - console.log(`Native artifact: ${binary}`); -} - -async function verifySourceTree() { - const temporary = await mkdtemp(join(buildRoot, 'source-audit-')); - const env = { ...process.env, GIT_INDEX_FILE: join(temporary, 'index') }; - const git = async (args, environment = process.env) => (await promisify(execFile)('git', args, { cwd: checkout, env: environment, maxBuffer: 16 * 1024 * 1024 })).stdout; - try { - await git(['read-tree', source.revision], env); - for (const patch of source.patches) await git(['apply', '--cached', join(definition, patch)], env); - const expected = await git(['diff', '--no-ext-diff', '--binary', '--cached', source.revision], env); - const actual = await git(['diff', '--no-ext-diff', '--binary', source.revision]); - const extra = await git(['ls-files', '--others', '--exclude-standard']); - if (actual !== expected || extra.trim()) throw new Error('Native checkout contains changes beyond the pinned patches. Use a clean dedicated build directory.'); - return (await git(['write-tree'], env)).trim(); - } finally { await rm(temporary, { recursive: true, force: true }); } -} - -async function files(directory) { - const entries = await readdir(directory, { withFileTypes: true }); - return (await Promise.all(entries.sort((a, b) => a.name.localeCompare(b.name)).map((entry) => entry.isDirectory() - ? files(join(directory, entry.name)) : [join(directory, entry.name)]))).flat(); -} +await writeFile(join(output, 'artifact.json'), JSON.stringify(manifest, null, 2) + '\n', { flag: 'wx' }); +await validateCodexRuntimeArtifact(output, { expectedTarget: target.nativeTarget }); +console.log(JSON.stringify({ directory: output, distribution: manifest.distribution.kind, version: source.upstreamVersion, binarySha256: manifest.sha256 })); diff --git a/scripts/build-desktop.mjs b/scripts/build-desktop.mjs index 72bc8ec..30ff462 100644 --- a/scripts/build-desktop.mjs +++ b/scripts/build-desktop.mjs @@ -1,9 +1,63 @@ import { spawnSync } from 'node:child_process'; -import { chmod, copyFile, cp, mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises'; -import { dirname, resolve } from 'node:path'; +import { chmod, copyFile, cp, lstat, mkdir, mkdtemp, readdir, rename, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { validateCodexRuntimeArtifact } from './codex-runtime-artifact.mjs'; +import { checkedFile } from './build-native-release.mjs'; const root = resolve(dirname(fileURLToPath(import.meta.url)), '..'); + +/** Copy only an admitted official macOS artifact, including its original signatures. */ +export async function stageDesktopCodexRuntime(artifactRoot, destination, admissionOptions = {}) { + const artifact = await validateCodexRuntimeArtifact(artifactRoot, { ...admissionOptions, expectedTarget: 'aarch64-apple-darwin' }); + await mkdir(destination); + for (const file of artifact.files) { + const filename = join(destination, file.path); + await mkdir(dirname(filename), { recursive: true }); + await checkedFile(file.source, { expected: file, destination: filename, mode: file.mode }); + } + const runtime = { directory: destination, files: artifact.files.map(({ path, sha256, mode }) => ({ path, sha256, mode })) }; + await verifyDesktopCodexRuntime(runtime); + return runtime; +} + +async function verifyDesktopCodexRuntime(runtime) { + const paths = []; + const visit = async (directory, prefix = '') => { + if (!(await lstat(directory)).isDirectory()) throw new Error('Desktop Codex directories must not be links.'); + const entries = await readdir(directory, { withFileTypes: true }); + if (!entries.length) throw new Error('Desktop Codex contains an unexpected empty directory.'); + for (const entry of entries) { + const path = prefix ? `${prefix}/${entry.name}` : entry.name; + if (entry.isDirectory()) await visit(join(directory, entry.name), path); + else if (entry.isFile()) paths.push(path); + else throw new Error('Desktop Codex contains a link or special file.'); + } + }; + await visit(runtime.directory); + if (JSON.stringify(paths.sort()) !== JSON.stringify(runtime.files.map(file => file.path).sort())) throw new Error('Desktop Codex inventory changed.'); + for (const file of runtime.files) { + const filename = join(runtime.directory, file.path); + await checkedFile(filename, { expected: file }); + if (((await lstat(filename)).mode & 0o777) !== file.mode) throw new Error('Desktop Codex file permissions changed.'); + } +} + +/** + * Outer signing must leave every upstream executable and resource unchanged. + * @param {string} bundle + * @param {{ directory: string, files: Array<{path: string, sha256: string, mode: number}> }} runtime + * @param {(command: string, args: string[], options: {stdio: 'inherit'}) => {status: number | null, error?: Error}} [run] + */ +export async function signDesktopBundle(bundle, runtime, run = spawnSync) { + await verifyDesktopCodexRuntime(runtime); + const signature = await run('codesign', ['--force', '--sign', '-', bundle], { stdio: 'inherit' }); + if (signature.error) throw signature.error; + if (signature.status !== 0) throw new Error('Desktop application signing failed'); + await verifyDesktopCodexRuntime(runtime); +} + async function replaceExecutable(source, destination) { const temporary = `${destination}.${process.pid}.tmp`; try { @@ -13,41 +67,40 @@ async function replaceExecutable(source, destination) { await rm(temporary, { force: true }); } } -const metadata = spawnSync('cargo', ['metadata', '--format-version', '1', '--no-deps', '--manifest-path', 'desktop/Cargo.toml'], { cwd: root, encoding: 'utf8' }); -if (metadata.error) throw metadata.error; -if (metadata.status !== 0) throw new Error(metadata.stderr || 'Could not locate Cargo output directory'); -const targetDirectory = JSON.parse(metadata.stdout).target_directory; -const result = spawnSync('cargo', ['build', '--locked', '--release', '--manifest-path', 'desktop/Cargo.toml'], { cwd: root, stdio: 'inherit' }); -if (result.error) throw result.error; -if (result.status !== 0) process.exit(result.status ?? 1); -const name = `rat-things-desktop${process.platform === 'win32' ? '.exe' : ''}`; -await mkdir(resolve(root, 'dist'), { recursive: true }); -await replaceExecutable(resolve(targetDirectory, 'release', name), resolve(root, 'dist', name)); -if (process.platform === 'darwin') { - const bundle = resolve(root, 'dist/Rat Things.app/Contents'); - await mkdir(`${bundle}/MacOS`, { recursive: true }); - await mkdir(`${bundle}/Resources`, { recursive: true }); - await replaceExecutable(resolve(root, 'dist', name), `${bundle}/MacOS/${name}`); - await replaceExecutable(process.execPath, `${bundle}/MacOS/node`); - await chmod(`${bundle}/MacOS/node`, 0o755); - await copyFile(resolve(root, 'dist/console-server.mjs'), `${bundle}/Resources/console-server.mjs`); - await copyFile(resolve(root, 'THIRD_PARTY_NOTICES.md'), `${bundle}/Resources/THIRD_PARTY_NOTICES.md`); - await copyFile(resolve(root, 'assets/brand/rat-things.icns'), `${bundle}/Resources/AppIcon.icns`); - const target = `${process.arch === 'arm64' ? 'aarch64' : 'x86_64'}-apple-darwin`; - const runtime = resolve(root, `node_modules/@openai/codex-darwin-${process.arch}/vendor/${target}`); - const descriptor = JSON.parse(await readFile(`${runtime}/codex-package.json`, 'utf8')); - const source = JSON.parse(await readFile(resolve(root, 'runtime/codex/source.json'), 'utf8')); - if (descriptor.version !== source.upstreamVersion || descriptor.target !== target) throw new Error('The bundled host Codex must match the pinned runtime version'); - await rm(`${bundle}/MacOS/codex-runtime`, { recursive: true, force: true }); - await rm(`${bundle}/Resources/codex-runtime`, { recursive: true, force: true }); - await cp(runtime, `${bundle}/Resources/codex-runtime`, { recursive: true }); - await cp(resolve(root, 'runtime/codex/licenses'), `${bundle}/Resources/codex-licenses`, { recursive: true }); - for (const executable of ['bin/codex', 'bin/codex-code-mode-host', 'codex-path/rg', 'codex-resources/zsh/bin/zsh']) { - const signature = spawnSync('codesign', ['--force', '--sign', '-', `${bundle}/Resources/codex-runtime/${executable}`], { stdio: 'inherit' }); - if (signature.error) throw signature.error; - if (signature.status !== 0) process.exit(signature.status ?? 1); - } - await writeFile(`${bundle}/Info.plist`, ` +async function buildDesktop() { + const metadata = spawnSync('cargo', ['metadata', '--format-version', '1', '--no-deps', '--manifest-path', 'desktop/Cargo.toml'], { cwd: root, encoding: 'utf8' }); + if (metadata.error) throw metadata.error; + if (metadata.status !== 0) throw new Error(metadata.stderr || 'Could not locate Cargo output directory'); + const targetDirectory = JSON.parse(metadata.stdout).target_directory; + const result = spawnSync('cargo', ['build', '--locked', '--release', '--manifest-path', 'desktop/Cargo.toml'], { cwd: root, stdio: 'inherit' }); + if (result.error) throw result.error; + if (result.status !== 0) process.exit(result.status ?? 1); + const name = `rat-things-desktop${process.platform === 'win32' ? '.exe' : ''}`; + await mkdir(resolve(root, 'dist'), { recursive: true }); + await replaceExecutable(resolve(targetDirectory, 'release', name), resolve(root, 'dist', name)); + if (process.platform === 'darwin') { + const bundle = resolve(root, 'dist/Rat Things.app/Contents'); + await mkdir(`${bundle}/MacOS`, { recursive: true }); + await mkdir(`${bundle}/Resources`, { recursive: true }); + await replaceExecutable(resolve(root, 'dist', name), `${bundle}/MacOS/${name}`); + await replaceExecutable(process.execPath, `${bundle}/MacOS/node`); + await chmod(`${bundle}/MacOS/node`, 0o755); + await copyFile(resolve(root, 'dist/console-server.mjs'), `${bundle}/Resources/console-server.mjs`); + await copyFile(resolve(root, 'THIRD_PARTY_NOTICES.md'), `${bundle}/Resources/THIRD_PARTY_NOTICES.md`); + await copyFile(resolve(root, 'assets/brand/rat-things.icns'), `${bundle}/Resources/AppIcon.icns`); + const temporary = await mkdtemp(join(tmpdir(), 'rat-desktop-codex-')); + let codex; + try { + const artifactRoot = join(temporary, 'artifact'); + const packaged = spawnSync(process.execPath, [resolve(root, 'scripts/build-codex-runtime.mjs'), artifactRoot], { cwd: root, stdio: 'inherit' }); + if (packaged.error) throw packaged.error; + if (packaged.status !== 0) throw new Error('Official desktop Codex packaging failed'); + await rm(`${bundle}/MacOS/codex-runtime`, { recursive: true, force: true }); + await rm(`${bundle}/Resources/codex-runtime`, { recursive: true, force: true }); + codex = await stageDesktopCodexRuntime(artifactRoot, `${bundle}/Resources/codex-runtime`); + } finally { await rm(temporary, { recursive: true, force: true }); } + await cp(resolve(root, 'runtime/codex/licenses'), `${bundle}/Resources/codex-licenses`, { recursive: true }); + await writeFile(`${bundle}/Info.plist`, ` CFBundleExecutable${name} @@ -58,9 +111,10 @@ if (process.platform === 'darwin') { CFBundleShortVersionString0.1.0 NSHighResolutionCapable \n`); - const signature = spawnSync('codesign', ['--force', '--sign', '-', resolve(bundle, '..')], { stdio: 'inherit' }); - if (signature.error) throw signature.error; - if (signature.status !== 0) process.exit(signature.status ?? 1); - console.log('Built dist/Rat Things.app. Launch through rat-things console to inherit your API and AWS profile settings.'); + await signDesktopBundle(resolve(bundle, '..'), codex); + console.log('Built dist/Rat Things.app. Launch through rat-things console to inherit your API and AWS profile settings.'); + } + console.log(`Built dist/${name}`); } -console.log(`Built dist/${name}`); + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) await buildDesktop(); diff --git a/scripts/build-native-app.mjs b/scripts/build-native-app.mjs index 768c2e9..0541286 100644 --- a/scripts/build-native-app.mjs +++ b/scripts/build-native-app.mjs @@ -85,7 +85,7 @@ export async function assembleNativeApp(options, { repositoryRoot = root, const npmLock = JSON.parse((await input(join(repository, 'package-lock.json'), 'package-lock.json')).data); if (npmLock.packages?.['node_modules/esbuild']?.version !== esbuildVersion) throw new Error('Installed app bundling requires the locked esbuild version.'); const protocol = JSON.parse((await input(join(repository, 'runtime/native/protocol.json'), 'runtime/native/protocol.json')).data); - for (const name of ['scripts/build-native-app.mjs', 'scripts/native-app-inputs.mjs', 'scripts/build-native-release.mjs', 'scripts/codex-runtime-artifact.mjs', 'scripts/codex-runtime-targets.mjs', 'scripts/native-guest-inputs.mjs', 'scripts/javascript-notices.mjs', 'scripts/native-build-files.mjs', 'scripts/native-guest-archive.mjs']) { + for (const name of ['runtime/native/host-code-policy.json', 'scripts/build-native-app.mjs', 'scripts/native-app-inputs.mjs', 'scripts/build-native-release.mjs', 'scripts/codex-runtime-artifact.mjs', 'scripts/codex-runtime-targets.mjs', 'scripts/native-guest-inputs.mjs', 'scripts/javascript-notices.mjs', 'scripts/native-build-files.mjs', 'scripts/native-guest-archive.mjs']) { await input(join(repository, name), name); } await mkdir(output, { mode: 0o700 }); diff --git a/scripts/build-native-guest.mjs b/scripts/build-native-guest.mjs index a83725e..b91c684 100644 --- a/scripts/build-native-guest.mjs +++ b/scripts/build-native-guest.mjs @@ -71,9 +71,8 @@ async function nativeGuest(options) { const definitionRoot = resolve(options.definitionRoot ?? join(root, 'runtime/codex')); const artifactRoot = resolve(options.guestArtifact); const artifactBytes = await readInput(join(artifactRoot, 'artifact.json')); - const artifact = await validateCodexRuntimeArtifact(artifactRoot, { expectedTarget: 'aarch64-unknown-linux-gnu', definitionRoot }); - const sourceDefinition = JSON.parse(await readInput(join(definitionRoot, 'source.json'), 'codex-definition/source.json')); - for (const name of ['artifact-policy.json', ...sourceDefinition.patches]) await readInput(join(definitionRoot, name), `codex-definition/${name}`); + const artifact = await validateCodexRuntimeArtifact(artifactRoot, { expectedTarget: 'aarch64-unknown-linux-musl', definitionRoot }); + await readInput(join(definitionRoot, 'source.json'), 'codex-definition/source.json'); await readInput(join(root, 'runtime/codex/companion-packages.json'), 'runtime/codex/companion-packages.json'); const kernelBytes = await readInput(join(root, 'runtime/native/kernel-inputs.json'), 'runtime/native/kernel-inputs.json'); const kernelInputs = JSON.parse(kernelBytes.toString('utf8')); diff --git a/scripts/build-native-release.mjs b/scripts/build-native-release.mjs index 9138b46..6cbe5bb 100644 --- a/scripts/build-native-release.mjs +++ b/scripts/build-native-release.mjs @@ -2,7 +2,8 @@ import { execFile } from 'node:child_process'; import { createHash } from 'node:crypto'; import { constants } from 'node:fs'; import { chmod, lstat, mkdir, open, readdir, realpath, writeFile } from 'node:fs/promises'; -import { dirname, join, resolve } from 'node:path'; +import { dirname, join, posix, resolve } from 'node:path'; +import hostCodePolicy from '../runtime/native/host-code-policy.json' with { type: 'json' }; import { fileURLToPath } from 'node:url'; import { isDeepStrictEqual, promisify } from 'node:util'; import { validateCodexRuntimeArtifact } from './codex-runtime-artifact.mjs'; @@ -25,15 +26,9 @@ const manifestPath = 'Contents/Resources/runtime-manifest.json'; const signaturePaths = ['CodeDirectory', 'CodeRequirements', 'CodeResources', 'CodeSignature'].map(path => `Contents/_CodeSignature/${path}`); const limits = { entries: 4096, fileBytes: 768 * 1024 ** 2, totalBytes: 2 * 1024 ** 3, manifestBytes: 128 * 1024 }; const guestArtifacts = ['Image', 'worker-initramfs.gz', 'archive-inventory.json', 'worker-dependencies.json', 'worker-notices.json', 'worker-metafile.json', 'worker-build.json', 'build.log']; -const jitEntitlements = ['com.apple.security.cs.allow-jit', 'com.apple.security.cs.allow-unsigned-executable-memory']; -export const nativeExecutablePolicy = Object.freeze([ - { path: 'Contents/Resources/codex/bin/codex', role: 'host-codex', identifier: 'dev.rat-things.runtime.codex', entitlements: 'jit' }, - { path: 'Contents/Resources/codex/bin/codex-code-mode-host', role: 'host-companion', identifier: 'dev.rat-things.runtime.code-mode-host', entitlements: 'jit' }, - { path: 'Contents/Resources/codex/codex-path/rg', role: 'host-companion', identifier: 'dev.rat-things.runtime.ripgrep' }, - { path: 'Contents/Resources/codex/codex-resources/zsh/bin/zsh', role: 'host-companion', identifier: 'dev.rat-things.runtime.zsh' }, - { path: 'Contents/Helpers/rat-native-worker', role: 'vm-helper', identifier: 'dev.rat-things.runtime.vm-helper', entitlements: 'virtualization' }, -]); +export const nativeExecutablePolicy = Object.freeze(hostCodePolicy.map(policy => Object.freeze(policy))); const executables = new Map(nativeExecutablePolicy.map(value => [value.path, value])); +const upstreamSignerBlocker = 'Native macOS upstream-signer admission requires design and on-Mac validation'; export function parseNativeReleaseOptions(args) { const options = {}; @@ -59,12 +54,14 @@ function maximumVersion(versions) { }).at(-1); } -export function inspectMachO(bytes) { +export function inspectMachO(bytes, policy = { machOType: 2 }) { if (bytes.length < 32 || bytes.readUInt32LE(0) !== 0xfeedfacf || bytes.readUInt32LE(4) !== 0x100000c - || bytes.readUInt32LE(12) !== 2) throw new Error('Expected a thin ARM64 Mach-O executable.'); + || ![2, 6].includes(policy.machOType) || bytes.readUInt32LE(12) !== policy.machOType) { + throw new Error('Expected a thin ARM64 Mach-O matching the fixed host code policy.'); + } const count = bytes.readUInt32LE(16); const end = 32 + bytes.readUInt32LE(20); if (!count || count > 4096 || end > bytes.length) throw new Error('Mach-O load commands exceed their bounds.'); - const versions = []; const libraries = []; + const versions = []; const libraries = []; const rpaths = []; let offset = 32; for (let index = 0; index < count; index++) { if (offset + 8 > end) throw new Error('Truncated Mach-O load command.'); @@ -76,17 +73,29 @@ export function inspectMachO(bytes) { } else if (command === 0x24) { if (size < 16) throw new Error('Invalid macOS deployment command.'); versions.push(version(bytes.readUInt32LE(offset + 8))); - } else if ([0xc, 0x80000018, 0x8000001f, 0x20, 0x80000023].includes(command)) { - if (size < 24) throw new Error('Invalid linked library command.'); + } else if ([0xc, 0x80000018, 0x8000001f, 0x20, 0x80000023, 0x8000001c].includes(command)) { + const minimum = command === 0x8000001c ? 12 : 24; + if (size < minimum) throw new Error('Invalid linked library command.'); const start = offset + bytes.readUInt32LE(offset + 8); const terminator = bytes.indexOf(0, start); - if (start < offset + 24 || start >= offset + size || terminator < start || terminator >= offset + size) throw new Error('Invalid linked library name.'); + if (start < offset + minimum || start >= offset + size || terminator < start || terminator >= offset + size) throw new Error('Invalid linked library name.'); const library = bytes.toString('utf8', start, terminator); - if (!library.startsWith('/usr/lib/') && !library.startsWith('/System/Library/')) throw new Error(`Runtime executable requires an unbundled library: ${library}.`); - libraries.push(library); + if (command === 0x8000001c) rpaths.push(library); + else libraries.push(library); } offset += size; } if (offset !== end || !versions.length || versions.some(value => Number(value.split('.')[0]) < 11)) throw new Error('Missing or inconsistent macOS deployment commands.'); + if (!isDeepStrictEqual(rpaths, policy.rpaths ?? [])) throw new Error('Mach-O search paths differ from the fixed host code policy.'); + for (const library of libraries) { + const system = (library.startsWith('/usr/lib/') || library.startsWith('/System/Library/')) && posix.normalize(library) === library; + const local = name => policy.path && name.startsWith('@loader_path/') + ? posix.normalize(posix.join(posix.dirname(policy.path), name.slice('@loader_path/'.length))) : undefined; + const candidates = library.startsWith('@rpath/') + ? rpaths.map(path => local(`${path}/${library.slice('@rpath/'.length)}`)) : [local(library)]; + if (!system && !candidates.some(path => executables.get(path)?.machOType === 6)) { + throw new Error(`Runtime code requires an unbundled library: ${library}.`); + } + } return { minimumMacOS: maximumVersion(versions), libraries }; } @@ -103,6 +112,10 @@ export function nativeCompatibility(protocol, machOs) { } export function signingArguments(identity, policy, filename, entitlementPaths) { + if (!isDeepStrictEqual(policy, nativeExecutablePolicy.find(value => value.role === 'vm-helper')) + && !isDeepStrictEqual(policy, { identifier: 'dev.rat-things.runtime' })) { + throw new Error('Only the Rat Things VM helper and outer bundle may be signed.'); + } return ['--force', '--sign', identity, '--identifier', policy.identifier, '--options', 'runtime', '--timestamp', ...(policy.entitlements ? ['--entitlements', entitlementPaths[policy.entitlements]] : []), filename]; } @@ -220,9 +233,8 @@ async function guestEvidence(directory, readInput, host, companions) { if (inputs.codexArtifactSha256 !== (await checkedFile(join(directory, 'evidence/codex-artifact.json'))).sha256 || !isDeepStrictEqual(artifact.source, host.source) || artifact.platform !== 'linux' || artifact.arch !== 'arm64' || !isDeepStrictEqual(artifact.upstreamPackage, { name: '@openai/codex-linux-arm64', version: companions.version }) - || artifact.build?.patchedTree !== host.build.patchedTree || artifact.build.profile !== 'release' || artifact.build.debug !== 'none' - || artifact.build.incremental !== false || artifact.build.locked !== true || !Number.isSafeInteger(artifact.build.jobs) || artifact.build.jobs < 1 - || artifact.build.nativeTarget !== 'aarch64-unknown-linux-gnu') throw new Error('Guest Codex evidence differs from the admitted release definition.'); + || 'build' in artifact || artifact.sha256 !== companions.files.find(file => file.path === 'bin/codex')?.sha256 + || !isDeepStrictEqual(artifact.distribution, { kind: 'official-npm', target: 'aarch64-unknown-linux-musl', integrity: companions.integrity })) throw new Error('Guest Codex evidence differs from the admitted release definition.'); const archive = await readJson('archive-inventory.json'); if (archive.format !== 'newc' || archive.sourceDateEpoch !== inputs.sourceDateEpoch || !Array.isArray(archive.entries) || archive.entries.length > 100_000 || !Number.isSafeInteger(archive.uncompressedBytes) || archive.uncompressedBytes > 1400 * 1024 ** 2 @@ -244,8 +256,7 @@ async function guestEvidence(directory, readInput, host, companions) { } for (const file of manifestFiles) { const member = members.find(entry => entry.path === `opt/codex-runtime/${file.path}`); const companion = companions.files.find(entry => entry.path === file.path); - const descriptor = { ...companions.descriptor, target: 'aarch64-unknown-linux-gnu' }; - const expected = file.path === 'bin/codex' ? artifact.sha256 : file.path === 'codex-package.json' ? digest(jsonBytes(descriptor)) : companion.sha256; + const expected = companion.sha256; if (file.sha256 !== expected || !hashPattern.test(file.sha256) || member.sha256 !== file.sha256 || member.mode !== (0o100000 | (companion.executable ? 0o755 : 0o444))) throw new Error(`Guest Codex member differs from the admitted package: ${file.path}.`); } @@ -254,7 +265,14 @@ async function guestEvidence(directory, readInput, host, companions) { } const infoPlist = version => `\n\nCFBundleIdentifierdev.rat-things.runtimeCFBundlePackageTypeBNDLCFBundleVersion${version}\n`; -const entitlementPlist = keys => `\n\n${keys.map(key => `${key}`).join('')}\n`; +function requireOfficialPayload(inventory, files) { + for (const file of files) { + const path = `Contents/Resources/codex/${file.path}`; + if (inventory.find(value => value.path === path)?.sha256 !== file.sha256) { + throw new Error(`Official Codex payload changed during native assembly: ${file.path}.`); + } + } +} export async function assembleNativeRelease(options, { repositoryRoot = root, run = async (command, args) => execute(command, args, { timeout: 180_000, maxBuffer: 4 * 1024 ** 2 }) } = {}) { const { hostArtifact, guestOutput, definitionRoot, output, identity } = options; @@ -265,15 +283,10 @@ export async function assembleNativeRelease(options, { repositoryRoot = root, ru sources.push({ path: label, bytes: result.bytes, sha256: result.sha256 }); return result; }; - const builderSources = ['scripts/build-native-release.mjs', 'scripts/codex-runtime-artifact.mjs', 'scripts/codex-runtime-targets.mjs', 'scripts/native-guest-inputs.mjs', 'scripts/javascript-notices.mjs', 'scripts/native-build-files.mjs', 'scripts/native-guest-archive.mjs']; + const builderSources = ['runtime/native/host-code-policy.json', 'scripts/build-native-release.mjs', 'scripts/codex-runtime-artifact.mjs', 'scripts/codex-runtime-targets.mjs', 'scripts/native-guest-inputs.mjs', 'scripts/javascript-notices.mjs', 'scripts/native-build-files.mjs', 'scripts/native-guest-archive.mjs']; for (const path of builderSources) await readInput(join(root, path), path); const originalHost = await readInput(join(hostArtifact, 'artifact.json'), 'host/artifact.json'); - const sourceDefinition = JSON.parse((await readInput(join(definitionRoot, 'source.json'), 'definition/source.json')).data); - await readInput(join(definitionRoot, 'artifact-policy.json'), 'definition/artifact-policy.json'); - for (const path of sourceDefinition.patches) { - if (!safePath(path)) throw new Error('Invalid trusted patch path.'); - await readInput(join(definitionRoot, path), `definition/${path}`); - } + await readInput(join(definitionRoot, 'source.json'), 'definition/source.json'); const companionInventoryPath = join(repositoryRoot, 'runtime/codex/companion-packages.json'); const companions = JSON.parse((await readInput(companionInventoryPath, 'runtime/codex/companion-packages.json')).data); await readInput(join(repositoryRoot, 'package-lock.json'), 'package-lock.json'); @@ -314,7 +327,7 @@ export async function assembleNativeRelease(options, { repositoryRoot = root, ru }; try { await write(join(evidence, 'host-artifact.json'), originalHost.data); - for (const path of ['source.json', 'artifact-policy.json', ...sourceDefinition.patches]) { + for (const path of ['source.json']) { await checkedFile(join(definitionRoot, path), { expected: tracked.get(join(definitionRoot, path)), destination: join(evidence, 'definition', path) }); } for (const path of ['package-lock.json', 'runtime/codex/companion-packages.json']) { @@ -328,14 +341,13 @@ export async function assembleNativeRelease(options, { repositoryRoot = root, ru } await write(join(evidence, 'native-inputs/main.swift'), helperSource.data); await write(join(evidence, 'native-inputs/entitlements.plist'), helperEntitlements.data); - await write(join(evidence, 'native-inputs/jit-entitlements.plist'), entitlementPlist(jitEntitlements)); - const machOs = []; const unsigned = []; + const machOs = []; const originalExecutables = []; for (const file of host.files) { const path = `Contents/Resources/codex/${file.path}`; expectedPaths.push(path); const copied = await checkedFile(file.source, { expected: file, destination: join(bundle, path), mode: file.mode, capture: executables.has(path) }); tracked.set(file.source, { bytes: copied.bytes, sha256: copied.sha256 }); - if (!executables.has(path)) staged.set(path, { bytes: copied.bytes, sha256: copied.sha256 }); - if (executables.has(path)) { machOs.push({ path, ...inspectMachO(copied.data) }); unsigned.push({ path, bytes: copied.bytes, sha256: copied.sha256 }); } + staged.set(path, { bytes: copied.bytes, sha256: copied.sha256 }); + if (executables.has(path)) { machOs.push({ path, ...inspectMachO(copied.data, executables.get(path)) }); originalExecutables.push({ path, bytes: copied.bytes, sha256: copied.sha256 }); } } for (const name of ['Image', 'worker-initramfs.gz']) { const path = `Contents/Resources/linux/${name}`; expectedPaths.push(path); @@ -353,41 +365,48 @@ export async function assembleNativeRelease(options, { repositoryRoot = root, ru await mkdir(dirname(join(bundle, helperPath)), { recursive: true }); await run('/usr/bin/xcrun', compileArguments); await chmod(join(bundle, helperPath), 0o755); expectedPaths.push(helperPath); const helper = await checkedFile(join(bundle, helperPath), { capture: true }); - machOs.push({ path: helperPath, ...inspectMachO(helper.data) }); unsigned.push({ path: helperPath, bytes: helper.bytes, sha256: helper.sha256 }); + machOs.push({ path: helperPath, ...inspectMachO(helper.data) }); originalExecutables.push({ path: helperPath, bytes: helper.bytes, sha256: helper.sha256 }); const compatibility = nativeCompatibility(protocol, machOs); - const entitlements = { jit: join(evidence, 'native-inputs/jit-entitlements.plist'), virtualization: join(evidence, 'native-inputs/entitlements.plist') }; - const signatures = []; + const entitlements = { virtualization: join(evidence, 'native-inputs/entitlements.plist') }; + const signatures = []; const preservedSignatures = []; const mappings = []; for (const policy of nativeExecutablePolicy) { const filename = join(bundle, policy.path); const prior = await run('/usr/bin/codesign', ['--display', '--verbose=4', '--entitlements', ':-', filename]).then(result => ({ status: 0, ...result }), error => ({ status: error.code, stdout: error.stdout, stderr: error.stderr })); + const original = originalExecutables.find(file => file.path === policy.path); + if (policy.role !== 'vm-helper') { + const packaged = await checkedFile(filename, { expected: original }); + preservedSignatures.push({ path: policy.path, metadata: prior }); + mappings.push({ path: policy.path, kind: 'official-unmodified', input: original, packaged, + signature: { action: 'preserved', signerAdmission: 'pending' } }); + continue; + } const args = signingArguments(identity, policy, filename, entitlements); await run('/usr/bin/codesign', args); await run('/usr/bin/codesign', ['--verify', '--strict', '--all-architectures', '--test-requirement', `=${requirement(policy.identifier)}`, filename]); const signed = await checkedFile(filename, { capture: true }); const { path: _path, ...deployment } = machOs.find(value => value.path === policy.path); - if (!isDeepStrictEqual(inspectMachO(signed.data), deployment)) throw new Error('Signing changed executable deployment requirements.'); + if (!isDeepStrictEqual(inspectMachO(signed.data, policy), deployment)) throw new Error('Signing changed executable deployment requirements.'); const signedMetadata = await run('/usr/bin/codesign', ['--display', '--verbose=4', '--entitlements', ':-', filename]); staged.set(policy.path, { bytes: signed.bytes, sha256: signed.sha256 }); signatures.push({ path: policy.path, identifier: policy.identifier, prior, arguments: args, signedMetadata, signed: { bytes: signed.bytes, sha256: signed.sha256 } }); + mappings.push({ path: policy.path, kind: 'rat-things-signed', input: original, + packaged: { bytes: signed.bytes, sha256: signed.sha256 }, identifier: policy.identifier, entitlements: ['com.apple.security.virtualization'] }); } - const mappings = signatures.map(value => ({ path: value.path, unsigned: unsigned.find(file => file.path === value.path), signed: value.signed, - identifier: value.identifier, entitlements: nativeExecutablePolicy.find(policy => policy.path === value.path).entitlements === 'jit' ? jitEntitlements - : value.path === helperPath ? ['com.apple.security.virtualization'] : [] })); const blockers = [...new Set([...guest.build.outstandingDistributionInputs.filter(value => value !== 'signed runtime assembly and installed acceptance'), - 'Complete host and guest Codex linked dependency sources and companion notices', 'Installed runtime and application acceptance', 'Notarization and Gatekeeper distribution assessment'])]; + upstreamSignerBlocker, 'Complete host and guest Codex linked dependency sources and companion notices', 'Installed runtime and application acceptance', 'Notarization and Gatekeeper distribution assessment'])]; await resource('Contents/Info.plist', infoPlist('1')); for (const notice of notices) await resource(`Contents/Resources/notices/codex-${notice.name}`, notice.data); await resource('Contents/Resources/notices/rat-things-LICENSE', firstParty.data); - await resource('Contents/Resources/notices/NOTICE.txt', `Rat Things native runtime candidate\n\nThis bundle contains modified OpenAI Codex ${host.manifest.source.upstreamVersion}.\nThe source revision and ordered patches are recorded in provenance.json.\nOpenAI Codex LICENSE and NOTICE are included beside this file.\nThe guest retains its packaged license files inside the initramfs.\n\nThis signed candidate is not cleared for public distribution.\nOutstanding distribution inputs:\n${blockers.map(value => `- ${value}`).join('\n')}\n`); + await resource('Contents/Resources/notices/NOTICE.txt', `Rat Things native runtime candidate\n\nThis bundle preserves the official OpenAI Codex ${host.manifest.source.upstreamVersion} release bytes and original signatures.\nOnly the Rat Things VM helper and outer bundle are signed during assembly.\nOfficial input hashes, preserved signatures and helper signing are recorded in provenance.json.\nOpenAI Codex LICENSE and NOTICE are included beside this file.\nThe guest retains its packaged license files inside the initramfs.\n\nThis signed candidate is not cleared for public distribution.\nOutstanding distribution inputs:\n${blockers.map(value => `- ${value}`).join('\n')}\n`); await resource('Contents/Resources/provenance.json', jsonBytes({ schemaVersion: 1, distributionReady: false, reproducibility: 'not-established', - source: host.manifest.source, build: { host: host.manifest.build, guest: guest.artifact.build }, - unsignedInputs: { hostArtifactSha256: originalHost.sha256, guestArtifactSha256: guest.inputs.codexArtifactSha256, guestLockSha256: guest.build.lockSha256, + source: host.manifest.source, distribution: { host: host.manifest.distribution, guest: guest.artifact.distribution }, + inputs: { hostArtifactSha256: originalHost.sha256, guestArtifactSha256: guest.inputs.codexArtifactSha256, guestLockSha256: guest.build.lockSha256, guestInputsSha256: guest.build.inputsSha256, nativeSourceSha256: helperSource.sha256, nativeEntitlementsSha256: helperEntitlements.sha256, protocolSha256: protocolInput.sha256 }, definitionInputs: ordered(sources.filter(value => value.path.startsWith('definition/'))), assemblerSources: ordered(sources.filter(value => builderSources.includes(value.path))), helperBuild: { swift: toolchain.swift, sdkVersion: toolchain.sdkVersion, sdkBuild: toolchain.sdkBuild, deploymentTarget }, executables: mappings, deployment: machOs, outstandingDistributionInputs: blockers })); await resource('Contents/Resources/component-inventory.json', jsonBytes({ schemaVersion: 1, distributionReady: false, - codex: { source: host.manifest.source, patchedTree: host.manifest.build.patchedTree, license: 'Apache-2.0', notice: 'notices/codex-NOTICE', + codex: { source: host.manifest.source, distribution: 'official-npm', license: 'Apache-2.0', notice: 'notices/codex-NOTICE', host: { package: host.manifest.upstreamPackage, files: mappings.filter(value => value.path !== helperPath) }, guest: { package: guest.artifact.upstreamPackage, archive: 'linux/worker-initramfs.gz', files: guest.artifact.files.map(file => ({ ...file, path: `opt/codex-runtime/${file.path}` })) }, companions: Object.values(companions).map(value => ({ name: value.name, version: value.version, origin: value.resolved, integrity: value.integrity, @@ -399,21 +418,23 @@ export async function assembleNativeRelease(options, { repositoryRoot = root, ru notices: { index: 'opt/rat-things/notices/javascript/index.json', sha256: guest.inputs.javascriptNoticesSha256 } }, nativeHelper: { source: 'runtime/native/main.swift', sha256: helperSource.sha256, license: 'MIT', notice: 'notices/rat-things-LICENSE' }, outstandingDistributionInputs: blockers })); const beforeManifest = await nativeBundleInventory(bundle); + requireOfficialPayload(beforeManifest, host.files); if (staged.size !== expectedPaths.length || beforeManifest.some(file => !isDeepStrictEqual(staged.get(file.path), { bytes: file.bytes, sha256: file.sha256 }))) { throw new Error('Staged runtime payload changed before manifest creation.'); } - const manifest = nativeManifest(`${host.manifest.source.upstreamVersion}-native-${host.manifest.build.patchedTree.slice(0, 12)}`, compatibility, beforeManifest, expectedPaths); + const manifest = nativeManifest(`${host.manifest.source.upstreamVersion}-native-${host.manifest.sha256.slice(0, 12)}`, compatibility, beforeManifest, expectedPaths); await write(join(bundle, manifestPath), manifest.bytes); for (const [filename, expected] of tracked) await checkedFile(filename, { expected }); await validateCodexRuntimeArtifact(hostArtifact, admissionOptions); await write(join(evidence, 'input-inventory.json'), jsonBytes(ordered(sources))); - await write(join(evidence, 'signing.json'), jsonBytes({ identity, team: 'PB8NAVKU57', toolchain, compileArguments, signatures })); + await write(join(evidence, 'signing.json'), jsonBytes({ identity, team: 'PB8NAVKU57', toolchain, compileArguments, signatures, preservedSignatures })); await chmod(join(bundle, 'Contents/Info.plist'), 0o644); await run('/usr/bin/codesign', signingArguments(identity, { identifier: 'dev.rat-things.runtime' }, bundle, entitlements)); await chmod(join(bundle, 'Contents/Info.plist'), 0o444); await run('/usr/bin/codesign', ['--verify', '--deep', '--strict', '--all-architectures', '--test-requirement', `=${requirement('dev.rat-things.runtime')}`, bundle]); const finalInventory = await nativeBundleInventory(bundle); + requireOfficialPayload(finalInventory, host.files); if (!samePaths(finalInventory.map(file => file.path), [...expectedPaths, manifestPath, ...signaturePaths]) || !isDeepStrictEqual(finalInventory.filter(file => expectedPaths.includes(file.path)), beforeManifest) || (await checkedFile(join(bundle, manifestPath))).sha256 !== manifest.releaseId) throw new Error('Outer signing changed the payload or generated unexpected signature files.'); diff --git a/scripts/codex-runtime-artifact.mjs b/scripts/codex-runtime-artifact.mjs index ac8d2c2..ec80ad7 100644 --- a/scripts/codex-runtime-artifact.mjs +++ b/scripts/codex-runtime-artifact.mjs @@ -17,20 +17,18 @@ const readJson = async path => JSON.parse(await readFile(path, 'utf8')); * @typedef {object} CodexArtifactOptions * @property {string} [expectedTarget] * @property {string} [definitionRoot] - * @property {string} [expectedPatchedTree] Trusted expected tree, never taken from the artifact. * @property {string} [packageLockPath] * @property {string} [companionInventoryPath] */ /** - * Validate an unsigned runtime against a trusted source definition and npm companion inventory. + * Admit only the unchanged official runtime and companions bound to the npm lock. * @param {string} artifactRoot * @param {CodexArtifactOptions} options */ export async function validateCodexRuntimeArtifact(artifactRoot, { expectedTarget, definitionRoot = definition, - expectedPatchedTree, packageLockPath = join(root, 'package-lock.json'), companionInventoryPath = join(definition, 'companion-packages.json'), } = {}) { @@ -39,24 +37,9 @@ export async function validateCodexRuntimeArtifact(artifactRoot, { const manifestPath = join(artifactRoot, 'artifact.json'); const manifest = JSON.parse((await readRegularFile(manifestPath)).toString('utf8')); const source = await readJson(join(definitionRoot, 'source.json')); - const patches = await Promise.all(source.patches.map(async name => { - if (!validPath(name)) throw new Error('Invalid Codex source patch path.'); - return { name, sha256: digest(await readFile(join(definitionRoot, name))) }; - })); - const sourceIdentity = { ...source, patches }; - if (!isDeepStrictEqual(manifest.source, sourceIdentity)) throw new Error('Codex runtime does not match the pinned source and ordered patches.'); - if (expectedPatchedTree === undefined) { - const policy = await readJson(join(definitionRoot, 'artifact-policy.json')); - if (policy.sourceSha256 !== digest(JSON.stringify(sourceIdentity))) throw new Error('Codex artifact policy does not match the source and ordered patches.'); - expectedPatchedTree = policy.patchedTree; - } - if (!/^[a-f0-9]{40}$/.test(expectedPatchedTree ?? '') - || manifest.platform !== target.platform || manifest.arch !== target.arch - || manifest.build?.nativeTarget !== expectedTarget || manifest.build.patchedTree !== expectedPatchedTree - || manifest.build.profile !== 'release' || manifest.build.debug !== 'none' - || manifest.build.incremental !== false || manifest.build.locked !== true - || !Number.isSafeInteger(manifest.build.jobs) || manifest.build.jobs < 1) { - throw new Error('Codex runtime build provenance is incomplete or incompatible.'); + if ('patches' in source || !isDeepStrictEqual(manifest.source, source)) throw new Error('Codex runtime does not match the pinned unmodified upstream source.'); + if (manifest.platform !== target.platform || manifest.arch !== target.arch || 'build' in manifest) { + throw new Error('Codex runtime distribution provenance is incompatible.'); } const lock = await readJson(packageLockPath); const packageLock = lock.packages?.[`node_modules/${target.packageName}`]; @@ -68,8 +51,10 @@ export async function validateCodexRuntimeArtifact(artifactRoot, { || !isDeepStrictEqual(manifest.upstreamPackage, { name: target.packageName, version: packageLock.version })) { throw new Error('Codex runtime companions do not match the pinned npm package.'); } - const expectedDescriptor = { ...companions.descriptor, target: expectedTarget }; - if (expectedDescriptor.layoutVersion !== 1 || expectedDescriptor.version !== source.upstreamVersion + const expectedDistribution = { kind: 'official-npm', target: target.vendorTarget, integrity: packageLock.integrity }; + if (!isDeepStrictEqual(manifest.distribution, expectedDistribution)) throw new Error('Codex runtime distribution provenance differs from the official package.'); + const expectedDescriptor = companions.descriptor; + if (expectedDescriptor.target !== target.vendorTarget || expectedDescriptor.layoutVersion !== 1 || expectedDescriptor.version !== source.upstreamVersion || expectedDescriptor.variant !== 'codex' || expectedDescriptor.entrypoint !== 'bin/codex' || expectedDescriptor.resourcesDir !== 'codex-resources' || expectedDescriptor.pathDir !== 'codex-path') { throw new Error('Pinned Codex package descriptor is incompatible.'); @@ -88,15 +73,14 @@ export async function validateCodexRuntimeArtifact(artifactRoot, { if (digest(bytes) !== file.sha256) throw new Error(`Codex runtime checksum mismatch: ${path}`); if (path === 'bin/codex' && file.sha256 !== manifest.sha256) throw new Error('Codex runtime entrypoint digest does not match its manifest.'); const companion = pinned.get(path); + if (path === 'bin/codex' && !isExecutable(bytes, target)) throw new Error(`Codex runtime executable is not ${target.platform} ${target.arch.toUpperCase()}: ${path}`); if (path === 'codex-package.json') { - if (!isDeepStrictEqual(JSON.parse(bytes.toString('utf8')), expectedDescriptor)) throw new Error('Codex runtime package descriptor does not match the compiled runtime.'); - const expectedHash = target.platform === 'linux' ? digest(JSON.stringify(expectedDescriptor, null, 2) + '\n') : companion.sha256; - if (file.sha256 !== expectedHash) throw new Error('Codex runtime package descriptor differs from the pinned package.'); - } else if (path !== 'bin/codex' && file.sha256 !== companion.sha256) { - throw new Error(`Codex runtime companion differs from the pinned npm package: ${path}`); + if (!isDeepStrictEqual(JSON.parse(bytes.toString('utf8')), expectedDescriptor)) throw new Error('Codex runtime package descriptor does not match the official runtime.'); + if (file.sha256 !== companion.sha256) throw new Error('Codex runtime package descriptor differs from the pinned package.'); + } else if (file.sha256 !== companion.sha256) { + throw new Error(`Codex runtime file differs from the official npm package: ${path}`); } if (path === 'bin/codex' || companion.executable) { - if (!isExecutable(bytes, target)) throw new Error(`Codex runtime executable is not ${target.platform} ${target.arch.toUpperCase()}: ${path}`); if (!((await lstat(filename)).mode & 0o111)) throw new Error(`Codex runtime executable permission is missing: ${path}`); } files.push({ path, source: filename, sha256: file.sha256, mode: companion.executable ? 0o755 : 0o444 }); @@ -110,8 +94,8 @@ export async function validateCodexRuntimeArtifact(artifactRoot, { * @param {CodexArtifactOptions} options */ export async function codexRuntimeEntries(artifactRoot = process.env.CODEX_RUNTIME_ARTIFACT ?? join(root, '.runtime/codex/linux-arm64'), options = {}) { - const artifact = await validateCodexRuntimeArtifact(artifactRoot, { ...options, expectedTarget: 'aarch64-unknown-linux-gnu' }).catch(error => { - if (error.code === 'ENOENT' && error.path === join(artifactRoot, 'artifact.json')) throw new Error('Build the Linux ARM64 Codex runtime first; set CODEX_RUNTIME_ARTIFACT to its exported directory. See testing/README.md.'); + const artifact = await validateCodexRuntimeArtifact(artifactRoot, { ...options, expectedTarget: 'aarch64-unknown-linux-musl' }).catch(error => { + if (error.code === 'ENOENT' && error.path === join(artifactRoot, 'artifact.json')) throw new Error('Prepare the official Linux ARM64 Codex runtime first; set CODEX_RUNTIME_ARTIFACT to its exported directory. See testing/README.md.'); throw error; }); return [...artifact.files.map(file => ({ source: file.source, target: `codex-runtime/${file.path}`, mode: file.mode })), diff --git a/scripts/codex-runtime-targets.mjs b/scripts/codex-runtime-targets.mjs index cb4a8bc..1fe7106 100644 --- a/scripts/codex-runtime-targets.mjs +++ b/scripts/codex-runtime-targets.mjs @@ -4,11 +4,11 @@ export const codexRuntimeTargets = { vendorTarget: 'aarch64-apple-darwin', packageName: '@openai/codex-darwin-arm64', }, 'linux-arm64': { - platform: 'linux', arch: 'arm64', nativeTarget: 'aarch64-unknown-linux-gnu', + platform: 'linux', arch: 'arm64', nativeTarget: 'aarch64-unknown-linux-musl', vendorTarget: 'aarch64-unknown-linux-musl', packageName: '@openai/codex-linux-arm64', elfMachine: 183, }, 'linux-x64': { - platform: 'linux', arch: 'x64', nativeTarget: 'x86_64-unknown-linux-gnu', + platform: 'linux', arch: 'x64', nativeTarget: 'x86_64-unknown-linux-musl', vendorTarget: 'x86_64-unknown-linux-musl', packageName: '@openai/codex-linux-x64', elfMachine: 62, }, }; diff --git a/scripts/ensure-worker-runtime.mjs b/scripts/ensure-worker-runtime.mjs index 695c52e..7ab0d84 100644 --- a/scripts/ensure-worker-runtime.mjs +++ b/scripts/ensure-worker-runtime.mjs @@ -7,7 +7,7 @@ import { validateCodexRuntimeArtifact } from './codex-runtime-artifact.mjs'; * @param {{ root: string, artifact?: string | undefined, run: (name: string, args: string[]) => Promise, validate?: (path: string) => Promise }} options */ export async function ensureWorkerRuntime({ root, artifact, run, - validate = path => validateCodexRuntimeArtifact(path, { expectedTarget: 'aarch64-unknown-linux-gnu' }), + validate = path => validateCodexRuntimeArtifact(path, { expectedTarget: 'aarch64-unknown-linux-musl' }), }) { const existing = resolve(root, artifact ?? '.runtime/codex/linux-arm64'); try { await validate(existing); return existing; } diff --git a/src/adapters/codex-environment-files.ts b/src/adapters/codex-environment-files.ts index a769457..1b10063 100644 --- a/src/adapters/codex-environment-files.ts +++ b/src/adapters/codex-environment-files.ts @@ -5,6 +5,8 @@ import { CodexRpcClient } from './codex-rpc.js'; import environmentFileWorkerSource from './environment-file-worker-source.json' with { type: 'json' }; import { isolatedCodexArguments } from '../domain/local-codex-arguments.js'; import type { EnvironmentFileOperation } from '../core/environment-file-ports.js'; +import type { ManagedCodexPolicy } from '../domain/codex-managed-policy.js'; +import { verifyManagedCodexPolicy } from './codex-managed-policy.js'; export class EnvironmentFilesCleanupUnconfirmedError extends Error {} @@ -13,7 +15,7 @@ export async function codexEnvironmentFiles(options: { registryURL?: string; environmentId?: string; harnessKey?: string; workspace: string; operation: EnvironmentFileOperation | readonly EnvironmentFileOperation[]; binary?: string; signal?: AbortSignal; - managed?: { binaryArguments: string[] }; + managed?: { binaryArguments: string[]; policy: ManagedCodexPolicy }; environment?: Readonly; temporaryHomeParent?: string; identity?: { uid: number; gid: number }; @@ -33,6 +35,7 @@ export async function codexEnvironmentFiles(options: { }, ...(options.signal ? { signal: options.signal } : {}), ...(options.identity ? { identity: options.identity } : {}) }); try { await rpc.initialize(); + if (options.managed) await verifyManagedCodexPolicy({ call: rpc.call.bind(rpc), policy: options.managed.policy }); const started = await rpc.call('thread/start', { cwd: options.workspace, environments: [{ environmentId: remote ? 'remote' : 'local', cwd: options.workspace }], ephemeral: true, approvalPolicy: 'never', ...(options.managed ? { permissions: 'rat_managed' } : { sandbox: 'danger-full-access' }), selectedCapabilityRoots: [], diff --git a/src/adapters/codex-managed-policy.ts b/src/adapters/codex-managed-policy.ts new file mode 100644 index 0000000..79780c9 --- /dev/null +++ b/src/adapters/codex-managed-policy.ts @@ -0,0 +1,37 @@ +import { readdir } from 'node:fs/promises'; +import { dirname, isAbsolute, join } from 'node:path'; +import { managedCodexPolicyMatches, type ManagedCodexPolicy } from '../domain/codex-managed-policy.js'; + +type Call = (method: string, params: Record, timeoutMs?: number) => Promise; + +/** Check the running executor before any thread or tool can use its merged policy. */ +export async function verifyManagedCodexPolicy(options: { call: Call; policy: ManagedCodexPolicy }): Promise { + try { + const config = await options.call('config/read', { cwd: options.policy.workspace, includeLayers: true }); + const requirements = await options.call('configRequirements/read', {}); + if (!managedCodexPolicyMatches(config, requirements, options.policy) || !record(config) || !Array.isArray(config.layers)) throw new Error(); + const inspected = new Set(); + for (const layer of config.layers) { + if (!record(layer) || !record(layer.name) || !record(layer.config)) throw new Error(); + if (typeof layer.disabledReason === 'string' && layer.disabledReason.length > 0) continue; + if (layer.name.type === 'sessionFlags') { inspected.add('sessionFlags'); continue; } + // Other managed layers and project policy are not part of the host-owned envelope. + if (!['user', 'system'].includes(String(layer.name.type)) || typeof layer.name.file !== 'string' || !isAbsolute(layer.name.file)) throw new Error(); + if (layer.name.type === 'system' && Object.keys(layer.config).length) throw new Error(); + inspected.add(String(layer.name.type)); + // Native network_rule grants can widen the proxy without appearing in config/read. + // Refuse unfamiliar rule files instead of trying to emulate the upstream policy parser. + const directory = join(dirname(layer.name.file), 'rules'); + const entries = await readdir(directory).catch((error: unknown) => { + if (record(error) && error.code === 'ENOENT') return []; + throw error; + }); + if (entries.some(name => name.endsWith('.rules'))) throw new Error(); + } + if (!['sessionFlags', 'user', 'system'].every(source => inspected.has(source))) throw new Error(); + } catch { + throw new Error('Managed executor policy could not be verified. Refusing to expose execution tools.'); + } +} + +function record(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); } diff --git a/src/adapters/codex-rpc.ts b/src/adapters/codex-rpc.ts index 36855ac..7b5c680 100644 --- a/src/adapters/codex-rpc.ts +++ b/src/adapters/codex-rpc.ts @@ -23,6 +23,7 @@ export class CodexRpcClient { private nextId = 0; private readonly waiting = new Map; stopStartupProbe?: () => void }>(); private closed = false; + private checkpointDiagnosticFailure = false; private closing: Promise | undefined; private readonly spawnedAt = performance.now(); private readonly processOrdinal = ++nextProcessOrdinal; @@ -39,6 +40,7 @@ export class CodexRpcClient { const category = /database is locked|database is busy|SQLITE_BUSY|SQLITE_LOCKED/i.test(line) ? 'sqlite_contention' : /disk I\/O error|SQLITE_IOERR/i.test(line) ? 'sqlite_io' : /pool timed out while waiting for an open connection/i.test(line) ? 'sqlite_pool_timeout' : undefined; + if (category === 'sqlite_io' || /failed to (?:flush|persist|shutdown (?:the )?thread store)|error flushing (?:rollout|thread)/i.test(line)) this.checkpointDiagnosticFailure = true; if (category && !reported.has(category)) { reported.add(category); console.error(JSON.stringify({ message: 'Native harness storage diagnostic', category })); @@ -48,8 +50,8 @@ export class CodexRpcClient { this.child.stdout.once('data', () => { this.firstOutputAt = performance.now(); }); this.reader.on('line', (line) => { let value: Record; - try { value = JSON.parse(line) as Record; } catch { return; } - if (!record(value)) return; + try { value = JSON.parse(line) as Record; } catch { this.checkpointDiagnosticFailure = true; return; } + if (!record(value)) { this.checkpointDiagnosticFailure = true; return; } if (typeof value.method === 'string') { const event: CodexRpcEvent = { method: value.method, params: record(value.params) ? value.params : {}, ...(typeof value.id === 'string' || typeof value.id === 'number' ? { requestId: value.id } : {}) }; try { options.onEvent?.(event); } catch { void this.close(); return; } @@ -82,9 +84,9 @@ export class CodexRpcClient { this.child.once('error', () => { console.error(JSON.stringify({ message: 'Native harness process failed to start' })); fail(); }); this.child.once('exit', (code, signal) => { if (!this.closed) console.error(JSON.stringify({ message: 'Native harness process exited unexpectedly', code, signal })); - errors.close(); fail(); }); + this.child.once('close', () => errors.close()); if (options.signal) { const abort = () => { void this.close(); }; options.signal.addEventListener('abort', abort, { once: true }); @@ -135,17 +137,18 @@ export class CodexRpcClient { if (this.child.exitCode === null && this.child.signalCode === null) { this.child.kill('SIGKILL'); await exited; } } - /** After every thread acknowledged persistence, require a clean process exit as well. */ + /** Ordinary EOF with drained protocol output; this is not a persistence acknowledgement. */ public async closeCheckpoint(): Promise { - if (this.closed) throw new Error('Native process closed before checkpoint acknowledgement'); + if (this.closed) throw new Error('Native process closed before checkpoint EOF'); this.closed = true; - const exited = once(this.child, 'exit'); + const exited = once(this.child, 'close'); this.child.stdin.end(); let timer: ReturnType | undefined; const timeout = new Promise((_resolve, reject) => { timer = setTimeout(() => reject(new Error('Native checkpoint exit timed out')), 30_000); }); try { const [code, signal] = await Promise.race([exited, timeout]); if (code !== 0 || signal) throw new Error('Native checkpoint exit was not clean'); + if (this.checkpointDiagnosticFailure) throw new Error('Native checkpoint observed a storage or protocol failure'); } catch (error) { if (this.child.exitCode === null && this.child.signalCode === null) { this.child.kill('SIGKILL'); await exited.catch(() => {}); } throw error; diff --git a/src/adapters/local-native-verifier.ts b/src/adapters/local-native-verifier.ts index 93c2539..0a89fbf 100644 --- a/src/adapters/local-native-verifier.ts +++ b/src/adapters/local-native-verifier.ts @@ -1,4 +1,5 @@ import { spawn } from 'node:child_process'; +import hostCodePolicy from '../../runtime/native/host-code-policy.json' with { type: 'json' }; import { lstat, realpath } from 'node:fs/promises'; import { isAbsolute, relative, resolve, sep } from 'node:path'; import type { NativeBundleVerification, NativeBundleVerifier, NativeCopyLimits } from '../core/native-installation-ports.js'; @@ -9,13 +10,7 @@ export const NATIVE_VERIFIER_LIMITS: NativeCopyLimits = Object.freeze({ manifestBytes: 128 * 1024, depth: 32, pathBytes: 1024, }); const limitNames = ['entries', 'fileBytes', 'totalBytes', 'manifestBytes', 'depth', 'pathBytes'] as const; -const executablePaths = Object.freeze([ - 'Contents/Helpers/rat-native-worker', - 'Contents/Resources/codex/bin/codex', - 'Contents/Resources/codex/bin/codex-code-mode-host', - 'Contents/Resources/codex/codex-path/rg', - 'Contents/Resources/codex/codex-resources/zsh/bin/zsh', -]); +const executablePaths = Object.freeze(hostCodePolicy.map(policy => policy.path)); const stages = new Set(['arguments', 'bounds', 'filesystem', 'manifest', 'bundle-signature', 'executable-signature']); const reasons = new Set(['unsigned', 'ad_hoc', 'wrong_team', 'wrong_identifier', 'invalid']); const maximumDeadlineMs = 60_000; diff --git a/src/app/local-native-service.ts b/src/app/local-native-service.ts index cebf807..805e698 100644 --- a/src/app/local-native-service.ts +++ b/src/app/local-native-service.ts @@ -126,7 +126,7 @@ export class LocalNativeService { const release = status.selected; const companionRoot = join(release.bundlePath, 'Contents/Resources/codex'); const environment = Object.freeze({ ...this.environment, CODEX_BINARY: release.hostCodexPath, - PATH: [join(companionRoot, 'codex-path'), join(companionRoot, 'codex-resources/zsh/bin'), '/usr/bin', '/bin', '/usr/sbin', '/sbin'].join(':') }); + PATH: [join(companionRoot, 'codex-path'), '/usr/bin', '/bin', '/usr/sbin', '/sbin'].join(':') }); this.state = { kind: 'ready', release, host: Object.freeze({ binary: release.hostCodexPath, environment }) }; } else this.state = { kind: status.change.kind === 'failed' ? 'failed' : 'needs_setup' }; } diff --git a/src/core/native-checkpoint-planning.ts b/src/core/native-checkpoint-planning.ts new file mode 100644 index 0000000..65a930f --- /dev/null +++ b/src/core/native-checkpoint-planning.ts @@ -0,0 +1,61 @@ +import { canonicalJson, sha256Hex } from '../domain/json.js'; +import { nativeThreadHistory, type NativeCheckpointManifest } from '../domain/native-checkpoint.js'; +import type { SessionRuntimeState } from './session-runtime-planning.js'; +import { projectSessionItems } from './session-run-projection.js'; + +/** Native disk data alone cannot certify that acknowledged public work survived. */ +export function planNativeCheckpoint(state: SessionRuntimeState, histories: unknown[]): NativeCheckpointManifest { + const expectedIds = new Set([state.rootThreadId, ...state.subagents.map(agent => agent.id), ...state.turns.map(turn => turn.threadId)]); + if (!state.turns.length || state.requiredActions.length || state.turns.some(binding => binding.turn.status !== 'completed')) throw new Error('Native checkpoint requires completed Session history'); + const threads = histories.map(nativeThreadHistory); + for (const thread of threads) { + const agent = state.subagents.find(agent => agent.id === thread.id); + const parent = agent?.parent_agent_id === state.agentId ? state.rootThreadId : agent?.parent_agent_id; + if (thread.id !== state.rootThreadId && (!agent || thread.parentThreadId !== parent)) throw new Error('Native checkpoint parent differs from the Session journal'); + } + if (threads.length !== expectedIds.size || new Set(threads.map(thread => thread.id)).size !== expectedIds.size || threads.some(thread => !expectedIds.has(String(thread.id)))) throw new Error('Native checkpoint thread identities differ from the Session journal'); + const agentIds = { [state.rootThreadId]: state.agentId, ...Object.fromEntries(state.subagents.map(agent => [agent.id, agent.id])) }; + for (const binding of state.turns) { + const thread = threads.find(thread => thread.id === binding.threadId)!; + const turn = (thread.turns as Array>).find(turn => turn.id === binding.nativeTurnId); + if (!turn) throw new Error('Native checkpoint is missing an acknowledged Session turn'); + const projected = projectSessionItems(binding.turn.id, (turn.items as unknown[]).map(item => ({ method: 'item/completed', params: { item } })), undefined, { includeUser: true, agentIds }); + let projectedIndex = 0; + for (const item of binding.items) { + if (item.type === 'agent_message') { + // Raw inter-agent payloads are omitted by public thread/read. They remain + // in the host journal; cold evidence does not claim hidden mailbox fidelity. + if (![state.agentId, ...expectedIds].includes(item.sender_agent_id) || ![state.agentId, ...expectedIds].includes(item.recipient_agent_id)) throw new Error('Native checkpoint has an unknown coordination recipient'); + continue; + } + if (['create_subagent_call', 'send_subagent_input_call', 'interrupt_subagent_call'].includes(item.type)) { + const call = item.id ? state.coordinationCalls?.[item.id] : undefined; + const activity = (turn.items as Array>).find(native => native.type === 'subAgentActivity' && native.id === item.id); + if (call && activity) { + const kind = item.type === 'create_subagent_call' ? 'started' : item.type === 'interrupt_subagent_call' ? 'interrupted' : 'interacted'; + if (call.threadId !== binding.threadId || call.turnId !== binding.nativeTurnId || call.recipients.length !== 1 || activity.agentThreadId !== call.recipients[0] || activity.kind !== kind) throw new Error('Native checkpoint coordination recipients differ from the Session journal'); + if (item.type === 'create_subagent_call' && ((item.model !== null && activity.model !== item.model) || (item.reasoning_effort !== null && activity.reasoningEffort !== item.reasoning_effort))) throw new Error('Native checkpoint coordination settings differ from the Session journal'); + continue; + } + } + // Legacy rollout hydration synthesizes display Item IDs, while preserving + // native Turn IDs and content. Paginated history retains its Item IDs. + const semantic = (value: unknown) => thread.historyMode === 'legacy' && typeof value === 'object' && value !== null ? Object.fromEntries(Object.entries(value).filter(([key]) => key !== 'id')) : value; + const index = projected.findIndex((native, index) => index >= projectedIndex && canonicalJson(semantic(native)) === canonicalJson(semantic(item))); + if (index < 0) throw new Error('Native checkpoint is missing acknowledged Session item content'); + projectedIndex = index + 1; + } + } + const ordered: Record[] = []; + const pending = [...threads]; + while (pending.length) { + const index = pending.findIndex(thread => thread.id === state.rootThreadId ? thread.parentThreadId === null || thread.parentThreadId === undefined : ordered.some(parent => parent.id === thread.parentThreadId)); + if (index < 0) throw new Error('Native checkpoint parent identities are incomplete'); + ordered.push(pending.splice(index, 1)[0]!); + } + return { rootThreadId: state.rootThreadId, hostJournalDigest: nativeHostJournalDigest(state), threads: ordered }; +} + +export function nativeHostJournalDigest(state: SessionRuntimeState): string { + return sha256Hex(canonicalJson({ rootThreadId: state.rootThreadId, turns: state.turns, subagents: state.subagents.map(agent => ({ id: agent.id, parent_agent_id: agent.parent_agent_id })), coordinationCalls: state.coordinationCalls ?? {} })); +} diff --git a/src/core/session-stream.ts b/src/core/session-stream.ts index b2f498a..bc08959 100644 --- a/src/core/session-stream.ts +++ b/src/core/session-stream.ts @@ -62,9 +62,10 @@ export function planSessionStream(previous: SessionStreamSnapshot | undefined, c }); } else if (output_index !== null && item.type !== 'message' && item.type !== 'function_call_output' && 'status' in item) { if (item.type === 'command_execution') { - const before = old?.type === 'command_execution' ? old.output ?? '' : ''; + // Native live output can miss an early chunk. Only the completed aggregate + // can produce an ordered append-only stream without replacing prior text. const output = item.output ?? ''; - if (output !== before && output.startsWith(before)) events.push({ type: 'agent.output.command_execution_output.delta', session_id, turn_id: item.turn_id, item_id: item.id, output_index, delta: output.slice(before.length) }); + if (output && itemFinished(item) && !itemFinished(old)) events.push({ type: 'agent.output.command_execution_output.delta', session_id, turn_id: item.turn_id, item_id: item.id, output_index, delta: output }); } if (item.type === 'reasoning') { item.summary.forEach((part, summary_index) => { diff --git a/src/domain/codex-managed-policy.ts b/src/domain/codex-managed-policy.ts new file mode 100644 index 0000000..d399d11 --- /dev/null +++ b/src/domain/codex-managed-policy.ts @@ -0,0 +1,80 @@ +import type { SessionLaunch } from './session-execution.js'; + +export interface ManagedCodexPolicy { + workspace: string; + network: Extract['network']; + credentialProxy: boolean; +} + +/** Public network intent is enforced by the proxy; its empty allowlist denies all egress. */ +export function managedCodexConfiguration(policy: ManagedCodexPolicy) { + const proxy = policy.network.access !== 'enabled'; + const domains = Object.fromEntries((policy.network.access === 'restricted' ? policy.network.allowed_domains : []).map(host => [host.toLowerCase(), 'allow'])); + return { + approval_policy: 'never', default_permissions: 'rat_managed', + features: { network_proxy: { enabled: proxy, credential_broker: false } }, + projects: { [policy.workspace]: { trust_level: 'untrusted' } }, + permissions: { rat_managed: { + filesystem: { ':root': 'read', [policy.workspace]: 'write' }, + network: { enabled: true, mode: 'full', domains, unix_sockets: {}, + proxy_url: 'http://127.0.0.1:0', socks_url: 'http://127.0.0.1:0', + enable_socks5: true, enable_socks5_udp: false, allow_local_binding: false, + allow_upstream_proxy: policy.credentialProxy && policy.network.access !== 'disabled', + dangerously_allow_non_loopback_proxy: false, dangerously_allow_all_unix_sockets: false }, + } }, + }; +} + +/** Validate the merged public configuration, not merely the flags we requested. */ +export function managedCodexPolicyMatches(value: unknown, requirements: unknown, policy: ManagedCodexPolicy): boolean { + if (!record(value) || !record(value.config) || !managedCodexRequirementsSafe(requirements)) return false; + const config = value.config; + const expected = managedCodexConfiguration(policy); + return config.approval_policy === expected.approval_policy + && config.default_permissions === expected.default_permissions + && record(config.permissions) && equivalent(config.permissions.rat_managed, expected.permissions.rat_managed) + && record(config.features) && equivalent(config.features.network_proxy, expected.features.network_proxy) + && equivalent(config.projects, expected.projects); +} + +// These public 0.164 requirements can affect execution, startup or persistence. +// Absence is understood; an active requirement needs its own policy review. +const inactiveRequirementFields = new Set([ + 'modelProvider', 'modelProviders', 'cliAuthCredentialsStore', 'chatgptBaseUrl', + 'additionalDeveloperInstructions', 'allowedApprovalPolicies', 'allowedApprovalsReviewers', + 'allowedSandboxModes', 'allowedWindowsSandboxImplementations', 'allowedPermissionProfiles', + 'defaultPermissions', 'allowedWebSearchModes', 'allowManagedHooksOnly', 'allowBrowserAndComputerUse', + 'allowAppshots', 'allowRemoteControl', 'computerUse', 'browserUse', 'inAppBrowser', + 'featureRequirements', 'hooks', 'enforceResidency', 'network', 'application', 'autoReview', + 'models', 'sqliteHome', 'logDir', 'modelCatalogJson', 'checkForUpdateOnStartup', 'allowLoginShell', 'feedback', +]); + +/** Authentication restrictions do not grant execution authority or change the selected profile. */ +export function managedCodexRequirementsSafe(value: unknown): boolean { + if (!record(value) || !Object.hasOwn(value, 'requirements')) return false; + if (Object.keys(value).some(key => key !== 'requirements' && key !== 'supportsIndependentSpeedModes')) return false; + if (value.supportsIndependentSpeedModes != null && typeof value.supportsIndependentSpeedModes !== 'boolean') return false; + if (value.requirements === null) return true; + if (!record(value.requirements)) return false; + return Object.entries(value.requirements).every(([key, restriction]) => { + if (key !== 'allowedLoginMethods') return inactiveRequirementFields.has(key) && restriction === null; + // Upstream permits an empty list (no login), and serializes known methods in lowercase. + return restriction === null || Array.isArray(restriction) + && restriction.every(method => method === 'api' || method === 'chatgpt') + && new Set(restriction).size === restriction.length; + }); +} + +function equivalent(actual: unknown, expected: unknown): boolean { + return JSON.stringify(canonical(actual)) === JSON.stringify(canonical(expected)); +} + +// The public config reader serializes absent profile fields as null. Null adds no grant. +function canonical(value: unknown): unknown { + if (Array.isArray(value)) return value.map(canonical); + if (record(value)) return Object.fromEntries(Object.entries(value).filter(([, child]) => child !== null) + .sort(([left], [right]) => left.localeCompare(right)).map(([key, child]) => [key, canonical(child)])); + return value; +} + +function record(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); } diff --git a/src/domain/native-checkpoint.ts b/src/domain/native-checkpoint.ts new file mode 100644 index 0000000..6e9d7f3 --- /dev/null +++ b/src/domain/native-checkpoint.ts @@ -0,0 +1,50 @@ +import { canonicalJson, sha256Hex } from './json.js'; + +/** Evidence of an observed cold continuation, never an upstream flush acknowledgement. */ +export interface NativeCheckpointEvidence { + kind: 'codex-cold-native'; + threadIds: string[]; + historyDigest: string; + /** Separately persisted public items, including coordination payloads absent from native read. */ + hostJournalDigest: string; + historyModes: Array<'legacy' | 'paginated'>; +} +export interface NativeCheckpointManifest { + rootThreadId: string; + hostJournalDigest: string; + /** Parent-before-child order. Runtime-only fields are excluded explicitly below. */ + threads: Record[]; +} + +export function nativeThreadHistory(value: unknown): Record { + if (!record(value) || typeof value.id !== 'string' || !value.id || value.ephemeral !== false + || !['legacy', 'paginated'].includes(String(value.historyMode)) || !Array.isArray(value.turns) + || !value.turns.length || value.turns.length > 10_000) throw new Error('Native checkpoint history is unavailable or unsupported'); + const ids = new Set(); + for (const turn of value.turns) { + if (!record(turn) || typeof turn.id !== 'string' || !turn.id || ids.has(turn.id) + || turn.status !== 'completed' || !Array.isArray(turn.items)) throw new Error('Native checkpoint requires complete unique successful turns'); + ids.add(turn.id); + } + if (JSON.stringify(value).length > 32 * 1024 * 1024) throw new Error('Native checkpoint history exceeds its bound'); + // sessionId depends on whether the tree is loaded. Callers validate it separately; + // updatedAt/status/environments/canAcceptDirectInput describe the current process. + const fields = ['id', 'parentThreadId', 'forkedFromId', 'ephemeral', 'historyMode', 'modelProvider', 'model', + 'reasoningEffort', 'createdAt', 'cwd', 'cliVersion', 'source', 'agentNickname', 'agentRole', 'name', 'turns']; + return Object.fromEntries(fields.filter(key => value[key] !== undefined).map(key => [key, structuredClone(value[key])])); +} + +export function assertNativeHistory(expected: Record, actual: unknown, sessionId: string): void { + if (!record(actual) || actual.sessionId !== sessionId || canonicalJson(nativeThreadHistory(actual)) !== canonicalJson(expected)) { + throw new Error('Cold native history differs from the completed Session manifest'); + } +} + +export function nativeCheckpointEvidence(manifest: NativeCheckpointManifest): NativeCheckpointEvidence { + return { kind: 'codex-cold-native', threadIds: manifest.threads.map(thread => thread.id as string), + historyDigest: sha256Hex(canonicalJson(manifest.threads)), hostJournalDigest: manifest.hostJournalDigest, historyModes: [...new Set(manifest.threads.map(thread => thread.historyMode as 'legacy' | 'paginated'))] }; +} +export function sameNativeThreadIds(left: string[], right: string[]): boolean { + return left.length === right.length && new Set(left).size === left.length && new Set(right).size === right.length && left.every(id => right.includes(id)); +} +function record(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); } diff --git a/src/domain/session-errors.ts b/src/domain/session-errors.ts index 72d0c62..dd6415c 100644 --- a/src/domain/session-errors.ts +++ b/src/domain/session-errors.ts @@ -22,32 +22,40 @@ const messages: Record = { }; const nativeCodes: Record = { contextWindowExceeded: 'context_length_exceeded', sessionBudgetExceeded: 'session_budget_exceeded', - usageLimitExceeded: 'usage_limit_exceeded', creditBalanceExhausted: 'credit_balance_exhausted', - rateLimitExceeded: 'rate_limit_exceeded', serverOverloaded: 'server_overloaded', - requestTimeout: 'request_timeout', + usageLimitExceeded: 'usage_limit_exceeded', + rateLimitExceeded: 'rate_limit_exceeded', serverOverloaded: 'server_overloaded', flexUnavailable: 'server_overloaded', cyberPolicy: 'cyber_policy', misalignmentPolicyViolation: 'cyber_policy', internalServerError: 'server_error', unauthorized: 'authentication_error', badRequest: 'invalid_request', - sandboxError: 'sandbox_error', executorVersionIncompatible: 'executor_version_incompatible', + sandboxError: 'sandbox_error', activeTurnNotSteerable: 'active_turn_not_steerable', }; -/** Preserve typed failures without exposing provider bodies, headers or host diagnostics. */ +/** Preserve official typed failures without inferring precision from private diagnostics. */ export function sessionTurnError(value: unknown, previous?: TurnError | null): TurnError { const error = record(value) ? value : {}; const info = error.codexErrorInfo; - const variant = typeof info === 'string' ? info : record(info) ? Object.keys(info)[0] : undefined; + const variant = typeof info === 'string' ? info : record(info) && Object.keys(info).length === 1 ? Object.keys(info)[0] : undefined; const details = record(info) && variant && record(info[variant]) ? info[variant] : undefined; const status = details?.httpStatusCode; const publicCode = typeof error.code === 'string' && Object.hasOwn(messages, error.code) ? error.code as TurnError['code'] : undefined; const nativeCode = variant && Object.hasOwn(nativeCodes, variant) ? nativeCodes[variant] : undefined; const transportCode = variant && ['httpConnectionFailed', 'responseStreamConnectionFailed', 'responseStreamDisconnected', 'responseTooManyFailedAttempts'].includes(variant) - ? status === 401 || status === 403 ? 'authentication_error' : status === 404 ? 'resource_not_found' - : status === 408 || status === 504 ? 'request_timeout' : status === 429 ? 'rate_limit_exceeded' - : status === 503 ? 'server_overloaded' : typeof status === 'number' && status >= 500 ? 'server_error' - : typeof status === 'number' && status >= 400 ? 'invalid_request' : 'connection_failed' + ? nativeHttpError(status) : undefined; - const code = publicCode ?? nativeCode ?? transportCode; - return code ? { code, message: messages[code] } : previous ?? { code: 'internal_error', message: messages.internal_error }; + const code = publicCode ?? nativeCode ?? transportCode ?? previous?.code ?? 'internal_error'; + return { code, message: messages[code] }; +} + +function nativeHttpError(status: unknown): TurnError['code'] { + if (typeof status !== 'number' || !Number.isInteger(status) || status < 400 || status > 599) return 'connection_failed'; + if (status === 401 || status === 403) return 'authentication_error'; + // Payment Required cannot distinguish credits, billing setup or another policy. + if (status === 402) return 'internal_error'; + if (status === 404) return 'resource_not_found'; + if (status === 408 || status === 504) return 'request_timeout'; + if (status === 429) return 'rate_limit_exceeded'; + if (status === 503) return 'server_overloaded'; + return status >= 500 ? 'server_error' : 'invalid_request'; } function record(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); } diff --git a/src/domain/session-model-validation.ts b/src/domain/session-model-validation.ts index 351994c..edef018 100644 --- a/src/domain/session-model-validation.ts +++ b/src/domain/session-model-validation.ts @@ -5,6 +5,36 @@ import capabilities from '../../runtime/codex/model-capabilities.json' with { ty /** Provider aliases do not change the capability envelope of a known model. */ export function canonicalModelName(model: string): string { return model.replace(/^openai\./, ''); } +export type NativeServiceTier = 'default' | 'priority' | 'flex'; + +/** An explicit standard tier clears a prior per-thread tier in the official protocol. */ +export function nativeServiceTier(tier: string | undefined): NativeServiceTier { + if (tier === undefined || tier === 'auto' || tier === 'default') return 'default'; + if (tier === 'fast' || tier === 'priority') return 'priority'; + if (tier === 'flex') return 'flex'; + return invalid('The selected service tier is not supported by Codex', 'agent.service_tier'); +} + +/** Admit explicit tiers using the running provider's model/list, never another provider's catalog. */ +export function admitNativeServiceTier(tier: string | undefined, capability: { + model: string | undefined; + provider: string; + models: unknown; +}): NativeServiceTier { + const resolved = nativeServiceTier(tier); + if (resolved === 'default') return resolved; + const model = typeof capability.model === 'string' && capability.model.length > 0 && Array.isArray(capability.models) ? capability.models.find((entry: unknown) => record(entry) + && (entry.model === capability.model || entry.id === capability.model)) : undefined; + if (!record(model) || !Array.isArray(model.serviceTiers)) { + return invalid('The selected model has no verified service-tier capabilities', 'agent.service_tier'); + } + // Stock Codex accepts flex for catalog models on Responses providers. Bedrock + // additionally requires the tier to be advertised by its provider catalog. + if (resolved === 'flex' && capability.provider !== 'amazon-bedrock') return resolved; + if (model.serviceTiers.some((entry: unknown) => record(entry) && entry.id === resolved)) return resolved; + return invalid('The selected model and provider do not support this service tier', 'agent.service_tier'); +} + export function validateSessionModelSettings(agent: AgentSession['agent']): void { const models: Record = capabilities.models; const model = models[canonicalModelName(agent.model)]; @@ -13,3 +43,5 @@ export function validateSessionModelSettings(agent: AgentSession['agent']): void invalid('The selected model does not support the resulting reasoning effort', 'agent.reasoning.effort'); } } + +function record(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); } diff --git a/src/domain/workspace-checkpoint.ts b/src/domain/workspace-checkpoint.ts index 7830443..4609807 100644 --- a/src/domain/workspace-checkpoint.ts +++ b/src/domain/workspace-checkpoint.ts @@ -1,3 +1,4 @@ +import type { NativeCheckpointEvidence } from './native-checkpoint.js'; import type { SessionLaunch } from './session-execution.js'; import { canonicalJson, sha256Hex } from './json.js'; @@ -8,6 +9,7 @@ export interface WorkspaceCompatibility { sandboxId: string; setupDigest: string export interface WorkspaceCheckpoint extends WorkspaceCompatibility { id: string; piContextId?: string; + nativeDurability?: NativeCheckpointEvidence; source: WorkspaceExecution; rootThreadId: string; lastRootTurnId: string | null; diff --git a/src/runner/codex-app-server.ts b/src/runner/codex-app-server.ts index eded26d..f9c85a4 100644 --- a/src/runner/codex-app-server.ts +++ b/src/runner/codex-app-server.ts @@ -9,6 +9,8 @@ import type { SandboxMode } from '../domain/contracts.js'; import { emitMetric } from '../core/metrics.js'; import type { AgentSessionInputMessageParam, AgentSessionItem } from '../domain/agents-api.js'; import { projectSessionItems } from '../core/session-run-projection.js'; +import type { ManagedCodexPolicy } from '../domain/codex-managed-policy.js'; +import { verifyManagedCodexPolicy } from '../adapters/codex-managed-policy.js'; const CODEX_INVALID_REQUEST = -32600; const MISSING_ROLLOUT_MESSAGE = 'no rollout found for thread id '; @@ -59,6 +61,7 @@ export interface CodexAppServerRequest { prompt: string; sandbox: SandboxMode; permissions?: string; + managedPolicy?: ManagedCodexPolicy; persistent: boolean; modelProvider: string; model?: string; @@ -334,11 +337,14 @@ export async function runCodexAppServer( version: process.env.npm_package_version ?? '0.1.0', }, capabilities: { - experimentalApi: Boolean(request.captureTraces || request.sessionConfig || request.dynamicTools?.length), + experimentalApi: Boolean(request.managedPolicy || request.captureTraces || request.sessionConfig || request.dynamicTools?.length), requestAttestation: false, }, }); notify('initialized'); + if (request.permissions === 'rat_managed' && !request.managedPolicy) throw new Error('Managed executor policy intent is missing'); + if (request.managedPolicy && request.permissions !== 'rat_managed') throw new Error('Managed executor permission profile is not selected'); + if (request.managedPolicy) await verifyManagedCodexPolicy({ call, policy: request.managedPolicy }); const skillInputs = request.skills?.length ? await resolveSkillInputs(call, request.workspace, request.skills) @@ -350,7 +356,7 @@ export async function runCodexAppServer( // Never ask for a mid-Run human authorization decision. approvalPolicy: 'never', approvalsReviewer: 'user', - sandbox: request.sandbox, + ...(request.permissions ? { permissions: request.permissions } : { sandbox: request.sandbox }), modelProvider: request.modelProvider, ...(request.model ? { model: request.model } : {}), ...(request.personality ? { personality: request.personality } : {}), @@ -416,7 +422,7 @@ export async function runCodexAppServer( cwd: executionWorkspace, approvalPolicy: 'never', approvalsReviewer: 'user', - sandboxPolicy: sandboxPolicyFor(request.sandbox, executionWorkspace, request.networkAccess), + ...(request.permissions ? { permissions: request.permissions } : { sandboxPolicy: sandboxPolicyFor(request.sandbox, executionWorkspace, request.networkAccess) }), ...(request.model ? { model: request.model } : {}), ...(request.reasoningEffort ? { effort: request.reasoningEffort } : {}), ...(request.reasoningSummary ? { summary: request.reasoningSummary } : {}), @@ -511,11 +517,11 @@ export function sandboxPolicyFor( } function appServerConfig(request: CodexAppServerRequest): Record { - const config: Record = { ...request.sessionConfig }; + const config: Record = { ...request.sessionConfig, ...environmentToolConfig(request.environments) }; if (request.onServerRequest && !request.sessionConfig) { config['features.default_mode_request_user_input'] = true; } - if (request.sandbox === 'workspace-write') { + if (!request.permissions && request.sandbox === 'workspace-write') { config.sandbox_workspace_write = { network_access: request.networkAccess }; } if (request.webSearch) config.web_search = request.webSearch; @@ -529,6 +535,10 @@ function appServerConfig(request: CodexAppServerRequest): Record 0 ? { config } : {}; } +export function environmentToolConfig(environments: CodexAppServerRequest['environments']): Record { + return environments?.length === 0 ? { 'features.shell_tool': false, 'features.view_image': false } : {}; +} + interface SkillInput { type: 'skill'; name: string; diff --git a/src/runner/codex-cold-verification.ts b/src/runner/codex-cold-verification.ts new file mode 100644 index 0000000..0d42bed --- /dev/null +++ b/src/runner/codex-cold-verification.ts @@ -0,0 +1,191 @@ +import { createHash } from 'node:crypto'; +import { spawn } from 'node:child_process'; +import { access, chmod, chown, copyFile, lstat, mkdir, mkdtemp, readdir, readFile, readlink, realpath, rm, symlink, writeFile } from 'node:fs/promises'; +import { constants } from 'node:fs'; +import { dirname, isAbsolute, join, relative, resolve } from 'node:path'; +import { tmpdir } from 'node:os'; +import { CodexRpcClient } from '../adapters/codex-rpc.js'; +import { assertNativeHistory, nativeCheckpointEvidence, type NativeCheckpointEvidence, type NativeCheckpointManifest } from '../domain/native-checkpoint.js'; +import type { CodexAppServerRequest } from './codex-app-server.js'; +import { stopCheckpointWriters } from './checkpoint-writers.js'; + +export interface PreparedColdVerification { + verify(manifest: NativeCheckpointManifest): Promise; + close(): Promise; +} + +/** Reject ambient effects rather than interpreting arbitrary configuration as safe. */ +export function assertCheckpointStartupSafe(value: unknown, sessionConfig: unknown): void { + if (!record(value) || !record(value.config)) throw new Error('Native checkpoint configuration is unavailable'); + for (const config of [value.config, sessionConfig ?? {}]) { + if (!record(config)) throw new Error('Native checkpoint configuration is unsupported'); + for (const key of ['hooks', 'mcp_servers', 'plugins', 'notify', 'apps', 'browser_use', 'computer_use', 'desktop']) { + const item = config[key]; + if (item !== undefined && item !== null && !(record(item) && Object.keys(item).length === 0) && !(Array.isArray(item) && item.length === 0)) throw new Error('Native checkpoint does not support active startup integrations'); + } + for (const key of ['experimental_thread_store', 'experimental_thread_store_endpoint', 'sqlite_home']) { + if (config[key] !== undefined && config[key] !== null) throw new Error('Native checkpoint storage configuration is unsupported'); + } + } +} + +/** Only the dedicated Linux worker may stop this UID and bind exact native paths. */ +export async function prepareCodexColdVerification(options: { request: CodexAppServerRequest; stateRoot: string; manifest: NativeCheckpointManifest }): Promise { + const { request, stateRoot, manifest } = options; + if (manifest.threads.some(thread => thread.cwd !== '/workspace' || typeof thread.modelProvider !== 'string' || !/^[a-zA-Z0-9_-]{1,100}$/.test(thread.modelProvider))) throw new Error('Cold native checkpoint thread paths or providers are unsupported'); + if (manifest.threads.some(thread => ['amazon-bedrock', 'amazon-bedrock-runtime', 'ollama', 'lmstudio'].includes(String(thread.modelProvider)))) throw new Error('Cold native checkpoint provider is not yet supported'); + const uid = request.identity?.uid; + const gid = request.identity?.gid; + const home = request.environment.CODEX_HOME; + const sqlite = request.environment.CODEX_SQLITE_HOME; + if (process.platform !== 'linux' || process.getuid?.() !== 0 || !Number.isSafeInteger(uid) || !Number.isSafeInteger(gid) || !uid || !gid + || !request.managedPolicy || request.executionWorkspace !== '/workspace' || !home || !within(stateRoot, home) + || !within(stateRoot, request.workspace) || (sqlite && !within(home, sqlite)) || !isAbsolute(request.binary)) throw new Error('Cold native checkpoint requires an isolated Linux guest with durable native storage'); + for (const path of [stateRoot, home, request.workspace]) if (await realpath(path) !== resolve(path)) throw new Error('Cold native checkpoint paths must not be symbolic links'); + const identity = { uid: uid!, gid: gid! }; + const binary = await realpath(request.binary); + const bwrap = resolve(dirname(binary), '../codex-resources/bwrap'); + await access(bwrap, constants.X_OK); + const directory = await mkdtemp(join(tmpdir(), 'rat-cold-native-')); + // Root owns every staging name. A live agent may traverse the preflight + // directories but cannot replace them or plant a candidate symlink. + await chmod(directory, 0o711); + const candidate = join(directory, 'candidate'); + const disposable = join(directory, 'verify'); + let closed = false; + let attempted = false; + let rpc: CodexRpcClient | undefined; + const close = async () => { + if (closed) return; + closed = true; + await rpc?.close(); + // The UID belongs exclusively to this stopped worker; this also reaps any + // verifier descendants after a failed protocol request or a clean EOF. + if (attempted) await stopCheckpointWriters(identity.uid); + await rm(directory, { recursive: true, force: true }); + }; + try { + await mkdir(disposable, { mode: 0o711 }); + await ownedDirectory(join(disposable, 'home'), identity); + await ownedDirectory(join(disposable, 'workspace'), identity); + const args = namespaceArguments({ binary, home, sqlite, workspace: request.workspace, disposable, identity }); + // Check namespace support before sealing durable authority or stopping Codex. + await checkedProcess(bwrap, [...args, '--', '/usr/bin/true'], identity); + return { + close, + verify: async manifest => { + if (closed || attempted) throw new Error('Cold native verifier is unavailable'); + attempted = true; + const custodian = { uid: 0, gid: 0 }; + await ownedDirectory(candidate, custodian); + await ownedDirectory(join(candidate, 'home'), custodian); + await ownedDirectory(join(candidate, 'workspace'), custodian); + await copyPersistence(home, join(candidate, 'home'), custodian); + await copyTree(request.workspace, join(candidate, 'workspace'), custodian); + const candidateDigest = await treeDigest(candidate); + await rm(disposable, { recursive: true, force: true }); + await copyTree(candidate, disposable, identity); + await chown(disposable, 0, 0); await chmod(disposable, 0o711); + // Never copy auth/config/rules/plugins from CODEX_HOME. This complete + // replacement configuration disables integrations before process startup. + await rm(join(disposable, 'workspace', '.codex'), { recursive: true, force: true }); + await writeFile(join(disposable, 'home', 'config.toml'), verifierConfiguration(manifest), { mode: 0o600 }); + await chown(join(disposable, 'home', 'config.toml'), identity.uid, identity.gid); + let activity = false; + rpc = new CodexRpcClient({ binary: bwrap, binaryArguments: [...args, '--', binary, 'app-server'], cwd: '/', environment: {}, identity, + onEvent: event => { + if (event.requestId !== undefined || /^(turn\/(started|completed)|item\/|error$)/.test(event.method)) activity = true; + }, onServerRequest: async () => { activity = true; throw new Error('Cold native verifier cannot perform tools'); }, + }); + try { + await rpc.initialize(); + for (const expected of manifest.threads) { + const result = await rpc.call('thread/read', { threadId: expected.id, includeTurns: true }); + assertNativeHistory(expected, record(result) ? result.thread : undefined, String(expected.id)); + } + for (const expected of manifest.threads) { + await rpc.call('thread/resume', { threadId: expected.id, + ...(expected.id === manifest.rootThreadId ? { cwd: '/workspace', approvalPolicy: 'never', sandbox: 'read-only', + config: { 'features.goals': false, 'features.hooks': false, 'features.plugins': false, 'features.apps': false, 'mcp_servers': {} } } : {}) }); + const result = await rpc.call('thread/read', { threadId: expected.id, includeTurns: true }); + assertNativeHistory(expected, record(result) ? result.thread : undefined, manifest.rootThreadId); + } + await rpc.closeCheckpoint(); + await stopCheckpointWriters(identity.uid); + if (activity) throw new Error('Cold native verification attempted activity'); + if (await treeDigest(candidate) !== candidateDigest) throw new Error('Cold native candidate changed during verification'); + return nativeCheckpointEvidence(manifest); + } finally { await rpc.close(); await stopCheckpointWriters(identity.uid); } + }, + }; + } catch (error) { await close(); throw error; } +} + +function namespaceArguments(options: { binary: string; home: string; sqlite: string | undefined; workspace: string; disposable: string; identity: { uid: number; gid: number } }): string[] { + const { binary, home, sqlite, workspace, disposable } = options; + return ['--unshare-all', '--die-with-parent', '--new-session', '--cap-drop', 'ALL', '--clearenv', + '--ro-bind', '/usr', '/usr', '--symlink', 'usr/bin', '/bin', '--symlink', 'usr/lib', '/lib', '--symlink', 'usr/lib64', '/lib64', + '--proc', '/proc', '--dev', '/dev', '--tmpfs', '/tmp', '--dir', '/home/agent', + '--ro-bind', binary, binary, + '--bind', join(disposable, 'home'), home, + '--ro-bind', join(disposable, 'workspace'), workspace, + ...(workspace === '/workspace' ? [] : ['--ro-bind', join(disposable, 'workspace'), '/workspace']), + '--setenv', 'PATH', '/usr/bin:/bin', '--setenv', 'HOME', '/home/agent', '--setenv', 'CODEX_HOME', home, + ...(sqlite ? ['--setenv', 'CODEX_SQLITE_HOME', sqlite] : []), + '--setenv', 'AWS_EC2_METADATA_DISABLED', 'true', '--setenv', 'AWS_CONFIG_FILE', '/dev/null', '--setenv', 'AWS_SHARED_CREDENTIALS_FILE', '/dev/null', + '--setenv', 'LOG_FORMAT', 'json', '--setenv', 'RUST_LOG', 'warn', '--chdir', '/workspace']; +} +function verifierConfiguration(manifest: NativeCheckpointManifest): string { + // Keep built-in OpenAI intact. Its prewarm has no credentials and can only + // reach a closed loopback address in the private network namespace. + const providers = [...new Set(manifest.threads.map(thread => String(thread.modelProvider)))].filter(provider => provider !== 'openai').map(provider => `\n[model_providers.${JSON.stringify(provider)}]\nname = "Offline checkpoint verifier"\nbase_url = "http://127.0.0.1:1"\nwire_api = "responses"\nrequires_openai_auth = false\nsupports_websockets = false\nrequest_max_retries = 0\nstream_max_retries = 0\n`).join(''); + return 'openai_base_url = "http://127.0.0.1:1"\napproval_policy = "never"\nsandbox_mode = "read-only"\ncli_auth_credentials_store = "file"\nnotify = []\nweb_search = "disabled"\n[features]\nplugins = false\napps = false\nhooks = false\ngoals = false\nmulti_agent = true\nmulti_agent_v2 = true\nresponses_websockets = false\nresponses_websockets_v2 = false\ncode_mode_prewarm = false\nshell_snapshot = false\nshell_snapshot_v2 = false\n' + providers; +} +function within(root: string, path: string): boolean { const part = relative(root, path); return isAbsolute(root) && isAbsolute(path) && part !== '' && part !== '..' && !part.startsWith('../') && !isAbsolute(part); } +async function ownedDirectory(path: string, identity: { uid: number; gid: number }): Promise { + try { await mkdir(path, { mode: 0o700 }); } catch (error) { if (!record(error) || error.code !== 'EEXIST') throw error; } + const stat = await lstat(path); + if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error('Cold native destination must be a real directory'); + await chmod(path, 0o700); await chown(path, identity.uid, identity.gid); +} +async function copyPersistence(source: string, destination: string, identity: { uid: number; gid: number }): Promise { + for (const name of await readdir(source)) { + if (['sessions', 'archived_sessions', 'sqlite'].includes(name) || /^(state|thread_history|goals|memories|queue)_\d+\.sqlite(?:-wal|-shm)?$/.test(name)) { + await copyTree(join(source, name), join(destination, name), identity); + } + } +} +async function copyTree(source: string, destination: string, identity: { uid: number; gid: number }, root = source, budget = { entries: 0, bytes: 0 }): Promise { + const stat = await lstat(source); + if (++budget.entries > 100_000 || (budget.bytes += stat.isFile() ? stat.size : 0) > 512 * 1024 * 1024) throw new Error('Cold native candidate exceeds copy bounds'); + if (stat.isDirectory()) { + await ownedDirectory(destination, identity); + for (const name of await readdir(source)) await copyTree(join(source, name), join(destination, name), identity, root, budget); + } else if (stat.isFile()) { + await copyFile(source, destination, constants.COPYFILE_EXCL); await chmod(destination, 0o600); await chown(destination, identity.uid, identity.gid); + } else if (stat.isSymbolicLink()) { + const target = await readlink(source); + if (!within(root, resolve(dirname(source), target))) throw new Error('Cold native candidate has an external symbolic link'); + await symlink(target, destination); + } else throw new Error('Cold native candidate contains a non-regular file'); +} +async function treeDigest(root: string): Promise { + const digest = createHash('sha256'); + const visit = async (path: string): Promise => { + const stat = await lstat(path); digest.update(JSON.stringify([relative(root, path), stat.isFile() ? stat.size : null])); + if (stat.isDirectory()) { digest.update('directory'); for (const name of (await readdir(path)).sort()) await visit(join(path, name)); } + else if (stat.isFile()) { digest.update('file'); digest.update(await readFile(path)); } + else if (stat.isSymbolicLink()) { digest.update('link'); digest.update(await readlink(path)); } + else throw new Error('Cold native candidate contains a non-regular file'); + }; + await visit(root); return digest.digest('hex'); +} +async function checkedProcess(binary: string, args: string[], identity: { uid: number; gid: number }): Promise { + await new Promise((resolve, reject) => { + const child = spawn(binary, args, { cwd: '/', env: {}, ...identity, stdio: 'ignore' }); + const timer = setTimeout(() => { child.kill('SIGKILL'); }, 10_000); + child.once('error', () => { clearTimeout(timer); reject(new Error('Cold native namespace could not start')); }); + child.once('close', code => { clearTimeout(timer); code === 0 ? resolve() : reject(new Error('Cold native namespace is unavailable')); }); + }); +} +function record(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); } diff --git a/src/runner/hosted-environment-planning.ts b/src/runner/hosted-environment-planning.ts index 82084ab..e7c4db7 100644 --- a/src/runner/hosted-environment-planning.ts +++ b/src/runner/hosted-environment-planning.ts @@ -1,13 +1,16 @@ import type { SessionLaunch } from '../domain/session-execution.js'; +import { managedCodexConfiguration } from '../domain/codex-managed-policy.js'; /** The hosted discriminator is the upstream wire name; execution remains in an isolated AWS worker. */ export function hostedCodexArguments(base: string[] | undefined, network: Extract['network'], credentialProxy = false, workspace = '/workspace'): string[] { const arguments_ = base ?? ['app-server']; - const domains = network.allowed_domains.map((host) => `${JSON.stringify(host)} = "allow"`).join(', '); - const proxy = network.access === 'restricted'; - return ['-c', 'default_permissions = "rat_managed"', '-c', `features.network_proxy = ${proxy}`, '-c', - `permissions.rat_managed = { filesystem = { ":root" = "read", ${JSON.stringify(workspace)} = "write" }, network = { enabled = ${network.access !== 'disabled'}, mode = "full", domains = { ${domains} }, proxy_url = "http://127.0.0.1:0", socks_url = "http://127.0.0.1:0", enable_socks5 = true, enable_socks5_udp = false, allow_local_binding = false, allow_upstream_proxy = ${credentialProxy} } }`, - ...arguments_]; + const config = managedCodexConfiguration({ workspace, network, credentialProxy }); + return [...Object.entries(config).flatMap(([key, value]) => ['-c', `${key} = ${toml(value)}`]), ...arguments_]; +} + +function toml(value: unknown): string { + if (typeof value === 'object' && value !== null) return `{ ${Object.entries(value).map(([key, child]) => `${/^[A-Za-z0-9_-]+$/.test(key) ? key : JSON.stringify(key)} = ${toml(child)}`).join(', ')} }`; + return JSON.stringify(value); } diff --git a/src/runner/hosted-environment.ts b/src/runner/hosted-environment.ts index ffe3ef6..c714442 100644 --- a/src/runner/hosted-environment.ts +++ b/src/runner/hosted-environment.ts @@ -14,6 +14,7 @@ import type { ManagedSandboxGeneration } from '../core/managed-environment-plann import { resetPersistentWorkspace } from './workspace.js'; import { workspaceDigests, type WorkspaceCheckpoint } from '../domain/workspace-checkpoint.js'; import { verifyManagedExecutor } from './managed-executor-verification.js'; +import { verifyManagedCodexPolicy } from '../adapters/codex-managed-policy.js'; /** Map a private persistent directory to the standard workspace path before guest code starts. */ export async function bindHostedWorkspace(workspace: string, visible = '/workspace'): Promise { @@ -126,6 +127,7 @@ export async function prepareHostedEnvironment(options: { }); try { await rpc.initialize(); + await verifyManagedCodexPolicy({ call: rpc.call.bind(rpc), policy: { workspace: executionWorkspace, network: launch.environment.network, credentialProxy: Boolean(options.credentials) } }); for (const [index, command] of commands.entries()) { const execution = await rpc.call('command/exec', { ...command, env: { ...hostedProcessEnvironment(configuration.env ?? {}, plan.environment.PATH, executionWorkspace), ...options.credentials?.shellEnvironment }, permissionProfile: 'rat_managed', timeoutMs: 600_000, outputBytesCap: 64 * 1024, diff --git a/src/runner/local-managed-environment.ts b/src/runner/local-managed-environment.ts index 5b71030..1fa4de6 100644 --- a/src/runner/local-managed-environment.ts +++ b/src/runner/local-managed-environment.ts @@ -58,6 +58,7 @@ export class LocalManagedEnvironment { environment: { type: 'self_hosted', id: session.environment.id, workspace_directory: '/workspace', capability_directories: [...new Set(roots)], remote_url: connection.url }, }, connection.harnessKey); return { ...result, permissions: 'rat_managed', binaryArguments: hostedCodexArguments(base.binaryArguments, session.environment.network), + managedPolicy: { workspace: '/workspace', network: session.environment.network, credentialProxy: false }, sessionConfig: { ...result.sessionConfig, shell_environment_policy: { inherit: 'core', set: hostedProcessEnvironment(configuration?.env ?? {}, '/usr/local/bin:/usr/bin:/bin') } } }; } public async initialize(owner: string, session: AgentSession) { if (session.environment.type === 'openai_hosted') await this.ensure(owner, session); } diff --git a/src/runner/main.ts b/src/runner/main.ts index 4282161..791026f 100644 --- a/src/runner/main.ts +++ b/src/runner/main.ts @@ -56,6 +56,7 @@ import { planCodexLaunch } from './agent-planning.js'; import { codexEnvironmentFiles } from '../adapters/codex-environment-files.js'; import type { EnvironmentFileOperations } from '../core/environment-file-ports.js'; import { WorkspaceHandoffService } from '../core/workspace-handoff-service.js'; +import { prepareCodexColdVerification } from './codex-cold-verification.js'; import { WorkspaceAdmission } from './workspace-handoff.js'; import { workspaceDigests, type WorkspaceCheckpoint } from '../domain/workspace-checkpoint.js'; import { canonicalJson, sha256Hex } from '../domain/json.js'; @@ -229,7 +230,7 @@ export async function runAgentWorker(): Promise { }); launch = prepared.launch; if (!prepared.sandbox) throw new Error('Managed sandbox generation is missing'); - environmentFiles = { execute: async (_environmentId, _reference, operation) => { const result = await codexEnvironmentFiles({ workspace: '/workspace', operation, binary: plan.binary, ...(launch.runtime?.kind === 'pi' && launch.environment.type === 'openai_hosted' ? { managed: { binaryArguments: hostedCodexArguments(['-c', 'features.plugins=false', 'app-server'], launch.environment.network) } } : {}), ...(plan.identity ? { identity: plan.identity } : {}), signal: abort.signal }); if (launch.runtime?.kind === 'pi') await syncWorkspaceStorage(workspace); return result; } }; + environmentFiles = { execute: async (_environmentId, _reference, operation) => { const result = await codexEnvironmentFiles({ workspace: '/workspace', operation, binary: plan.binary, ...(launch.runtime?.kind === 'pi' && launch.environment.type === 'openai_hosted' ? { managed: { binaryArguments: hostedCodexArguments(['-c', 'features.plugins=false', 'app-server'], launch.environment.network), policy: { workspace: '/workspace', network: launch.environment.network, credentialProxy: false } } } : {}), ...(plan.identity ? { identity: plan.identity } : {}), signal: abort.signal }); if (launch.runtime?.kind === 'pi') await syncWorkspaceStorage(workspace); return result; } }; runnerControl?.setEnvironmentFiles((operation) => environmentFiles!.execute(environmentId, '', operation as import('../core/environment-file-ports.js').EnvironmentFileOperation)); managedReady = true; if (launch.runtime?.kind === 'pi') await syncWorkspaceStorage(workspace); @@ -258,6 +259,7 @@ export async function runAgentWorker(): Promise { }; checkpointOptions = { admission, + prepareNativeVerification: (request, manifest) => prepareCodexColdVerification({ request, manifest, stateRoot: durableStateRoot! }), begin: async () => { checkpointAttempted = true; await managedHeartbeat?.stop(); @@ -267,10 +269,10 @@ export async function runAgentWorker(): Promise { stopRemainingWriters: stopWriters, flushJournal: async () => { await sessionJournal!.flush(); await stopWriters(); }, syncStorage: () => syncWorkspaceStorage(durableStateRoot!), - commit: async state => { + commit: async (state, nativeDurability) => { if (abort.signal.aborted || state.requiredActions.length || state.turns.some(binding => !['completed', 'failed', 'cancelled'].includes(binding.turn.status))) throw new Error('Session changed during checkpoint shutdown'); const root = state.turns.filter(binding => binding.turn.subagent_id === null).at(-1); - const checkpoint: WorkspaceCheckpoint = { id: checkpointId, ...(piContextId ? { piContextId } : {}), source, sandboxId: prepared.sandbox!.id, ...digests, authorityDigest, + const checkpoint: WorkspaceCheckpoint = { ...(nativeDurability ? { nativeDurability } : {}), id: checkpointId, ...(piContextId ? { piContextId } : {}), source, sandboxId: prepared.sandbox!.id, ...digests, authorityDigest, rootThreadId: state.rootThreadId, lastRootTurnId: root?.turn.id ?? null, lastNativeTurnId: root?.nativeTurnId ?? null, reason: 'hibernate', createdAt: Math.floor(Date.now() / 1000) }; await handoffs.commit(sessionOwner, environmentId, checkpoint); diff --git a/src/runner/managed-executor-verification.ts b/src/runner/managed-executor-verification.ts index 2a2c5ab..2f7b0f4 100644 --- a/src/runner/managed-executor-verification.ts +++ b/src/runner/managed-executor-verification.ts @@ -6,6 +6,7 @@ import type { SessionLaunch } from '../domain/session-execution.js'; import type { CodexLaunchPlan } from './agent-planning.js'; import { hostedCodexArguments } from './hosted-environment-planning.js'; import { verifyManagedSandbox } from './managed-sandbox.js'; +import { verifyManagedCodexPolicy } from '../adapters/codex-managed-policy.js'; export async function verifyManagedExecutor(options: { plan: CodexLaunchPlan; workspace: string; @@ -24,6 +25,7 @@ export async function verifyManagedExecutor(options: { onServerRequest: async () => { throw new Error('Sandbox verification cannot request approval'); }, }); await rpc.initialize(); + await verifyManagedCodexPolicy({ call: rpc.call.bind(rpc), policy: { workspace: options.workspace, network: options.network, credentialProxy: options.credentialProxy } }); await verifyManagedSandbox({ call: rpc.call.bind(rpc), workspace: options.workspace, home, ...(plan.identity ? { identity: plan.identity } : {}) }); } finally { try { await rpc?.close(); } finally { await rm(home, { recursive: true, force: true }); } diff --git a/src/runner/pi-effect-broker.ts b/src/runner/pi-effect-broker.ts index d7d0a71..1bd82e8 100644 --- a/src/runner/pi-effect-broker.ts +++ b/src/runner/pi-effect-broker.ts @@ -3,7 +3,7 @@ import { mkdtemp, chown, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { Type } from 'typebox'; -import { PI_MANAGED_TOOLS, parsePiManagedOperation, piManagedCommand } from './pi-managed-tools.js'; +import { PI_MANAGED_TOOLS, parsePiManagedOperation, piManagedCommand, piManagedResult } from './pi-managed-tools.js'; import AjvModule from 'ajv'; import { CodexRpcClient } from '../adapters/codex-rpc.js'; import type { NativeLease } from '../core/session-native-store.js'; @@ -15,6 +15,7 @@ import { hostedCodexArguments, hostedProcessEnvironment } from './hosted-environ import type { ToolExecutionResult } from '@earendil-works/pi-durable'; import type { PiEffectTool, PiToolEffect } from './pi/protocol.js'; import { verifyManagedSandbox } from './managed-sandbox.js'; +import { verifyManagedCodexPolicy } from '../adapters/codex-managed-policy.js'; const Ajv = AjvModule.default; @@ -39,6 +40,9 @@ export async function createPiEffectBroker(options: { launch: SessionLaunch; pla let threadId: string | undefined; try { await rpc.initialize(); + if (launch.environment.type === 'openai_hosted') await verifyManagedCodexPolicy({ call: rpc.call.bind(rpc), policy: { + workspace: plan.workspace, network: launch.environment.network, credentialProxy: Boolean(options.credentials), + } }); if (managed) await verifyManagedSandbox({ call: rpc.call.bind(rpc), workspace: plan.workspace, home, ...(plan.identity ? { identity: plan.identity } : {}) }); if (launch.agent.tools.some(tool => tool.type === 'mcp')) { const result = await rpc.call('thread/start', { cwd: managed ? plan.workspace : home, ephemeral: true, approvalPolicy: 'never', @@ -84,7 +88,8 @@ export async function createPiEffectBroker(options: { launch: SessionLaunch; pla }), isError: response.isError === true }; } if (!managed) throw new Error('Managed tools require an environment'); - const command = piManagedCommand(parsePiManagedOperation(request), process.execPath, plan.workspace); + const operation = parsePiManagedOperation(request); + const command = piManagedCommand(operation, process.execPath, plan.workspace); const processId = `pi-${randomUUID()}`; const invocation = AbortSignal.any([signal, shutdown.signal]); invocation.throwIfAborted(); @@ -104,9 +109,10 @@ export async function createPiEffectBroker(options: { launch: SessionLaunch; pla try { const response = await Promise.race([execution, failedCancellation]); await cancellation; - if (!record(response)) throw new Error('Invalid managed command response'); + if (!record(response) || typeof response.stdout !== 'string' || typeof response.stderr !== 'string' + || typeof response.exitCode !== 'number' || !Number.isInteger(response.exitCode)) throw new Error('Invalid managed command response'); await options.flushWorkspace?.(); - return { output: [{ type: 'text' as const, text: `${String(response.stdout ?? '')}${String(response.stderr ?? '')}` }], isError: response.exitCode !== 0 }; + return piManagedResult(operation, { stdout: response.stdout, stderr: response.stderr, exitCode: response.exitCode }); } catch (error) { shutdown.abort(error); throw error; } finally { invocation.removeEventListener('abort', cancel); } })(); diff --git a/src/runner/pi-managed-tools.ts b/src/runner/pi-managed-tools.ts index efdce11..1c8a86a 100644 --- a/src/runner/pi-managed-tools.ts +++ b/src/runner/pi-managed-tools.ts @@ -1,5 +1,6 @@ import { Type } from 'typebox'; import AjvModule from 'ajv'; +import type { ToolExecutionResult } from '@earendil-works/pi-durable'; import type { PiEffectTool, PiToolEffect } from './pi/protocol.js'; export const PI_MANAGED_TOOLS: readonly PiEffectTool[] = [ @@ -12,6 +13,12 @@ export type PiManagedOperation = | { kind: 'read' | 'list'; path: string } | { kind: 'write'; path: string; content: string }; +export interface PiManagedCommandResponse { + readonly stdout: string; + readonly stderr: string; + readonly exitCode: number; +} + const ajv = new AjvModule.default({ strict: false }); const validators = new Map(PI_MANAGED_TOOLS.map(tool => [tool.name, ajv.compile(tool.parameters)])); const FILE_PROGRAM = `const fs=require('node:fs');const [op,path,data]=process.argv.slice(1);if(op==='read')process.stdout.write(fs.readFileSync(path,'utf8'));else if(op==='write'){fs.writeFileSync(path,data);console.log('Written');}else if(op==='list')console.log(JSON.stringify(fs.readdirSync(path,{withFileTypes:true}).map(e=>({name:e.name,directory:e.isDirectory()}))));else throw Error('Invalid file operation');`; @@ -33,3 +40,10 @@ export function piManagedCommand(operation: PiManagedOperation, nodeBinary: stri ? { command: [...operation.command], cwd: operation.cwd ?? workspace } : { command: [nodeBinary, '-e', FILE_PROGRAM, operation.kind, operation.path, operation.kind === 'write' ? operation.content : ''], cwd: workspace }; } + +export function piManagedResult(operation: PiManagedOperation, response: PiManagedCommandResponse): ToolExecutionResult { + const isError = response.exitCode !== 0; + // File stdout is data; successful runtime warnings must not alter its bytes. + const text = operation.kind === 'command' || isError ? response.stdout + response.stderr : response.stdout; + return { output: [{ type: 'text', text }], isError }; +} diff --git a/src/runner/session-launch-planning.ts b/src/runner/session-launch-planning.ts index 137b86c..d39ea51 100644 --- a/src/runner/session-launch-planning.ts +++ b/src/runner/session-launch-planning.ts @@ -6,6 +6,7 @@ import { sessionRecoveryItems } from './session-recovery-planning.js'; import type { SessionMcpRuntime } from './session-mcp.js'; import { hostedCodexArguments, hostedProcessEnvironment } from './hosted-environment-planning.js'; import type { SessionEnvironmentCredentialsRuntime } from './session-environment-credentials.js'; +import { nativeServiceTier } from '../domain/session-model-validation.js'; /** Translate public agent settings once, without changing the requested model or widening authority. */ export function planSessionLaunch(base: CodexLaunchPlan, launch: SessionLaunch, environmentToken?: string, mcp?: Pick, credentials?: Pick): Partial { @@ -23,7 +24,7 @@ export function planSessionLaunch(base: CodexLaunchPlan, launch: SessionLaunch, ...(launch.history?.length ? { recoveryItems: sessionRecoveryItems(launch.history) } : {}), model: agent.model, developerInstructions: agent.instructions, - serviceTier: agent.service_tier, + serviceTier: nativeServiceTier(agent.service_tier), ...(agent.reasoning.effort !== null ? { reasoningEffort: agent.reasoning.effort } : {}), ...(agent.reasoning.summary !== null ? { reasoningSummary: agent.reasoning.summary } : {}), ...(agent.text.format.type === 'json_schema' ? { outputSchema: agent.text.format.schema as Record } : {}), @@ -31,6 +32,7 @@ export function planSessionLaunch(base: CodexLaunchPlan, launch: SessionLaunch, ...(launch.environment.type === 'none' ? { environments: [] } : {}), ...(launch.environment.type === 'openai_hosted' ? { permissions: 'rat_managed', + managedPolicy: { workspace: '/workspace', network: launch.environment.network, credentialProxy: Boolean(credentials) }, binaryArguments: hostedCodexArguments(base.binaryArguments, launch.environment.network, Boolean(credentials)), environments: [{ environmentId: 'local', cwd: '/workspace' }], executionWorkspace: '/workspace', // Keep credential placeholders in the app-server process environment as a diff --git a/src/runner/session-runtime.ts b/src/runner/session-runtime.ts index cb50355..84b7cef 100644 --- a/src/runner/session-runtime.ts +++ b/src/runner/session-runtime.ts @@ -1,13 +1,20 @@ +import { verifyManagedCodexPolicy } from '../adapters/codex-managed-policy.js'; +import { admitNativeServiceTier, nativeServiceTier } from '../domain/session-model-validation.js'; +import { nativeHostJournalDigest, planNativeCheckpoint } from '../core/native-checkpoint-planning.js'; +import type { NativeCheckpointManifest, NativeCheckpointEvidence } from '../domain/native-checkpoint.js'; +import { assertCheckpointStartupSafe } from './codex-cold-verification.js'; +import type { PreparedColdVerification } from './codex-cold-verification.js'; import type { AgentSessionInputMessageParam, AgentSessionInputParam, Turn } from '../domain/agents-api.js'; import { CodexRpcClient, CodexRpcError, type CodexRpcEvent } from '../adapters/codex-rpc.js'; import { bindSessionTurn, initialSessionRuntime, reduceSessionRuntime, resolveSessionFunction, rootTurnBusy, stoppedSessionRuntime, type SessionRuntimeState } from '../core/session-runtime-planning.js'; import type { CodexAppServerRequest } from './codex-app-server.js'; -import { sandboxPolicyFor } from './codex-app-server.js'; +import { environmentToolConfig, sandboxPolicyFor } from './codex-app-server.js'; import type { SessionModelSettings } from '../domain/session-execution.js'; import { checkpointWorkspace, type WorkspaceHandoffPorts } from './workspace-handoff.js'; -export type SessionCheckpointOptions = Omit & { - commit(state: SessionRuntimeState): ReturnType; +export type SessionCheckpointOptions = Omit & { + prepareNativeVerification?(request: CodexAppServerRequest, manifest: NativeCheckpointManifest): Promise; + commit(state: SessionRuntimeState, evidence?: NativeCheckpointEvidence): ReturnType; }; type ToolResult = Extract; @@ -18,6 +25,7 @@ export class SessionRuntime { private readonly rpc: Pick & Partial>; private state: SessionRuntimeState | undefined; private readonly requests = new Map(); + private readonly terminalWaiters = new Set<() => void>(); private readonly started = new Map>(); private readonly operations = new Map }>(); private readonly earlyEvents: CodexRpcEvent[] = []; @@ -28,6 +36,7 @@ export class SessionRuntime { private closed = false; private expired = false; private checkpointing = false; + private checkpointActivity = false; private checkpointFailure: unknown; private idleTimer: ReturnType | undefined; private lifetimeTimer: ReturnType | undefined; @@ -71,8 +80,16 @@ export class SessionRuntime { if (!ports || !this.state || this.closed || this.checkpointing) throw new Error('Session checkpoint is unavailable'); // Set synchronously so a native disconnect cannot finish the Run during shutdown. this.checkpointing = true; + this.checkpointActivity = false; let transitionStarted = false; let threadIds: string[] = []; + let manifest: NativeCheckpointManifest | undefined; + let verifier: PreparedColdVerification | undefined; + const rootThreadId = this.state.rootThreadId; + const assertQuiescent = () => { + if (this.checkpointActivity || manifest && nativeHostJournalDigest(this.snapshot()) !== manifest.hostJournalDigest || this.requests.size || this.state!.requiredActions.length + || this.state!.turns.some(binding => !terminal(binding.turn))) throw new Error('Native activity changed during checkpoint shutdown'); + }; try { return await checkpointWorkspace({ ...ports, inspectIdle: async () => { @@ -80,64 +97,86 @@ export class SessionRuntime { && !this.state!.requiredActions.length && !this.state!.turns.some(binding => !terminal(binding.turn)); // Empty native threads may never materialize a rollout. Reject these before // beginning shutdown so we cannot publish an unresumable checkpoint. - if (idle) await this.checkPersistedHistory(this.state!.rootThreadId); - return { idle, threadIds: [this.state!.rootThreadId] }; + if (idle) { + if (!ports.prepareNativeVerification) throw new Error('Cold native checkpoint is unavailable on this host'); + const loaded = await this.loadedCheckpointThreads(); + if (!loaded.includes(rootThreadId)) throw new Error('Root thread is not loaded'); + threadIds = checkpointThreadIds([...new Set([rootThreadId, ...this.state!.subagents.map(agent => agent.id), ...this.state!.turns.map(binding => binding.threadId)])]); + if (loaded.some(id => !threadIds.includes(id))) throw new Error('Native threads differ from the Session journal'); + const histories: unknown[] = []; + for (const threadId of threadIds) { + const result = await this.rpc.call('thread/read', { threadId, includeTurns: true }); + if (!record(result) || !record(result.thread) || result.thread.id !== threadId) throw new Error('Native checkpoint history is unavailable'); + histories.push(result.thread); + } + manifest = planNativeCheckpoint(this.snapshot(), histories); + assertCheckpointStartupSafe(await this.rpc.call('config/read', { includeLayers: true }), this.options.request.sessionConfig); + const requirements = await this.rpc.call('configRequirements/read', {}); + if (!record(requirements) || requirements.requirements !== null) throw new Error('Cold native checkpoint requirements are unsupported'); + verifier = await ports.prepareNativeVerification(this.options.request, manifest); + assertQuiescent(); + } + return { idle, threadIds }; }, begin: async () => { transitionStarted = true; clearTimeout(this.idleTimer); clearTimeout(this.lifetimeTimer); + assertQuiescent(); await ports.begin(); }, shutdownNative: async () => { - const loaded = async () => { - const ids: string[] = []; - let cursor: string | undefined; - const cursors = new Set(); - do { - const page = await this.rpc.call('thread/loaded/list', { limit: 100, ...(cursor ? { cursor } : {}) }); - if (!record(page) || !Array.isArray(page.data) || page.data.some(id => typeof id !== 'string') || ids.length + page.data.length > 1000 - || page.nextCursor != null && typeof page.nextCursor !== 'string') throw new Error('Invalid native thread list'); - ids.push(...page.data as string[]); - cursor = page.nextCursor as string | undefined; - if (cursor && cursors.has(cursor)) throw new Error('Native thread pagination repeated'); - if (cursor) cursors.add(cursor); - } while (cursor); - return [...new Set(ids)]; - }; - threadIds = await loaded(); - if (!threadIds.includes(this.state!.rootThreadId)) throw new Error('Root thread is not loaded'); - // Stop descendants before the root so parent shutdown cannot hide a failed child flush. - for (const threadId of [...threadIds.filter(id => id !== this.state!.rootThreadId), this.state!.rootThreadId]) { - const result = await this.rpc.call('thread/checkpoint', { threadId }, 35_000); - if (!record(result) || result.threadId !== threadId || result.flushed !== true) throw new Error('Native thread checkpoint was not acknowledged'); - await this.checkPersistedHistory(threadId); - } - if ((await loaded()).some(id => !threadIds.includes(id))) throw new Error('Native threads changed during checkpoint shutdown'); + assertQuiescent(); + if ((await this.loadedCheckpointThreads()).some(id => !threadIds.includes(id))) throw new Error('Native threads changed during checkpoint shutdown'); + assertQuiescent(); if (!this.rpc.closeCheckpoint) throw new Error('Native checkpoint shutdown is unsupported'); await this.rpc.closeCheckpoint(); this.disconnected(); - // Every loaded thread was checked above; the outer barrier identifies the root. - return { threadIds: [this.state!.rootThreadId], flushed: true, forced: false }; + assertQuiescent(); + return { kind: 'codex-eof', threadIds }; }, - commit: () => ports.commit(this.snapshot()), + verifyNative: async () => { + assertQuiescent(); + if (!manifest || !verifier) throw new Error('Cold native verification was not prepared'); + const evidence = await verifier.verify(manifest); + assertQuiescent(); + return evidence; + }, + commit: evidence => { assertQuiescent(); return ports.commit(this.snapshot(), evidence); }, settle: () => {}, }); } catch (error) { if (transitionStarted) this.checkpointFailure = error; throw error; } finally { - if (transitionStarted) await this.close(); - this.checkpointing = false; - if (this.closed) this.finish(); + try { await verifier?.close(); } + catch (error) { if (transitionStarted) this.checkpointFailure = error; throw error; } + finally { + if (transitionStarted) await this.close(); + this.checkpointing = false; + if (this.closed) this.finish(); + } } } - private async checkPersistedHistory(threadId: string): Promise { - const history = await this.rpc.call('thread/turns/list', { threadId, limit: 1, itemsView: 'summary' }); - if (!record(history) || !Array.isArray(history.data)) throw new Error('Native checkpoint history is unavailable'); + private async loadedCheckpointThreads(): Promise { + let ids: string[] = []; + let cursor: string | undefined; + const cursors = new Set(); + do { + const page = checkpointThreadPage(await this.rpc.call('thread/loaded/list', { limit: 100, ...(cursor ? { cursor } : {}) })); + ids = checkpointThreadIds([...ids, ...page.ids]); + cursor = page.nextCursor; + if (cursor && cursors.has(cursor)) throw new Error('Native thread pagination repeated'); + if (cursor) cursors.add(cursor); + } while (cursor); + return ids; } public async initialize(): Promise { const request = this.options.request; await this.rpc.initialize(); + if (request.permissions === 'rat_managed' && !request.managedPolicy) throw new Error('Managed executor policy intent is missing'); + if (request.managedPolicy && request.permissions !== 'rat_managed') throw new Error('Managed executor permission profile is not selected'); + if (request.managedPolicy) await verifyManagedCodexPolicy({ call: this.rpc.call.bind(this.rpc), policy: request.managedPolicy }); + const serviceTier = await this.admitTier(request.serviceTier, request.model); const parameters = { cwd: request.executionWorkspace ?? request.workspace, modelProvider: request.modelProvider, model: request.model, @@ -146,10 +185,10 @@ export class SessionRuntime { experimentalRawEvents: true, selectedCapabilityRoots: request.selectedCapabilityRoots ?? [], developerInstructions: request.developerInstructions, - serviceTier: request.serviceTier, + serviceTier, environments: request.environments, dynamicTools: request.dynamicTools, - config: { ...request.sessionConfig, web_search: request.webSearch ?? 'disabled', + config: { ...request.sessionConfig, 'features.fast_mode': true, ...environmentToolConfig(request.environments), web_search: request.webSearch ?? 'disabled', 'features.default_mode_request_user_input': false, 'tools.experimental_request_user_input.enabled': false, 'orchestrator.skills.enabled': false, 'skills.bundled.enabled': false, 'features.goals': false }, }; @@ -217,12 +256,40 @@ export class SessionRuntime { // Cache only attempts that cross the native boundary, including ambiguity. if (!this.state || this.closed || this.checkpointing) return Promise.reject(new Error('Session runtime is unavailable')); if (rootTurnBusy(this.state, this.starting)) return Promise.reject(new Error('The root agent already has an active turn')); - const promise = this.startTurn(turn, input, this.state.rootThreadId, settings); + // Reserve admission while the read-only capability lookup runs, but cache only + // native attempts. Unsupported tier requests can be corrected and retried. + this.starting = true; + let nativeAttempt = false; + const promise = this.admitTier(settings?.service_tier ?? this.options.request.serviceTier, settings?.model ?? this.options.request.model).then(serviceTier => { + if (!this.state || this.closed || this.checkpointing) throw new Error('Session runtime is unavailable'); + nativeAttempt = true; + return this.startTurn(turn, input, this.state.rootThreadId, serviceTier, settings); + }).catch(error => { if (!nativeAttempt && this.started.get(turn.id) === promise) this.started.delete(turn.id); throw error; }) + .finally(() => { this.starting = false; }); this.started.set(turn.id, promise); return promise; } - private async startTurn(turn: Turn, input: AgentSessionInputMessageParam[], rootThreadId: string, settings?: SessionModelSettings): Promise { + private async admitTier(tier: string | undefined, model: string | undefined) { + if (nativeServiceTier(tier) === 'default') return 'default' as const; + const configuration = await this.rpc.call('config/read', { includeLayers: false }); + if (!record(configuration) || !record(configuration.config) || (configuration.config.model_provider ?? 'openai') !== this.options.request.modelProvider) throw new Error('Native model catalogue provider differs from the Session provider'); + const models: unknown[] = []; + const cursors = new Set(); + let cursor: string | undefined; + do { + const page = await this.rpc.call('model/list', { includeHidden: true, limit: 100, ...(cursor ? { cursor } : {}) }); + if (!record(page) || !Array.isArray(page.data) || models.length + page.data.length > 10_000 + || (page.nextCursor !== undefined && page.nextCursor !== null && (typeof page.nextCursor !== 'string' || !page.nextCursor))) throw new Error('Native model capabilities are unavailable'); + models.push(...page.data); + cursor = typeof page.nextCursor === 'string' ? page.nextCursor : undefined; + if (cursor && (cursors.has(cursor) || cursors.size >= 100)) throw new Error('Native model pagination repeated'); + if (cursor) cursors.add(cursor); + } while (cursor); + return admitNativeServiceTier(tier, { model, provider: this.options.request.modelProvider, models }); + } + + private async startTurn(turn: Turn, input: AgentSessionInputMessageParam[], rootThreadId: string, serviceTier: 'default' | 'priority' | 'flex', settings?: SessionModelSettings): Promise { this.starting = true; this.startingTurn = turn; clearTimeout(this.idleTimer); @@ -241,7 +308,7 @@ export class SessionRuntime { ...(settings?.reasoning.effort === null ? { collaborationMode: { mode: 'default', settings: { model: settings.model, reasoning_effort: null, developer_instructions: null }, } } : {}), - serviceTier: settings?.service_tier ?? request.serviceTier, summary: request.reasoningSummary, + serviceTier, summary: request.reasoningSummary, ...(request.outputSchema ? { outputSchema: request.outputSchema } : {}), }); if (!record(result) || !record(result.turn) || typeof result.turn.id !== 'string') throw new Error('Native session returned no turn'); @@ -287,7 +354,29 @@ export class SessionRuntime { public async cancel(turnId: string): Promise { const binding = this.binding(turnId); if (terminal(binding.turn)) return; - await this.rpc.call('turn/interrupt', { threadId: binding.threadId, turnId: binding.nativeTurnId }); + // The stock interrupt acknowledgement can precede its terminal notification. + // Observe the exact existing binding before admitting another public Turn. + const stopped = this.waitForTerminal(binding.threadId, binding.nativeTurnId); + try { await this.rpc.call('turn/interrupt', { threadId: binding.threadId, turnId: binding.nativeTurnId }); await stopped.promise; } + finally { stopped.release(); } + } + + private waitForTerminal(threadId: string, nativeTurnId: string) { + let timer: ReturnType | undefined; + let observe = () => {}; + const release = () => { clearTimeout(timer); this.terminalWaiters.delete(observe); }; + const promise = new Promise((resolve, reject) => { + observe = () => { + const binding = this.state?.turns.find(value => value.threadId === threadId && value.nativeTurnId === nativeTurnId); + if (this.closed) { release(); reject(new Error('Executor closed before interruption settled')); } + else if (binding && terminal(binding.turn)) { release(); resolve(); } + }; + this.terminalWaiters.add(observe); + timer = setTimeout(() => { release(); reject(new Error('Native interruption terminal notification timed out')); }, 30_000); + observe(); + }); + void promise.catch(() => {}); + return { promise, release }; } public async toolResult(result: ToolResult): Promise { @@ -318,6 +407,7 @@ export class SessionRuntime { } private event(event: CodexRpcEvent): void { + if (this.checkpointing && checkpointHasActivity(event)) this.checkpointActivity = true; if (!this.state) { this.earlyEvents.push(event); return; } const nativeTurn = record(event.params.turn) ? event.params.turn.id : event.params.turnId; // Hydration notifications can arrive after thread/fork returns, even while @@ -369,6 +459,7 @@ export class SessionRuntime { } private async serverRequest(event: CodexRpcEvent & { requestId: string | number }): Promise { + if (this.checkpointing) { this.checkpointActivity = true; throw new Error('Native interaction is unavailable during checkpoint shutdown'); } if (event.method === 'currentTime/read') return { currentTimeAt: this.now() }; if (!this.declaredFunction(event)) { // Approval-shaped requests fail closed; a guest never widens its own envelope. @@ -407,6 +498,7 @@ export class SessionRuntime { private publish() { if (!this.state) return; this.options.changed?.(this.state); + for (const observe of [...this.terminalWaiters]) observe(); if (this.closed) return; if (this.state.turns.some((binding) => !terminal(binding.turn))) { clearTimeout(this.idleTimer); this.idleTimer = undefined; } // Connected environments receive host keep-alives between turns. Only an @@ -415,6 +507,31 @@ export class SessionRuntime { } } +function checkpointThreadIds(value: unknown): string[] { + if (!Array.isArray(value) || value.length > 1000 + || value.some(id => typeof id !== 'string' || !id.length || id.length > 256 || id.trim() !== id) + || new Set(value).size !== value.length) throw new Error('Invalid native checkpoint thread identities'); + return [...value] as string[]; +} + + + +function checkpointThreadPage(value: unknown): { ids: string[]; nextCursor?: string } { + if (!record(value)) throw new Error('Invalid native thread list'); + const ids = checkpointThreadIds(value.data); + const cursor = value.nextCursor; + if (ids.length > 100 || cursor != null && (typeof cursor !== 'string' || !cursor.length || cursor.length > 256 || !ids.length)) { + throw new Error('Invalid native thread pagination'); + } + return { ids, ...(typeof cursor === 'string' ? { nextCursor: cursor } : {}) }; +} + +function checkpointHasActivity(event: CodexRpcEvent): boolean { + return event.requestId !== undefined || event.method.startsWith('item/') + || ['thread/started', 'turn/started', 'turn/completed', 'rawResponseItem/completed', 'rawResponse/completed', 'error'].includes(event.method) + || event.method === 'thread/status/changed' && record(event.params.status) && event.params.status.type === 'active'; +} + function nativeInput(input: AgentSessionInputMessageParam[]) { return input.flatMap((message) => message.content.map((part) => part.type === 'input_text' ? { type: 'text', text: part.text } : { type: 'image', url: part.image_url })); } function terminal(turn: Turn): boolean { return ['completed', 'cancelled', 'failed'].includes(turn.status); } function record(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); } diff --git a/src/runner/workspace-handoff.ts b/src/runner/workspace-handoff.ts index ee40d68..9bd32f5 100644 --- a/src/runner/workspace-handoff.ts +++ b/src/runner/workspace-handoff.ts @@ -1,3 +1,5 @@ +import type { NativeCheckpointEvidence } from '../domain/native-checkpoint.js'; +import { sameNativeThreadIds } from '../domain/native-checkpoint.js'; import type { WorkspaceCheckpoint } from '../domain/workspace-checkpoint.js'; /** Shared by native starts, tool responses and file operations when the host enables handoff. */ @@ -35,11 +37,12 @@ export interface WorkspaceHandoffPorts { admission: WorkspaceAdmission; inspectIdle(): { idle: boolean; threadIds: string[] } | Promise<{ idle: boolean; threadIds: string[] }>; begin(): Promise; - shutdownNative(): Promise; + shutdownNative(): Promise; + verifyNative?(): Promise; stopRemainingWriters(): Promise; flushJournal(): Promise; syncStorage(): Promise; - commit(): Promise; + commit(evidence?: NativeCheckpointEvidence): Promise; /** Must retain an already committed checkpoint if the commit acknowledgement was lost. */ abort(): Promise; /** Runtime completion must be held until this is called, including on failure. */ @@ -58,13 +61,15 @@ export async function checkpointWorkspace(ports: WorkspaceHandoffPorts): Promise await ports.begin(); nativeStarted = true; const acknowledgement = await ports.shutdownNative(); - if (!acknowledgement.flushed || acknowledgement.forced || acknowledgement.threadIds.length !== idle.threadIds.length - || new Set(acknowledgement.threadIds).size !== idle.threadIds.length - || idle.threadIds.some(id => !acknowledgement.threadIds.includes(id))) throw new Error('Native shutdown did not acknowledge every thread flush'); + const cold = 'kind' in acknowledgement && acknowledgement.kind === 'codex-eof'; + if (!sameNativeThreadIds(acknowledgement.threadIds, idle.threadIds) || (!cold && (!('flushed' in acknowledgement) || acknowledgement.flushed !== true || acknowledgement.forced !== false || 'kind' in acknowledgement))) throw new Error('Native shutdown did not acknowledge every thread flush'); + if (cold && !ports.verifyNative) throw new Error('Cold native verification is unavailable'); await ports.stopRemainingWriters(); await ports.flushJournal(); await ports.syncStorage(); - return await ports.commit(); + const evidence = cold ? await ports.verifyNative!() : undefined; + if (cold && (!evidence || evidence.kind !== 'codex-cold-native' || !Array.isArray(evidence.threadIds) || !sameNativeThreadIds(evidence.threadIds, idle.threadIds) || !/^[a-f0-9]{64}$/.test(evidence.historyDigest) || !/^[a-f0-9]{64}$/.test(evidence.hostJournalDigest) || !Array.isArray(evidence.historyModes) || !evidence.historyModes.length || evidence.historyModes.some(mode => !['legacy', 'paginated'].includes(mode)))) throw new Error('Cold native verification evidence is invalid'); + return await ports.commit(evidence); } catch (error) { if (beginAttempted) { // If this fails, the durable lease/tombstone remains recoverable only after diff --git a/testing/README.md b/testing/README.md index bf4d360..5b95912 100644 --- a/testing/README.md +++ b/testing/README.md @@ -21,27 +21,30 @@ using a customer's third-party account. Keep it out of ordinary deployments. ## Agents API native compatibility gate -`npm run test:agents:parity` runs the Agents suites with strict native interruption +`npm run test:agents:parity:host` runs the Agents suites with strict native interruption assertions. A stock harness that rejects an interrupted child's follow-up at the configured limit fails this gate, even when the ordinary diagnostic suite passes. The fixture model servers are local and do not make paid model calls. +The worker-image harness runs `npm run test:agents:parity`, which also verifies +the installed Pi child at `/opt/agent-runtime/pi-child.mjs` under worker UID 10001. +That packaging check requires the image; host parity excludes only that test. -The ordered native patches and exact upstream revision are in `runtime/codex/`. -See the [Codex runtime maintenance plan](../runtime/codex/README.md) for patch -contracts, platform upgrade gates and rollback. -Prepare or build them in a dedicated directory with sufficient free space: +The official release version, upstream revision and full package inventory are in +`runtime/codex/`. See the [Codex runtime maintenance guide](../runtime/codex/README.md) +for adapter contracts, upgrade gates and rollback. Package the installed official +release into a new directory and test those exact bytes: ```bash -npm run codex:build -- /path/to/empty/build-directory --prepare-only -npm run codex:build -- /path/to/empty/build-directory -CODEX_CONFORMANCE_BINARY=/path/to/empty/build-directory/runtime/bin/codex npm run test:agents:parity +npm run codex:build -- /path/to/new-runtime-directory +CODEX_CONFORMANCE_BINARY=/path/to/new-runtime-directory/runtime/bin/codex npm run test:agents:parity:host ``` -The builder never resets an existing checkout. It records the upstream commit, -patch hashes and every packaged file hash. The complete runtime preserves the -matching upstream companion host and resources. Source preparation alone is not -verification of the patch. Build the same source for the worker's Linux ARM64 -baseline, then package the image inputs: +The historical command name packages the official binary; it does not compile +Rust or apply patches. It refuses an existing output directory. The export +contains `artifact.json` and `runtime/`, including all matching companions and +resources. Validation rejects modified bytes even if their manifest hashes agree. + +Prepare the Linux ARM64 artifact for Lambda and worker-image packaging: ```bash docker buildx build --platform linux/arm64 --file runtime/codex/Dockerfile \ @@ -49,20 +52,15 @@ docker buildx build --platform linux/arm64 --file runtime/codex/Dockerfile \ npm run package ``` -The builder defaults to one Cargo job; increase `CARGO_BUILD_JOBS` only on a host -with sufficient memory. Compilation caches are disposable; retain the exported -`runtime/` and `artifact.json` together. `CODEX_RUNTIME_ARTIFACT` can select another -export directory. Packaging verifies source, patches, file digests and Linux ARM64 -executables, including the code-mode host and bundled bwrap. The image has no stock -runtime fallback. `npm run package` and therefore `npm run check` require this -artifact. Run the image canary after each accepted artifact or runner change. +`CODEX_RUNTIME_ARTIFACT` can select another export directory. Keep its manifest +and runtime together. `npm run package` and therefore `npm run check` require a +matching ARM64 artifact. Run the image canary after each runtime or runner change. -For an x86 Linux local worker, build the same patched source against the Amazon -Linux 2023 baseline. Keep this artifact separate from the ARM64 cloud artifact: +For an x86 Linux local worker, package the matching official Linux x64 release. +Keep it separate from the ARM64 cloud artifact: ```bash -docker buildx build --platform linux/amd64 --file runtime/codex/Dockerfile \ - --output type=local,dest=.runtime/codex/linux-x64 . +npm run codex:build -- .runtime/codex/linux-x64 npm run runtime:local:build ``` @@ -89,6 +87,14 @@ local model fixtures with Linux dependencies. `MICROVM_E2E_IMAGE_TAG` selects a different worker image. The container enables namespaces for bubblewrap; native processes remain unprivileged and receive no model credentials. +The image harness also runs `tests/runner/codex-cold-verification.test.ts` with +`CODEX_COLD_NATIVE_E2E=true` in a separate root-controlled disposable container. +This exercises the production cold-recovery verifier, writer cleanup, isolated +filesystem copies and corruption rejection for both native history modes. Its +agent processes still run as UID 10001. Never enable this opt-in on the host: +writer cleanup terminates that dedicated UID. CI runs this image check on ARM64 +and x64. + Local readiness also requires `npm run check`, `npm run smoke:local`, [native console checks](native-console/README.md), LocalStack and the ARM64 image canary below. The native gate covers worker behavior; API compatibility and live AWS wiring require their @@ -172,25 +178,19 @@ in a paid tier. Its current [official coverage data](https://github.com/localsta does not document the Lambda MicroVM APIs, so `RunMicrovm`, lifecycle hooks, managed networking, and isolation remain live-AWS-only checks. -CI builds the native artifact on ARM64 and caches the exported runtime by source/build inputs. -The cache is saved after compilation so a later acceptance failure does not force -another cold build. Every restored runtime still passes all acceptance checks. -The pinned companion version is part of `runtime/codex/source.json`; ordinary TypeScript -dependency updates do not invalidate this native cache. -A cold build requires at least 20 GiB free for source, compilation and container layers; the -builder itself checks for 16 GiB before compiling. Standard hosted runners advertise only 14 GB -of storage, so the job removes unused preinstalled SDKs and verifies free space before setup. -If that does not provide enough space, configure the repository variable `RAT_NATIVE_RUNNER` -with an available ARM64 runner label that does. The job's 240-minute limit includes the cold -native build, repository checks and actual worker-image tests. A cached artifact still passes -source, package and checksum validation before packaging. +CI packages the official ARM64 and x64 distributions directly from their pinned +npm packages. It caches complete exports by source, package and packager inputs; +every restored export still passes artifact and behavior checks. There is no +custom Codex compilation or Rust build-space requirement. The ARM64 job uses +`RAT_NATIVE_RUNNER` when configured, otherwise the hosted ARM64 runner, and runs +the repository checks and actual worker-image tests. After the native worker image passes, CI exports `codex-linux-arm64-` as a downloadable artifact containing `linux-arm64.tar.gz`. Extract that tarball into an empty directory and set `CODEX_RUNTIME_ARTIFACT` to that directory, which must contain `artifact.json` and `runtime/`. The tarball preserves executable permissions and hidden companion resources. Packaging verifies source identity, companions and every digest -before writing Lambda archives; an export from a different source or patch is +before writing Lambda archives; an export from another release or with changed official bytes is rejected. Cache presence alone is not native image acceptance. Host-native sandbox checks require an OS that supports the pinned Codex sandbox diff --git a/testing/aws/README.md b/testing/aws/README.md index 49622fc..abb9bb7 100644 --- a/testing/aws/README.md +++ b/testing/aws/README.md @@ -70,7 +70,7 @@ thread after worker termination and verifies fallback to saved public history. T tests ordinary replacement, not adoption of a committed workspace checkpoint. The cases delete their own Sessions and preserve the supplied Agent and deployment. -Build and verify the patched Linux ARM64 artifact before packaging (see +Package and verify the official Linux ARM64 artifact before packaging (see `testing/README.md`). EC2 deployment additionally requires an immutable `AWS_E2E_EC2_WORKER_AMI_ID` and digest-pinned `AWS_E2E_EC2_WORKER_IMAGE`. To exercise prepared AMIs, deploy with `AWS_E2E_ENABLE_EC2_WORKER_AMI_PIPELINE=true` diff --git a/testing/local-runtime/codex-parity.ts b/testing/local-runtime/codex-parity.ts index fc36f17..19c5caf 100644 --- a/testing/local-runtime/codex-parity.ts +++ b/testing/local-runtime/codex-parity.ts @@ -20,7 +20,7 @@ import { nodeSubprocessStdioProbe, restrictedNetworkSocketProbe } from './subpro const { values } = parseArgs({ options: { directory: { type: 'string' }, 'timeout-seconds': { type: 'string', default: '300' }, help: { type: 'boolean' } } }); if (values.help) { - console.log('Usage: CODEX_BINARY=/absolute/patched/codex RAT_THINGS_LINUX_KVM_RUNTIME=/absolute/assets npx tsx testing/local-runtime/codex-parity.ts [--directory EVIDENCE_ROOT] [--timeout-seconds 300]\nReal KVM worker and patched host Codex with a local Responses fixture; includes broken-sandbox refusal for Codex and Pi. No live inference.'); + console.log('Usage: CODEX_BINARY=/absolute/official/codex RAT_THINGS_LINUX_KVM_RUNTIME=/absolute/assets npx tsx testing/local-runtime/codex-parity.ts [--directory EVIDENCE_ROOT] [--timeout-seconds 300]\nReal KVM worker and official host Codex with a local Responses fixture; includes broken-sandbox refusal for Codex and Pi. No live inference.'); process.exit(0); } const timeoutMs = Number(values['timeout-seconds']) * 1000; @@ -146,9 +146,9 @@ function safeError(error: unknown) { let text = error instanceof Error ? `${erro try { assert(sourceAssets && isAbsolute(sourceAssets), 'Set RAT_THINGS_LINUX_KVM_RUNTIME to the real prepared Linux KVM assets'); - assert(binary && isAbsolute(binary), 'Set CODEX_BINARY to the actual patched host Codex executable'); - const artifact = await validateCodexRuntimeArtifact(resolve(process.env.CODEX_RUNTIME_ARTIFACT ?? '.runtime/codex/linux-x64'), { expectedTarget: 'x86_64-unknown-linux-gnu' }); - assert.equal(await hashFile(binary), artifact.manifest.sha256, 'Host binary must match the admitted patched x64 artifact'); + assert(binary && isAbsolute(binary), 'Set CODEX_BINARY to the actual official host Codex executable'); + const artifact = await validateCodexRuntimeArtifact(resolve(process.env.CODEX_RUNTIME_ARTIFACT ?? '.runtime/codex/linux-x64'), { expectedTarget: 'x86_64-unknown-linux-musl' }); + assert.equal(await hashFile(binary), artifact.manifest.sha256, 'Host binary must match the admitted official x64 artifact'); await launch(normalProfile); const outsidePath = `/home/agent/rat-codex-outside-${nonce}`, escapedPath = `/home/agent/rat-codex-symlink-${nonce}`; const content = `Real Codex remote workspace ${nonce}\n`; @@ -229,7 +229,7 @@ finally { if (previousSecret === undefined) delete process.env.RAT_CODEX_PARITY_SECRET; else process.env.RAT_CODEX_PARITY_SECRET = previousSecret; const passed = completed && failure === undefined && cleanupErrors.length === 0; const evidence = join(directory, 'evidence.json'); - await writeFile(evidence, JSON.stringify({ version: 1, status: passed ? 'passed' : 'failed', fixture: 'deterministic local Responses fixture; real patched host Codex and KVM worker', + await writeFile(evidence, JSON.stringify({ version: 1, status: passed ? 'passed' : 'failed', fixture: 'deterministic local Responses fixture; real official host Codex and KVM worker', coverage: 'HTTP Session, remote commands, confinement, cancellation, graceful restart, and shared startup refusal; no Rust UI or live inference claim', phase, runtimeObservations, turns, proofs, restartCalls, refusals, modelCalls: requests.length, piModelCalls: faux.state.callCount, ...(failure === undefined ? {} : { failure: safeError(failure) }), cleanupErrors }, null, 2) + '\n', { mode: 0o600 }); diff --git a/testing/local-runtime/linux-kvm/README.md b/testing/local-runtime/linux-kvm/README.md index 2a9244c..e04d1d1 100644 --- a/testing/local-runtime/linux-kvm/README.md +++ b/testing/local-runtime/linux-kvm/README.md @@ -17,7 +17,7 @@ requires usable `/dev/kvm`, `/dev/vhost-vsock`, a user systemd manager, and enou memory for the runtime's admission checks. The preparation command does not change host permissions or install host packages. -First build the patched Linux x64 Codex artifact and local worker image using the +First package the official Linux x64 Codex artifact and local worker image using the [contributor instructions](../../README.md). Set `DOCKER_HOST` to the local engine socket and unset `DOCKER_CONTEXT` when selecting an endpoint explicitly. Only the immutable image ID resolved at the start of preparation is used. Every @@ -64,20 +64,20 @@ harnesses. Both use the actual default KVM runtime and preserve evidence in a new `run-...` directory: ```bash -CODEX_BINARY=/absolute/path/to/patched-artifact/runtime/bin/codex \ +CODEX_BINARY=/absolute/path/to/official-artifact/runtime/bin/codex \ RAT_THINGS_LINUX_KVM_RUNTIME=/absolute/path/to/image-HASH \ npx tsx testing/local-runtime/pi-parity.ts \ --directory test-results/local-pi-parity -CODEX_BINARY=/absolute/path/to/patched-artifact/runtime/bin/codex \ -CODEX_RUNTIME_ARTIFACT=/absolute/path/to/patched-artifact \ +CODEX_BINARY=/absolute/path/to/official-artifact/runtime/bin/codex \ +CODEX_RUNTIME_ARTIFACT=/absolute/path/to/official-artifact \ RAT_THINGS_LINUX_KVM_RUNTIME=/absolute/path/to/image-HASH \ npx tsx testing/local-runtime/codex-parity.ts \ --directory test-results/local-codex-parity ``` The Pi harness uses a deterministic fixture model. The Codex harness uses a -loopback Responses fixture and verifies the host executable against the patched +loopback Responses fixture and verifies the host executable against the official artifact. Both exercise managed file and command effects, same-UID writes outside the workspace, symlink escape denial, disabled command networking, cancellation of command descendants, and a new Turn after restarting the local service. Pi diff --git a/testing/local-runtime/linux-kvm/init.sh b/testing/local-runtime/linux-kvm/init.sh index c2ee9c0..8193255 100644 --- a/testing/local-runtime/linux-kvm/init.sh +++ b/testing/local-runtime/linux-kvm/init.sh @@ -11,6 +11,10 @@ if [ "${1:-}" != rat-root-mounted ]; then exec chroot . /init rat-root-mounted fi mount -t proc proc /proc +# Resolve the guest's own hostname locally before executor configuration discovery. +printf '%s\n' rat-things-worker > /proc/sys/kernel/hostname +printf '%s\n' rat-things-worker > /etc/hostname +printf '%s\n' '127.0.0.1 localhost rat-things-worker' '::1 localhost rat-things-worker' > /etc/hosts mount -t sysfs sysfs /sys mount -t devtmpfs devtmpfs /dev mkdir -p /dev/pts /dev/shm /run /tmp /workspace /mnt/rat-disk diff --git a/testing/local-runtime/linux-kvm/prepare.ts b/testing/local-runtime/linux-kvm/prepare.ts index 91f4698..6c30869 100644 --- a/testing/local-runtime/linux-kvm/prepare.ts +++ b/testing/local-runtime/linux-kvm/prepare.ts @@ -171,7 +171,7 @@ function meter(limit: number) { export async function prepareLinuxKvm(options: Options): Promise { if (process.platform !== 'linux' || process.arch !== 'x64') throw new Error('Linux KVM preparation requires x86-64 Linux'); - const artifact = await validateCodexRuntimeArtifact(resolve(options.artifact), { expectedTarget: target }); + const artifact = await validateCodexRuntimeArtifact(resolve(options.artifact), { expectedTarget: 'x86_64-unknown-linux-musl' }); const endpoint = !process.env.DOCKER_CONTEXT && process.env.DOCKER_HOST || (JSON.parse((await execute('docker', ['context', 'inspect'])).stdout)[0]?.Endpoints?.docker?.Host as unknown); if (typeof endpoint !== 'string' || !endpoint.startsWith('unix:///')) throw new Error('Preparation requires a local Docker Unix socket'); @@ -275,7 +275,7 @@ function options(arguments_: string[]): Options { } if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { - if (process.argv.includes('--help')) console.log('Development only: prepare a Linux x64 KVM worker from a verified patched local image.\nOptions: --output DIR --image IMAGE --artifact DIR --kernel-inputs DIR --qemu-evidence FILE\nNo VM is started. Existing image generations are never overwritten.'); + if (process.argv.includes('--help')) console.log('Development only: prepare a Linux x64 KVM worker from a verified official-Codex local image.\nOptions: --output DIR --image IMAGE --artifact DIR --kernel-inputs DIR --qemu-evidence FILE\nNo VM is started. Existing image generations are never overwritten.'); else prepareLinuxKvm(options(process.argv.slice(2))).then(directory => console.log(JSON.stringify({ directory, kind: 'linux-kvm-development' }))) .catch(error => { console.error(error instanceof Error ? error.message : 'Linux KVM preparation failed'); process.exitCode = 1; }); } diff --git a/testing/local-runtime/native-guest-build.md b/testing/local-runtime/native-guest-build.md index e027eff..5b2b67b 100644 --- a/testing/local-runtime/native-guest-build.md +++ b/testing/local-runtime/native-guest-build.md @@ -1,7 +1,7 @@ # Build a maintained native guest -Build the Linux ARM64 Codex release artifact first. The artifact must match the -source definition, patched tree policy, and complete companion inventory accepted +Package the official Linux ARM64 Codex release first. The artifact must match the +source definition, npm lock and complete official file inventory accepted by `scripts/codex-runtime-artifact.mjs`. Capture an initial package candidate into a new directory outside `dist`. @@ -14,8 +14,8 @@ node scripts/build-native-guest.mjs record-candidate \ ``` Use `--definition-root /absolute/path/to/runtime/codex` when the checked source -definition is staged separately. That directory must include `source.json`, the -ordered patches, and `artifact-policy.json`. Use `--kernel-input-directory PATH` +definition is staged separately. That directory must include `source.json`. +Use `--kernel-input-directory PATH` to supply `vmlinuz-virt` and `modloop-virt` explicitly. Their locked checksums still apply. Otherwise the builder downloads the two URLs in `runtime/native/kernel-inputs.json`. diff --git a/testing/local-runtime/native-signatures/README.md b/testing/local-runtime/native-signatures/README.md index 0af895a..cc7c5a5 100644 --- a/testing/local-runtime/native-signatures/README.md +++ b/testing/local-runtime/native-signatures/README.md @@ -15,7 +15,7 @@ python3 testing/local-runtime/native-signatures/fixtures.py \ npx vitest run tests/adapters/local-native-verifier.test.ts --no-cache ``` -Read the final JSON line for the result. `evidence/latest-run.txt` names the directory containing `results.json` and `commands.json`. Every accepted fixture must return the exact manifest bytes and all five executable paths. +Read the final JSON line for the result. `evidence/latest-run.txt` names the directory containing `results.json` and `commands.json`. Every accepted fixture must return the exact manifest bytes and every path in the fixed host code policy. The suite preserves the complete Codex package at `Contents/Resources/codex`. Its descriptor stays beside `bin`, `codex-path`, and `codex-resources`. The VM helper stays at `Contents/Helpers/rat-native-worker`. The outer `Runtime.bundle` uses `CFBundlePackageType=BNDL` without `CFBundleExecutable`. diff --git a/testing/local-runtime/native-signatures/fixtures.py b/testing/local-runtime/native-signatures/fixtures.py index 1ddb3a2..0b3abd0 100644 --- a/testing/local-runtime/native-signatures/fixtures.py +++ b/testing/local-runtime/native-signatures/fixtures.py @@ -14,13 +14,8 @@ IDENTITY = 'Developer ID Application: Gregory Pazo (PB8NAVKU57)' BUNDLE_ID = 'dev.rat-things.runtime' -EXECUTABLES = { - 'Contents/Helpers/rat-native-worker': ('vm-helper', BUNDLE_ID + '.vm-helper'), - 'Contents/Resources/codex/bin/codex': ('host-codex', BUNDLE_ID + '.codex'), - 'Contents/Resources/codex/bin/codex-code-mode-host': ('host-companion', BUNDLE_ID + '.code-mode-host'), - 'Contents/Resources/codex/codex-path/rg': ('host-companion', BUNDLE_ID + '.ripgrep'), - 'Contents/Resources/codex/codex-resources/zsh/bin/zsh': ('host-companion', BUNDLE_ID + '.zsh'), -} +POLICY = json.loads((Path(__file__).resolve().parents[3] / 'runtime/native/host-code-policy.json').read_text()) +EXECUTABLES = {item['path']: (item['role'], item['identifier']) for item in POLICY} SIGNATURES = {'CodeDirectory', 'CodeRequirements', 'CodeResources', 'CodeSignature'} MANIFEST = 'Contents/Resources/runtime-manifest.json' LIMITS = [4096, 768 * 1024 * 1024, 2 * 1024 * 1024 * 1024, 128 * 1024, 32, 1024] @@ -98,7 +93,9 @@ def verify(name, bundle, accepted, stage=None, reason=None, limits=LIMITS): source = root / 'fixture.c' source.write_text('int main(void) { return 0; }\n') binary = root / 'fixture' + library = root / 'fixture.dylib' command(['xcrun', 'clang', '-arch', 'arm64', '-arch', 'x86_64', source, '-o', binary]) + command(['xcrun', 'clang', '-dynamiclib', '-arch', 'arm64', '-arch', 'x86_64', source, '-o', library]) base = root / 'good' / 'Runtime.bundle' (base / 'Contents/Resources/licenses').mkdir(parents=True) (base / 'Contents/Info.plist').write_bytes(plistlib.dumps({ @@ -107,7 +104,7 @@ def verify(name, bundle, accepted, stage=None, reason=None, limits=LIMITS): for path, (_, identifier) in EXECUTABLES.items(): target = base / path target.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(binary, target) + shutil.copyfile(library if path.endswith('.dylib') else binary, target) target.chmod(0o755) sign(target, identifier) for path in ['Contents/Resources/kernel', 'Contents/Resources/initramfs', 'Contents/Resources/licenses/LICENSE', @@ -120,7 +117,7 @@ def verify(name, bundle, accepted, stage=None, reason=None, limits=LIMITS): command(['/usr/bin/codesign', '--display', '--requirements', '-', '--verbose=4', base]) for path in EXECUTABLES: command(['/usr/bin/codesign', '--display', '--requirements', '-', '--verbose=4', base / path]) - verify('valid-five-executable-layout', base, True) + verify('valid-pinned-host-code-layout', base, True) verify('production-ceilings-clamp', base, True, limits=[9_007_199_254_740_991] * 6) source_copy = clone('copy-source') installed = root / 'installed' / 'Runtime.bundle' diff --git a/testing/local-runtime/native-vm/worker/README.md b/testing/local-runtime/native-vm/worker/README.md index 1e7a03d..f23b1ec 100644 --- a/testing/local-runtime/native-vm/worker/README.md +++ b/testing/local-runtime/native-vm/worker/README.md @@ -1,6 +1,6 @@ # Native Rat Things worker proof -This opt-in Apple silicon experiment runs the real Rat Things worker and patched Codex +This opt-in Apple silicon experiment runs the real Rat Things worker and official Codex executor inside a Linux VM created by Apple's Virtualization framework. It uses the same `LocalRuntime` port, Session service, environment relay and remote file protocol as the container path. The production runtime factory does not select diff --git a/testing/local-runtime/pi-parity.ts b/testing/local-runtime/pi-parity.ts index 4e5fb9e..06aa203 100644 --- a/testing/local-runtime/pi-parity.ts +++ b/testing/local-runtime/pi-parity.ts @@ -60,7 +60,7 @@ const catalog: PiModelDescriptor[] = [{ id: model, name: 'Deterministic acceptan api: 'openai-codex-responses', input: ['text'], contextWindow: 100000, maxTokens: 10000, reasoning: ['low', 'medium', 'high'] }]; const requestOptions = { maxRetries: 0, timeout: timeoutMs }; const digest = (value: string) => createHash('sha256').update(value).digest('hex'); -const command = (program: string) => ({ command: ['node', '-e', program], cwd: '/workspace' }); +const command = (program: string) => ({ command: ['node', '--disable-warning=UNDICI-EHPA', '-e', program], cwd: '/workspace' }); const proofPath = '/workspace/parity.txt'; const proofText = `Real worker Pi file ${nonce}\n`; const deniedPath = `/home/agent/parity-denied-${nonce}`; diff --git a/testing/local-runtime/subprocess-fixtures.ts b/testing/local-runtime/subprocess-fixtures.ts index d888974..917b8f4 100644 --- a/testing/local-runtime/subprocess-fixtures.ts +++ b/testing/local-runtime/subprocess-fixtures.ts @@ -1,18 +1,21 @@ -/** Exact programs shared with the pinned Codex local-subprocess-stdio regression tests. */ +/** Programs exercising subprocess behavior in the official Codex sandbox. */ export const nodeSubprocessStdioProbe = String.raw` const assert = require('node:assert/strict'); const { spawnSync, spawn } = require('node:child_process'); const child = "let input='';process.stdin.setEncoding('utf8');process.stdin.on('data',chunk=>input+=chunk);process.stdin.on('end',()=>{process.stdout.write('out:'+input);process.stderr.write('err:'+input)});"; -const sync = spawnSync(process.execPath, ['-e', child], { input: 'sync', encoding: 'utf8', timeout: 3000 }); +// Newer Node 22 releases announce their environment proxy on stderr at startup. +// Exclude that known warning while still asserting every byte from the child. +const nodeArgs = program => ['--disable-warning=UNDICI-EHPA', '-e', program]; +const sync = spawnSync(process.execPath, nodeArgs(child), { input: 'sync', encoding: 'utf8', timeout: 3000 }); assert.equal(sync.error, undefined); assert.equal(sync.status, 0); assert.equal(sync.stdout, 'out:sync'); assert.equal(sync.stderr, 'err:sync'); -const failed = spawnSync(process.execPath, ['-e', "console.error('intentional-failure');process.exitCode=7"], { encoding: 'utf8', timeout: 3000 }); +const failed = spawnSync(process.execPath, nodeArgs("console.error('intentional-failure');process.exitCode=7"), { encoding: 'utf8', timeout: 3000 }); assert.equal(failed.error, undefined); assert.equal(failed.status, 7); assert.equal(failed.stderr, 'intentional-failure\n'); -const asyncChild = spawn(process.execPath, ['-e', child]); +const asyncChild = spawn(process.execPath, nodeArgs(child)); let stdout = '', stderr = ''; asyncChild.stdout.on('data', chunk => stdout += chunk); asyncChild.stderr.on('data', chunk => stderr += chunk); @@ -27,9 +30,9 @@ asyncChild.on('close', code => { asyncChild.stdin.end('async'); `; -/** Require precise syscall denials; missing Python, socket families, or sandbox support fail. */ +/** Proxy isolation permits local metadata; actual egress and named Unix sockets remain denied. */ export const restrictedNetworkSocketProbe = String.raw` -import errno, os, socket +import errno, os, socket, urllib.request, urllib.error def denied(operation): try: @@ -46,23 +49,29 @@ os.write(left.fileno(), b'local') assert os.read(right.fileno(), 5) == b'local' left.shutdown(socket.SHUT_WR) assert os.read(right.fileno(), 1) == b'' -# Each mismatching field independently denies, including both mismatching. -denied(lambda: right.getsockopt(socket.IPPROTO_TCP, socket.SO_TYPE)) -denied(lambda: right.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)) -denied(lambda: right.getsockopt(socket.IPPROTO_TCP, socket.SO_ERROR)) -denied(lambda: right.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 65536)) -denied(lambda: right.getpeername()) -denied(lambda: right.shutdown(socket.SHUT_RD)) -denied(lambda: right.shutdown(socket.SHUT_RDWR)) left.close() right.close() -for family in (socket.AF_INET, socket.AF_INET6, socket.AF_VSOCK, socket.AF_NETLINK): +for family in (socket.AF_UNIX, socket.AF_VSOCK, socket.AF_NETLINK): denied(lambda: socket.socket(family, socket.SOCK_STREAM)) +for family in (socket.AF_INET, socket.AF_INET6, socket.AF_VSOCK, socket.AF_NETLINK): denied(lambda: socket.socketpair(family, socket.SOCK_STREAM)) -with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as endpoint: - denied(lambda: endpoint.bind('/tmp/rat-network-denied')) - denied(lambda: endpoint.connect('/tmp/rat-network-denied')) - denied(lambda: endpoint.listen(1)) - denied(lambda: endpoint.accept()) +with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as endpoint: + endpoint.settimeout(1) + try: + endpoint.connect(('192.0.2.1', 443)) + except OSError: + pass + else: + raise AssertionError('direct external connection was allowed') +proxy = os.environ.get('HTTP_PROXY') +assert proxy, 'managed proxy was not installed' +opener = urllib.request.build_opener(urllib.request.ProxyHandler({'http': proxy})) +try: + opener.open('http://example.invalid/', timeout=2) +except urllib.error.HTTPError as error: + assert error.code == 403, error.code + assert b'"reason":"not_allowed"' in error.read() +else: + raise AssertionError('proxy egress was allowed') print('local-socket-only-ok') `; diff --git a/testing/run-native-image-parity.sh b/testing/run-native-image-parity.sh index 94b3ad6..7a66b26 100755 --- a/testing/run-native-image-parity.sh +++ b/testing/run-native-image-parity.sh @@ -16,3 +16,8 @@ docker buildx build --platform linux/arm64 --load \ # native child processes run as the same non-root UID used by the AWS worker. # Fixture HTTP servers are local; no model credentials are forwarded. docker run --rm --platform linux/arm64 --privileged --init "$parity_image" +# Cold recovery is a trusted-controller operation: stop the dedicated guest UID, +# synchronize native storage, then verify a disposable copy in a fresh namespace. +docker run --rm --platform linux/arm64 --privileged --init --user 0 \ + --env CODEX_COLD_NATIVE_E2E=true --entrypoint npx "$parity_image" \ + --no-install vitest run tests/runner/codex-cold-verification.test.ts --no-file-parallelism diff --git a/tests/adapters/codex-managed-policy.test.ts b/tests/adapters/codex-managed-policy.test.ts new file mode 100644 index 0000000..8b2bbed --- /dev/null +++ b/tests/adapters/codex-managed-policy.test.ts @@ -0,0 +1,62 @@ +import { expect, it } from 'vitest'; +import { verifyManagedCodexPolicy } from '../../src/adapters/codex-managed-policy.js'; +import { managedCodexConfiguration, managedCodexPolicyMatches, managedCodexRequirementsSafe, type ManagedCodexPolicy } from '../../src/domain/codex-managed-policy.js'; + +const policy: ManagedCodexPolicy = { workspace: '/workspace', network: { access: 'disabled', allowed_domains: [] }, credentialProxy: false }; + +it('keeps no-egress intent independent from the upstream proxy-enabled bit', () => { + const config = managedCodexConfiguration({ ...policy, credentialProxy: true }); + expect(config.permissions.rat_managed.network).toMatchObject({ enabled: true, domains: {}, allow_upstream_proxy: false, allow_local_binding: false }); + expect(config.features.network_proxy).toEqual({ enabled: true, credential_broker: false }); + expect(managedCodexPolicyMatches({ config }, { requirements: null }, policy)).toBe(true); +}); + +it.each([ + null, {}, { allowedLoginMethods: null }, { allowedLoginMethods: [] }, + { allowedLoginMethods: ['api'] }, { allowedLoginMethods: ['chatgpt'] }, { allowedLoginMethods: ['api', 'chatgpt'] }, + { allowedLoginMethods: ['chatgpt'], network: null, allowedPermissionProfiles: null, modelProvider: null, hooks: null }, +])('accepts absent execution requirements and authentication-only restrictions: %j', requirements => { + const response = { supportsIndependentSpeedModes: true, requirements }; + expect(managedCodexRequirementsSafe(response)).toBe(true); + expect(managedCodexPolicyMatches({ config: managedCodexConfiguration(policy) }, response, policy)).toBe(true); +}); + +it.each([ + { allowedLoginMethods: 'chatgpt' }, { allowedLoginMethods: ['saml'] }, { allowedLoginMethods: ['Chatgpt'] }, + { allowedLoginMethods: ['chatgpt', 'chatgpt'] }, { allowedLoginMethods: [null] }, + { allowedLoginMethods: ['chatgpt'], network: { allowedDomains: ['example.com'] } }, + { allowedLoginMethods: ['chatgpt'], allowedPermissionProfiles: {} }, + { allowedLoginMethods: ['chatgpt'], allowLoginShell: false }, + { allowedLoginMethods: ['chatgpt'], unknownAuthority: null }, + { allowedLoginMethods: ['chatgpt'], unknownAuthority: true }, +])('rejects malformed authentication or additional requirements: %j', requirements => { + expect(managedCodexRequirementsSafe({ requirements })).toBe(false); + expect(managedCodexPolicyMatches({ config: managedCodexConfiguration(policy) }, { requirements }, policy)).toBe(false); +}); + +it.each([{}, { requirements: [] }, { requirements: null, supportsIndependentSpeedModes: 'true' }, { requirements: null, unknownAuthority: null }])( + 'rejects an unknown or malformed requirements response: %j', response => { + expect(managedCodexRequirementsSafe(response)).toBe(false); + }, +); + +it.each([ + { name: 'managed requirements', requirements: { requirements: { network: { allowedDomains: ['example.com'] } } } }, + { name: 'authentication mixed with execution requirements', requirements: { requirements: { allowedLoginMethods: ['chatgpt'], network: { allowedDomains: ['example.com'] } } } }, + { name: 'authentication mixed with unknown requirements', requirements: { requirements: { allowedLoginMethods: ['chatgpt'], unknownAuthority: null } } }, + { name: 'missing requirements response', requirements: {} }, + { name: 'missing policy-layer inventory' }, + { name: 'unknown effective grants', extend: { permissions: { rat_managed: { filesystem: { '/': 'write' } } } } }, + { name: 'unknown policy layer', layer: { name: { type: 'enterpriseManaged' }, config: {} } }, + { name: 'unexpected system config', layer: { name: { type: 'system', file: '/etc/codex/config.toml' }, config: { features: { network_proxy: true } } } }, +])('refuses $name before running a command', async test => { + const methods: string[] = []; + const config = { ...managedCodexConfiguration(policy), ...('extend' in test ? test.extend : {}) }; + await expect(verifyManagedCodexPolicy({ policy, call: async method => { + methods.push(method); + if (method === 'config/read') return { config, layers: 'layer' in test ? [test.layer] : [] }; + if (method === 'configRequirements/read') return 'requirements' in test ? test.requirements : { requirements: null }; + throw new Error('Execution must not start'); + } })).rejects.toThrow('Refusing to expose execution tools'); + expect(methods).toEqual(['config/read', 'configRequirements/read']); +}); diff --git a/tests/adapters/codex-rpc-checkpoint.test.ts b/tests/adapters/codex-rpc-checkpoint.test.ts index ddd9599..5302b7f 100644 --- a/tests/adapters/codex-rpc-checkpoint.test.ts +++ b/tests/adapters/codex-rpc-checkpoint.test.ts @@ -51,3 +51,12 @@ it('does not treat a process already closed by normal cancellation as checkpoint await client.close(); await expect(client.closeCheckpoint()).rejects.toThrow('before checkpoint'); }); + +it.each(['storage', 'protocol'])('rejects late %s failure even when ordinary EOF exits zero', async kind => { + const source = `const reader = require('readline').createInterface({ input: process.stdin }); + reader.on('line', line => { const value = JSON.parse(line); if (value.id) process.stdout.write(JSON.stringify({ id: value.id, result: {} }) + '\\n'); }); + reader.on('close', () => { ${kind === 'storage' ? "process.stderr.write('failed to flush native thread store\\n');" : "process.stdout.write('malformed protocol output\\n');"} });`; + const client = new CodexRpcClient({ binary: process.execPath, binaryArguments: ['-e', source], cwd: process.cwd(), environment: {} }); + try { await client.initialize(); await expect(client.closeCheckpoint()).rejects.toThrow('storage or protocol'); } + finally { await client.close(); } +}); diff --git a/tests/adapters/local-native-verifier.test.ts b/tests/adapters/local-native-verifier.test.ts index 2f312ba..c2af072 100644 --- a/tests/adapters/local-native-verifier.test.ts +++ b/tests/adapters/local-native-verifier.test.ts @@ -2,14 +2,11 @@ import { chmod, mkdtemp, mkdir, readFile, realpath, rm, symlink, writeFile } fro import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, describe, expect, it } from 'vitest'; +import hostCodePolicy from '../../runtime/native/host-code-policy.json' with { type: 'json' }; import { LocalNativeBundleVerifier, NATIVE_VERIFIER_LIMITS } from '../../src/adapters/local-native-verifier.js'; const directories: string[] = []; -const executablePaths = [ - 'Contents/Helpers/rat-native-worker', 'Contents/Resources/codex/bin/codex', - 'Contents/Resources/codex/bin/codex-code-mode-host', 'Contents/Resources/codex/codex-path/rg', - 'Contents/Resources/codex/codex-resources/zsh/bin/zsh', -]; +const executablePaths = hostCodePolicy.map(policy => policy.path); const receipt = (manifest = Buffer.from([0, 255, 128, 13, 10])) => ({ schemaVersion: 1, kind: 'authenticated', manifestBase64: manifest.toString('base64'), executablePaths, }); @@ -43,6 +40,14 @@ describe('trusted native verifier subprocess boundary', () => { expect(await test.verifier.verify(test.input)).toEqual({ kind: 'authenticated', manifestBytes: Buffer.alloc(0), executablePaths }); }); + it('rejects a successful receipt that omits a pinned dylib or adds an unreviewed executable', async () => { + const dylib = executablePaths.find(path => path.endsWith('.dylib'))!; + for (const paths of [executablePaths.filter(path => path !== dylib), [...executablePaths, 'Contents/Resources/codex/codex-resources/zsh/bin/zsh']]) { + const test = await fixture(response({ ...receipt(), executablePaths: paths })); + expect(await test.verifier.verify(test.input)).toEqual(unavailable('Runtime verifier returned an invalid response')); + } + }); + it('uses fixed arguments, clamps production ceilings and clears ambient environment', async () => { const test = await fixture(); const observation = join(test.directory, 'arguments.json'); diff --git a/tests/agents/local-codex-selection.test.ts b/tests/agents/local-codex-selection.test.ts index 76b27fb..e677ff0 100644 --- a/tests/agents/local-codex-selection.test.ts +++ b/tests/agents/local-codex-selection.test.ts @@ -6,6 +6,7 @@ import { createLocalServices, LOCAL_OWNER } from '../../src/app/local-compositio import type { LocalRuntime } from '../../src/core/local-runtime-ports.js'; import { AgentsApiError } from '../../src/domain/agents-api-validation.js'; import { developmentLocalCodexHost } from '../../src/runner/local-codex.js'; +import { managedCodexConfiguration } from '../../src/domain/codex-managed-policy.js'; afterEach(() => vi.unstubAllEnvs()); @@ -28,7 +29,12 @@ lines.on('line', line => { fs.appendFileSync(${JSON.stringify(log)}, JSON.stringify({ method: request.method, binary: process.argv[1], path: process.env.PATH }) + '\\n'); if (request.id === undefined) return; let result = {}; - if (request.method === 'config/read') result = { config: {} }; + if (request.method === 'config/read') result = ${JSON.stringify({ + config: managedCodexConfiguration({ workspace: '/workspace', network: { access: 'disabled', allowed_domains: [] }, credentialProxy: false }), + layers: [{ name: { type: 'sessionFlags' }, config: {} }, + ...['user', 'system'].map(type => ({ name: { type, file: join(directory, type, 'config.toml') }, config: {} }))], + })}; + if (request.method === 'configRequirements/read') result = { requirements: null }; if (request.method === 'account/read') result = { account: null }; if (request.method === 'model/list') result = { data: [{ id: 'fixture-model', isDefault: true }] }; if (request.method === 'thread/start' || request.method === 'thread/resume') result = { thread: { id: 'fixture-thread' } }; diff --git a/tests/agents/local-native-startup.test.ts b/tests/agents/local-native-startup.test.ts index 3ea8a95..42c6095 100644 --- a/tests/agents/local-native-startup.test.ts +++ b/tests/agents/local-native-startup.test.ts @@ -125,7 +125,7 @@ it('pins the copied host and companion paths without accepting ambient executabl const host = service.hostCodex(); const bundle = join(test.profile, 'runtime/versions', test.bundle.id, 'Runtime.bundle'); expect(host.binary).toBe(join(bundle, 'Contents/Helpers/codex')); - expect(host.environment.PATH).toBe(`${bundle}/Contents/Resources/codex/codex-path:${bundle}/Contents/Resources/codex/codex-resources/zsh/bin:/usr/bin:/bin:/usr/sbin:/sbin`); + expect(host.environment.PATH).toBe(`${bundle}/Contents/Resources/codex/codex-path:/usr/bin:/bin:/usr/sbin:/sbin`); expect(host.environment.HTTPS_PROXY).toBe('http://proxy.example'); expect(Object.isFrozen(host)).toBe(true); expect(Object.isFrozen(host.environment)).toBe(true); diff --git a/tests/agents/local-turn-admission.test.ts b/tests/agents/local-turn-admission.test.ts index 9b0e343..a03756e 100644 --- a/tests/agents/local-turn-admission.test.ts +++ b/tests/agents/local-turn-admission.test.ts @@ -9,6 +9,7 @@ import { SessionService } from '../../src/core/session-service.js'; import type { SessionState } from '../../src/core/session-ports.js'; import type { SessionRuntimeState } from '../../src/core/session-runtime-planning.js'; import type { Turn } from '../../src/domain/agents-api.js'; +import { managedCodexConfiguration } from '../../src/domain/codex-managed-policy.js'; import { LocalManagedEnvironment } from '../../src/runner/local-managed-environment.js'; import { LocalSessionExecution } from '../../src/runner/local-session-execution.js'; import { developmentLocalCodexHost } from '../../src/runner/local-codex.js'; @@ -26,6 +27,12 @@ vi.mock('../../src/runner/local-codex.js', async original => ({ ...await origina emit: (event: CodexRpcEvent) => options.onEvent?.(event), call: async (method: string) => { rpc.calls.push(method); + if (method === 'config/read') return { + config: managedCodexConfiguration({ workspace: '/workspace', network: { access: 'disabled', allowed_domains: [] }, credentialProxy: false }), + layers: [{ name: { type: 'sessionFlags' }, config: {} }, + ...['user', 'system'].map(type => ({ name: { type, file: join(options.cwd!, type, 'config.toml') }, config: {} }))], + }; + if (method === 'configRequirements/read') return { requirements: null }; if (method === 'turn/start') { await rpc.turn?.(); return { turn: { id: `native_turn_${rpc.calls.length}` } }; } if (method === 'turn/interrupt') await rpc.interrupt?.(); if (method === 'thread/start' || method === 'thread/resume') return { thread: { id: `thread_${index}` } }; diff --git a/tests/agents/native-checkpoint.test.ts b/tests/agents/native-checkpoint.test.ts new file mode 100644 index 0000000..d555ca5 --- /dev/null +++ b/tests/agents/native-checkpoint.test.ts @@ -0,0 +1,284 @@ +import { createServer, type ServerResponse } from 'node:http'; +import { once } from 'node:events'; +import { cp, mkdtemp, rename, rm, readFile, readdir } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; +import { expect, it } from 'vitest'; +import { CodexRpcClient, type CodexRpcEvent } from '../../src/adapters/codex-rpc.js'; +import type { SessionRuntimeState } from '../../src/core/session-runtime-planning.js'; +import type { Turn } from '../../src/domain/agents-api.js'; +import type { WorkspaceCheckpoint } from '../../src/domain/workspace-checkpoint.js'; +import { SessionRuntime, type SessionCheckpointOptions } from '../../src/runner/session-runtime.js'; +import { WorkspaceAdmission } from '../../src/runner/workspace-handoff.js'; +import { assertNativeHistory, nativeCheckpointEvidence } from '../../src/domain/native-checkpoint.js'; +import { syncWorkspaceStorage } from '../../src/runner/workspace-sync.js'; +import { createHash } from 'node:crypto'; + +// Uses the official unmodified binary. Model traffic stays on loopback. +// Host-only copies prove native continuation; the privileged guest test covers isolation. +const native = it.skipIf(process.env.CODEX_REQUIRE_PARITY !== 'true'); +const binary = process.env.CODEX_CONFORMANCE_BINARY ?? resolve('node_modules/.bin/codex'); +const sessionId = 'native_checkpoint_session'; +const agentId = 'native_checkpoint_agent'; +type ModelRequest = { input: Array<{ type?: string; recipient?: string; role?: string; content?: unknown }> }; + +native.each([{ paginated: false }, { paginated: true }])('checkpoints real root and child histories and continues them in a fresh process (paginated=$paginated)', async ({ paginated }) => { + const fixture = await checkpointFixture({ paginated }); + try { + const source = fixture.create(); + await source.initialize(); + await source.start(turn('first'), input('ROOT_INPUT_MARKER: delegate the fixture task.')); + await until(() => { + const state = source.snapshot(); + return state.subagents.length === 1 && state.turns.length === 2 && state.turns.every(binding => binding.turn.status === 'completed') + && state.subagents.every(child => fixture.completedChildActivities.has(`${state.rootThreadId}:${child.id}`)); + }, 'root and child completion'); + const before = source.snapshot(); + const childId = before.subagents[0]!.id; + const expectedIds = [before.rootThreadId, childId].sort(); + + const checkpoint = await source.checkpoint() as WorkspaceCheckpoint; + await source.finished; + source.assertSuccessfulShutdown(); + expect(checkpoint.rootThreadId).toBe(before.rootThreadId); + expect(checkpoint.nativeDurability?.historyModes).toEqual(paginated ? ['paginated'] : ['legacy', 'paginated']); + expect(fixture.acknowledgedIds).toEqual([expectedIds]); + // These host ports record ordering; this proves native journal handoff, not + // an AWS filesystem snapshot, host journal implementation or worker replacement. + expect(fixture.order).toEqual(['begin', 'native-exit', 'writers', 'journal', 'storage-sync', 'cold-verified', 'commit']); + await expect(fixture.clients[0]!.call('thread/loaded/list', {})).rejects.toThrow('unavailable'); + + const successor = fixture.create(before); + const resumed = await successor.initialize(); + expect(resumed.rootThreadId).toBe(before.rootThreadId); + expect(fixture.clients).toHaveLength(2); + const freshClient = fixture.clients[1]!; + const rootHistory = await freshClient.call('thread/turns/list', { threadId: before.rootThreadId, limit: 100, itemsView: 'full' }); + const childHistory = await freshClient.call('thread/turns/list', { threadId: childId, limit: 100, itemsView: 'full' }); + expect(JSON.stringify(rootHistory)).toContain('ROOT_DURABLE_MARKER'); + expect(JSON.stringify(childHistory)).toContain('CHILD_DURABLE_MARKER'); + + await successor.start(turn('continued'), input('CONTINUE_CHECKPOINT: continue the existing child and root.')); + await until(() => { + const state = successor.snapshot(); + return state.turns.length === 4 && state.turns.every(binding => binding.turn.status === 'completed'); + }, 'resumed root and child completion'); + const after = successor.snapshot(); + expect(after.subagents.map(child => child.id)).toEqual([childId]); + expect(after.rootThreadId).toBe(before.rootThreadId); + expect(fixture.rootRequests).toHaveLength(4); + expect(fixture.childRequests).toHaveLength(2); + // The model inputs in the successor process must contain disk-restored + // native history, independently of the host's saved SessionRuntimeState. + expect(JSON.stringify(fixture.rootRequests[2])).toContain('ROOT_DURABLE_MARKER'); + expect(JSON.stringify(fixture.childRequests[1])).toContain('CHILD_DURABLE_MARKER'); + expect(JSON.stringify(after.turns.filter(binding => binding.turn.id === 'continued'))).toContain('ROOT_CONTINUED'); + expect(JSON.stringify(after.turns.filter(binding => binding.threadId === childId))).toContain('CHILD_CONTINUED'); + } finally { await fixture.close(); } +}, 60_000); + +native('rejects a busy child without shutting down either thread', async () => { + const fixture = await checkpointFixture({ paginated: false, holdChild: true }); + try { + const runtime = fixture.create(); + const rootId = (await runtime.initialize()).rootThreadId; + await runtime.start(turn('first'), input('ROOT_INPUT_MARKER: delegate the fixture task.')); + await until(() => { + const state = runtime.snapshot(); + return fixture.childRequests.length === 1 && state.subagents.length === 1 + && state.turns.some(binding => binding.turn.id === 'first' && binding.turn.status === 'completed') + && state.turns.some(binding => binding.turn.subagent_id !== null && binding.turn.status === 'in_progress'); + }, 'busy child with idle root'); + const childId = runtime.snapshot().subagents[0]!.id; + const rpc = fixture.clients[0]!; + await expect(runtime.checkpoint()).rejects.toThrow('active turns'); + expect(fixture.order).toEqual([]); + const loaded = await rpc.call('thread/loaded/list', { limit: 100 }) as { data: string[] }; + expect([...loaded.data].sort()).toEqual([rootId, childId].sort()); + expect(runtime.snapshot().turns.find(binding => binding.threadId === childId)?.turn.status).toBe('in_progress'); + fixture.releaseChild(); + await until(() => runtime.snapshot().turns.every(binding => binding.turn.status === 'completed') + && fixture.completedChildActivities.has(`${rootId}:${childId}`), 'child completion after refused shutdown'); + await expect(runtime.checkpoint()).resolves.toMatchObject({ rootThreadId: rootId }); + expect(fixture.acknowledgedIds).toEqual([[rootId, childId].sort()]); + expect(fixture.order.at(-1)).toBe('commit'); + } finally { await fixture.close(); } +}, 45_000); + +async function checkpointFixture(options: { paginated: boolean; holdChild?: boolean }) { + const historyMode = options.paginated ? 'paginated' : 'legacy'; + const home = await mkdtemp(join(tmpdir(), 'rat-native-checkpoint-')); + const rootRequests: ModelRequest[] = []; + const childRequests: ModelRequest[] = []; + const order: string[] = []; + const acknowledgedIds: string[][] = []; + const clients: CodexRpcClient[] = []; + const runtimes: SessionRuntime[] = []; + const completedChildActivities = new Set(); + let releaseChild = () => {}; + const server = createServer(async (request, response) => { + try { + const chunks: Buffer[] = []; for await (const chunk of request) chunks.push(Buffer.from(chunk)); + if (!request.url?.endsWith('/responses')) { response.writeHead(404).end(); return; } + const body = JSON.parse(Buffer.concat(chunks).toString()) as ModelRequest; + const child = body.input.some(item => item.type === 'agent_message' && item.recipient === '/root/checkpoint_child'); + const requests = child ? childRequests : rootRequests; + requests.push(body); + const count = requests.length; + const id = `${child ? 'child' : 'root'}_${count}`; + const item = !child && count === 1 + ? tool(id, 'spawn_agent', { task_name: 'checkpoint_child', message: 'CHILD_INPUT_MARKER: complete the child task.', fork_turns: 'none' }) + : !child && count === 3 + ? tool(id, 'followup_task', { target: 'checkpoint_child', message: 'Continue the persisted child task.' }) + : { type: 'message', role: 'assistant', id: `message_${id}`, phase: 'final_answer', content: [{ type: 'output_text', + text: child ? count === 1 ? 'CHILD_DURABLE_MARKER' : 'CHILD_CONTINUED' : count === 2 ? 'ROOT_DURABLE_MARKER' : 'ROOT_CONTINUED' }] }; + response.writeHead(200, { 'content-type': 'text/event-stream' }); + response.write(`data: ${JSON.stringify({ type: 'response.created', response: { id } })}\n\n`); + const finish = () => completeResponse(response, id, item); + if (child && count === 1 && options.holdChild) releaseChild = finish; + else finish(); + } catch { response.destroy(new Error('Local checkpoint model fixture failed')); } + }); + server.listen(0, '127.0.0.1'); await once(server, 'listening'); + const address = server.address(); if (!address || typeof address === 'string') throw new Error('Missing fixture address'); + const create = (previous?: SessionRuntimeState) => { + const checkpointPorts: SessionCheckpointOptions = { + admission: new WorkspaceAdmission(), + prepareNativeVerification: async () => ({ + close: async () => {}, + verify: async manifest => { + const copyRoot = await mkdtemp(join(tmpdir(), 'rat-cold-checkpoint-copy-')); + const candidate = join(copyRoot, 'candidate'); + const parked = join(copyRoot, 'original'); + const count = rootRequests.length + childRequests.length; + await cp(home, candidate, { recursive: true }); + const digest = await fileDigest(candidate); + await rename(home, parked); + let verifier: CodexRpcClient | undefined; + let activity = false; + try { + await cp(candidate, home, { recursive: true }); + verifier = new CodexRpcClient({ binary, cwd: home, environment: { PATH: process.env.PATH, HOME: home, CODEX_HOME: home }, + onEvent: event => { if (event.requestId !== undefined || /^(turn\/(started|completed)|item\/|error$)/.test(event.method)) activity = true; } }); + await verifier.initialize(); + for (const expected of manifest.threads) { + const result = await verifier.call('thread/read', { threadId: expected.id, includeTurns: true }) as { thread: unknown }; + assertNativeHistory(expected, result.thread, String(expected.id)); + } + for (const expected of manifest.threads) { + await verifier.call('thread/resume', { threadId: expected.id, deferGoalContinuation: true, + ...(expected.id === manifest.rootThreadId ? { config: { 'model_providers.fixture': providerConfig(address.port), 'features.multi_agent': true, 'features.multi_agent_v2': true, 'features.goals': false } } : {}) }); + const result = await verifier.call('thread/read', { threadId: expected.id, includeTurns: true }) as { thread: unknown }; + assertNativeHistory(expected, result.thread, manifest.rootThreadId); + } + await verifier.closeCheckpoint(); + expect(activity).toBe(false); + expect(rootRequests.length + childRequests.length).toBe(count); + expect(await fileDigest(candidate)).toBe(digest); + acknowledgedIds.push(manifest.threads.map(thread => String(thread.id)).sort()); + order.push('cold-verified'); + return nativeCheckpointEvidence(manifest); + } finally { + await verifier?.close(); await rm(home, { recursive: true, force: true }); await rename(parked, home); + await rm(copyRoot, { recursive: true, force: true }); + } + }, + }), + begin: async () => { order.push('begin'); }, + stopRemainingWriters: async () => { order.push('writers'); }, + flushJournal: async () => { order.push('journal'); }, + syncStorage: async () => { await syncWorkspaceStorage(home); order.push('storage-sync'); }, + commit: async (state, nativeDurability) => { + order.push('commit'); + return { ...(nativeDurability ? { nativeDurability } : {}), id: 'native-checkpoint', source: { runId: 'native-source', generation: 'generation' }, sandboxId: 'native-sandbox', + setupDigest: 'a'.repeat(64), policyDigest: 'b'.repeat(64), rootThreadId: state.rootThreadId, + lastRootTurnId: state.turns.filter(binding => binding.turn.subagent_id === null).at(-1)?.turn.id ?? null, + lastNativeTurnId: state.turns.filter(binding => binding.turn.subagent_id === null).at(-1)?.nativeTurnId ?? null, + createdAt: 1, reason: 'hibernate' }; + }, + abort: async () => { order.push('abort'); }, + }; + const runtime = new SessionRuntime({ sessionId, agentId, lifetime: 'host-managed', checkpoint: checkpointPorts, + ...(previous ? { previous, requiredResumeThreadId: previous.rootThreadId } : {}), + request: { binary, workspace: home, environment: { PATH: process.env.PATH, HOME: home, CODEX_HOME: home }, + timeoutMs: 45_000, persistent: true, prompt: '', sandbox: 'read-only', networkAccess: false, environments: [], model: 'gpt-5.5', modelProvider: 'fixture', + sessionConfig: { 'model_providers.fixture': { name: 'local checkpoint fixture', base_url: `http://127.0.0.1:${address.port}`, + wire_api: 'responses', requires_openai_auth: false, supports_websockets: false, request_max_retries: 0, stream_max_retries: 0 }, + 'features.multi_agent': true, 'features.multi_agent_v2': { enabled: true, max_concurrent_threads_per_session: 2 }, + 'features.goals': false }, + }, + client: options => { + const rpc = new CodexRpcClient({ ...options, onEvent: event => { + // A child's turn/completed can precede the parent's final subAgentActivity + // item/completed. Starting checkpoint between them correctly fails closed. + // Wait for that real notification rather than assuming terminal Turns + // imply that every native writer/event has settled. + const activity = completedChildActivity(event); + if (activity) completedChildActivities.add(activity); + options.onEvent?.(event); + } }); + clients.push(rpc); + // Observe the real subprocess transport; never substitute RPC results. + return { initialize: () => rpc.initialize(), close: () => rpc.close(), + call: async (method, params, timeout) => { + if (method === 'thread/checkpoint') throw new Error('Private checkpoint RPC is forbidden'); + return rpc.call(method, method === 'thread/start' ? { ...(params as object), historyMode } : params, timeout); + }, + closeCheckpoint: async () => { await rpc.closeCheckpoint(); order.push('native-exit'); }, + }; + }, + }); + runtimes.push(runtime); return runtime; + }; + return { create, clients, rootRequests, childRequests, order, acknowledgedIds, completedChildActivities, releaseChild: () => releaseChild(), + close: async () => { + releaseChild(); + for (const runtime of runtimes) await runtime.close(); + server.closeAllConnections(); await new Promise(done => server.close(() => done())); + await rm(home, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + }, + }; +} + +function completedChildActivity(event: CodexRpcEvent): string | undefined { + const { threadId, item } = event.params; + if (event.method !== 'item/completed' || typeof threadId !== 'string' || !item || typeof item !== 'object' + || !('type' in item) || item.type !== 'subAgentActivity' || !('kind' in item) || item.kind !== 'completed' + || !('agentThreadId' in item) || typeof item.agentThreadId !== 'string') return undefined; + return `${threadId}:${item.agentThreadId}`; +} + +function completeResponse(response: ServerResponse, id: string, item: unknown): void { + if (response.writableEnded || response.destroyed) return; + response.end([{ type: 'response.output_item.done', item }, { type: 'response.completed', response: { id } }] + .map(event => `data: ${JSON.stringify(event)}\n\n`).join('')); +} + +function tool(id: string, name: string, args: Record) { + return { type: 'function_call', id: `function_${id}`, call_id: `call_${id}`, namespace: 'collaboration', name, arguments: JSON.stringify(args) }; +} + +function turn(id: string): Turn { + return { id, object: 'agent.session.turn', session_id: sessionId, agent_id: agentId, subagent_id: null, + status: 'queued', created_at: 1, started_at: null, completed_at: null, error: null, usage: null }; +} + +function input(text: string): import('../../src/domain/agents-api.js').AgentSessionInputMessageParam[] { + return [{ role: 'user', content: [{ type: 'input_text', text }] }]; +} + +async function until(predicate: () => boolean, description: string): Promise { + for (let attempt = 0; attempt < 1_500; attempt++) { if (predicate()) return; await delay(10); } + throw new Error(`Timed out waiting for ${description}`); +} + +function providerConfig(port: number) { return { name: 'local checkpoint fixture', base_url: `http://127.0.0.1:${port}`, wire_api: 'responses', requires_openai_auth: false, supports_websockets: false, request_max_retries: 0, stream_max_retries: 0 }; } +async function fileDigest(root: string): Promise { + const hash = createHash('sha256'); + const visit = async (directory: string): Promise => { for (const entry of (await readdir(directory, { withFileTypes: true })).sort((a, b) => a.name.localeCompare(b.name))) { + const path = join(directory, entry.name); hash.update(path.slice(root.length)); + if (entry.isDirectory()) await visit(path); else if (entry.isFile()) hash.update(await readFile(path)); else throw new Error('Unexpected fixture file'); + } }; + await visit(root); return hash.digest('hex'); +} diff --git a/tests/agents/native-item-events.test.ts b/tests/agents/native-item-events.test.ts index f4d7d8d..95965d3 100644 --- a/tests/agents/native-item-events.test.ts +++ b/tests/agents/native-item-events.test.ts @@ -27,12 +27,15 @@ it.each([false, true])('retains native Item boundaries with interruption=%s', as { type: 'response.output_item.done', output_index: 2, item: message('answer_fixture', 'final_answer', 'Done') }, { type: 'response.completed', response: { id: 'response_fixture' } }, ]; + let requests = 0; const server = createServer(async (request, response) => { for await (const _chunk of request) { /* Drain the fixture request. */ } if (!request.url?.endsWith('/responses')) { response.writeHead(404).end(); return; } + requests++; + const holdOpen = interrupted && requests === 1; response.writeHead(200, { 'content-type': 'text/event-stream' }); - const body = (interrupted ? wireEvents.slice(0, 7) : wireEvents).map(event => `data: ${JSON.stringify(event)}\n\n`).join(''); - if (interrupted) response.write(body); + const body = (holdOpen ? wireEvents.slice(0, 7) : wireEvents).map(event => `data: ${JSON.stringify(event)}\n\n`).join(''); + if (holdOpen) response.write(body); else response.end(body); }); server.listen(0, '127.0.0.1'); await once(server, 'listening'); @@ -55,6 +58,8 @@ it.each([false, true])('retains native Item boundaries with interruption=%s', as if (interrupted) { for (let attempt = 0; attempt < 1_000 && !runtime.snapshot().turns[0]?.items.some(item => item.type === 'message' && item.content.some(part => part.type === 'output_text' && part.text === 'Working')); attempt++) await delay(10); await runtime.cancel('turn'); + // Awaited cancellation includes projection of the terminal public state. + expect(runtime.snapshot().turns[0]?.turn.status).toBe('cancelled'); } const status = interrupted ? 'cancelled' : 'completed'; for (let attempt = 0; attempt < 1_000 && runtime.snapshot().turns[0]?.turn.status !== status; attempt++) await delay(10); @@ -81,6 +86,16 @@ it.each([false, true])('retains native Item boundaries with interruption=%s', as expect(done[1]).toMatchObject({ item: { phase: 'commentary' } }); if (!interrupted) expect(done[2]).toMatchObject({ item: { phase: 'final_answer' } }); expect(events.indexOf(done.at(-1)!)).toBeLessThan(events.findIndex(event => event.type === `agent.session.turn.${status}`)); + if (interrupted) { + const thread = runtime.snapshot().rootThreadId; + await runtime.start({ id: 'followup', object: 'agent.session.turn', session_id: 'session', agent_id: 'agent', subagent_id: null, + status: 'queued', created_at: 2, started_at: null, completed_at: null, error: null, usage: null }, + [{ role: 'user', content: [{ type: 'input_text', text: 'Continue after the acknowledged cancellation.' }] }]); + await expect.poll(() => runtime.snapshot().turns[1]?.turn.status, { timeout: 10_000, interval: 10 }).toBe('completed'); + expect(runtime.snapshot().rootThreadId).toBe(thread); + expect(runtime.snapshot().turns.map(binding => binding.turn.status)).toEqual(['cancelled', 'completed']); + expect(requests).toBe(2); + } } finally { await runtime.close(); server.closeAllConnections(); await new Promise(resolve => server.close(() => resolve())); diff --git a/tests/agents/native-managed-policy.test.ts b/tests/agents/native-managed-policy.test.ts new file mode 100644 index 0000000..8cdc296 --- /dev/null +++ b/tests/agents/native-managed-policy.test.ts @@ -0,0 +1,65 @@ +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { expect, it } from 'vitest'; +import { CodexRpcClient } from '../../src/adapters/codex-rpc.js'; +import { verifyManagedCodexPolicy } from '../../src/adapters/codex-managed-policy.js'; +import { hostedCodexArguments } from '../../src/runner/hosted-environment-planning.js'; +import type { ManagedCodexPolicy } from '../../src/domain/codex-managed-policy.js'; +import { localCodexArguments } from '../../src/runner/local-codex.js'; + +it.each(['clean', 'restricted-uppercase', 'untrusted-project', 'inherited-domain', 'inherited-filesystem', 'inherited-credential-broker', 'private-rules'] as const)( + 'checks the actual official executor policy before tools: %s', async mode => { + const root = await mkdtemp(join(tmpdir(), 'rat-native-policy-')); + const home = join(root, 'home'), workspace = join(root, 'workspace'); + await mkdir(home); await mkdir(workspace); + const policy: ManagedCodexPolicy = { workspace, network: mode === 'restricted-uppercase' + ? { access: 'restricted', allowed_domains: ['API.Example.com'] } : { access: 'disabled', allowed_domains: [] }, credentialProxy: false }; + if (mode === 'untrusted-project') { + await mkdir(join(workspace, '.codex', 'rules'), { recursive: true }); + await writeFile(join(home, 'config.toml'), `[projects.${JSON.stringify(workspace)}]\ntrust_level="trusted"\n`); + await writeFile(join(workspace, '.codex', 'config.toml'), '[features.network_proxy.domains]\n"1.1.1.1"="allow"\n'); + await writeFile(join(workspace, '.codex', 'rules', 'default.rules'), 'network_rule(host="1.1.1.1", protocol="http", decision="allow")\n'); + } + if (mode === 'inherited-domain') await writeFile(join(home, 'config.toml'), '[features.network_proxy.domains]\n"1.1.1.1"="allow"\n'); + if (mode === 'inherited-filesystem') await writeFile(join(home, 'config.toml'), `[permissions.rat_managed.filesystem]\n${JSON.stringify(root)}="write"\n`); + if (mode === 'inherited-credential-broker') await writeFile(join(home, 'config.toml'), '[features.network_proxy]\ncredential_broker=true\ndangerously_allow_all_unix_sockets=true\n'); + if (mode === 'private-rules') { + await mkdir(join(home, 'rules')); + await writeFile(join(home, 'rules', 'default.rules'), 'network_rule(host="1.1.1.1", protocol="http", decision="allow")\n'); + } + const rpc = new CodexRpcClient({ binary: process.env.CODEX_CONFORMANCE_BINARY ?? resolve('node_modules/.bin/codex'), + cwd: workspace, binaryArguments: hostedCodexArguments(undefined, policy.network, false, workspace), + environment: { PATH: process.env.PATH, HOME: home, CODEX_HOME: home, CODEX_API_KEY: 'no-inference-policy-fixture' }, + onServerRequest: async () => { throw new Error('Unexpected approval'); }, + }); + try { + await rpc.initialize(); + const verified = verifyManagedCodexPolicy({ call: rpc.call.bind(rpc), policy }); + if (mode === 'clean' || mode === 'restricted-uppercase' || mode === 'untrusted-project') await expect(verified).resolves.toBeUndefined(); + else await expect(verified).rejects.toThrow('Refusing to expose execution tools'); + } finally { await rpc.close(); await rm(root, { recursive: true, force: true }); } + }, 15_000); + +it.each(['chatgpt', 'api'] as const)('accepts the official %s login restriction with a separate guest workspace', async login => { + const root = await mkdtemp(join(tmpdir(), 'rat-native-remote-policy-')); + const home = join(root, 'home'); + // The host harness has no guest workspace mounted locally. Policy verification + // must preserve its remote path rather than substituting the host process cwd. + const workspace = join(root, 'absent-guest-workspace'); + await mkdir(home); + const policy: ManagedCodexPolicy = { workspace, network: { access: 'disabled', allowed_domains: [] }, credentialProxy: false }; + const arguments_ = hostedCodexArguments(localCodexArguments(['app-server']), policy.network, false, workspace); + // Keep the final override last: the production local arguments intentionally force ChatGPT. + if (login === 'api') arguments_.splice(arguments_.length - 1, 0, '-c', 'forced_login_method="api"'); + const rpc = new CodexRpcClient({ binary: process.env.CODEX_CONFORMANCE_BINARY ?? resolve('node_modules/.bin/codex'), + cwd: home, binaryArguments: arguments_, environment: { PATH: process.env.PATH, HOME: home, CODEX_HOME: home }, + onServerRequest: async () => { throw new Error('Unexpected approval'); }, + }); + try { + await rpc.initialize(); + const requirements = await rpc.call('configRequirements/read', {}); + expect(requirements).toMatchObject({ requirements: { allowedLoginMethods: [login] } }); + await expect(verifyManagedCodexPolicy({ call: rpc.call.bind(rpc), policy })).resolves.toBeUndefined(); + } finally { await rpc.close(); await rm(root, { recursive: true, force: true }); } +}, 15_000); diff --git a/tests/agents/native-process-lifetime.test.ts b/tests/agents/native-process-lifetime.test.ts index 804097f..3082601 100644 --- a/tests/agents/native-process-lifetime.test.ts +++ b/tests/agents/native-process-lifetime.test.ts @@ -49,6 +49,7 @@ it('keeps an environment command process alive across completed root Turns', asy const runtime = new SessionRuntime({ sessionId: 'sess_process', agentId: 'agent_process', request: { ...planned, binary: process.env.CODEX_CONFORMANCE_BINARY ?? resolve('node_modules/.bin/codex'), binaryArguments: hostedCodexArguments(['app-server'], { access: 'enabled', allowed_domains: [] }, false, home), + managedPolicy: { workspace: home, network: { access: 'enabled', allowed_domains: [] }, credentialProxy: false }, workspace: home, executionWorkspace: home, environments: [{ environmentId: 'local', cwd: home }], environment: { PATH: process.env.PATH, HOME: home, CODEX_HOME: home }, timeoutMs: 20000, persistent: true, prompt: '', sandbox: 'read-only', model: 'gpt-5.4', modelProvider: 'fixture', networkAccess: true, diff --git a/tests/agents/native-protocol-exports.test.ts b/tests/agents/native-protocol-exports.test.ts new file mode 100644 index 0000000..4495a07 --- /dev/null +++ b/tests/agents/native-protocol-exports.test.ts @@ -0,0 +1,70 @@ +import { execFile } from 'node:child_process'; +import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { promisify } from 'node:util'; +import { expect, it } from 'vitest'; + +const native = it.skipIf(process.env.CODEX_REQUIRE_PARITY !== 'true'); +const binary = process.env.CODEX_CONFORMANCE_BINARY ?? resolve('node_modules/.bin/codex'); +const execute = promisify(execFile); +type Schema = { + definitions?: Record; + properties?: Record; + oneOf?: Schema[]; + anyOf?: Schema[]; + enum?: string[]; + required?: string[]; + type?: string; + items?: Schema; + $ref?: string; +}; + +native.each([false, true])('exports the official public protocol with experimental=%s', async experimental => { + const home = await mkdtemp(join(tmpdir(), 'rat-native-protocol-exports-')); + const jsonDir = join(home, 'json'); + const tsDir = join(home, 'typescript'); + const flags = experimental ? ['--experimental'] : []; + try { + for (const [command, directory] of [['generate-json-schema', jsonDir], ['generate-ts', tsDir]] as const) { + await execute(binary, ['app-server', command, '--out', directory, ...flags], { + cwd: home, env: { PATH: process.env.PATH, HOME: home, CODEX_HOME: home }, + timeout: 15_000, killSignal: 'SIGKILL', maxBuffer: 1024 * 1024, + }); + } + const requests = await schema(join(jsonDir, 'ClientRequest.json')); + const notifications = await schema(join(jsonDir, 'ServerNotification.json')); + const flatBundle = await schema(join(jsonDir, 'codex_app_server_protocol.v2.schemas.json')); + const bundleText = await readFile(join(jsonDir, 'codex_app_server_protocol.schemas.json'), 'utf8'); + const requestTypes = compactTypeScript(await readFile(join(tsDir, 'ClientRequest.ts'), 'utf8')); + const errorTypes = await readFile(join(tsDir, 'v2/CodexErrorInfo.ts'), 'utf8'); + const checkpointRequests = requests.oneOf?.filter(request => request.properties?.method?.enum?.includes('thread/checkpoint')); + + for (const errorSchema of [notifications.definitions?.CodexErrorInfo, flatBundle.definitions?.CodexErrorInfo]) { + const errors = errorSchema?.anyOf?.flatMap(variant => variant.enum ?? []); + expect(errors).toEqual(expect.arrayContaining(['usageLimitExceeded', 'contextWindowExceeded', 'other'])); + for (const privateError of ['requestTimeout', 'creditBalanceExhausted', 'executorVersionIncompatible']) expect(errors).not.toContain(privateError); + } + for (const value of ['usageLimitExceeded', 'contextWindowExceeded', 'other']) expect(errorTypes).toContain(`"${value}"`); + const methods = requests.oneOf?.flatMap(request => request.properties?.method?.enum ?? []); + expect(methods).toEqual(expect.arrayContaining(['model/list', 'thread/start', 'thread/read', 'thread/resume', 'thread/loaded/list', 'thread/turns/list', 'turn/start', 'turn/interrupt'])); + expect(checkpointRequests).toEqual([]); + for (const text of [JSON.stringify(requests), JSON.stringify(flatBundle), bundleText, requestTypes]) { + for (const privateName of ['thread/checkpoint', 'ThreadCheckpointParams', 'ThreadCheckpointResponse']) expect(text).not.toContain(privateName); + } + for (const name of ['ThreadCheckpointParams', 'ThreadCheckpointResponse']) { + await expect(readFile(join(jsonDir, `v2/${name}.json`))).rejects.toMatchObject({ code: 'ENOENT' }); + await expect(readFile(join(tsDir, `v2/${name}.ts`))).rejects.toMatchObject({ code: 'ENOENT' }); + } + } finally { + await rm(home, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } +}, 45_000); + +async function schema(path: string): Promise { + return JSON.parse(await readFile(path, 'utf8')) as Schema; +} + +function compactTypeScript(value: string): string { + return value.replace(/\/\/[^\n]*|\/\*[\s\S]*?\*\//g, '').replace(/\s+/g, ''); +} diff --git a/tests/agents/native-session-settings.test.ts b/tests/agents/native-session-settings.test.ts index 8ec9e28..2a2c254 100644 --- a/tests/agents/native-session-settings.test.ts +++ b/tests/agents/native-session-settings.test.ts @@ -8,8 +8,7 @@ import { expect, it } from 'vitest'; import { SessionRuntime } from '../../src/runner/session-runtime.js'; import type { SessionModelSettings } from '../../src/domain/session-execution.js'; -// This contract requires the repository's patched runtime, not the npm CLI. -it.skipIf(process.env.CODEX_REQUIRE_PARITY !== 'true')('changes model, reasoning effort and service tier across Turns on the same native thread', async () => { +it.each(['auto', 'priority'] as const)('changes supported settings on the same official native thread from initial tier %s', async initialTier => { const requests: Array> = []; const server = createServer(async (request, response) => { const chunks: Buffer[] = []; for await (const chunk of request) chunks.push(Buffer.from(chunk)); @@ -26,19 +25,26 @@ it.skipIf(process.env.CODEX_REQUIRE_PARITY !== 'true')('changes model, reasoning server.listen(0, '127.0.0.1'); await once(server, 'listening'); const address = server.address(); if (!address || typeof address === 'string') throw new Error('Missing fixture address'); const home = await mkdtemp(join(tmpdir(), 'rat-native-settings-')); + const provider = { name: 'local fixture', base_url: `http://127.0.0.1:${address.port}`, wire_api: 'responses', requires_openai_auth: false, supports_websockets: false }; const runtime = new SessionRuntime({ sessionId: 'session', agentId: 'agent', request: { binary: process.env.CODEX_CONFORMANCE_BINARY ?? resolve('node_modules/.bin/codex'), workspace: home, + // model/list uses startup provider configuration, before thread/start overrides. + binaryArguments: ['-c', 'model_provider="fixture"', ...Object.entries(provider).flatMap(([key, value]) => ['-c', `model_providers.fixture.${key}=${JSON.stringify(value)}`]), 'app-server'], environment: { PATH: process.env.PATH, HOME: home, CODEX_HOME: home }, timeoutMs: 20_000, - prompt: '', persistent: true, sandbox: 'read-only', networkAccess: false, environments: [], model: 'gpt-5.4', modelProvider: 'fixture', - sessionConfig: { 'model_providers.fixture': { name: 'local fixture', base_url: `http://127.0.0.1:${address.port}`, wire_api: 'responses', requires_openai_auth: false, supports_websockets: false } }, + prompt: '', persistent: true, sandbox: 'read-only', networkAccess: false, environments: [], model: 'gpt-5.5', modelProvider: 'fixture', serviceTier: initialTier, + // The host's explicit capability enables a later admitted priority tier even + // when inherited settings disabled fast mode; it does not select a paid tier. + sessionConfig: { 'model_providers.fixture': provider, 'features.fast_mode': false }, } }); const settings: SessionModelSettings[] = [ - { model: 'gpt-5.4', reasoning: { effort: 'high' }, service_tier: 'priority' }, + { model: 'gpt-5.5', reasoning: { effort: 'high' }, service_tier: 'priority' }, { model: 'gpt-6-astra', reasoning: { effort: 'low' }, service_tier: 'flex' }, { model: 'gpt-6-astra', reasoning: { effort: 'low' }, service_tier: 'auto' }, + { model: 'gpt-6-astra', reasoning: { effort: 'high' }, service_tier: 'fast' }, { model: 'gpt-6-astra', reasoning: { effort: 'high' }, service_tier: 'default' }, - { model: 'gpt-6-astra', reasoning: { effort: null }, service_tier: 'fast' }, - ...(['none', 'minimal', 'medium', 'xhigh', 'max'] as const).map(effort => ({ model: 'gpt-6-astra', reasoning: { effort }, service_tier: 'auto' as const })), + { model: 'gpt-6-astra', reasoning: { effort: 'high' }, service_tier: 'priority' }, + { model: 'gpt-6-astra', reasoning: { effort: null }, service_tier: 'auto' }, + ...(['medium', 'xhigh', 'max'] as const).map(effort => ({ model: 'gpt-6-astra', reasoning: { effort }, service_tier: 'default' as const })), ]; const turnRequests: Array> = []; try { @@ -56,13 +62,30 @@ it.skipIf(process.env.CODEX_REQUIRE_PARITY !== 'true')('changes model, reasoning turnRequests.push(requests.at(-1)!); } expect(turnRequests.map(request => ({ model: request.model, reasoning: request.reasoning, service_tier: request.service_tier }))).toEqual([ - { model: 'gpt-5.4', reasoning: expect.objectContaining({ effort: 'high' }), service_tier: 'priority' }, + { model: 'gpt-5.5', reasoning: expect.objectContaining({ effort: 'high' }), service_tier: 'priority' }, { model: 'gpt-6-astra', reasoning: expect.objectContaining({ effort: 'low' }), service_tier: 'flex' }, - { model: 'gpt-6-astra', reasoning: expect.objectContaining({ effort: 'low' }), service_tier: 'auto' }, - { model: 'gpt-6-astra', reasoning: expect.objectContaining({ effort: 'high' }), service_tier: 'default' }, - { model: 'gpt-6-astra', reasoning: expect.objectContaining({ effort: 'low' }), service_tier: 'fast' }, - ...(['none', 'minimal', 'medium', 'xhigh', 'max'] as const).map(effort => ({ model: 'gpt-6-astra', reasoning: expect.objectContaining({ effort }), service_tier: 'auto' })), + { model: 'gpt-6-astra', reasoning: expect.objectContaining({ effort: 'low' }), service_tier: undefined }, + { model: 'gpt-6-astra', reasoning: expect.objectContaining({ effort: 'high' }), service_tier: 'priority' }, + { model: 'gpt-6-astra', reasoning: expect.objectContaining({ effort: 'high' }), service_tier: undefined }, + { model: 'gpt-6-astra', reasoning: expect.objectContaining({ effort: 'high' }), service_tier: 'priority' }, + { model: 'gpt-6-astra', reasoning: expect.objectContaining({ effort: 'low' }), service_tier: undefined }, + ...(['medium', 'xhigh', 'max'] as const).map(effort => ({ model: 'gpt-6-astra', reasoning: expect.objectContaining({ effort }), service_tier: undefined })), ]); + const count = requests.length; + const rejectedTurn = { id: 'retry-after-admission', object: 'agent.session.turn' as const, session_id: 'session', agent_id: 'agent', subagent_id: null, + status: 'queued' as const, created_at: 1, started_at: null, completed_at: null, error: null, usage: null }; + const input = [{ role: 'user' as const, content: [{ type: 'input_text' as const, text: 'Continue after a rejected setting.' }] }]; + await expect(runtime.start(rejectedTurn, input, { model: 'custom-uncatalogued-model', reasoning: { effort: 'low' }, service_tier: 'priority' })) + .rejects.toMatchObject({ status: 400, code: 'invalid_request', param: 'agent.service_tier' }); + expect(requests).toHaveLength(count); + expect(runtime.snapshot().rootThreadId).toBe(thread); + expect(runtime.snapshot().turns.some(binding => binding.turn.id === rejectedTurn.id)).toBe(false); + // Admission made no native attempt, so correcting that same public Turn is safe. + await runtime.start(rejectedTurn, input, { model: 'gpt-6-astra', reasoning: { effort: 'low' }, service_tier: 'priority' }); + for (let n = 0; n < 1_000 && runtime.snapshot().turns.find(binding => binding.turn.id === rejectedTurn.id)?.turn.status !== 'completed'; n++) await delay(10); + expect(runtime.snapshot().turns.find(binding => binding.turn.id === rejectedTurn.id)?.turn.status).toBe('completed'); + expect(requests.at(-1)).toMatchObject({ model: 'gpt-6-astra', service_tier: 'priority' }); + expect(runtime.snapshot().rootThreadId).toBe(thread); } finally { await runtime.close(); server.closeAllConnections(); await new Promise(resolve => server.close(() => resolve())); diff --git a/tests/agents/native-tools.test.ts b/tests/agents/native-tools.test.ts index 12124df..20edaf6 100644 --- a/tests/agents/native-tools.test.ts +++ b/tests/agents/native-tools.test.ts @@ -59,10 +59,14 @@ it.each((['fixture', 'amazon-bedrock'] as const).flatMap(provider => (['programm const home = await mkdtemp(join(tmpdir(), 'rat-native-tools-')); const tokenFile = join(home, 'fixture-token'); await writeFile(tokenFile, 'local-test-token', { mode: 0o600 }); - const model = provider === 'amazon-bedrock' ? 'openai.gpt-5.6-terra' : 'gpt-5.4'; + // This fixture exercises native web/tool search and optional code mode. The + // active gpt-5.5 catalogue entry supports these together; code-mode-only models do not. + const model = provider === 'amazon-bedrock' ? 'openai.gpt-5.6-terra' : 'gpt-5.5'; const planned = planSessionLaunch(planCodexLaunch({ version: '1', prompt: '', agent: { sandbox: 'read-only', capabilities: { webSearch: 'cached' } } }, home, 20_000, {}), { sessionId: 'sess_tools', turnId: 'turn_tools', input: [], environment: { type: 'none' }, - agent: sessionAgent({ model, reasoning: { effort: 'high', summary: 'concise' }, service_tier: 'priority', + // Tool conformance uses standard execution on both providers. Explicit tier + // admission and cross-Turn resets are covered by native-session-settings. + agent: sessionAgent({ model, reasoning: { effort: 'high', summary: 'concise' }, service_tier: 'auto', text: { verbosity: 'low', format: { type: 'json_schema', schema: { type: 'object', properties: { answer: { type: 'string' } }, required: ['answer'], additionalProperties: false } } }, tools: [ { type: 'web_search', mode: 'cached', context_size: 'low', allowed_domains: ['example.com'], location: { country: 'US', city: 'Seattle' } }, mode === 'programmatic' ? { type: 'programmatic_tool_calling', enabled: true } : { type: 'tool_search' }, @@ -100,9 +104,10 @@ it.each((['fixture', 'amazon-bedrock'] as const).flatMap(provider => (['programm expect(JSON.stringify(requests[1]?.input)).toContain('tool_search_output'); expect(JSON.stringify(requests[1]?.input)).toContain('lookup'); } - expect(requests[0]).toMatchObject({ model, ...(provider === 'fixture' || process.env.CODEX_REQUIRE_PARITY === 'true' ? { service_tier: 'priority' } : {}), reasoning: { effort: 'high', summary: 'concise' }, text: { + expect(requests[0]).toMatchObject({ model, reasoning: { effort: 'high', summary: 'concise' }, text: { verbosity: 'low', format: { type: 'json_schema', schema: { type: 'object', required: ['answer'] } }, } }); + for (const request of requests) expect(request).not.toHaveProperty('service_tier'); expect(requests[0]?.tools).toEqual(expect.arrayContaining([expect.objectContaining({ type: 'web_search', external_web_access: false, search_context_size: 'low', filters: { allowed_domains: ['example.com'] }, user_location: expect.objectContaining({ country: 'US', city: 'Seattle' }), })])); diff --git a/tests/agents/native-turn-errors.test.ts b/tests/agents/native-turn-errors.test.ts index f6f1dc9..4fab553 100644 --- a/tests/agents/native-turn-errors.test.ts +++ b/tests/agents/native-turn-errors.test.ts @@ -8,29 +8,33 @@ import { expect, it } from 'vitest'; import { SessionRuntime } from '../../src/runner/session-runtime.js'; import type { Turn } from '../../src/domain/agents-api.js'; -const cases: Array<{ code: string; expected: NonNullable['code']; status?: number; patched?: boolean }> = [ +const cases: Array<{ code: string; expected: NonNullable['code']; status?: number; native?: unknown }> = [ { code: 'context_length_exceeded', expected: 'context_length_exceeded' }, { code: 'insufficient_quota', expected: 'usage_limit_exceeded' }, { code: 'cyber_policy', expected: 'cyber_policy' }, { code: 'rate_limit_exceeded', expected: 'rate_limit_exceeded' }, + { code: 'slow_down', expected: 'rate_limit_exceeded' }, + { code: 'slow_down', expected: 'rate_limit_exceeded', status: 503 }, { code: 'server_is_overloaded', expected: 'server_overloaded' }, { code: 'rate_limit_exceeded', expected: 'rate_limit_exceeded', status: 429 }, { code: 'server_error', expected: 'server_error', status: 500 }, - { code: 'credit_balance_exhausted', expected: 'credit_balance_exhausted', patched: true }, - { code: 'credit_balance_exhausted', expected: 'credit_balance_exhausted', status: 402, patched: true }, - { code: 'invalid_api_key', expected: 'authentication_error', status: 401, patched: true }, - { code: 'model_not_found', expected: 'resource_not_found', status: 404, patched: true }, - { code: 'invalid_request_error', expected: 'invalid_request', status: 400, patched: true }, - { code: 'context_length_exceeded', expected: 'context_length_exceeded', status: 400, patched: true }, - { code: 'insufficient_quota', expected: 'usage_limit_exceeded', status: 429, patched: true }, - { code: 'access_denied', expected: 'authentication_error', status: 403, patched: true }, - { code: 'service_unavailable', expected: 'server_overloaded', status: 503, patched: true }, - { code: 'timeout', expected: 'request_timeout', status: 408, patched: true }, - { code: 'gateway_timeout', expected: 'request_timeout', status: 504, patched: true }, + // Stock Codex deliberately collapses detailed quota causes into usageLimitExceeded. + { code: 'credit_balance_exhausted', expected: 'usage_limit_exceeded', native: 'usageLimitExceeded' }, + { code: 'credit_balance_exhausted', expected: 'internal_error', status: 402, native: { httpConnectionFailed: { httpStatusCode: 402 } } }, + { code: 'invalid_api_key', expected: 'authentication_error', status: 401 }, + { code: 'model_not_found', expected: 'resource_not_found', status: 404 }, + { code: 'invalid_request_error', expected: 'internal_error', status: 400, native: 'other' }, + { code: 'context_length_exceeded', expected: 'internal_error', status: 400, native: 'other' }, + { code: 'insufficient_quota', expected: 'usage_limit_exceeded', status: 429 }, + { code: 'access_denied', expected: 'authentication_error', status: 403 }, + { code: 'service_unavailable', expected: 'server_overloaded', status: 503 }, + { code: 'timeout', expected: 'request_timeout', status: 408 }, + { code: 'gateway_timeout', expected: 'request_timeout', status: 504 }, + // Unknown provider aliases remain opaque; do not parse diagnostics for their names. ...(['server_error', 'request_timeout', 'session_budget_exceeded', 'usage_limit_exceeded', 'server_overloaded'] as const) - .map(code => ({ code, expected: code, patched: true })), + .map(code => ({ code, expected: 'internal_error' as const, native: 'other' })), ]; -for (const scenario of cases) it.skipIf(scenario.patched && process.env.CODEX_REQUIRE_PARITY !== 'true')(`preserves provider failure ${scenario.code} over ${scenario.status ?? 'SSE'}`, async () => { +for (const scenario of cases) it(`projects official provider failure ${scenario.code} over ${scenario.status ?? 'SSE'} without inventing detail`, async () => { let requests = 0; const nativeErrors: unknown[] = []; const server = createServer(async (request, response) => { @@ -48,7 +52,7 @@ for (const scenario of cases) it.skipIf(scenario.patched && process.env.CODEX_RE const runtime = new SessionRuntime({ sessionId: 'session', agentId: 'agent', request: { binary: process.env.CODEX_CONFORMANCE_BINARY ?? resolve('node_modules/.bin/codex'), workspace: home, environment: { PATH: process.env.PATH, HOME: home, CODEX_HOME: home }, timeoutMs: 15_000, - prompt: '', persistent: true, sandbox: 'read-only', networkAccess: false, environments: [], model: 'gpt-5.4', modelProvider: 'fixture', + prompt: '', persistent: true, sandbox: 'read-only', networkAccess: false, environments: [], model: 'gpt-5.5', modelProvider: 'fixture', onEvent: event => { if (event.method === 'error' || event.method === 'turn/completed') nativeErrors.push(event); }, sessionConfig: { 'model_providers.fixture': { name: 'local fixture', base_url: `http://127.0.0.1:${address.port}`, wire_api: 'responses', requires_openai_auth: false, supports_websockets: false, stream_max_retries: 0, request_max_retries: 0 } }, } }); @@ -61,6 +65,10 @@ for (const scenario of cases) it.skipIf(scenario.patched && process.env.CODEX_RE const turn = runtime.snapshot().turns[0]?.turn; expect(turn, JSON.stringify(nativeErrors)).toMatchObject({ status: 'failed', error: { code: scenario.expected } }); expect(turn?.error?.message).not.toContain('PRIVATE_PROVIDER_DIAGNOSTIC'); + if (scenario.native !== undefined) expect(nativeErrors).toContainEqual(expect.objectContaining({ + method: 'turn/completed', params: expect.objectContaining({ turn: expect.objectContaining({ error: expect.objectContaining({ codexErrorInfo: scenario.native }) }) }), + })); + // These fixtures explicitly disable native retries; this is not a retry-policy assertion. expect(requests).toBe(1); } finally { await runtime.close(); server.closeAllConnections(); diff --git a/tests/agents/sdk-conformance.test.ts b/tests/agents/sdk-conformance.test.ts index 65fc46e..a2b9798 100644 --- a/tests/agents/sdk-conformance.test.ts +++ b/tests/agents/sdk-conformance.test.ts @@ -95,7 +95,9 @@ describe('OpenAI SDK Agents API conformance', () => { expect(replaced.reasoning).toEqual({ effort: 'low', summary: 'auto' }); const reset = await alice.beta.agents.update(agent.id, { reasoning: null }); expect(reset.reasoning).toEqual(agent.reasoning); - expect((await alice.beta.agents.create({ model: 'gpt-5.4' })).reasoning.effort).toBe('medium'); + expect((await alice.beta.agents.create({ model: 'gpt-6-sol' })).reasoning.effort).toBe('medium'); + // Models absent from the pinned native catalogue retain the provider default. + expect((await alice.beta.agents.create({ model: 'gpt-5.4' })).reasoning.effort).toBeNull(); }); it('isolates owners on reads, updates, deletes, and pagination cursors', async () => { diff --git a/tests/agents/session-item-events.test.ts b/tests/agents/session-item-events.test.ts index bf529b1..e2a5b6d 100644 --- a/tests/agents/session-item-events.test.ts +++ b/tests/agents/session-item-events.test.ts @@ -34,12 +34,27 @@ describe('native Item updates projected into public events', () => { const f = fixture(); const item = { id: 'command', type: 'commandExecution', command: 'work', cwd: '/workspace' }; f.emit('item/started', { item }); - f.emit('item/commandExecution/outputDelta', { itemId: item.id, delta: 'first\n' }); - f.emit('item/commandExecution/outputDelta', { itemId: item.id, delta: 'second\n' }); - f.emit('item/completed', { item: { ...item, status: 'completed', exitCode: 0, aggregatedOutput: null } }); + expect(f.emit('item/commandExecution/outputDelta', { itemId: item.id, delta: 'first\n' })).toEqual([]); + expect(f.emit('item/commandExecution/outputDelta', { itemId: item.id, delta: 'second\n' })).toEqual([]); + const events = f.emit('item/completed', { item: { ...item, status: 'completed', exitCode: 0, aggregatedOutput: null } }); + expect(events[0]).toMatchObject({ type: 'agent.output.command_execution_output.delta', delta: 'first\nsecond\n' }); expect(f.snapshot().items[0]).toMatchObject({ status: 'completed', exit_code: 0, output: 'first\nsecond\n' }); }); + it('publishes the canonical completed command output once when live output missed its prefix', () => { + const f = fixture(); + const item = { id: 'command', type: 'commandExecution', command: 'work', cwd: '/workspace' }; + f.emit('item/started', { item }); + expect(f.emit('item/commandExecution/outputDelta', { itemId: item.id, delta: 'late\n' })).toEqual([]); + expect(f.snapshot().items[0]).toMatchObject({ status: 'in_progress', output: 'late\n' }); + const terminal = { item: { ...item, status: 'completed', exitCode: 0, aggregatedOutput: 'early\nlate\n' } }; + const events = f.emit('item/completed', terminal); + expect(events.map(event => event.type)).toEqual(['agent.output.command_execution_output.delta', 'agent.session.turn.item.done']); + expect(events[0]).toMatchObject({ delta: 'early\nlate\n' }); + expect(f.emit('item/completed', terminal)).toEqual([]); + expect(f.snapshot().items[0]).toMatchObject({ output: 'early\nlate\n' }); + }); + it('retains command output across a sparse terminal snapshot without duplicating it', () => { const f = fixture(); const item = { id: 'command', type: 'commandExecution', command: 'work', cwd: '/workspace', status: 'completed', exitCode: 0 }; diff --git a/tests/agents/session-lifecycle.test.ts b/tests/agents/session-lifecycle.test.ts index 09640a5..71145e6 100644 --- a/tests/agents/session-lifecycle.test.ts +++ b/tests/agents/session-lifecycle.test.ts @@ -89,7 +89,7 @@ describe('Agents API session lifecycle through the OpenAI SDK', () => { expect(unchanged.metadata).toEqual({}); await expect(f.api.sessions.update(session.id, { agent: { model: 'unknown' } })).rejects.toMatchObject({ status: 400 }); expect((await f.api.sessions.update(session.id, { agent: { model: 'gpt-5.4-mini-2026-03-17', reasoning: { effort: 'none' }, service_tier: 'priority' } })).agent).toMatchObject({ model: 'gpt-5.4-mini-2026-03-17', reasoning: { effort: 'none' }, service_tier: 'priority' }); - expect((await f.api.sessions.update(session.id, { agent: { model: 'openai.gpt-5.4', reasoning: { effort: null } } })).agent.reasoning.effort).toBe('medium'); + expect((await f.api.sessions.update(session.id, { agent: { model: 'openai.gpt-6-sol', reasoning: { effort: null } } })).agent.reasoning.effort).toBe('medium'); }); it('serves owner-scoped traces, validates cursors and preserves captured steps after runtime removal', async () => { diff --git a/tests/agents/session-model-validation.test.ts b/tests/agents/session-model-validation.test.ts new file mode 100644 index 0000000..4cb1b7c --- /dev/null +++ b/tests/agents/session-model-validation.test.ts @@ -0,0 +1,50 @@ +import { expect, it } from 'vitest'; +import { admitNativeServiceTier, nativeServiceTier } from '../../src/domain/session-model-validation.js'; + +const catalog = [ + { id: 'catalog-entry', model: 'gpt-test', serviceTiers: [{ id: 'priority' }] }, + { id: 'standard-only', model: 'standard-only', serviceTiers: [] }, +]; + +it.each([ + [undefined, 'default'], ['auto', 'default'], ['default', 'default'], + ['fast', 'priority'], ['priority', 'priority'], ['flex', 'flex'], +])('normalizes public tier %s to an explicit native %s', (tier, expected) => { + expect(nativeServiceTier(tier)).toBe(expected); +}); + +it.each([undefined, 'auto', 'default'])('admits standard tier %s without a model catalog', tier => { + expect(admitNativeServiceTier(tier, { model: 'custom-model', provider: 'amazon-bedrock', models: undefined })).toBe('default'); +}); + +it('uses model/list identifiers and does not mutate its provider capability evidence', () => { + const before = structuredClone(catalog); + expect(admitNativeServiceTier('fast', { model: 'gpt-test', provider: 'openai', models: catalog })).toBe('priority'); + expect(admitNativeServiceTier('priority', { model: 'catalog-entry', provider: 'openai', models: catalog })).toBe('priority'); + expect(catalog).toEqual(before); +}); + +it('accepts stock Responses flex for a known model, while requiring advertised Bedrock tiers', () => { + expect(admitNativeServiceTier('flex', { model: 'standard-only', provider: 'openai', models: catalog })).toBe('flex'); + expect(() => admitNativeServiceTier('flex', { model: 'standard-only', provider: 'amazon-bedrock', models: catalog })) + .toThrow('The selected model and provider do not support this service tier'); + expect(admitNativeServiceTier('flex', { model: 'openai.gpt-test', provider: 'amazon-bedrock', models: [ + { id: 'openai.gpt-test', model: 'openai.gpt-test', serviceTiers: [{ id: 'flex' }] }, + ] })).toBe('flex'); +}); + +it.each([ + ['priority', { model: 'standard-only', provider: 'openai', models: catalog }], + ['priority', { model: 'openai.gpt-test', provider: 'amazon-bedrock', models: catalog }], + ['flex', { model: 'custom-model', provider: 'fixture', models: catalog }], + ['fast', { model: 'custom-model', provider: 'fixture', models: [] }], + ['priority', { model: 'gpt-test', provider: 'openai', models: [{ model: 'gpt-test', additionalSpeedTiers: ['priority'] }] }], + ['priority', { model: undefined, provider: 'openai', models: [{ serviceTiers: [{ id: 'priority' }] }] }], + ['PRIVATE_UNKNOWN_TIER', { model: 'gpt-test', provider: 'openai', models: catalog }], +])('rejects unsupported tier %s without exposing capability diagnostics', (tier, capability) => { + try { admitNativeServiceTier(tier, capability); throw new Error('Expected rejection'); } + catch (error) { + expect(error).toMatchObject({ status: 400, code: 'invalid_request', param: 'agent.service_tier' }); + expect((error as Error).message).not.toContain('PRIVATE'); + } +}); diff --git a/tests/agents/session-runtime.test.ts b/tests/agents/session-runtime.test.ts index b82eff5..2270b16 100644 --- a/tests/agents/session-runtime.test.ts +++ b/tests/agents/session-runtime.test.ts @@ -28,6 +28,8 @@ function fixture(lifetime?: 'bounded' | 'host-managed') { initialize: async () => {}, call: async (method, params) => { calls.push({ method, params }); + if (method === 'config/read') return { config: { model_provider: 'openai' } }; + if (method === 'model/list') return { data: [{ id: 'gpt-6-astra', model: 'gpt-6-astra', serviceTiers: [{ id: 'priority' }] }], nextCursor: null }; if (method === 'thread/start') return { thread: { id: 'root' } }; if (method === 'turn/start') { const id = `native-${++counter}`; @@ -70,14 +72,59 @@ describe('persistent native session runtime', () => { try { await f.runtime.initialize(); await f.runtime.start(rootTurn('first'), message); + expect(f.calls.some(({ method }) => method === 'model/list')).toBe(false); f.emit({ method: 'turn/completed', params: { threadId: 'root', turn: { id: 'native-1', status: 'completed' } } }); await f.controller().startSessionTurn!(rootTurn('second'), message, { model: 'gpt-6-astra', reasoning: { effort: 'low' }, service_tier: 'priority' }); const starts = f.calls.filter(({ method }) => method === 'turn/start'); expect(starts[0]!.params).toMatchObject({ model: 'fixture' }); expect(starts[1]!.params).toMatchObject({ model: 'gpt-6-astra', effort: 'low', serviceTier: 'priority' }); + expect(f.calls.filter(({ method }) => method === 'config/read' || method === 'model/list')).toEqual([ + { method: 'config/read', params: { includeLayers: false } }, + { method: 'model/list', params: { includeHidden: true, limit: 100 } }, + ]); expect(f.calls.filter(({ method }) => method === 'thread/start')).toHaveLength(1); } finally { await f.runtime.close(); } }); + + it('shares a pending tier admission for the same Turn while rejecting a different Turn', async () => { + const f = fixture(); + let release!: () => void; const blocked = new Promise(resolve => { release = resolve; }); + let entered!: () => void; const waiting = new Promise(resolve => { entered = resolve; }); + let first: Promise | undefined; + try { + await f.runtime.initialize(); + const call = f.rpc().call.bind(f.rpc()); + vi.spyOn(f.rpc(), 'call').mockImplementation(async (method, params) => { + if (method === 'model/list') { entered(); await blocked; } + return call(method, params); + }); + const settings = { model: 'gpt-6-astra', reasoning: { effort: 'low' as const }, service_tier: 'priority' as const }; + first = f.runtime.start(rootTurn('first'), message, settings); + await waiting; + const duplicate = f.runtime.start(rootTurn('first'), message, settings); + void duplicate.catch(() => {}); + expect(duplicate).toBe(first); + await expect(f.runtime.start(rootTurn('second'), message, settings)).rejects.toThrow('already has an active turn'); + expect(f.calls.filter(({ method }) => method === 'turn/start')).toEqual([]); + release(); + await Promise.all([first, duplicate]); + expect(f.calls.filter(({ method }) => method === 'model/list')).toHaveLength(1); + expect(f.calls.filter(({ method }) => method === 'turn/start')).toHaveLength(1); + } finally { release(); await first?.catch(() => {}); await f.runtime.close(); } + }); + + it('allows a corrected tier retry after unsupported admission without a native attempt', async () => { + const f = fixture(); + try { + await f.runtime.initialize(); + await expect(f.runtime.start(rootTurn('first'), message, { model: 'unknown-model', reasoning: { effort: 'low' }, service_tier: 'priority' })) + .rejects.toMatchObject({ code: 'invalid_request', param: 'agent.service_tier' }); + expect(f.calls.filter(({ method }) => method === 'turn/start')).toEqual([]); + expect(f.closed()).toBe(false); + await f.runtime.start(rootTurn('first'), message, { model: 'gpt-6-astra', reasoning: { effort: 'low' }, service_tier: 'priority' }); + expect(f.calls.filter(({ method }) => method === 'turn/start')).toHaveLength(1); + } finally { await f.runtime.close(); } + }); it.each([false, true])('retries a follow-up rejected before native admission, with start pending=%s', async (pending) => { const f = fixture(); let release!: () => void; const blocked = new Promise((resolve) => { release = resolve; }); @@ -148,7 +195,21 @@ describe('persistent native session runtime', () => { await expect(f.controller().interrupt()).rejects.toThrow('intended Session Turn'); expect(f.calls.filter(({ method }) => method === 'turn/interrupt' || method === 'turn/steer')).toEqual([]); await f.controller().steer('Current input', message, 'second'); - await f.controller().interrupt('second'); + let settled = false; + const interrupted = f.controller().interrupt('second').then(() => { settled = true; }); + // The fake RPC acknowledges immediately, but the public state must settle first. + await new Promise(resolve => setImmediate(resolve)); + expect(settled).toBe(false); + expect(f.runtime.snapshot().turns[1]?.turn.status).toBe('in_progress'); + await expect(f.runtime.start(rootTurn('third'), message)).rejects.toThrow('already has an active turn'); + f.emit({ method: 'turn/completed', params: { threadId: 'root', turn: { id: 'native-1', status: 'completed' } } }); + await new Promise(resolve => setImmediate(resolve)); + expect(settled).toBe(false); + f.emit({ method: 'turn/completed', params: { threadId: 'root', turn: { id: 'native-2', status: 'interrupted' } } }); + await interrupted; + expect(settled).toBe(true); + expect(f.runtime.snapshot().turns[1]?.turn.status).toBe('cancelled'); + await f.runtime.start(rootTurn('third'), message); expect(f.calls.filter(({ method }) => method === 'turn/interrupt' || method === 'turn/steer')).toEqual([ { method: 'turn/steer', params: { threadId: 'root', expectedTurnId: 'native-2', input: [{ type: 'text', text: 'Work' }] } }, { method: 'turn/interrupt', params: { threadId: 'root', turnId: 'native-2' } }, diff --git a/tests/agents/turn-errors.test.ts b/tests/agents/turn-errors.test.ts index 71399f3..9703723 100644 --- a/tests/agents/turn-errors.test.ts +++ b/tests/agents/turn-errors.test.ts @@ -4,29 +4,43 @@ import { bindSessionTurn, initialSessionRuntime, reduceSessionRuntime } from '.. it.each([ ['contextWindowExceeded', 'context_length_exceeded'], ['sessionBudgetExceeded', 'session_budget_exceeded'], - ['usageLimitExceeded', 'usage_limit_exceeded'], ['creditBalanceExhausted', 'credit_balance_exhausted'], + ['usageLimitExceeded', 'usage_limit_exceeded'], ['flexUnavailable', 'server_overloaded'], ['rateLimitExceeded', 'rate_limit_exceeded'], ['serverOverloaded', 'server_overloaded'], ['cyberPolicy', 'cyber_policy'], ['misalignmentPolicyViolation', 'cyber_policy'], ['internalServerError', 'server_error'], ['unauthorized', 'authentication_error'], - ['badRequest', 'invalid_request'], ['sandboxError', 'sandbox_error'], ['requestTimeout', 'request_timeout'], - ['executorVersionIncompatible', 'executor_version_incompatible'], ['unknown', 'internal_error'], + ['badRequest', 'invalid_request'], ['sandboxError', 'sandbox_error'], + ['creditBalanceExhausted', 'internal_error'], ['requestTimeout', 'internal_error'], + ['executorVersionIncompatible', 'internal_error'], ['unknown', 'internal_error'], ])('projects typed native %s as %s without its private diagnostics', (native, expected) => { const result = sessionTurnError({ codexErrorInfo: native, message: 'PRIVATE', additionalDetails: 'PRIVATE' }); expect(result.code).toBe(expected); expect(JSON.stringify(result)).not.toContain('PRIVATE'); }); -it.each([[401, 'authentication_error'], [403, 'authentication_error'], [404, 'resource_not_found'], [408, 'request_timeout'], +it.each([[401, 'authentication_error'], [402, 'internal_error'], [403, 'authentication_error'], [404, 'resource_not_found'], [408, 'request_timeout'], [429, 'rate_limit_exceeded'], [500, 'server_error'], [503, 'server_overloaded'], [504, 'request_timeout'], [422, 'invalid_request'], [null, 'connection_failed']])('maps typed HTTP status %s to %s', (status, expected) => { expect(sessionTurnError({ codexErrorInfo: { responseStreamConnectionFailed: { httpStatusCode: status } }, message: 'PRIVATE' }).code).toBe(expected); }); +it.each(['credit_balance_exhausted', 'request_timeout', 'executor_version_incompatible'] as const)('preserves richer explicit public failure %s from another runtime', code => { + expect(sessionTurnError({ code, message: 'PRIVATE' })).toMatchObject({ code }); + expect(JSON.stringify(sessionTurnError({ code, message: 'PRIVATE' }))).not.toContain('PRIVATE'); +}); + +it('never infers a code from diagnostic text or ambiguous native objects', () => { + for (const codexErrorInfo of ['other', { usageLimitExceeded: {}, creditBalanceExhausted: {} }]) { + expect(sessionTurnError({ codexErrorInfo, message: 'credit_balance_exhausted: PRIVATE', additionalDetails: 'request_timeout' }).code).toBe('internal_error'); + } + expect(sessionTurnError({ codexErrorInfo: 'other' }, { code: 'usage_limit_exceeded', message: 'PRIVATE' })) + .toEqual({ code: 'usage_limit_exceeded', message: 'The model provider usage limit has been reached.' }); +}); + it('ignores retry notifications and retains a specific failure through a sparse terminal notification', () => { const state = bindSessionTurn(initialSessionRuntime('session', 'agent', 'root'), 'native', { id: 'turn', object: 'agent.session.turn', session_id: 'session', agent_id: 'agent', subagent_id: null, status: 'in_progress', created_at: 1, started_at: 1, completed_at: null, error: null, usage: null }); - const event = { method: 'error', observedAt: 2, params: { threadId: 'root', turnId: 'native', error: { codexErrorInfo: 'creditBalanceExhausted', message: 'PRIVATE' } } }; + const event = { method: 'error', observedAt: 2, params: { threadId: 'root', turnId: 'native', error: { codexErrorInfo: 'usageLimitExceeded', message: 'PRIVATE' } } }; expect(reduceSessionRuntime(state, { ...event, params: { ...event.params, willRetry: true } })).toEqual(state); const failed = reduceSessionRuntime(state, event); const completed = reduceSessionRuntime(failed, { method: 'turn/completed', observedAt: 3, params: { threadId: 'root', turn: { id: 'native', status: 'failed', error: null } } }); - expect(completed.turns[0]?.turn.error?.code).toBe('credit_balance_exhausted'); + expect(completed.turns[0]?.turn.error?.code).toBe('usage_limit_exceeded'); expect(JSON.stringify(completed)).not.toContain('PRIVATE'); }); diff --git a/tests/domain/native-release.test.ts b/tests/domain/native-release.test.ts index 6c65eb5..752cfeb 100644 --- a/tests/domain/native-release.test.ts +++ b/tests/domain/native-release.test.ts @@ -43,7 +43,7 @@ describe('native release decisions', () => { it('preserves the Codex package layout inside sealed resources', () => { const files = [manifest.files[0], { ...manifest.files[1], path: 'Contents/Resources/codex/bin/codex' }, - ...['bin/codex-code-mode-host', 'codex-path/rg', 'codex-resources/zsh/bin/zsh'].map(path => ({ + ...['bin/codex-code-mode-host', 'codex-path/rg', 'codex-resources/voice/bin/codex-voice-host'].map(path => ({ ...manifest.files[1], role: 'host-companion', path: `Contents/Resources/codex/${path}`, }))]; expect(decodeAuthenticatedNativeManifest({ ...manifest, files }).files.map(file => file.path)).toEqual(files.map(file => file!.path)); diff --git a/tests/pi/managed-effects.test.ts b/tests/pi/managed-effects.test.ts index 08f337a..1ad9717 100644 --- a/tests/pi/managed-effects.test.ts +++ b/tests/pi/managed-effects.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { mkdtemp, rm, access, symlink, readFile } from 'node:fs/promises'; +import { mkdtemp, mkdir, rm, access, symlink, readFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { createServer } from 'node:http'; @@ -34,6 +34,15 @@ describe('Pi managed effect boundary', () => { const broker = await createPiEffectBroker({ launch, plan, lease }); const signal = new AbortController(); try { + const filesDirectory = join(workspace, 'files'); await mkdir(filesDirectory); + const file = join(filesDirectory, 'data.txt'); + const content = 'exact file data\n[UNDICI-EHPA] is valid file content é'; + const write = await broker.execute(effect('write_file', { path: file, content }), signal.signal); + expect(write).toEqual({ output: [{ type: 'text', text: 'Written\n' }], isError: false }); + const read = await broker.execute(effect('read_file', { path: file }), signal.signal); + expect(read).toEqual({ output: [{ type: 'text', text: content }], isError: false }); + const list = await broker.execute(effect('list_files', { path: filesDirectory }), signal.signal); + expect(list).toEqual({ output: [{ type: 'text', text: '[{"name":"data.txt","directory":false}]\n' }], isError: false }); const marker = join(workspace, 'started'); const child = `const fs=require('node:fs');const path=process.argv[1];fs.writeFileSync(path,'ready');setInterval(()=>fs.appendFileSync(path,'.'),20);setTimeout(()=>process.exit(0),3000);`; const parent = `require('node:child_process').spawn(process.execPath,['-e',process.argv[1],process.argv[2]],{stdio:'inherit'});setInterval(()=>{},1000);`; diff --git a/tests/pi/managed-subprocess.test.ts b/tests/pi/managed-subprocess.test.ts index 629a548..31d9446 100644 --- a/tests/pi/managed-subprocess.test.ts +++ b/tests/pi/managed-subprocess.test.ts @@ -10,15 +10,15 @@ import { planCodexLaunch } from '../../src/runner/agent-planning.js'; import { createPiEffectBroker } from '../../src/runner/pi-effect-broker.js'; import { nodeSubprocessStdioProbe, restrictedNetworkSocketProbe } from '../../testing/local-runtime/subprocess-fixtures.js'; -// Stock and non-Linux runs skip the patched artifact contract. Strict parity +// Non-strict and non-Linux runs skip the installed runtime contract. Strict parity // requires an explicitly selected executable; missing prerequisites never pass. -describe.skipIf(process.platform !== 'linux' || process.env.CODEX_REQUIRE_PARITY !== 'true')('patched managed subprocess compatibility', () => { +describe.skipIf(process.platform !== 'linux' || process.env.CODEX_REQUIRE_PARITY !== 'true')('official managed subprocess compatibility', () => { it.each([ - { name: 'captures Node subprocess stdin/stdout/stderr and preserves failures', command: [process.execPath, '-e', nodeSubprocessStdioProbe], output: 'local-subprocess-stdio-ok\n' }, - { name: 'permits local socket metadata while denying network and socket mutation', command: ['python3', '-c', restrictedNetworkSocketProbe], output: 'local-socket-only-ok\n' }, + { name: 'captures Node subprocess stdin/stdout/stderr and preserves failures', command: [process.execPath, '--disable-warning=UNDICI-EHPA', '-e', nodeSubprocessStdioProbe], output: 'local-subprocess-stdio-ok\n' }, + { name: 'permits anonymous socketpairs while denying external and named socket connections', command: ['python3', '-c', restrictedNetworkSocketProbe], output: 'local-socket-only-ok\n' }, ])('$name', async ({ command, output }) => { const binary = process.env.CODEX_CONFORMANCE_BINARY; - if (!binary) throw new Error('Strict subprocess parity requires CODEX_CONFORMANCE_BINARY to select the patched artifact'); + if (!binary) throw new Error('Strict subprocess parity requires CODEX_CONFORMANCE_BINARY to select the official executable'); await access(binary, constants.X_OK); const workspace = await mkdtemp(join(tmpdir(), 'rat-managed-subprocess-')); let broker: Awaited> | undefined; diff --git a/tests/pi/managed-tools.test.ts b/tests/pi/managed-tools.test.ts index 98b38db..3577295 100644 --- a/tests/pi/managed-tools.test.ts +++ b/tests/pi/managed-tools.test.ts @@ -1,5 +1,5 @@ import { expect, it } from 'vitest'; -import { parsePiManagedOperation, piManagedCommand } from '../../src/runner/pi-managed-tools.js'; +import { parsePiManagedOperation, piManagedCommand, piManagedResult } from '../../src/runner/pi-managed-tools.js'; import type { PiToolEffect } from '../../src/runner/pi/protocol.js'; const effect = (name: string, args: PiToolEffect['arguments']): PiToolEffect => ({ kind: 'managed', name, arguments: args, callId: 'one', taskId: 1, conversationId: 1 }); @@ -23,3 +23,32 @@ it.each([ ])('rejects operations outside the fixed capability envelope', request => { expect(() => parsePiManagedOperation(request)).toThrow('capability envelope'); }); + +const runtimeWarning = '(node:2) [UNDICI-EHPA] Warning: EnvHttpProxyAgent is experimental\n'; + +it.each([ + { name: 'read_file', args: { path: 'file' }, stdout: 'file content\n[UNDICI-EHPA] is also valid file data\0é' }, + { name: 'read_file', args: { path: 'empty' }, stdout: '' }, + { name: 'write_file', args: { path: 'file', content: 'data' }, stdout: 'Written\n' }, + { name: 'list_files', args: { path: '.' }, stdout: '[{"name":"file","directory":false}]\n' }, +])('keeps successful $name data exact when the runtime writes a warning', ({ name, args, stdout }) => { + const operation = parsePiManagedOperation(effect(name, args)); + const response = Object.freeze({ stdout, stderr: runtimeWarning, exitCode: 0 }); + expect(piManagedResult(operation, response)).toEqual({ output: [{ type: 'text', text: stdout }], isError: false }); +}); + +it.each([ + effect('read_file', { path: 'missing' }), + effect('write_file', { path: '/outside', content: 'denied' }), + effect('list_files', { path: 'missing' }), + effect('exec_command', { command: ['false'] }), +])('retains stdout and stderr for a failed $name operation', request => { + const stdout = 'partial output\n'; const stderr = `operation failed\n${runtimeWarning}`; + expect(piManagedResult(parsePiManagedOperation(request), { stdout, stderr, exitCode: 1 })) + .toEqual({ output: [{ type: 'text', text: stdout + stderr }], isError: true }); +}); + +it('retains successful command diagnostics', () => { + expect(piManagedResult(parsePiManagedOperation(effect('exec_command', { command: ['true'] })), { stdout: 'result\n', stderr: runtimeWarning, exitCode: 0 })) + .toEqual({ output: [{ type: 'text', text: `result\n${runtimeWarning}` }], isError: false }); +}); diff --git a/tests/runner/codex-app-server.test.ts b/tests/runner/codex-app-server.test.ts index 0e9d4e0..e1c2dbb 100644 --- a/tests/runner/codex-app-server.test.ts +++ b/tests/runner/codex-app-server.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it, vi } from 'vitest'; import { mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { managedCodexConfiguration } from '../../src/domain/codex-managed-policy.js'; import { runCodexAppServer, sandboxPolicyFor, @@ -123,6 +124,47 @@ describe('Codex app-server notifications', () => { }); }); + it.each([undefined, 'other-profile'])('rejects a managed policy without its selected profile (%s)', async permissions => { + const directory = await mkdtemp(join(tmpdir(), 'rat-app-managed-selection-')); + const server = join(directory, 'fake-app-server.mjs'); await writeFile(server, FAKE_APP_SERVER); + const events: string[] = []; + try { + await expect(runCodexAppServer({ binary: process.execPath, binaryArguments: [server], workspace: directory, environment: {}, timeoutMs: 3_000, + prompt: 'Must not start', sandbox: 'workspace-write', ...(permissions ? { permissions } : {}), + managedPolicy: { workspace: directory, network: { access: 'disabled', allowed_domains: [] }, credentialProxy: false }, + persistent: true, modelProvider: 'openai', networkAccess: false, onEvent: event => { events.push(event.method); }, + })).rejects.toThrow('permission profile is not selected'); + expect(events).not.toContain('test/threadParams'); + } finally { await rm(directory, { recursive: true, force: true }); } + }); + + it.each(['start', 'resume', 'fallback'] as const)('keeps the verified managed profile at every native boundary (%s)', async mode => { + const directory = await mkdtemp(join(tmpdir(), 'rat-app-managed-profile-')); + const fakeServer = join(directory, 'fake-app-server.mjs'); + await writeFile(fakeServer, FAKE_APP_SERVER); + const policy = { workspace: directory, network: { access: 'disabled' as const, allowed_domains: [] }, credentialProxy: false }; + const config = { config: managedCodexConfiguration(policy), layers: [ + { name: { type: 'sessionFlags' }, config: {} }, + { name: { type: 'user', file: join(directory, 'user/config.toml') }, config: {} }, + { name: { type: 'system', file: join(directory, 'system/config.toml') }, config: {} }, + ] }; + const events: Array<{ method: string; params: Record }> = []; + try { + await runCodexAppServer({ binary: process.execPath, binaryArguments: [fakeServer], workspace: directory, + environment: { FAKE_MANAGED_CONFIG: JSON.stringify(config), ...(mode === 'fallback' ? { FAKE_RESUME_ERROR: 'missing' } : {}) }, timeoutMs: 5_000, + prompt: 'Use the fixed managed policy', sandbox: 'workspace-write', permissions: 'rat_managed', managedPolicy: policy, + persistent: true, modelProvider: 'openai', networkAccess: false, + ...(mode !== 'start' ? { resumeThreadId: 'previous' } : {}), + onEvent: event => { events.push(event); }, onTurnStarted: controller => controller.steer('Complete'), + }); + const thread = events.find(event => event.method === 'test/threadParams')?.params; + const turn = events.find(event => event.method === 'test/turnParams')?.params; + expect(thread).toMatchObject({ permissions: 'rat_managed' }); expect(thread).not.toHaveProperty('sandbox'); + expect(thread?.config ?? {}).not.toHaveProperty('sandbox_workspace_write'); + expect(turn).toMatchObject({ permissions: 'rat_managed' }); expect(turn).not.toHaveProperty('sandboxPolicy'); + } finally { await rm(directory, { recursive: true, force: true }); } + }); + it('bridges events, fixed capabilities, and steering bidirectionally', async () => { const directory = await mkdtemp(join(tmpdir(), 'rat-app-server-test-')); const fakeServer = join(directory, 'fake-app-server.mjs'); @@ -281,6 +323,8 @@ input.on('line', (line) => { send({ method: 'test/initializeParams', params: message.params }); return; } + if (message.method === 'config/read') { send({ id: message.id, result: JSON.parse(process.env.FAKE_MANAGED_CONFIG) }); return; } + if (message.method === 'configRequirements/read') { send({ id: message.id, result: { requirements: null } }); return; } if (message.method === 'skills/list') { send({ id: message.id, result: { data: [{ cwd: message.params.cwds[0], skills: [{ name: 'test-skill', @@ -308,6 +352,7 @@ input.on('line', (line) => { } }); } else { send({ id: message.id, result: { thread: { id: 'thread-1' } } }); + send({ method: 'test/threadParams', params: message.params }); } return; } diff --git a/tests/runner/codex-cold-verification.test.ts b/tests/runner/codex-cold-verification.test.ts new file mode 100644 index 0000000..4a13984 --- /dev/null +++ b/tests/runner/codex-cold-verification.test.ts @@ -0,0 +1,134 @@ +import { spawn } from 'node:child_process'; +import { createServer } from 'node:http'; +import { once } from 'node:events'; +import { chown, mkdir, mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/promises'; +import { join, resolve } from 'node:path'; +import { tmpdir } from 'node:os'; +import { setTimeout as delay } from 'node:timers/promises'; +import { expect, it } from 'vitest'; +import { assertCheckpointStartupSafe, prepareCodexColdVerification } from '../../src/runner/codex-cold-verification.js'; +import { CodexRpcClient } from '../../src/adapters/codex-rpc.js'; +import { nativeThreadHistory, type NativeCheckpointManifest } from '../../src/domain/native-checkpoint.js'; +import { stopCheckpointWriters } from '../../src/runner/checkpoint-writers.js'; +import { syncWorkspaceStorage } from '../../src/runner/workspace-sync.js'; + +it.each(['hooks', 'mcp_servers', 'plugins', 'notify', 'apps', 'browser_use', 'computer_use', 'desktop'])('rejects active %s before shutdown', key => { + expect(() => assertCheckpointStartupSafe({ config: { [key]: { enabled: true } } }, {})).toThrow('startup integrations'); + expect(() => assertCheckpointStartupSafe({ config: {} }, { [key]: ['active'] })).toThrow('startup integrations'); +}); +it('accepts absent defaults but rejects unknown native storage configuration', () => { + expect(() => assertCheckpointStartupSafe({ config: { hooks: null, plugins: {}, mcp_servers: {}, notify: [] } }, {})).not.toThrow(); + expect(() => assertCheckpointStartupSafe({ config: { experimental_thread_store: true } }, {})).toThrow('storage'); +}); + +// Run this in the disposable worker image as root, with its dedicated agent UID. +// It must never run in the user's host service or against their native home. +const guest = it.skipIf(process.env.CODEX_COLD_NATIVE_E2E !== 'true'); +guest.each(['legacy', 'paginated'] as const)('verifies real root and child histories inside the offline namespace (%s)', async historyMode => { + const f = await guestFixture(historyMode); + try { + const stages = new Set((await readdir(tmpdir())).filter(name => name.startsWith('rat-cold-native-'))); + const verifier = await prepareCodexColdVerification({ request: f.request, stateRoot: f.root, manifest: f.manifest }); + try { + const stage = (await readdir(tmpdir())).find(name => name.startsWith('rat-cold-native-') && !stages.has(name)); + if (!stage) throw new Error('Missing private verifier staging directory'); + await hostileStagingAttempt(join(tmpdir(), stage), f.root); + await f.stop(); + // Invalid synthetic auth would fail a verifier that accidentally copied it. + await writeFile(join(f.home, 'auth.json'), '{"access_token":"SYNTHETIC_DO_NOT_USE"}'); + const before = f.requests(); + const evidence = await verifier.verify(f.manifest); + expect(evidence).toMatchObject({ kind: 'codex-cold-native', threadIds: f.manifest.threads.map(thread => thread.id), historyModes: historyMode === 'legacy' ? ['legacy', 'paginated'] : ['paginated'] }); + expect(f.requests()).toBe(before); + expect(await readFile(join(f.home, 'auth.json'), 'utf8')).toContain('SYNTHETIC_DO_NOT_USE'); + } finally { await verifier.close(); } + } finally { await f.close(); } +}, 60_000); + +guest.each(['legacy', 'paginated'] as const)('rejects a corrupt stopped candidate without removing its source (%s)', async historyMode => { + const f = await guestFixture(historyMode); + try { + const verifier = await prepareCodexColdVerification({ request: f.request, stateRoot: f.root, manifest: f.manifest }); + try { + await f.stop(); + if (historyMode === 'legacy') { + for (const path of await files(join(f.home, 'sessions'))) if (path.endsWith('.jsonl')) await writeFile(path, ''); + } else { + for (const path of await files(f.home)) if (/thread_history_\d+\.sqlite$/.test(path)) { + await writeFile(path, 'corrupt'); await rm(`${path}-wal`, { force: true }); await rm(`${path}-shm`, { force: true }); + } + } + await syncWorkspaceStorage(f.root); + await expect(verifier.verify(f.manifest)).rejects.toThrow(); + expect((await readdir(f.home)).length).toBeGreaterThan(0); + } finally { await verifier.close(); } + } finally { await f.close(); } +}, 60_000); + +async function guestFixture(historyMode: 'legacy' | 'paginated') { + if (process.getuid?.() !== 0) throw new Error('Run CODEX_COLD_NATIVE_E2E only as root in the disposable worker container'); + const root = await mkdtemp(join(tmpdir(), 'rat-native-cold-guest-')); + const home = join(root, 'codex-home'); const workspace = join(root, 'workspace'); + const identity = { uid: 10001, gid: 10001 }; + await chown(root, identity.uid, identity.gid); + for (const directory of [home, workspace, '/workspace']) { await mkdir(directory, { recursive: true }); await chown(directory, identity.uid, identity.gid); } + let requests = 0; let rootRequests = 0; + const server = createServer(async (request, response) => { + if (request.method !== 'POST') { response.writeHead(426, { Connection: 'close' }).end(); return; } + const parts: Buffer[] = []; for await (const part of request) parts.push(Buffer.from(part)); + if (!request.url?.endsWith('/responses')) { response.writeHead(404).end(); return; } + const body = JSON.parse(Buffer.concat(parts).toString()) as { input: Array<{ type?: string; recipient?: string }> }; + requests++; + const child = body.input.some(item => item.type === 'agent_message' && item.recipient === '/root/cold_child'); + if (!child) rootRequests++; + const id = `response_${requests}`; + const item = !child && rootRequests === 1 + ? { type: 'function_call', id: 'spawn', call_id: 'spawn', namespace: 'collaboration', name: 'spawn_agent', arguments: JSON.stringify({ task_name: 'cold_child', message: 'CHILD_INPUT', fork_turns: 'none' }) } + : { type: 'message', id: `message_${requests}`, role: 'assistant', phase: 'final_answer', content: [{ type: 'output_text', text: child ? 'CHILD_PERSISTED' : 'ROOT_PERSISTED' }] }; + response.writeHead(200, { 'content-type': 'text/event-stream' }); + response.end([{ type: 'response.created', response: { id } }, { type: 'response.output_item.done', item }, { type: 'response.completed', response: { id } }].map(event => `data: ${JSON.stringify(event)}\n\n`).join('')); + }); + server.on('upgrade', (_request, socket) => { socket.end('HTTP/1.1 426 Upgrade Required\r\nConnection: close\r\nContent-Length: 0\r\n\r\n'); }); + server.listen(0, '127.0.0.1'); await once(server, 'listening'); + const address = server.address(); if (!address || typeof address === 'string') throw new Error('Missing fixture port'); + const completed = new Set(); let childId: string | undefined; let childActivityDone = false; + const binary = process.env.CODEX_CONFORMANCE_BINARY ?? resolve('node_modules/.bin/codex'); + const rpc = new CodexRpcClient({ binary, cwd: workspace, environment: { PATH: '/usr/bin:/bin', HOME: home, CODEX_HOME: home, CODEX_API_KEY: 'synthetic-loopback-fixture-only' }, identity, + binaryArguments: ['-c', `openai_base_url="http://127.0.0.1:${address.port}"`, '-c', 'features.responses_websockets=false', '-c', 'features.responses_websockets_v2=false', 'app-server'], + onEvent: event => { + const value = event.params.item as { type?: string; agentThreadId?: string; kind?: string } | undefined; + if (value?.type === 'subAgentActivity' && value.agentThreadId) { childId = value.agentThreadId; if (value.kind === 'completed') childActivityDone = true; } + if (event.method === 'turn/completed') completed.add(String(event.params.threadId)); + } }); + let stopped = false; + const close = async () => { await rpc.close(); await stopCheckpointWriters(identity.uid); server.closeAllConnections(); await new Promise(done => server.close(() => done())); await rm(root, { recursive: true, force: true }); }; + try { + await rpc.initialize().catch(error => { throw new Error('Source fixture initialize failed', { cause: error }); }); + const call = (method: string, params: unknown) => rpc.call(method, params).catch(error => { throw new Error(`Source fixture ${method} failed`, { cause: error }); }); + const started = await call('thread/start', { cwd: '/workspace', modelProvider: 'openai', model: 'gpt-5.5', historyMode, approvalPolicy: 'never', sandbox: 'read-only', ephemeral: false, + config: { openai_base_url: `http://127.0.0.1:${address.port}`, 'features.responses_websockets': false, 'features.responses_websockets_v2': false, + 'features.multi_agent': true, 'features.multi_agent_v2': true, 'features.plugins': false, 'features.apps': false, 'features.hooks': false, 'features.goals': false } }) as { thread: { id: string } }; + await call('turn/start', { threadId: started.thread.id, input: [{ type: 'text', text: 'ROOT_INPUT: delegate one child.' }] }); + for (let count = 0; count < 1500 && (!childId || !completed.has(started.thread.id) || !completed.has(childId) || !childActivityDone); count++) await delay(10); + if (!childId || completed.size !== 2 || !childActivityDone) throw new Error('Native fixture did not complete'); + const threads: Record[] = []; + for (const threadId of [started.thread.id, childId]) { + const result = await call('thread/read', { threadId, includeTurns: true }) as { thread: unknown }; + threads.push(nativeThreadHistory(result.thread)); + } + const manifest: NativeCheckpointManifest = { rootThreadId: started.thread.id, hostJournalDigest: 'a'.repeat(64), threads }; + const request = { binary, workspace, executionWorkspace: '/workspace', environment: { CODEX_HOME: home }, identity, timeoutMs: 30_000, prompt: '', sandbox: 'read-only' as const, persistent: true, modelProvider: 'openai', networkAccess: false, + managedPolicy: { workspace: '/workspace', network: { access: 'disabled' as const, allowed_domains: [] }, credentialProxy: false } }; + return { root, home, request, manifest, requests: () => requests, close, + stop: async () => { if (stopped) return; await rpc.closeCheckpoint(); await stopCheckpointWriters(identity.uid); await syncWorkspaceStorage(root); stopped = true; } }; + } catch (error) { await close(); throw error; } +} +async function files(root: string): Promise { return (await Promise.all((await readdir(root, { withFileTypes: true })).map(async entry => entry.isDirectory() ? files(join(root, entry.name)) : [join(root, entry.name)]))).flat(); } + +async function hostileStagingAttempt(stage: string, root: string): Promise { + const victim = join(root, 'root-only-victim'); await mkdir(victim, { mode: 0o700 }); + const script = `const fs=require('node:fs');let blocked=0;for(const target of ['candidate','verify/home']){try{fs.symlinkSync(process.argv[2],process.argv[1]+'/'+target)}catch(error){if(['EACCES','EPERM','EEXIST'].includes(error.code))blocked++}};if(blocked!==2)process.exit(1);fs.writeFileSync(process.argv[1]+'/verify/home/config.toml','notify = ["false"]');`; + const child = spawn(process.execPath, ['-e', script, stage, victim], { uid: 10001, gid: 10001, env: {}, stdio: 'ignore' }); + const [code] = await once(child, 'close'); expect(code).toBe(0); + expect(await readdir(victim)).toEqual([]); +} diff --git a/tests/runner/managed-setup-admission.test.ts b/tests/runner/managed-setup-admission.test.ts index 5a6bdff..5f1513b 100644 --- a/tests/runner/managed-setup-admission.test.ts +++ b/tests/runner/managed-setup-admission.test.ts @@ -6,6 +6,7 @@ import { prepareHostedEnvironment } from '../../src/runner/hosted-environment.js import { planCodexLaunch } from '../../src/runner/agent-planning.js'; import { publicTemplate } from '../../src/domain/environment-planning.js'; import { sessionAgent } from '../../src/core/session-planning.js'; +import { managedCodexConfiguration } from '../../src/domain/codex-managed-policy.js'; it('refuses an unenforced executor before materializing input, running setup, or publishing readiness', async () => { const root = await mkdtemp(join(tmpdir(), 'rat-setup-admission-')); @@ -23,6 +24,12 @@ createInterface({input:process.stdin}).on('line', line => { appendFileSync(${JSON.stringify(invocations)},JSON.stringify(request.method)+'\\n'); const reply=result=>process.stdout.write(JSON.stringify({id:request.id,result})+'\\n'); if(request.method==='initialize')return reply({userAgent:'unconfined-test-executor'}); + if(request.method==='config/read')return reply(${JSON.stringify({ + config: managedCodexConfiguration({ workspace, network: { access: 'disabled', allowed_domains: [] }, credentialProxy: false }), + layers: [{ name: { type: 'sessionFlags' }, config: {} }, + ...['user', 'system'].map(type => ({ name: { type, file: join(root, type, 'config.toml') }, config: {} }))], + })}); + if(request.method==='configRequirements/read')return reply({requirements:null}); if(request.method==='command/exec')return execFile(request.params.command[0],request.params.command.slice(1),{cwd:request.params.cwd},(error,stdout,stderr)=>reply({exitCode:error?.code??0,stdout,stderr})); throw new Error('Unexpected inference or native operation'); }); @@ -36,7 +43,7 @@ createInterface({input:process.stdin}).on('line', line => { try { await expect(prepareHostedEnvironment({ launch, workspace, plan, stateDirectory, artifacts: { getBytes: async () => { throw new Error('Unexpected artifact read'); } } })) .rejects.toThrow('Refusing to expose execution tools'); - expect((await readFile(invocations, 'utf8')).trim().split('\n').map(line => JSON.parse(line))).toEqual(['initialize', 'command/exec']); + expect((await readFile(invocations, 'utf8')).trim().split('\n').map(line => JSON.parse(line))).toEqual(['initialize', 'config/read', 'configRequirements/read', 'command/exec']); expect(await readdir(workspace)).toEqual([]); expect(await readdir(stateDirectory)).toEqual([]); } finally { await rm(root, { recursive: true, force: true }); } diff --git a/tests/runner/native-checkpoint-planning.test.ts b/tests/runner/native-checkpoint-planning.test.ts new file mode 100644 index 0000000..c8f1673 --- /dev/null +++ b/tests/runner/native-checkpoint-planning.test.ts @@ -0,0 +1,60 @@ +import { expect, it } from 'vitest'; +import { initialSessionRuntime, type SessionRuntimeState } from '../../src/core/session-runtime-planning.js'; +import { planNativeCheckpoint } from '../../src/core/native-checkpoint-planning.js'; +import { projectSessionItems } from '../../src/core/session-run-projection.js'; +import type { Subagent, Turn } from '../../src/domain/agents-api.js'; + +const publicTurn = { id: 'public-turn', status: 'completed', subagent_id: null } as Turn; +const message = (id: string, text = 'acknowledged') => ({ type: 'agentMessage', id, text, phase: 'final_answer' }); +function fixture(items: unknown[], historyMode = 'paginated') { + const state: SessionRuntimeState = { ...initialSessionRuntime('session', 'agent', 'root'), turns: [{ threadId: 'root', nativeTurnId: 'native-turn', turn: publicTurn, + items: projectSessionItems(publicTurn.id, items.map(item => ({ method: 'item/completed', params: { item } }))) }] }; + const thread = { id: 'root', parentThreadId: null, sessionId: 'root', ephemeral: false, historyMode, turns: [{ id: 'native-turn', status: 'completed', items }] }; + return { state, thread }; +} +it('allows only known legacy display identity normalization while retaining ordered content', () => { + const f = fixture([message('live-first'), message('live-second', 'second')], 'legacy'); + f.thread.turns[0]!.items = [message('stored-1'), message('stored-2', 'second')]; + const planned = planNativeCheckpoint(f.state, [f.thread]); + expect(planned.hostJournalDigest).toMatch(/^[a-f0-9]{64}$/); + f.thread.turns[0]!.items.reverse(); + expect(() => planNativeCheckpoint(f.state, [f.thread])).toThrow('item content'); +}); +it('does not match two identical acknowledged legacy messages to one stored message', () => { + const f = fixture([message('first'), message('second')], 'legacy'); + f.thread.turns[0]!.items = [message('stored-1')]; + expect(() => planNativeCheckpoint(f.state, [f.thread])).toThrow('item content'); +}); +it('does not normalize paginated item identity or native turn identity', () => { + const f = fixture([message('original')]); + f.thread.turns[0]!.items = [message('replacement')]; + expect(() => planNativeCheckpoint(f.state, [f.thread])).toThrow('item content'); + f.thread.turns[0]!.id = 'different-turn'; + expect(() => planNativeCheckpoint(f.state, [f.thread])).toThrow('Session turn'); +}); +it.each(['text', 'tool-output'])('rejects missing acknowledged %s', kind => { + const item = kind === 'text' ? message('message') : { type: 'dynamicToolCall', id: 'tool', callId: 'call', tool: 'fixture', arguments: {}, status: 'completed', success: true, contentItems: [{ type: 'inputText', text: 'acknowledged tool output' }] }; + const f = fixture([item]); + f.thread.turns[0]!.items = kind === 'text' ? [message('message', 'different')] : [{ ...item, contentItems: [{ type: 'inputText', text: 'different' }] }]; + expect(() => planNativeCheckpoint(f.state, [f.thread])).toThrow('item content'); +}); +it('compares persisted coordination recipients while separately digesting host-only prompts', () => { + const f = fixture([]); + f.state.subagents = [{ id: 'child', parent_agent_id: 'agent' } as Subagent]; + f.state.coordinationCalls = { call: { threadId: 'root', turnId: 'native-turn', name: 'spawn_agent', arguments: { message: 'journal-owned prompt' }, recipients: ['child'] } }; + f.state.turns[0]!.items = [{ type: 'create_subagent_call', id: 'call', turn_id: publicTurn.id, status: 'completed', agent_id: 'agent', content: [{ type: 'output_text', text: 'journal-owned prompt' }], model: null, reasoning_effort: null }]; + f.thread.turns[0]!.items = [{ type: 'subAgentActivity', id: 'call', kind: 'started', agentThreadId: 'child' }]; + const child = { ...f.thread, id: 'child', parentThreadId: 'root', turns: [{ id: 'child-turn', status: 'completed', items: [message('child-output')] }] }; + const planned = planNativeCheckpoint(f.state, [f.thread, child]); + expect(planned.threads.map(thread => thread.id)).toEqual(['root', 'child']); + f.thread.turns[0]!.items = [{ type: 'subAgentActivity', id: 'call', kind: 'started', agentThreadId: 'different-child' }]; + expect(() => planNativeCheckpoint(f.state, [f.thread, child])).toThrow('recipients'); +}); + +it('rejects reparenting a known child under a known sibling', () => { + const f = fixture([message('root-output')]); + f.state.subagents = [{ id: 'child', parent_agent_id: 'agent' } as Subagent, { id: 'sibling', parent_agent_id: 'agent' } as Subagent]; + const child = { ...f.thread, id: 'child', parentThreadId: 'sibling', turns: [{ id: 'child-turn', status: 'completed', items: [message('child-output')] }] }; + const sibling = { ...f.thread, id: 'sibling', parentThreadId: 'root', turns: [{ id: 'sibling-turn', status: 'completed', items: [message('sibling-output')] }] }; + expect(() => planNativeCheckpoint(f.state, [f.thread, sibling, child])).toThrow('parent differs'); +}); diff --git a/tests/runner/session-checkpoint.test.ts b/tests/runner/session-checkpoint.test.ts index ffe67bc..3288886 100644 --- a/tests/runner/session-checkpoint.test.ts +++ b/tests/runner/session-checkpoint.test.ts @@ -1,17 +1,23 @@ import { describe, expect, it, vi } from 'vitest'; import { SessionRuntime, type SessionCheckpointOptions } from '../../src/runner/session-runtime.js'; import { WorkspaceAdmission } from '../../src/runner/workspace-handoff.js'; -import { CodexRpcError, type CodexRpcClient } from '../../src/adapters/codex-rpc.js'; +import type { CodexRpcClient, CodexRpcEvent } from '../../src/adapters/codex-rpc.js'; import type { Turn } from '../../src/domain/agents-api.js'; import type { WorkspaceCheckpoint } from '../../src/domain/workspace-checkpoint.js'; +import { nativeCheckpointEvidence } from '../../src/domain/native-checkpoint.js'; -const checkpoint: WorkspaceCheckpoint = { id: 'checkpoint', source: { runId: 'source', generation: 'generation' }, sandboxId: 'sandbox', setupDigest: 'a'.repeat(64), policyDigest: 'b'.repeat(64), rootThreadId: 'root', lastRootTurnId: null, lastNativeTurnId: null, createdAt: 1, reason: 'hibernate' }; +const checkpoint = { id: 'checkpoint', rootThreadId: 'root' } as WorkspaceCheckpoint; +const turn: Turn = { id: 'turn', session_id: 'session', agent_id: 'agent', subagent_id: null, object: 'agent.session.turn', status: 'queued', created_at: 1, started_at: null, completed_at: null, usage: null, error: null }; +const nativeItem = { id: 'message', type: 'agentMessage', text: 'durable output', phase: 'final_answer' }; +const history = (id = 'root') => ({ id, sessionId: id, parentThreadId: id === 'root' ? null : 'root', ephemeral: false, historyMode: 'paginated', turns: [{ id: id === 'root' ? 'native-turn' : 'child-turn', status: 'completed', items: [nativeItem] }] }); -function fixture(requiredResumeThreadId?: string, initialTurnId?: string) { +function fixture() { let handlers!: ConstructorParameters[0]; const order: string[] = []; + const verify = vi.fn(async manifest => { order.push('verify'); return nativeCheckpointEvidence(manifest); }); const ports: SessionCheckpointOptions = { admission: new WorkspaceAdmission(), + prepareNativeVerification: vi.fn(async () => ({ verify, close: vi.fn(async () => {}) })), begin: vi.fn(async () => { order.push('begin'); }), stopRemainingWriters: vi.fn(async () => { order.push('writers'); }), flushJournal: vi.fn(async () => { order.push('journal'); }), @@ -24,120 +30,94 @@ function fixture(requiredResumeThreadId?: string, initialTurnId?: string) { call: vi.fn(async (method: string, params: unknown): Promise => { if (method === 'thread/start' || method === 'thread/resume') return { thread: { id: 'root' } }; if (method === 'turn/start') return { turn: { id: 'native-turn' } }; - if (method === 'thread/loaded/list') return { data: ['root', 'child'], nextCursor: null }; - if (method === 'thread/turns/list') return { data: [], nextCursor: null }; - if (method === 'thread/checkpoint') { const { threadId } = params as { threadId: string }; order.push(threadId); return { threadId, flushed: true }; } + if (method === 'thread/loaded/list') return { data: ['root'], nextCursor: null }; + if (method === 'thread/read') return { thread: history((params as { threadId: string }).threadId) }; + if (method === 'config/read') return { config: {} }; + if (method === 'configRequirements/read') return { requirements: null }; throw new Error(`Unexpected method ${method}`); }), closeCheckpoint: vi.fn(async () => { order.push('native-exit'); handlers.onClose!(); }), close: vi.fn(async () => { handlers.onClose!(); }), }; const runtime = new SessionRuntime({ sessionId: 'session', agentId: 'agent', lifetime: 'host-managed', checkpoint: ports, - ...(requiredResumeThreadId ? { requiredResumeThreadId } : {}), - ...(initialTurnId ? { initialTurnId } : {}), request: { binary: 'unused', workspace: '/workspace', environment: {}, timeoutMs: 10_000, prompt: '', sandbox: 'read-only', persistent: true, modelProvider: 'openai', model: 'fixture', networkAccess: false }, client: options => { handlers = options; return rpc; }, }); - return { runtime, ports, rpc, order, disconnect: () => handlers.onClose!() }; + const emit = (event: CodexRpcEvent) => handlers.onEvent!(event); + return { runtime, ports, rpc, order, verify, emit, complete: async () => { + await runtime.initialize(); await runtime.start(turn, []); + emit({ method: 'item/completed', params: { threadId: 'root', turnId: 'native-turn', item: nativeItem } }); + emit({ method: 'turn/completed', params: { threadId: 'root', turn: { id: 'native-turn', status: 'completed' } } }); + } }; } -const turn: Turn = { id: 'turn', session_id: 'session', agent_id: 'agent', subagent_id: null, object: 'agent.session.turn', status: 'queued', created_at: 1, started_at: null, completed_at: null, usage: null, error: null }; - -describe('native Session checkpoint integration', () => { - it('keeps an empty unmaterialized thread alive instead of publishing an unresumable checkpoint', async () => { - const f = fixture(); - try { - await f.runtime.initialize(); - f.rpc.call.mockRejectedValueOnce(new Error('thread is not materialized yet')); - await expect(f.runtime.checkpoint()).rejects.toThrow('not materialized'); - expect(f.ports.begin).not.toHaveBeenCalled(); - expect(f.rpc.close).not.toHaveBeenCalled(); - await expect(f.ports.admission.run(async () => 'open')).resolves.toBe('open'); - } finally { await f.runtime.close(); } - }); - - it('does not mistake the gap before a replacement admits its accepted first Turn for idleness', async () => { - const f = fixture('root', turn.id); - try { - await f.runtime.initialize(); - await expect(f.runtime.checkpoint()).rejects.toThrow('active turns'); - expect(f.ports.begin).not.toHaveBeenCalled(); - await f.runtime.start(turn, []); - expect(f.rpc.call).toHaveBeenCalledWith('turn/start', expect.anything()); - } finally { await f.runtime.close(); } - }); - it('holds Run completion until every flush and the durable commit have finished', async () => { - const f = fixture(); - await f.runtime.initialize(); - let finished = false; - void f.runtime.finished.then(() => { finished = true; }); - vi.mocked(f.ports.commit).mockImplementationOnce(async state => { - expect(finished).toBe(false); - expect(state.rootThreadId).toBe('root'); +describe('stock native Session checkpoint', () => { + it('publishes only after EOF, stopped writers, journal, sync and cold verification', async () => { + const f = fixture(); await f.complete(); + let finished = false; void f.runtime.finished.then(() => { finished = true; }); + vi.mocked(f.ports.commit).mockImplementationOnce(async (_state, evidence) => { + expect(finished).toBe(false); expect(evidence).toMatchObject({ kind: 'codex-cold-native', threadIds: ['root'] }); f.order.push('commit'); return checkpoint; }); await expect(f.runtime.checkpoint()).resolves.toEqual(checkpoint); await f.runtime.finished; - expect(f.order).toEqual(['begin', 'child', 'root', 'native-exit', 'writers', 'journal', 'sync', 'commit']); - expect(finished).toBe(true); + expect(f.order).toEqual(['begin', 'native-exit', 'writers', 'journal', 'sync', 'verify', 'commit']); + expect(f.rpc.call.mock.calls.some(([method]) => method === 'thread/checkpoint')).toBe(false); expect(() => f.runtime.assertSuccessfulShutdown()).not.toThrow(); - await expect(f.ports.admission.run(async () => {})).rejects.toThrow(); }); - it('rejects busy checkpointing before durable transition and keeps accepting work', async () => { + it.each(['empty', 'busy', 'stale-items', 'missing-turn', 'missing-root', 'unknown-child', 'duplicate-loaded', 'unavailable-host', 'unsafe-config'])('rejects %s before destructive shutdown and reopens admission', async failure => { const f = fixture(); - try { - await f.runtime.initialize(); - await f.runtime.start(turn, []); - await expect(f.runtime.checkpoint()).rejects.toThrow(); - expect(f.ports.begin).not.toHaveBeenCalled(); - expect(f.rpc.close).not.toHaveBeenCalled(); - await expect(f.ports.admission.run(async () => 'accepted')).resolves.toBe('accepted'); - expect(() => f.runtime.assertSuccessfulShutdown()).not.toThrow(); - } finally { await f.runtime.close(); } + if (failure === 'empty') await f.runtime.initialize(); else await f.complete(); + const call = f.rpc.call.getMockImplementation()!; + f.rpc.call.mockImplementation(async (method, params) => { + if (method === 'thread/read' && failure === 'stale-items') return { thread: { ...history(), turns: [{ id: 'native-turn', status: 'completed', items: [] }] } }; + if (method === 'thread/read' && failure === 'missing-turn') return { thread: { ...history(), turns: [] } }; + if (method === 'thread/loaded/list' && failure === 'missing-root') return { data: [] }; + if (method === 'thread/loaded/list' && failure === 'unknown-child') return { data: ['root', 'unknown'] }; + if (method === 'thread/loaded/list' && failure === 'duplicate-loaded') return { data: ['root', 'root'] }; + if (method === 'config/read' && failure === 'unsafe-config') return { config: { hooks: { onStart: ['shell'] } } }; + return call(method, params); + }); + if (failure === 'unavailable-host') delete f.ports.prepareNativeVerification; + if (failure === 'busy') f.emit({ method: 'turn/started', params: { threadId: 'root', turn: { id: 'active', status: 'inProgress' } } }); + await expect(f.runtime.checkpoint()).rejects.toThrow(); + expect(f.ports.begin).not.toHaveBeenCalled(); expect(f.rpc.closeCheckpoint).not.toHaveBeenCalled(); + await expect(f.ports.admission.run(async () => 'open')).resolves.toBe('open'); + await f.runtime.close(); }); - it.each(['flush', 'exit', 'sync', 'commit'])('fails the Run and never publishes after %s failure', async failure => { - const f = fixture(); - await f.runtime.initialize(); - const error = new Error('injected failure'); - if (failure === 'flush') { - const call = f.rpc.call.getMockImplementation()!; - f.rpc.call.mockImplementation(async (method, params) => method === 'thread/checkpoint' ? { threadId: (params as { threadId: string }).threadId, flushed: false } : call(method, params)); - } else if (failure === 'exit') f.rpc.closeCheckpoint.mockRejectedValueOnce(error); - else vi.mocked(f.ports[failure === 'sync' ? 'syncStorage' : 'commit']).mockRejectedValueOnce(error); - await expect(f.runtime.checkpoint()).rejects.toThrow(); + it.each(['exit', 'writers', 'journal', 'sync', 'verify', 'commit'])('retains source authority and fails the run after %s failure', async failure => { + const f = fixture(); await f.complete(); + const error = new Error('injected'); + if (failure === 'exit') f.rpc.closeCheckpoint.mockRejectedValueOnce(error); + else if (failure === 'verify') f.verify.mockRejectedValueOnce(error); + else vi.mocked(f.ports[{ writers: 'stopRemainingWriters', journal: 'flushJournal', sync: 'syncStorage', commit: 'commit' }[failure] as 'commit']).mockRejectedValueOnce(error); + await expect(f.runtime.checkpoint()).rejects.toThrow('injected'); await f.runtime.finished; - expect(() => f.runtime.assertSuccessfulShutdown()).toThrow(); - expect(f.ports.abort).toHaveBeenCalledOnce(); + expect(f.ports.abort).toHaveBeenCalledOnce(); expect(() => f.runtime.assertSuccessfulShutdown()).toThrow(); if (failure !== 'commit') expect(f.ports.commit).not.toHaveBeenCalled(); await expect(f.ports.admission.run(async () => {})).rejects.toThrow(); }); - it('rejects a descendant that appeared after thread shutdown began', async () => { - const f = fixture(); - await f.runtime.initialize(); - let lists = 0; - const call = f.rpc.call.getMockImplementation()!; - f.rpc.call.mockImplementation(async (method, params) => method === 'thread/loaded/list' && ++lists > 1 ? { data: ['root', 'child', 'late-child'] } : call(method, params)); - await expect(f.runtime.checkpoint()).rejects.toThrow('threads changed'); - expect(f.ports.commit).not.toHaveBeenCalled(); - expect(f.rpc.closeCheckpoint).not.toHaveBeenCalled(); + it('includes a known unloaded child in the exact manifest', async () => { + const f = fixture(); await f.complete(); + f.emit({ method: 'thread/started', params: { thread: { id: 'child', source: { subAgent: { thread_spawn: { parent_thread_id: 'root' } } } } } }); + f.emit({ method: 'turn/completed', params: { threadId: 'child', turn: { id: 'child-turn', status: 'completed' } } }); + await f.runtime.checkpoint(); + expect(f.verify).toHaveBeenCalledWith(expect.objectContaining({ threads: [expect.objectContaining({ id: 'root' }), expect.objectContaining({ id: 'child' })] })); }); - it('never substitutes a new thread when a consumed checkpoint is missing native history', async () => { - const f = fixture('root'); - f.rpc.call.mockRejectedValueOnce(new CodexRpcError({ code: -32600, message: 'no rollout found for thread id root' })); - try { - await expect(f.runtime.initialize()).rejects.toThrow('protocol request'); - expect(f.rpc.call.mock.calls.map(call => call[0])).toEqual(['thread/resume']); - } finally { await f.runtime.close(); } + it.each(['turn/started', 'item/completed', 'error'])('rejects late %s during EOF before any publication', async method => { + const f = fixture(); await f.complete(); + f.rpc.closeCheckpoint.mockImplementationOnce(async () => { f.emit({ method, params: {} }); }); + await expect(f.runtime.checkpoint()).rejects.toThrow('activity changed'); + expect(f.ports.commit).not.toHaveBeenCalled(); expect(f.verify).not.toHaveBeenCalled(); }); - it('rejects native resume that acknowledges a different thread identity', async () => { - const f = fixture('root'); - f.rpc.call.mockResolvedValueOnce({ thread: { id: 'different-root' } }); - try { await expect(f.runtime.initialize()).rejects.toThrow('different thread'); } - finally { await f.runtime.close(); } + it.each([{ data: ['root'], nextCursor: 'repeat' }, { data: [], nextCursor: 'bad' }, { data: [42] }, { data: ['root'], nextCursor: 12 }])('rejects malformed or repeated loaded pagination %j', async page => { + const f = fixture(); await f.complete(); const call = f.rpc.call.getMockImplementation()!; + f.rpc.call.mockImplementation(async (method, params) => method === 'thread/loaded/list' ? page : call(method, params)); + await expect(f.runtime.checkpoint()).rejects.toThrow(); expect(f.ports.begin).not.toHaveBeenCalled(); await f.runtime.close(); }); }); diff --git a/tests/runner/workspace-handoff.test.ts b/tests/runner/workspace-handoff.test.ts index 53de987..0326dd4 100644 --- a/tests/runner/workspace-handoff.test.ts +++ b/tests/runner/workspace-handoff.test.ts @@ -75,3 +75,37 @@ it('holds completion while journal, storage or checkpoint commits are unresolved await operation; expect(f.ports.settle).toHaveBeenCalledOnce(); }); + +it('cold verification follows stopped writers and checked storage sync without pretending EOF acknowledged a flush', async () => { + const f = fixture(); + vi.mocked(f.ports.shutdownNative).mockImplementationOnce(async () => { f.trace.push('eof'); return { kind: 'codex-eof', threadIds: ['root', 'child'] }; }); + f.ports.verifyNative = vi.fn(async (): Promise => { f.trace.push('cold'); return { kind: 'codex-cold-native', threadIds: ['child', 'root'], historyDigest: 'a'.repeat(64), hostJournalDigest: 'b'.repeat(64), historyModes: ['paginated'] }; }); + await checkpointWorkspace(f.ports); + expect(f.trace).toEqual(['begin', 'eof', 'writers', 'journal', 'sync', 'cold', 'commit', 'settle']); + expect(f.ports.commit).toHaveBeenCalledWith(expect.objectContaining({ kind: 'codex-cold-native' })); +}); + +it.each(['missing', 'mismatch', 'failed'])('never publishes a cold checkpoint when verification is %s', async mode => { + const f = fixture(); + vi.mocked(f.ports.shutdownNative).mockResolvedValueOnce({ kind: 'codex-eof', threadIds: ['root', 'child'] }); + if (mode !== 'missing') f.ports.verifyNative = async () => { + if (mode === 'failed') throw new Error('corrupt history'); + return { kind: 'codex-cold-native', threadIds: ['root'], historyDigest: 'a'.repeat(64), hostJournalDigest: 'b'.repeat(64), historyModes: ['legacy'] }; + }; + await expect(checkpointWorkspace(f.ports)).rejects.toThrow(); expect(f.ports.commit).not.toHaveBeenCalled(); +}); + +it.each([undefined, null, {}, { kind: 'codex-cold-native', threadIds: ['root', 'child'], historyDigest: 'a'.repeat(64), hostJournalDigest: 'b'.repeat(64), historyModes: [] }])('rejects absent or malformed cold evidence %j', async evidence => { + const f = fixture(); + vi.mocked(f.ports.shutdownNative).mockResolvedValueOnce({ kind: 'codex-eof', threadIds: ['root', 'child'] }); + f.ports.verifyNative = async () => evidence as import('../../src/domain/native-checkpoint.js').NativeCheckpointEvidence; + await expect(checkpointWorkspace(f.ports)).rejects.toThrow('evidence'); + expect(f.ports.abort).toHaveBeenCalledOnce(); expect(f.ports.commit).not.toHaveBeenCalled(); +}); + +it.each([{ threadIds: ['root', 'child'] }, { kind: 'unknown', threadIds: ['root', 'child'], flushed: true, forced: false }, { threadIds: ['root', 'child'], flushed: 'yes', forced: false }])('rejects malformed shutdown acknowledgements %j', async acknowledgement => { + const f = fixture(); + f.ports.shutdownNative = async () => acknowledgement as Awaited>; + await expect(checkpointWorkspace(f.ports)).rejects.toThrow(); + expect(f.ports.commit).not.toHaveBeenCalled(); expect(f.ports.abort).toHaveBeenCalledOnce(); +}); diff --git a/tests/scripts/build-desktop.test.ts b/tests/scripts/build-desktop.test.ts new file mode 100644 index 0000000..c9e0aa5 --- /dev/null +++ b/tests/scripts/build-desktop.test.ts @@ -0,0 +1,80 @@ +import { createHash } from 'node:crypto'; +import { appendFileSync, chmodSync, writeFileSync } from 'node:fs'; +import { appendFile, lstat, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { afterEach, expect, it } from 'vitest'; +import { signDesktopBundle, stageDesktopCodexRuntime } from '../../scripts/build-desktop.mjs'; + +const digest = (value: Uint8Array | string) => createHash('sha256').update(value).digest('hex'); +const directories: string[] = []; +afterEach(async () => { await Promise.all(directories.splice(0).map(directory => rm(directory, { recursive: true, force: true }))); }); + +async function fixture() { + const directory = await mkdtemp(join(tmpdir(), 'rat-desktop-codex-test-')); directories.push(directory); + const artifactRoot = join(directory, 'artifact'); const definitionRoot = join(directory, 'definition'); + const bundle = join(directory, 'Rat Things.app'); const destination = join(bundle, 'Contents/Resources/codex-runtime'); + await mkdir(definitionRoot); await mkdir(dirname(destination), { recursive: true }); + const source = { repository: 'https://github.com/openai/codex.git', revision: 'a'.repeat(40), upstreamVersion: '0.164.0' }; + const descriptor = { layoutVersion: 1, version: source.upstreamVersion, target: 'aarch64-apple-darwin', variant: 'codex', + entrypoint: 'bin/codex', resourcesDir: 'codex-resources', pathDir: 'codex-path' }; + const executable = Buffer.alloc(64); executable.writeUInt32LE(0xfeedfacf, 0); executable.writeUInt32LE(0x100000c, 4); executable.writeUInt32LE(2, 12); + const contents = { 'bin/codex': Buffer.concat([executable, Buffer.from('original-signature')]), + 'codex-package.json': Buffer.from(JSON.stringify(descriptor)), 'codex-resources/example.txt': Buffer.from('official resource') }; + const files = Object.entries(contents).map(([path, bytes]) => ({ path, sha256: digest(bytes), executable: path === 'bin/codex' })); + for (const file of files) { + const path = join(artifactRoot, 'runtime', file.path); await mkdir(dirname(path), { recursive: true }); + await writeFile(path, contents[file.path as keyof typeof contents], { mode: file.executable ? 0o755 : 0o444 }); + } + const locked = { name: '@openai/codex', version: '0.164.0-darwin-arm64', resolved: 'https://registry.npmjs.org/@openai/codex/-/codex-0.164.0-darwin-arm64.tgz', integrity: `sha512-${'A'.repeat(86)}==` }; + const manifest = { source, platform: 'darwin', arch: 'arm64', upstreamPackage: { name: '@openai/codex-darwin-arm64', version: locked.version }, + distribution: { kind: 'official-npm', target: descriptor.target, integrity: locked.integrity }, sha256: files[0]!.sha256, + files: files.map(({ path, sha256 }) => ({ path, sha256 })) }; + const packageLockPath = join(directory, 'package-lock.json'); const companionInventoryPath = join(directory, 'companions.json'); + await writeFile(join(definitionRoot, 'source.json'), JSON.stringify(source)); + await writeFile(packageLockPath, JSON.stringify({ packages: { 'node_modules/@openai/codex-darwin-arm64': locked } })); + await writeFile(companionInventoryPath, JSON.stringify({ '@openai/codex-darwin-arm64': { ...locked, descriptor, files } })); + await writeFile(join(artifactRoot, 'artifact.json'), JSON.stringify(manifest)); + const stage = () => stageDesktopCodexRuntime(artifactRoot, destination, { definitionRoot, packageLockPath, companionInventoryPath }); + return { artifactRoot, destination, bundle, manifest, contents, stage }; +} + +it('preserves official bytes and signs only the outer development app', async () => { + const f = await fixture(); const runtime = await f.stage(); const calls: Array<{ command: string; args: string[] }> = []; + await signDesktopBundle(f.bundle, runtime, (command, args) => { + calls.push({ command, args }); return { status: 0 }; + }); + expect(calls).toEqual([{ command: 'codesign', args: ['--force', '--sign', '-', f.bundle] }]); + for (const [path, bytes] of Object.entries(f.contents)) expect(await readFile(join(f.destination, path))).toEqual(bytes); +}); + +it('rejects altered official input even when its descriptor and manifest agree', async () => { + const f = await fixture(); const changed = Buffer.concat([f.contents['bin/codex'], Buffer.from('changed')]); + await writeFile(join(f.artifactRoot, 'runtime/bin/codex'), changed); + f.manifest.sha256 = digest(changed); f.manifest.files[0]!.sha256 = digest(changed); + await writeFile(join(f.artifactRoot, 'artifact.json'), JSON.stringify(f.manifest)); + await expect(f.stage()).rejects.toThrow('differs from the official npm package'); + await expect(lstat(f.destination)).rejects.toThrow('ENOENT'); +}); + +it('rejects staged mutation before calling the signer', async () => { + const f = await fixture(); const runtime = await f.stage(); let called = false; + await appendFile(join(f.destination, 'bin/codex'), 'changed'); + await expect(signDesktopBundle(f.bundle, runtime, () => { called = true; return { status: 0 }; })).rejects.toThrow('checksum differs'); + expect(called).toBe(false); +}); + +it.each(['binary', 'resource', 'inventory'])('rejects outer signing that changes the official %s', async mutation => { + const f = await fixture(); const runtime = await f.stage(); + const run = () => { + if (mutation === 'inventory') writeFileSync(join(f.destination, 'extra'), 'unlisted'); + else { + const path = join(f.destination, mutation === 'binary' ? 'bin/codex' : 'codex-resources/example.txt'); + if (mutation === 'resource') chmodSync(path, 0o644); + appendFileSync(path, 'changed'); + if (mutation === 'resource') chmodSync(path, 0o444); + } + return { status: 0 }; + }; + await expect(signDesktopBundle(f.bundle, runtime, run)).rejects.toThrow(mutation === 'inventory' ? 'inventory changed' : 'checksum differs'); +}); diff --git a/tests/scripts/build-native-release.test.ts b/tests/scripts/build-native-release.test.ts index ae1ab40..6f3fa17 100644 --- a/tests/scripts/build-native-release.test.ts +++ b/tests/scripts/build-native-release.test.ts @@ -17,13 +17,21 @@ const protocol = { hostTarget: 'aarch64-apple-darwin', guestTarget: 'aarch64-unk const signaturePaths = ['CodeDirectory', 'CodeRequirements', 'CodeResources', 'CodeSignature'].map(path => `Contents/_CodeSignature/${path}`); afterEach(async () => { await Promise.all(directories.splice(0).map(path => rm(path, { recursive: true, force: true }))); }); -function machO(major = 15, minor = 2, command = 0x32) { - const bytes = Buffer.alloc(64); - bytes.writeUInt32LE(0xfeedfacf, 0); bytes.writeUInt32LE(0x100000c, 4); bytes.writeUInt32LE(2, 12); - bytes.writeUInt32LE(1, 16); bytes.writeUInt32LE(command === 0x32 ? 24 : 16, 20); +function machO(major = 15, minor = 2, command = 0x32, policy?: { machOType: number; rpaths?: string[] }, libraries: string[] = []) { + const linked = (name: string, command: number, minimum: number) => { + const bytes = Buffer.alloc(Math.ceil((minimum + Buffer.byteLength(name) + 1) / 8) * 8); + bytes.writeUInt32LE(command, 0); bytes.writeUInt32LE(bytes.length, 4); bytes.writeUInt32LE(minimum, 8); bytes.write(name, minimum); + return bytes; + }; + const extra = [...(policy?.rpaths ?? []).map(path => linked(path, 0x8000001c, 12)), ...libraries.map(path => linked(path, 0xc, 24))]; + const commandBytes = (command === 0x32 ? 24 : 16) + extra.reduce((total, value) => total + value.length, 0); + const bytes = Buffer.alloc(32 + commandBytes); + bytes.writeUInt32LE(0xfeedfacf, 0); bytes.writeUInt32LE(0x100000c, 4); bytes.writeUInt32LE(policy?.machOType ?? 2, 12); + bytes.writeUInt32LE(1 + extra.length, 16); bytes.writeUInt32LE(commandBytes, 20); bytes.writeUInt32LE(command, 32); bytes.writeUInt32LE(command === 0x32 ? 24 : 16, 36); if (command === 0x32) bytes.writeUInt32LE(1, 40); bytes.writeUInt32LE((major << 16) | (minor << 8), command === 0x32 ? 44 : 40); + Buffer.concat(extra).copy(bytes, 32 + (command === 0x32 ? 24 : 16)); return bytes; } @@ -38,26 +46,33 @@ async function fixture() { const write = async (path: string, bytes: string | Uint8Array, mode = 0o644) => { await mkdir(dirname(join(directory, path)), { recursive: true }); await writeFile(join(directory, path), bytes, { mode }); }; - const source = { repository: 'https://github.com/openai/codex.git', revision: 'a'.repeat(40), upstreamVersion: '0.154.0', patches: ['fixture.patch'] }; - const sourceIdentity = { ...source, patches: [{ name: 'fixture.patch', sha256: digest('patch') }] }; - await write('definition/source.json', json(source)); await write('definition/fixture.patch', 'patch'); - await write('definition/artifact-policy.json', json({ sourceSha256: digest(JSON.stringify(sourceIdentity)), patchedTree: 'b'.repeat(40) })); - const builds = { profile: 'release', debug: 'none', incremental: false, locked: true, jobs: 1, patchedTree: 'b'.repeat(40) }; - const paths = ['bin/codex', 'bin/codex-code-mode-host', 'codex-package.json', 'codex-path/rg', 'codex-resources/zsh/bin/zsh']; + const source = { repository: 'https://github.com/openai/codex.git', revision: 'a'.repeat(40), upstreamVersion: '0.164.0' }; + const sourceIdentity = source; + await write('definition/source.json', json(source)); + const paths = ['bin/codex', 'bin/codex-code-mode-host', 'codex-package.json', 'codex-path/rg']; const packages: Record = {}; const packageLocks: Record = {}; - const manifests = new Map(); for (const platform of ['darwin', 'linux']) { - const target = platform === 'darwin' ? protocol.hostTarget : protocol.guestTarget; + const target = platform === 'darwin' ? protocol.hostTarget : 'aarch64-unknown-linux-musl'; const descriptor = { layoutVersion: 1, version: source.upstreamVersion, target, variant: 'codex', entrypoint: 'bin/codex', resourcesDir: 'codex-resources', pathDir: 'codex-path' }; - const allPaths = [...paths, ...(platform === 'linux' ? ['codex-resources/bwrap'] : [])]; - const files = allPaths.map(path => ({ path, sha256: digest(path === 'codex-package.json' ? json(descriptor) : platform === 'darwin' ? machO(path.endsWith('zsh') ? 15 : 11, 0) : elf()), executable: path !== 'codex-package.json' })); + const policies = nativeExecutablePolicy.filter(policy => policy.role !== 'vm-helper'); + const allPaths = platform === 'darwin' + ? ['codex-package.json', ...policies.map(policy => policy.path.slice('Contents/Resources/codex/'.length)), 'codex-resources/voice/runtime.json', 'codex-resources/voice/NOTICE.md'] + : [...paths, 'codex-resources/bwrap']; + const content = (path: string) => { + if (path === 'codex-package.json') return json(descriptor); + if (path.endsWith('.json') || path.endsWith('.md')) return 'Pinned companion resource'; + const policy = policies.find(policy => policy.path === `Contents/Resources/codex/${path}`); + return platform === 'darwin' ? machO(path.endsWith('codex-voice-host') ? 15 : 11, 0, 0x32, policy) : elf(); + }; + const files = allPaths.map(path => ({ path, sha256: digest(content(path)), executable: !path.endsWith('.json') && !path.endsWith('.md') })); const packageName = `@openai/codex-${platform}-arm64`; const locked = { name: '@openai/codex', version: `${source.upstreamVersion}-${platform}-arm64`, resolved: `https://registry.npmjs.org/${platform}.tgz`, integrity: `sha512-${'A'.repeat(86)}==` }; packageLocks[`node_modules/${packageName}`] = locked; packages[packageName] = { ...locked, descriptor, files }; - for (const file of files) await write(`${platform}/runtime/${file.path}`, file.path === 'codex-package.json' ? json(descriptor) : platform === 'darwin' ? machO(file.path.endsWith('zsh') ? 15 : 11, 0) : elf(), file.executable ? 0o755 : 0o444); - const manifest = { source: sourceIdentity, build: { ...builds, nativeTarget: target }, binary: join(directory, platform, 'runtime/bin/codex'), + for (const file of files) await write(`${platform}/runtime/${file.path}`, content(file.path), file.executable ? 0o755 : 0o444); + const manifest = { source: sourceIdentity, distribution: { kind: 'official-npm', target, integrity: locked.integrity }, binary: join(directory, platform, 'runtime/bin/codex'), sha256: files.find(file => file.path === 'bin/codex')!.sha256, platform, arch: 'arm64', upstreamPackage: { name: packageName, version: locked.version }, files: files.map(({ path, sha256 }) => ({ path, sha256 })) }; manifests.set(platform, manifest); await write(`${platform}/artifact.json`, json(manifest)); @@ -119,28 +134,50 @@ async function fixture() { for (const path of signaturePaths) { await mkdir(dirname(join(filename, path)), { recursive: true }); await writeFile(join(filename, path), 'fixture-signature'); } } else await appendFile(filename, 'fixture-signature'); } + if (args.includes('--display')) return { stdout: '', stderr: `Executable=${args.at(-1)}` }; return { stdout: 'fixture', stderr: '' }; }; return { directory, write, options, repositoryRoot, run, calls, guestBuild, archive, inputs, contents }; } -it('assembles an exact signed candidate with untouched input provenance and hashes of signed bytes', async () => { +it('preserves every official byte and signs only the Rat Things helper and outer bundle', async () => { const f = await fixture(); const original = await readFile(join(f.options.hostArtifact, 'artifact.json')); const result = await assembleNativeRelease(f.options, { repositoryRoot: f.repositoryRoot, run: f.run }); const bundle = join(f.options.output, 'Runtime.bundle'); const manifestBytes = await readFile(join(bundle, 'Contents/Resources/runtime-manifest.json')); const manifest = decodeAuthenticatedNativeManifest(JSON.parse(manifestBytes.toString())); expect(result.releaseId).toBe(digest(manifestBytes)); expect(result.distributionReady).toBe(false); - expect(manifest.compatibility).toEqual({ ...nativeCompatibility(protocol, [11, 11, 11, 15, 15.2].map(value => ({ minimumMacOS: String(value) }))), minimumMacOS: '15.2.0' }); - expect(manifest.files.filter(file => file.kind === 'host-executable')).toHaveLength(5); + expect(manifest.compatibility).toEqual({ ...nativeCompatibility(protocol, nativeExecutablePolicy.map(() => ({ minimumMacOS: '15.2.0' }))), minimumMacOS: '15.2.0' }); + expect(manifest.files.filter(file => file.kind === 'host-executable')).toHaveLength(nativeExecutablePolicy.length); expect(await readFile(join(f.options.output, 'evidence/host-artifact.json'))).toEqual(original); expect(await readFile(join(f.options.hostArtifact, 'artifact.json'))).toEqual(original); const provenance = await readFile(join(bundle, 'Contents/Resources/provenance.json'), 'utf8'); expect(provenance).not.toContain(f.directory); expect(provenance).toContain('Corresponding sources'); - for (const mapping of JSON.parse(provenance).executables) expect(mapping.unsigned.sha256).not.toBe(mapping.signed.sha256); + for (const mapping of JSON.parse(provenance).executables) { + if (mapping.kind === 'official-unmodified') { + expect(mapping.packaged).toEqual({ bytes: mapping.input.bytes, sha256: mapping.input.sha256 }); + expect(mapping.signature).toMatchObject({ action: 'preserved', signerAdmission: 'pending' }); + expect(mapping).not.toHaveProperty('entitlements'); + } else { + expect(mapping.kind).toBe('rat-things-signed'); + expect(mapping.path).toBe('Contents/Helpers/rat-native-worker'); + expect(mapping.input.sha256).not.toBe(mapping.packaged.sha256); + } + } + for (const file of JSON.parse(original.toString()).files) { + expect(await readFile(join(bundle, 'Contents/Resources/codex', file.path))) + .toEqual(await readFile(join(f.options.hostArtifact, 'runtime', file.path))); + expect(manifest.files.find(entry => entry.path === `Contents/Resources/codex/${file.path}`)?.sha256).toBe(file.sha256); + } + const blocker = 'Native macOS upstream-signer admission requires design and on-Mac validation'; + expect(result.outstandingDistributionInputs).toContain(blocker); + expect(JSON.parse(provenance)).toMatchObject({ distributionReady: false, outstandingDistributionInputs: expect.arrayContaining([blocker]) }); const signedInventory = await readFile(join(f.options.output, 'evidence/signed-inventory.json')); expect(result.signedTreeSha256).toBe(nativeBundleFingerprint(JSON.parse(signedInventory.toString()))); + const signingEvidence = JSON.parse(await readFile(join(f.options.output, 'evidence/signing.json'), 'utf8')); + expect(signingEvidence.signatures.map((entry: { path: string }) => entry.path)).toEqual(['Contents/Helpers/rat-native-worker']); + expect(signingEvidence.preservedSignatures).toHaveLength(nativeExecutablePolicy.length - 1); const signing = f.calls.filter(call => call.args.includes('--sign')); - expect(signing).toHaveLength(6); expect(signing.at(-1)!.args.at(-1)).toBe(bundle); + expect(signing.map(call => call.args.at(-1))).toEqual([join(bundle, 'Contents/Helpers/rat-native-worker'), bundle]); expect(signing.every(call => call.args.includes('--timestamp') && call.args.includes('runtime') && !call.args.includes('--deep'))).toBe(true); expect(f.calls.find(call => call.args.includes('-target'))!.args).toContain('arm64-apple-macos15.0.0'); }); @@ -190,7 +227,7 @@ it('rejects changed source inputs before signing the outer bundle', async () => return result; }; await expect(assembleNativeRelease(f.options, { repositoryRoot: f.repositoryRoot, run })).rejects.toThrow('Input changed or checksum differs'); - expect(f.calls.filter(call => call.args.includes('--sign'))).toHaveLength(5); + expect(f.calls.filter(call => call.args.includes('--sign'))).toHaveLength(1); await expect(lstat(join(f.options.output, 'release.json'))).rejects.toThrow('ENOENT'); }); @@ -206,6 +243,20 @@ it('rejects an outer signer that mutates a payload after the manifest was writte await expect(assembleNativeRelease(f.options, { repositoryRoot: f.repositoryRoot, run })).rejects.toThrow('Outer signing changed'); }); +it.each(['helper', 'outer'])('rejects official payload changes during %s signing', async stage => { + const f = await fixture(); + const run = async (command: string, args: string[]) => { + const result = await f.run(command, args); + const suffix = stage === 'helper' ? 'rat-native-worker' : 'Runtime.bundle'; + if (args.includes('--sign') && args.at(-1)!.endsWith(suffix)) { + await appendFile(join(f.options.output, 'Runtime.bundle/Contents/Resources/codex/bin/codex'), 'changed signature'); + } + return result; + }; + await expect(assembleNativeRelease(f.options, { repositoryRoot: f.repositoryRoot, run })).rejects.toThrow('Official Codex payload changed'); + await expect(lstat(join(f.options.output, 'release.json'))).rejects.toThrow('ENOENT'); +}); + it('rejects staged archive changes during helper compilation before sealing the outer bundle', async () => { const f = await fixture(); const run = async (command: string, args: string[]) => { @@ -223,7 +274,39 @@ it('rejects staged archive changes during helper compilation before sealing the it('derives the deployment floor from modern and legacy Mach-O commands', () => { expect(inspectMachO(machO(15, 4))).toEqual({ minimumMacOS: '15.4.0', libraries: [] }); expect(inspectMachO(machO(12, 7, 0x24))).toEqual({ minimumMacOS: '12.7.0', libraries: [] }); - expect(nativeCompatibility(protocol, [machO(11), machO(12), machO(13), machO(15), machO(14)].map(inspectMachO)).minimumMacOS).toBe('15.2.0'); + expect(nativeCompatibility(protocol, nativeExecutablePolicy.map(() => inspectMachO(machO(15)))).minimumMacOS).toBe('15.2.0'); +}); + +it('accepts only the fixed bundled libraries and search paths for each Mach-O role', () => { + const voice = nativeExecutablePolicy.find(policy => policy.identifier === 'dev.rat-things.runtime.voice-host')!; + const library = nativeExecutablePolicy.find(policy => policy.path.endsWith('/lib/libgio-2.0.0.dylib'))!; + const plugin = nativeExecutablePolicy.find(policy => policy.path.endsWith('/plugins/libgstapp.dylib'))!; + for (const [policy, dependency] of [ + [voice, '@rpath/libgio-2.0.0.dylib'], + [library, '@loader_path/libglib-2.0.0.dylib'], + [plugin, '@loader_path/../lib/libgstapp-1.0.0.dylib'], + ] as const) { + expect(inspectMachO(machO(14, 0, 0x32, policy, [dependency]), policy).libraries).toEqual([dependency]); + } + for (const dependency of ['/tmp/injected.dylib', '/usr/lib/../../tmp/injected.dylib', '@rpath/unknown.dylib', '@loader_path/../bin/codex-voice-host']) { + expect(() => inspectMachO(machO(14, 0, 0x32, voice, [dependency]), voice)).toThrow('unbundled library'); + } + expect(() => inspectMachO(machO(14, 0, 0x32, { ...voice, rpaths: ['/tmp'] }), voice)).toThrow('search paths'); + expect(() => inspectMachO(machO(14, 0, 0x32, library))).toThrow('fixed host code policy'); + expect(() => inspectMachO(machO(14), library)).toThrow('fixed host code policy'); +}); + +it('keeps compiled signature identifiers aligned with the reviewed upstream native inventory', async () => { + const companions = JSON.parse(await readFile(new URL('../../runtime/codex/companion-packages.json', import.meta.url), 'utf8')); + const paths = companions['@openai/codex-darwin-arm64'].files.filter((file: { executable: boolean }) => file.executable) + .map((file: { path: string }) => `Contents/Resources/codex/${file.path}`); + expect(nativeExecutablePolicy.filter(policy => policy.role !== 'vm-helper').map(policy => policy.path).sort()).toEqual(paths.sort()); + const verifier = await readFile(new URL('../../runtime/local/verify-runtime.swift', import.meta.url), 'utf8'); + const compiledPolicy = [...verifier.matchAll(/ExecutablePolicy\(path: "([^"]+)", identifier: "([^"]+)"\)/g)] + .map(([, path, identifier]) => ({ path, identifier })); + expect(compiledPolicy).toEqual(nativeExecutablePolicy.map(({ path, identifier }) => ({ path, identifier }))); + expect(new Set(nativeExecutablePolicy.map(policy => policy.identifier)).size).toBe(nativeExecutablePolicy.length); + expect(nativeExecutablePolicy.some(policy => policy.path.endsWith('.json') || policy.path.includes('/zsh/'))).toBe(false); }); it('rejects missing, truncated, and non-macOS deployment commands', () => { @@ -232,11 +315,14 @@ it('rejects missing, truncated, and non-macOS deployment commands', () => { const platform = machO(); platform.writeUInt32LE(2, 40); expect(() => inspectMachO(platform)).toThrow('not macOS'); }); -it('limits JIT entitlements to the two Codex executables and virtualization to the helper', () => { - const commands = nativeExecutablePolicy.map(policy => signingArguments(identity, policy, policy.path, { jit: 'jit.plist', virtualization: 'vm.plist' })); - expect(commands.filter(args => args.includes('jit.plist')).map(args => args.at(-1))).toEqual(nativeExecutablePolicy.slice(0, 2).map(policy => policy.path)); - expect(commands.filter(args => args.includes('vm.plist')).map(args => args.at(-1))).toEqual(['Contents/Helpers/rat-native-worker']); - expect(commands.filter(args => !args.includes('--entitlements'))).toHaveLength(2); +it('refuses to sign upstream executables and limits Rat Things entitlements to virtualization', () => { + for (const policy of nativeExecutablePolicy.filter(value => value.role !== 'vm-helper')) { + expect(() => signingArguments(identity, policy, policy.path, { virtualization: 'vm.plist' })).toThrow('Only the Rat Things'); + } + const helper = nativeExecutablePolicy.find(value => value.role === 'vm-helper')!; + const helperArguments = signingArguments(identity, helper, helper.path, { virtualization: 'vm.plist' }); + expect(helperArguments.slice(-3)).toEqual(['--entitlements', 'vm.plist', helper.path]); + expect(signingArguments(identity, { identifier: 'dev.rat-things.runtime' }, 'Runtime.bundle', {})).not.toContain('--entitlements'); }); it('rejects unlisted executable files and links in the outer payload', async () => { diff --git a/tests/scripts/codex-runtime-artifact.test.ts b/tests/scripts/codex-runtime-artifact.test.ts index b607772..9f64d24 100644 --- a/tests/scripts/codex-runtime-artifact.test.ts +++ b/tests/scripts/codex-runtime-artifact.test.ts @@ -10,7 +10,7 @@ import { codexRuntimeEntries, validateCodexRuntimeArtifact } from '../../scripts const directories: string[] = []; afterEach(async () => { await Promise.all(directories.splice(0).map(path => rm(path, { recursive: true, force: true }))); }); const digest = (bytes: Uint8Array | string) => createHash('sha256').update(bytes).digest('hex'); -const targets = { linux: 'aarch64-unknown-linux-gnu', darwin: 'aarch64-apple-darwin' }; +const targets = { linux: 'aarch64-unknown-linux-musl', darwin: 'aarch64-apple-darwin' }; it('rejects missing native inputs before replacing any existing Lambda archive', async () => { const directory = await mkdtemp(join(tmpdir(), 'rat-package-preflight-')); directories.push(directory); @@ -20,17 +20,15 @@ it('rejects missing native inputs before replacing any existing Lambda archive', await writeFile(archive, 'previous archive'); await expect(promisify(execFile)(process.execPath, [resolve('scripts/package-lambdas.mjs')], { cwd: directory, env: { ...process.env, CODEX_RUNTIME_ARTIFACT: join(directory, 'missing') }, timeout: 5000, - })).rejects.toThrow('Build the Linux ARM64 Codex runtime first'); + })).rejects.toThrow('Prepare the official Linux ARM64 Codex runtime first'); expect(await readFile(archive, 'utf8')).toBe('previous archive'); }); async function fixture(platform: keyof typeof targets = 'linux', arch: 'arm64' | 'x64' = 'arm64') { const directory = await mkdtemp(join(tmpdir(), 'rat-runtime-artifact-')); directories.push(directory); - const source = JSON.parse(await readFile('runtime/codex/source.json', 'utf8')) as { repository: string; revision: string; upstreamVersion: string; patches: string[] }; - const policy = JSON.parse(await readFile('runtime/codex/artifact-policy.json', 'utf8')) as { patchedTree: string }; - const patches = await Promise.all(source.patches.map(async name => ({ name, sha256: digest(await readFile(join('runtime/codex', name))) }))); + const source = JSON.parse(await readFile('runtime/codex/source.json', 'utf8')) as { repository: string; revision: string; upstreamVersion: string }; const executable = Buffer.alloc(64); - const nativeTarget = arch === 'x64' ? 'x86_64-unknown-linux-gnu' : targets[platform]; + const nativeTarget = arch === 'x64' ? 'x86_64-unknown-linux-musl' : targets[platform]; if (platform === 'linux') { executable.write('7f454c46', 'hex'); executable[4] = 2; executable[5] = 1; executable[6] = 1; executable.writeUInt16LE(2, 16); executable.writeUInt16LE(arch === 'x64' ? 62 : 183, 18); executable.writeUInt32LE(1, 20); executable.writeUInt16LE(64, 52); @@ -42,10 +40,15 @@ async function fixture(platform: keyof typeof targets = 'linux', arch: 'arm64' | const pinned = packages[packageName]; const files: { path: string; sha256: string }[] = []; for (const file of pinned.files) { - const bytes = file.path === 'codex-package.json' ? Buffer.from(JSON.stringify(pinned.descriptor, null, 2) + '\n') : executable; + const nativeBytes = Buffer.from(executable); + if (/\.(?:so|dylib)(?:\.|$)/.test(file.path)) { + if (platform === 'darwin') nativeBytes.writeUInt32LE(6, 12); // MH_DYLIB + else nativeBytes.writeUInt16LE(3, 16); // ET_DYN + } + const bytes = file.path === 'codex-package.json' ? Buffer.from(JSON.stringify(pinned.descriptor, null, 2) + '\n') + : file.executable ? nativeBytes : Buffer.from('fixture data'); file.sha256 = digest(bytes); - const runtimeBytes = file.path === 'codex-package.json' - ? Buffer.from(JSON.stringify({ ...pinned.descriptor, target: nativeTarget }, null, 2) + '\n') : bytes; + const runtimeBytes = bytes; const path = join(directory, 'runtime', file.path); await mkdir(dirname(path), { recursive: true }); await writeFile(path, runtimeBytes, { mode: file.executable ? 0o755 : 0o644 }); @@ -53,9 +56,9 @@ async function fixture(platform: keyof typeof targets = 'linux', arch: 'arm64' | } const companionInventoryPath = join(directory, 'companions.json'); await writeFile(companionInventoryPath, JSON.stringify(packages)); - const manifest = { source: { ...source, patches }, platform, arch: arch as string, sha256: digest(executable), files, + const manifest = { source, platform, arch: arch as string, sha256: digest(executable), files, upstreamPackage: { name: packageName, version: `${source.upstreamVersion}-${platform}-${arch}` }, - build: { nativeTarget, patchedTree: policy.patchedTree, profile: 'release', debug: 'none', incremental: false, locked: true, jobs: 1 } }; + distribution: { kind: 'official-npm', target: nativeTarget, integrity: pinned.integrity } }; const save = () => writeFile(join(directory, 'artifact.json'), JSON.stringify(manifest)); const replaceFile = async (path: string, bytes: Uint8Array) => { await writeFile(join(directory, 'runtime', path), bytes); @@ -66,16 +69,15 @@ async function fixture(platform: keyof typeof targets = 'linux', arch: 'arm64' | const options = { expectedTarget: nativeTarget, companionInventoryPath }; const validate = () => validateCodexRuntimeArtifact(directory, options); await save(); - return { directory, executable, manifest, save, replaceFile, options, validate }; + return { directory, executable, manifest, save, replaceFile, options, validate, pinned }; } it.each(['linux', 'darwin'] as const)('admits a complete %s release and returns only verified runtime files', async platform => { const f = await fixture(platform); const admitted = await f.validate(); - expect(admitted.files.map(file => file.path)).toEqual([ - 'bin/codex', 'bin/codex-code-mode-host', 'codex-package.json', 'codex-path/rg', - ...(platform === 'linux' ? ['codex-resources/bwrap'] : []), 'codex-resources/zsh/bin/zsh', - ]); + expect(admitted.files.map(file => file.path)).toEqual(f.pinned.files.map((file: { path: string }) => file.path)); + expect(admitted.files.map(file => file.path)).toContain('codex-resources/voice/bin/codex-voice-host'); + expect(admitted.files.find(file => file.path === 'codex-resources/voice/runtime.json')!.mode).toBe(0o444); expect(admitted.files.find(file => file.path === 'bin/codex')!.mode).toBe(0o755); expect(admitted.files.find(file => file.path === 'codex-package.json')!.mode).toBe(0o444); }); @@ -84,18 +86,17 @@ it('keeps the packaging wrapper Linux-only and includes original provenance', as const f = await fixture(); const entries = await codexRuntimeEntries(f.directory, f.options); expect(entries.map(entry => entry.target)).toEqual([ - 'codex-runtime/bin/codex', 'codex-runtime/bin/codex-code-mode-host', 'codex-runtime/codex-package.json', - 'codex-runtime/codex-path/rg', 'codex-runtime/codex-resources/bwrap', 'codex-runtime/codex-resources/zsh/bin/zsh', + ...f.pinned.files.map((file: { path: string }) => `codex-runtime/${file.path}`), 'codex-runtime/artifact.json', ]); const host = await fixture('darwin'); - await expect(codexRuntimeEntries(host.directory, host.options)).rejects.toThrow('build provenance'); + await expect(codexRuntimeEntries(host.directory, host.options)).rejects.toThrow('distribution provenance'); }); it('admits Linux x64 explicitly while keeping Lambda packaging ARM64', async () => { const f = await fixture('linux', 'x64'); await expect(f.validate()).resolves.toMatchObject({ manifest: { platform: 'linux', arch: 'x64' } }); - await expect(codexRuntimeEntries(f.directory, f.options)).rejects.toThrow('build provenance'); + await expect(codexRuntimeEntries(f.directory, f.options)).rejects.toThrow('distribution provenance'); const otherArch = Buffer.from(f.executable); otherArch.writeUInt16LE(183, 18); await f.replaceFile('bin/codex', otherArch); await expect(f.validate()).rejects.toThrow('not linux X64'); @@ -104,57 +105,62 @@ it('admits Linux x64 explicitly while keeping Lambda packaging ARM64', async () it('requires an explicit supported target', async () => { const f = await fixture(); await expect(validateCodexRuntimeArtifact(f.directory)).rejects.toThrow('explicit supported'); - await expect(validateCodexRuntimeArtifact(f.directory, { ...f.options, expectedTarget: 'aarch64-unknown-linux-musl' })).rejects.toThrow('explicit supported'); + await expect(validateCodexRuntimeArtifact(f.directory, { ...f.options, expectedTarget: 'aarch64-unknown-linux-gnu' })).rejects.toThrow('explicit supported'); }); it.each([ - ['profile', 'debug'], ['debug', 'full'], ['incremental', true], ['locked', false], - ['jobs', 0], ['jobs', -1], ['jobs', 1.5], ['jobs', '1'], ['jobs', null], - ['patchedTree', 'a'.repeat(40)], ['patchedTree', ''], ['nativeTarget', 'aarch64-unknown-linux-musl'], -] as const)('rejects incompatible release provenance %s=%s', async (key, value) => { + ['kind', 'custom-build'], ['target', 'aarch64-unknown-linux-gnu'], ['integrity', 'sha512-untrusted'], +] as const)('rejects incompatible official distribution %s=%s', async (key, value) => { + const f = await fixture(); + Object.assign(f.manifest.distribution, { [key]: value }); await f.save(); + await expect(f.validate()).rejects.toThrow('distribution provenance'); +}); + +it.each(['kind', 'target', 'integrity'])('rejects missing official distribution evidence %s', async key => { const f = await fixture(); - Object.assign(f.manifest.build, { [key]: value }); await f.save(); - await expect(f.validate()).rejects.toThrow('build provenance'); + Reflect.deleteProperty(f.manifest.distribution, key); await f.save(); + await expect(f.validate()).rejects.toThrow('distribution provenance'); }); -it.each(['profile', 'debug', 'incremental', 'locked', 'jobs', 'patchedTree', 'nativeTarget'])('rejects missing build evidence %s', async key => { +it('rejects a custom build even when it declares matching official files', async () => { const f = await fixture(); - Reflect.deleteProperty(f.manifest.build, key); await f.save(); - await expect(f.validate()).rejects.toThrow('build provenance'); + Object.assign(f.manifest, { build: { profile: 'release' } }); await f.save(); + await expect(f.validate()).rejects.toThrow('distribution provenance'); }); it('rejects incorrect platform and architecture', async () => { const f = await fixture(); f.manifest.arch = 'x64'; await f.save(); - await expect(f.validate()).rejects.toThrow('build provenance'); + await expect(f.validate()).rejects.toThrow('distribution provenance'); f.manifest.arch = 'arm64'; f.manifest.platform = 'darwin'; await f.save(); - await expect(f.validate()).rejects.toThrow('build provenance'); + await expect(f.validate()).rejects.toThrow('distribution provenance'); }); -it('requires exact source identity and ordered patch hashes', async () => { +it('requires the exact unmodified source identity', async () => { const f = await fixture(); const original = structuredClone(f.manifest.source); f.manifest.source.revision = 'unrelated'; await f.save(); - await expect(f.validate()).rejects.toThrow('pinned source'); - f.manifest.source = structuredClone(original); f.manifest.source.patches.reverse(); await f.save(); - await expect(f.validate()).rejects.toThrow('ordered patches'); - f.manifest.source = structuredClone(original); f.manifest.source.patches[0]!.sha256 = '0'.repeat(64); await f.save(); - await expect(f.validate()).rejects.toThrow('ordered patches'); + await expect(f.validate()).rejects.toThrow('pinned unmodified upstream source'); + f.manifest.source = original; + Object.assign(f.manifest.source, { patches: [] }); await f.save(); + await expect(f.validate()).rejects.toThrow('pinned unmodified upstream source'); }); -it('admits a staged definition only with its independently supplied expected tree', async () => { +it('rejects a staged definition containing patches even if its manifest agrees', async () => { const f = await fixture(); const definitionRoot = join(f.directory, 'definition'); - await cp('runtime/codex', definitionRoot, { recursive: true }); - const source = JSON.parse(await readFile(join(definitionRoot, 'source.json'), 'utf8')); - source.patches.push('staged.patch'); - await writeFile(join(definitionRoot, 'source.json'), JSON.stringify(source)); - await writeFile(join(definitionRoot, 'staged.patch'), 'staged patch bytes'); - f.manifest.source.patches.push({ name: 'staged.patch', sha256: digest('staged patch bytes') }); - f.manifest.build.patchedTree = 'f'.repeat(40); await f.save(); - await expect(validateCodexRuntimeArtifact(f.directory, { ...f.options, definitionRoot })).rejects.toThrow('artifact policy'); - await expect(validateCodexRuntimeArtifact(f.directory, { ...f.options, definitionRoot, expectedPatchedTree: 'f'.repeat(40) })).resolves.toMatchObject({ manifest: f.manifest }); - await expect(validateCodexRuntimeArtifact(f.directory, { ...f.options, definitionRoot, expectedPatchedTree: 'e'.repeat(40) })).rejects.toThrow('build provenance'); + await mkdir(definitionRoot); + const patched = { ...f.manifest.source, patches: ['fixture.patch'] }; + await writeFile(join(definitionRoot, 'source.json'), JSON.stringify(patched)); + Object.assign(f.manifest.source, patched); await f.save(); + await expect(validateCodexRuntimeArtifact(f.directory, { ...f.options, definitionRoot })).rejects.toThrow('pinned unmodified upstream source'); +}); + +it('rejects a patched binary even if its manifest hashes are updated', async () => { + const f = await fixture(); + const modified = Buffer.from(f.executable); modified[63] = 1; + await f.replaceFile('bin/codex', modified); + await expect(f.validate()).rejects.toThrow('differs from the official npm package'); }); it.each(['name', 'version'] as const)('rejects mismatched upstream package %s', async field => { @@ -175,24 +181,24 @@ it('binds the checked companion inventory to the npm tarball integrity', async ( it('rejects a changed companion even if its declared checksum is changed too', async () => { const f = await fixture(); const executable = Buffer.from(f.executable); executable[63] = 1; - await f.replaceFile('codex-resources/zsh/bin/zsh', executable); - await expect(f.validate()).rejects.toThrow('companion differs'); + await f.replaceFile('codex-resources/voice/bin/codex-voice-host', executable); + await expect(f.validate()).rejects.toThrow('differs from the official npm package'); }); it.each(['linux', 'darwin'] as const)('rejects missing %s companions', async platform => { const f = await fixture(platform); - f.manifest.files = f.manifest.files.filter(file => file.path !== 'codex-resources/zsh/bin/zsh'); await f.save(); - await rm(join(f.directory, 'runtime/codex-resources/zsh'), { recursive: true }); + f.manifest.files = f.manifest.files.filter(file => file.path !== 'codex-resources/voice/bin/codex-voice-host'); await f.save(); + await rm(join(f.directory, 'runtime/codex-resources/voice/bin/codex-voice-host')); await expect(f.validate()).rejects.toThrow('companions are incomplete'); }); -it('rejects descriptor fields outside the permitted Linux target adjustment', async () => { +it('rejects descriptor changes including the Linux target', async () => { const f = await fixture(); const path = 'codex-package.json'; const descriptor = JSON.parse(await readFile(join(f.directory, 'runtime', path), 'utf8')); await f.replaceFile(path, Buffer.from(JSON.stringify({ ...descriptor, extra: 'untrusted' }, null, 2) + '\n')); await expect(f.validate()).rejects.toThrow('package descriptor'); - await f.replaceFile(path, Buffer.from(JSON.stringify({ ...descriptor, target: 'aarch64-unknown-linux-musl' }, null, 2) + '\n')); + await f.replaceFile(path, Buffer.from(JSON.stringify({ ...descriptor, target: 'aarch64-unknown-linux-gnu' }, null, 2) + '\n')); await expect(f.validate()).rejects.toThrow('package descriptor'); }); diff --git a/tests/scripts/native-app-inputs.test.ts b/tests/scripts/native-app-inputs.test.ts index 517cff1..85f165f 100644 --- a/tests/scripts/native-app-inputs.test.ts +++ b/tests/scripts/native-app-inputs.test.ts @@ -34,7 +34,7 @@ async function fixture() { const write = async (path: string, bytes: string | Uint8Array, mode = 0o444) => { await mkdir(dirname(path), { recursive: true }); await writeFile(path, bytes, { mode }); }; - for (const path of ['scripts/build-native-app.mjs', 'scripts/native-app-inputs.mjs', 'scripts/build-native-release.mjs', 'scripts/codex-runtime-artifact.mjs', 'scripts/codex-runtime-targets.mjs', 'scripts/native-guest-inputs.mjs', 'scripts/javascript-notices.mjs', 'scripts/native-build-files.mjs', 'scripts/native-guest-archive.mjs', 'src/domain/native-release.ts']) { + for (const path of ['runtime/native/host-code-policy.json', 'scripts/build-native-app.mjs', 'scripts/native-app-inputs.mjs', 'scripts/build-native-release.mjs', 'scripts/codex-runtime-artifact.mjs', 'scripts/codex-runtime-targets.mjs', 'scripts/native-guest-inputs.mjs', 'scripts/javascript-notices.mjs', 'scripts/native-build-files.mjs', 'scripts/native-guest-archive.mjs', 'src/domain/native-release.ts']) { await write(join(repositoryRoot, path), await readFile(resolve(path))); } await write(join(repositoryRoot, 'package.json'), json({ version: '0.1.0' })); diff --git a/tests/scripts/native-guest-hostname.test.ts b/tests/scripts/native-guest-hostname.test.ts new file mode 100644 index 0000000..92f8f5a --- /dev/null +++ b/tests/scripts/native-guest-hostname.test.ts @@ -0,0 +1,40 @@ +import { execFile } from 'node:child_process'; +import { mkdtemp, mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; +import { expect, it } from 'vitest'; + +it.each(['runtime/native/init.sh', 'testing/local-runtime/linux-kvm/init.sh'])('bootstraps a locally resolvable guest identity after archive scrubbing: %s', async script => { + const source = await readFile(script, 'utf8'); + await promisify(execFile)('bash', ['-n', script]); + const start = source.indexOf('mount -t proc proc /proc\n') + 'mount -t proc proc /proc\n'.length; + const end = source.indexOf('mount -t sysfs sysfs /sys', start); + expect(start).toBeGreaterThan('mount -t proc proc /proc\n'.length); + expect(end).toBeGreaterThan(start); + expect(end).toBeLessThan(source.indexOf('exec python3 /opt/rat-things/native-guest.py')); + const root = await mkdtemp(join(tmpdir(), 'rat-guest-hostname-')); + try { + const files = ['/proc/sys/kernel/hostname', '/etc/hostname', '/etc/hosts']; + await mkdir(join(root, 'proc/sys/kernel'), { recursive: true }); + await mkdir(join(root, 'etc')); + for (const file of files) await writeFile(join(root, file), '', { mode: 0o644 }); + // Execute the actual boot assignments against an empty disposable filesystem; + // never write the test host's /etc or kernel hostname. + let bootstrap = source.slice(start, end); + expect([...bootstrap.matchAll(/>\s*(\S+)/g)].map(match => match[1]).sort()).toEqual([...files].sort()); + for (const file of files) bootstrap = bootstrap.replaceAll(file, `'${join(root, file)}'`); + await promisify(execFile)('bash', ['-eu', '-c', bootstrap]); + const hostname = (await readFile(join(root, '/proc/sys/kernel/hostname'), 'utf8')).trim(); + expect(hostname).toBe('rat-things-worker'); + expect((await readFile(join(root, '/etc/hostname'), 'utf8')).trim()).toBe(hostname); + const addresses = (await readFile(join(root, '/etc/hosts'), 'utf8')).trim().split('\n').map(line => { + const [address, ...names] = line.trim().split(/\s+/); + expect(names).toContain(hostname); + expect(names).toContain('localhost'); + return address; + }); + expect(addresses.sort()).toEqual(['127.0.0.1', '::1']); + for (const file of ['/etc/hostname', '/etc/hosts']) expect((await stat(join(root, file))).mode & 0o777).toBe(0o644); + } finally { await rm(root, { recursive: true, force: true }); } +});