From 4d8796a843a7de6cb63b82858738e61b7bd0d7fb Mon Sep 17 00:00:00 2001 From: vladmesh <16962535+vladmesh@users.noreply.github.com> Date: Sat, 18 Jul 2026 19:09:17 +0300 Subject: [PATCH 1/3] feat: add Codex developer worker runtime --- .env.example | 9 ++ Makefile | 30 +++++-- docker-compose.yml | 4 + docs/CHANGELOG.md | 11 ++- docs/CONTRACTS.md | 18 +++- docs/STATUS.md | 9 ++ docs/coding-agents.md | 44 +++++++++- ...n-orchestrator-642-codex-worker-runtime.md | 73 ++++++++++++++++ .../src/worker_wrapper/runners/codex.py | 11 +++ .../src/worker_wrapper/wrapper.py | 34 +++++--- .../tests/component/test_full_cycle.py | 34 ++++++++ .../tests/unit/test_http_integration.py | 26 ++++++ .../worker-wrapper/tests/unit/test_runners.py | 14 ++++ services/langgraph/src/config/settings.py | 3 +- services/langgraph/src/nodes/developer.py | 8 +- .../tests/unit/test_developer_node.py | 43 ++++++++++ services/telegram_bot/src/config.py | 3 +- .../images/worker-base-codex/Dockerfile | 23 +++++ .../worker-manager/src/agents/__init__.py | 3 +- services/worker-manager/src/agents/codex.py | 16 ++++ services/worker-manager/src/codex_auth.py | 54 ++++++++++++ services/worker-manager/src/config.py | 6 ++ services/worker-manager/src/consumer.py | 1 + .../worker-manager/src/container_config.py | 16 +++- services/worker-manager/src/image_builder.py | 20 +++-- services/worker-manager/src/manager.py | 16 +++- .../tests/unit/test_agent_factories.py | 8 ++ .../tests/unit/test_codex_auth.py | 83 +++++++++++++++++++ .../tests/unit/test_container_config.py | 56 +++++++++++++ .../tests/unit/test_image_builder.py | 17 +++- shared/config.py | 4 +- shared/contracts/queues/worker.py | 1 + shared/contracts/vocab.py | 8 +- shared/tests/test_worker_contracts.py | 16 ++++ shared/tests/unit/test_vocab.py | 8 +- 35 files changed, 679 insertions(+), 51 deletions(-) create mode 100644 docs/reports/codegen-orchestrator-642-codex-worker-runtime.md create mode 100644 packages/worker-wrapper/src/worker_wrapper/runners/codex.py create mode 100644 services/worker-manager/images/worker-base-codex/Dockerfile create mode 100644 services/worker-manager/src/agents/codex.py create mode 100644 services/worker-manager/src/codex_auth.py create mode 100644 services/worker-manager/tests/unit/test_codex_auth.py diff --git a/.env.example b/.env.example index e237e9fe..c6215958 100644 --- a/.env.example +++ b/.env.example @@ -156,6 +156,15 @@ COMPOSE_PROJECT_NAME=codegen_orchestrator # Use an ISOLATED COPY of ~/.claude, not your live session — workers get rw access. # mkdir -p ~/.claude-worker && cp ~/.claude/.credentials.json ~/.claude-worker/ # HOST_CLAUDE_DIR=/home/youruser/.claude-worker +# Host path to a dedicated Codex profile. Do not use ~/.codex: Codex workers +# need read-write access so the CLI can refresh ChatGPT subscription tokens. +# One-time setup (run on the Docker host): +# install -d -m 0700 ~/.codex-worker +# printf 'cli_auth_credentials_store = "file"\n' > ~/.codex-worker/config.toml +# chmod 0600 ~/.codex-worker/config.toml +# CODEX_HOME=~/.codex-worker codex login --device-auth +# chmod 0600 ~/.codex-worker/auth.json +# HOST_CODEX_HOME=/home/youruser/.codex-worker # Per-agent-process timeout inside the worker (seconds). Code default is 900, # which fits live LLM engineering; override here only to tune. # WORKER_SUBPROCESS_TIMEOUT_SECONDS=900 diff --git a/Makefile b/Makefile index 1ac76262..c705885a 100644 --- a/Makefile +++ b/Makefile @@ -62,7 +62,7 @@ help: @echo " make rebuild - Rebuild everything (services + worker images), restart stack" @echo "" @echo "Worker Images:" - @echo " make rebuild-worker-images - Rebuild worker base images (common → claude)" + @echo " make rebuild-worker-images - Rebuild worker base images (common → claude/factory/codex)" @echo " make rebuild-worker-images-hard - Rebuild with --no-cache (when cache is stale)" # === Dependency Lock Files === @@ -122,7 +122,7 @@ cleanup-agents: @echo "✅ Agent containers cleaned up" # === Worker Base Images === -# Build the worker image chain: common -> claude/factory +# Build the worker image chain: common -> claude/factory/codex # Use rebuild-worker-images after changing worker-wrapper or worker-base Dockerfiles rebuild-worker-images: @@ -133,6 +133,12 @@ rebuild-worker-images: @echo "🔨 Building worker-base-claude..." docker build -t worker-base-claude:latest \ -f services/worker-manager/images/worker-base-claude/Dockerfile . + @echo "🔨 Building worker-base-factory..." + docker build -t worker-base-factory:latest \ + -f services/worker-manager/images/worker-base-factory/Dockerfile . + @echo "🔨 Building worker-base-codex..." + docker build -t worker-base-codex:latest \ + -f services/worker-manager/images/worker-base-codex/Dockerfile . @echo "🧹 Cleaning cached worker:* images..." @docker images -q 'worker:*' | xargs -r docker rmi 2>/dev/null || true @echo "✅ Worker images rebuilt!" @@ -146,6 +152,12 @@ rebuild-worker-images-hard: @echo "🔨 Building worker-base-claude (no-cache)..." docker build --no-cache -t worker-base-claude:latest \ -f services/worker-manager/images/worker-base-claude/Dockerfile . + @echo "🔨 Building worker-base-factory (no-cache)..." + docker build --no-cache -t worker-base-factory:latest \ + -f services/worker-manager/images/worker-base-factory/Dockerfile . + @echo "🔨 Building worker-base-codex (no-cache)..." + docker build --no-cache -t worker-base-codex:latest \ + -f services/worker-manager/images/worker-base-codex/Dockerfile . @echo "🧹 Cleaning cached worker:* images..." @docker images -q 'worker:*' | xargs -r docker rmi 2>/dev/null || true @echo "✅ Worker images rebuilt (no-cache)!" @@ -153,13 +165,17 @@ rebuild-worker-images-hard: # Check if worker images are stale and rebuild if needed check-worker-images: @CURRENT=$(WORKER_SOURCE_HASH); \ - STORED=$$(docker inspect worker-base-common:latest \ - --format '{{index .Config.Labels "org.codegen.worker_source_hash"}}' 2>/dev/null || echo "none"); \ - if [ "$$CURRENT" != "$$STORED" ]; then \ - echo "⚠️ Worker source files changed ($$STORED → $$CURRENT) — rebuilding images..."; \ + STALE=""; \ + for IMAGE in worker-base-common worker-base-claude worker-base-factory worker-base-codex; do \ + STORED=$$(docker inspect "$$IMAGE:latest" \ + --format '{{index .Config.Labels "org.codegen.worker_source_hash"}}' 2>/dev/null || echo "missing"); \ + if [ "$$CURRENT" != "$$STORED" ]; then STALE="$$STALE $$IMAGE($$STORED)"; fi; \ + done; \ + if [ -n "$$STALE" ]; then \ + echo "⚠️ Worker base images stale:$$STALE; rebuilding for hash $$CURRENT..."; \ $(MAKE) rebuild-worker-images; \ else \ - echo "✅ Worker images up to date (hash: $$CURRENT)"; \ + echo "✅ Worker base images up to date (hash: $$CURRENT)"; \ fi # === Quality === diff --git a/docker-compose.yml b/docker-compose.yml index 78181a29..3ffed2b6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -264,6 +264,9 @@ services: SERVICE_NAME: worker-manager # Host path for mounting .claude dir into workers HOST_CLAUDE_DIR: ${HOST_CLAUDE_DIR:-/home/user/.claude} + # Dedicated file-backed Codex profile. Never point this at the live ~/.codex. + HOST_CODEX_HOME: ${HOST_CODEX_HOME:-} + HOST_CODEX_VALIDATION_PATH: /host-codex SCAFFOLDED_WORKSPACE_PATH: /data/workspaces INTERNAL_NETWORK: codegen_internal WORKER_NETWORK: codegen_worker @@ -272,6 +275,7 @@ services: - /var/run/docker.sock:/var/run/docker.sock - ${WORKSPACE_HOST_PATH:-/data/workspaces}:/data/workspaces - ${HOST_CLAUDE_DIR:-/home/user/.claude}:/host-claude:ro + - ${HOST_CODEX_HOME:-./secrets/codex-unconfigured}:/host-codex:ro depends_on: redis: condition: service_healthy diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index fa2ec20d..1656c613 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -2,6 +2,15 @@ ## 2026-07-18 +- Add OpenAI Codex CLI as a developer-worker type alongside Claude Code and + Factory Droid. Project config now routes `agent_type=codex` without changing + the worker queue envelope, and unknown values fail explicitly. The dedicated + `worker-base-codex` image pins Codex CLI 0.144.6 and runs as the existing + non-root worker with the same workspace and resource limits. Host-session + mode validates and mounts an isolated file-backed `HOST_CODEX_HOME` read-write + for token refresh, without requiring an API key or defaulting to `~/.codex`. + Codex runs non-interactively with workspace-only writes and reports through + the existing HTTP bridge; its stdout and stderr are not persisted as results. - Make live harness API reads fail loud before body parsing. Parsed `tests/live` API responses now call `raise_for_status()` first, including scaffold/project polling and auth-gated server, ssh-key and port-allocation checks, with a mock-transport regression for a rejected polling call. @@ -88,7 +97,7 @@ ### Changed - **Typed `Run.result` (Sprint 002 Phase 2 keystone, `codegen_orchestrator-440`)**: `RunDTO.result` is no longer `dict | None`. Added `shared/contracts/dto/run_result.py` with one `extra="forbid"` model per `RunType` — `EngineeringRunResult` (`engineering_status` + commit/modules/tests), `DeployRunResult` (`deploy_outcome` + deployed_url/application_id/bot_username/deploy_fix_attempt/error_details/action + opaque deployment/smoke blobs), `QARunResult` (`qa_outcome` + summary/`failed_checks: list[QAFailedCheck]`/report/qa_attempt/deployed_url/error). `RunDTO` binds the union to `type` (`_check_result_matches_type`): a payload of the wrong type, an unknown field, an unknown outcome, or a missing required field is rejected at the boundary; `result=None` stays valid for runs with no result yet. Producers (`deploy`, `deploy_result_handler`, `deploy_failure_handler`, `engineering_result_handler`, `qa`) now build the typed model and `model_dump(mode="json")` — one wire form, and the duplicated inline classification→outcome dict in `deploy_result_handler` is replaced by the shared `_classification_to_outcome`. The scheduler supervisor reads `run.result.deploy_outcome` / `.qa_outcome` / typed attributes instead of `run.result or {}` + `.get()` + `DeployOutcome(str)`. `result=None` is allowed only before the outcome exists (`QUEUED`/`RUNNING`/`CANCELLED`); a `COMPLETED`/`FAILED` run without a result is rejected, so a terminal run that lost its outcome surfaces loudly instead of wedging a story forever. All producer failure paths that reach a terminal status now write a typed result — deploy outcomes, `QAOutcome.ERROR` when QA can't resolve its server (previously the run stayed `QUEUED` and the story sat in `TESTING` forever), and `EngineeringStatus.FAILED`/`GAVE_UP` on engineering failure/give-up paths. Migration: `Run.result` stays a JSON column and the API's `RunRead` stays dict-typed (passthrough), so no DB migration and no break on historical reads; in-flight runs parse by construction. The scheduler validates only the latest run per story (`get_latest_run_by_story` parses `rows[0]` alone), so an older legacy/corrupt run can't fail a story whose current run is valid; a latest run that fails validation (wrong-type/corrupt, or terminal-without-result) is routed to a terminal, visible state (`supervisor._fail_story_on_invalid_result`: fail story once + notify admins) instead of poison-looping. `deployed_url`/`application_id` stay optional on `DeployRunResult` (a standalone deploy, or a success where the app record didn't resolve, legitimately lacks `application_id`), but the QA handoff needs both — so `_handle_deploy_success_story` validates them *before* transitioning the story or creating a QA run, routing a `SUCCESS` that can't reach QA to a visible failure instead of crashing the tick after partial state. Contract tests in `shared/tests/unit/test_run_result.py` (each type, `result=None` per non-terminal status, terminal-without-result rejection, cross-type/unknown-outcome/unknown-field/missing-field rejection, optional-field round-trip); scheduler routing/invalid-result/CANCELLED-skip/terminal-no-result/missing-handoff-fields tests split across `services/scheduler/tests/unit/test_supervisor.py` and `test_supervisor_run_routing.py` (shared factories in `_run_routing_factories.py`); latest-only validation tests in `test_api_client.py`; QA server-resolve terminal-result test in `test_qa_consumer.py`. Closes the last slice of Phase 2; next is Phase 3 typed Redis consume. -- **Unified contract vocabularies (Sprint 002 Phase 2, `codegen_orchestrator-436`)**: added `shared/contracts/vocab.py` with one canonical `StrEnum` per cross-service concept — `AgentType` (moved here, re-exported from `queues.worker`), `ActionType`, `ResultStatus`, `LifecycleEvent`. Replaced the competing inline `Literal[...]` sets: `BaseResult.status` (dropped the `error` failure synonym, now `success`/`failed`/`timeout`), `EngineeringMessage.action` (`ActionType`), and `AgentConfigDTO.type` (`AgentType`). `LifecycleEvent` is the canonical member set, but each wire keeps its own supported slice as an explicit `Literal` subset over the enum members — `TaskProgressKind` (`started`/`progress`/`completed`/`failed`) for `ProgressEvent.type` and `WorkerEvent.event_type`, `WorkerLifecycleKind` (`started`/`completed`/`failed`/`stopped`) for `WorkerLifecycleEvent.event` — so the historically different per-field vocabularies are preserved, not merged (a progress stream still rejects `stopped`, the worker-lifecycle stream still rejects `progress`). Worker-side comparisons now use the enum instead of raw `"claude"`/`"factory"` strings (`worker-wrapper` config + wrapper, `worker-manager` container_config/manager/`_get_agent`), and `worker-manager` consumer stops passing `agent_type.value`. The provisioner-result / infra-service producer/consumer and the telegram admin notifier use `ResultStatus`. Because `BaseResult.status` no longer accepts `error`, the scheduler `provisioner:results` consumer now treats a message that fails validation as terminal (`handle_provisioner_entry`: logs it and ACKs) instead of leaving it unacked to poison-loop the `claim_pending` reclaim; transient processing errors still stay unacked for retry. Two historically-mismatched vocabularies are kept distinct on purpose, not merged: `DeployAction` (adds `stop`/`undeploy`) and `TaskType` (adds `refactor`) stay separate from `ActionType`, and `WorkerEvent.worker_type` is now `WorkerCliKind` (`droid`/`claude_code`/`codex`), explicitly disjoint from `AgentType`. Contract tests in `shared/tests/unit/test_vocab.py` (accepted values + rejection of unknowns and of out-of-slice lifecycle values per field) and `services/scheduler/tests/unit/test_provisioner_entry.py` (poison message ACKed, valid processed+ACKed, transient error left unacked). Out of scope (still open in Phase 2/3): `Run.result` union, typed Redis consume / raw-dict consumers. +- **Unified contract vocabularies (Sprint 002 Phase 2, `codegen_orchestrator-436`)**: added `shared/contracts/vocab.py` with one canonical `StrEnum` per cross-service concept — `AgentType` (moved here, re-exported from `queues.worker`), `ActionType`, `ResultStatus`, `LifecycleEvent`. Replaced the competing inline `Literal[...]` sets: `BaseResult.status` (dropped the `error` failure synonym, now `success`/`failed`/`timeout`), `EngineeringMessage.action` (`ActionType`), and `AgentConfigDTO.type` (`AgentType`). `LifecycleEvent` is the canonical member set, but each wire keeps its own supported slice as an explicit `Literal` subset over the enum members — `TaskProgressKind` (`started`/`progress`/`completed`/`failed`) for `ProgressEvent.type` and `WorkerEvent.event_type`, `WorkerLifecycleKind` (`started`/`completed`/`failed`/`stopped`) for `WorkerLifecycleEvent.event` — so the historically different per-field vocabularies are preserved, not merged (a progress stream still rejects `stopped`, the worker-lifecycle stream still rejects `progress`). Worker-side comparisons now use the enum instead of raw `"claude"`/`"factory"` strings (`worker-wrapper` config + wrapper, `worker-manager` container_config/manager/`_get_agent`), and `worker-manager` consumer stops passing `agent_type.value`. The provisioner-result / infra-service producer/consumer and the telegram admin notifier use `ResultStatus`. Because `BaseResult.status` no longer accepts `error`, the scheduler `provisioner:results` consumer now treats a message that fails validation as terminal (`handle_provisioner_entry`: logs it and ACKs) instead of leaving it unacked to poison-loop the `claim_pending` reclaim; transient processing errors still stay unacked for retry. Two historically-mismatched vocabularies are kept distinct on purpose, not merged: `DeployAction` (adds `stop`/`undeploy`) and `TaskType` (adds `refactor`) stay separate from `ActionType`, while `WorkerEvent.worker_type` remains the separate `WorkerCliKind` vocabulary (`droid`/`claude_code`/`codex`). Contract tests in `shared/tests/unit/test_vocab.py` (accepted values + rejection of unknowns and of out-of-slice lifecycle values per field) and `services/scheduler/tests/unit/test_provisioner_entry.py` (poison message ACKed, valid processed+ACKed, transient error left unacked). Out of scope (still open in Phase 2/3): `Run.result` union, typed Redis consume / raw-dict consumers. - **Typed response-DTO lifecycle fields (B7 slice of Sprint 002 Phase 2)**: lifecycle fields on the read-side DTOs now declare their existing `StrEnum` instead of bare `str`, so Pydantic rejects unknown values at the read boundary. `TaskDTO.status/type` and `TaskEventDTO.event_type/from_status/to_status`, `StoryDTO.status/type`, `ServerDTO.status` (+ `ServerCreate.status`), `ApplicationDTO/Create/Update.status`, `IncidentDTO.incident_type/status` (+ `IncidentCreate.incident_type`, `IncidentUpdate.status`), and `ServiceDeploymentDTO.status` (`DeploymentResult`). Dropped the "use str for flexibility" comments. Added accept-valid / reject-unknown unit tests per DTO. Only the B7 response-DTO slice; the duplicated vocabularies and `Run.result` typing from Phase 2 remain open. ### Fixed diff --git a/docs/CONTRACTS.md b/docs/CONTRACTS.md index 87db5ff3..30010a30 100644 --- a/docs/CONTRACTS.md +++ b/docs/CONTRACTS.md @@ -16,7 +16,7 @@ instead of restating a `Literal[...]` or a local enum: | Enum | Values | Used by | |------|--------|---------| -| `AgentType` | `claude`, `factory`, `noop` | `WorkerConfig.agent_type`, `AgentConfigDTO.type`, worker-manager/worker-wrapper agent branching (re-exported from `queues.worker`) | +| `AgentType` | `claude`, `factory`, `codex`, `noop` | `WorkerConfig.agent_type`, `AgentConfigDTO.type`, worker-manager/worker-wrapper agent branching (re-exported from `queues.worker`) | | `ActionType` | `create`, `feature`, `fix` | `EngineeringMessage.action` | | `ResultStatus` | `success`, `failed`, `timeout` | `BaseResult.status` (and its subclasses) | | `LifecycleEvent` | `started`, `progress`, `completed`, `failed`, `stopped` (canonical member set) | via the field-specific subsets below | @@ -34,8 +34,18 @@ enums do not, so merging them would broaden a field past what the wire supports: - `DeployAction` (`create`/`feature`/`fix` **plus** `stop`/`undeploy`) — deploy operations, a superset of `ActionType`. `TaskType` (**plus** `refactor`) — the planning-layer task kind. -- `WorkerCliKind` (`droid`/`claude_code`/`codex`) — the CLI's self-reported wire - identity on `worker:events`, which does **not** map onto `AgentType`. +- `WorkerCliKind` (`droid`/`claude_code`/`codex`) is the CLI's self-reported wire + identity on `worker:events`. It remains a separate concept from `AgentType`; + only the `codex` spelling currently overlaps. + +`WorkerConfig.host_codex_home` adds no new queue shape beyond an optional field. +For `agent_type=codex` and `auth_mode=host_session`, worker-manager validates a +dedicated file-backed ChatGPT profile before image resolution. It then mounts +that host directory read-write at `/home/worker/.codex` and sets `CODEX_HOME` +to the same path. The profile must contain access and refresh tokens so the +non-interactive worker can refresh its session. Unknown agent values fail +Pydantic validation or explicit +LangGraph/image-routing checks; there is no fallback to Claude. `ResultStatus` dropped the old `error` failure synonym: a failed result is `failed`, never `error`. The `provisioner:results` consumer treats a message @@ -1116,7 +1126,7 @@ The Orchestrator (LangGraph) listens to **one** stream for all worker results: # shared/contracts/queues/worker.py # AgentType is the canonical enum (shared/contracts/vocab.py), re-exported here. -from shared.contracts.vocab import AgentType # claude / factory / noop +from shared.contracts.vocab import AgentType # claude / factory / codex / noop class WorkerCapability(StrEnum): diff --git a/docs/STATUS.md b/docs/STATUS.md index 29e34fae..393e8bf6 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -15,6 +15,14 @@ Typed environment/secrets migration proposal: [typed env contract MVP](plans/typ ## Current Facts +- `codegen_orchestrator-642` adds Codex as a developer-worker type end to end. + `agent_type=codex` selects `worker-base-codex`, runs pinned Codex CLI 0.144.6 + through `codex exec --sandbox workspace-write`, and reports only through the + existing localhost HTTP bridge. Host-session auth uses a dedicated validated + read-write `HOST_CODEX_HOME`; unknown agent types fail instead of falling back + to Claude. Image-chain and container smoke evidence is in + [the task report](reports/codegen-orchestrator-642-codex-worker-runtime.md). + - Deploy resolves service-template 0.3.1 `POSTGRES_HOST_PORT` and `REDIS_HOST_PORT` from application allocations. Existing ports are reused and only missing infrastructure services are allocated. - Runtime deploy connections now use the selected server's required `ssh_user` together with the @@ -85,6 +93,7 @@ Typed environment/secrets migration proposal: [typed env contract MVP](plans/typ | Stabilization sequence (`codegen_orchestrator-434`) | COMPLETE | [stabilization plan v1](plans/codegen-stabilization-v1.md) | | B7 response-DTO enums (`codegen_orchestrator-435`) | COMPLETE | lifecycle fields on task/story/server/application/incident/service-deployment DTOs now use their `StrEnum`; slice of Phase 2 only | | Unified contract vocabularies (`codegen_orchestrator-436`) | COMPLETE | `shared/contracts/vocab.py` canonical `AgentType`/`ActionType`/`ResultStatus`/`LifecycleEvent`; inline `Literal` sets removed, `error` synonym dropped; `WorkerCliKind`/`DeployAction`/`TaskType` kept distinct; tests in `shared/tests/unit/test_vocab.py` | +| Codex developer worker (`codegen_orchestrator-642`) | COMPLETE | `AgentType.CODEX`, strict project routing, dedicated image and host-session profile, non-interactive runner, full image-chain build and container smoke; [report](reports/codegen-orchestrator-642-codex-worker-runtime.md) | | Typed `Run.result` union (`codegen_orchestrator-440`) | COMPLETE | `shared/contracts/dto/run_result.py` per-`RunType` models bound to `type`; producers emit typed models, scheduler reads typed attributes; invalid result → visible terminal state; tests in `shared/tests/unit/test_run_result.py` + `test_supervisor.py`. Closes Sprint 002 Phase 2 | | Typed engineering consume + dead-layer removal (`codegen_orchestrator-457`) | COMPLETE | Engineering consumer on `EngineeringMessage.model_validate`; deleted `langgraph/src/tools/` (allocator → `allocations.py`), second `agent_config_cache`, `scaffold_phase.py`, `worker:lifecycle` stream+contract, shared compat-shims; tests in `test_engineering_validation.py`, `test_dead_layer_removed.py`, `test_phase3_shims_removed.py`. Closes Sprint 002 Phase 3. PR #42 | | B3 incident journal reconciliation (`codegen_orchestrator-466`) | COMPLETE | Successful provisioning writes `READY` before journal closure. An unavailable journal remains observable and gets one warning; scheduler retries only active `provisioning_failed` entries for `READY` servers, idempotently and without recovery actions or per-tick notifications. | diff --git a/docs/coding-agents.md b/docs/coding-agents.md index ae273e29..339b1d1e 100644 --- a/docs/coding-agents.md +++ b/docs/coding-agents.md @@ -38,6 +38,46 @@ cat error.log | claude -p "Fix this error" **Цена:** Pro/Max подписка (~$20-100/мес), дешевле чем API. +## OpenAI Codex CLI + +Codex is available only for developer workers. The image pins Codex CLI +`0.144.6`; the wrapper runs it non-interactively: + +```bash +codex exec --sandbox workspace-write \ + "Read TASK.md and AGENTS.md, then complete the task described in TASK.md." +``` + +The task is in `/workspace/TASK.md`, and the shared developer instructions are +in `/workspace/AGENTS.md`. The agent must report success or failure through +`POST http://localhost:9090/result`. CLI stdout and stderr are diagnostics and +are neither accepted as the business result nor persisted for Codex workers. + +### Dedicated ChatGPT session profile + +Do not mount the operator's live `~/.codex`. Create a separate profile on the +Docker host and log in once with device authentication: + +```bash +install -d -m 0700 "$HOME/.codex-worker" +printf 'cli_auth_credentials_store = "file"\n' > "$HOME/.codex-worker/config.toml" +chmod 0600 "$HOME/.codex-worker/config.toml" +CODEX_HOME="$HOME/.codex-worker" codex login --device-auth +chmod 0600 "$HOME/.codex-worker/auth.json" +``` + +Set `HOST_CODEX_HOME=/home/youruser/.codex-worker` in `.env`, then rebuild the +worker images. Worker-manager requires directory mode `0700`, file modes +`0600`, access and refresh tokens in a valid `auth.json`, and +`cli_auth_credentials_store = "file"`. A missing or unsuitable profile stops +Codex worker creation before image resolution. The profile is mounted +read-write only into Codex containers at `/home/worker/.codex` so refreshed +tokens persist. Claude, Factory, and noop workers do not receive this mount. + +See the official [authentication](https://learn.chatgpt.com/docs/auth) and +[non-interactive mode](https://learn.chatgpt.com/docs/non-interactive-mode) +documentation for the upstream behavior. + --- ## Интеграция в проект @@ -49,7 +89,7 @@ Developer node в Engineering Subgraph использует coding agents чер 3. Worker-manager creates/checks out story feature branch (`story/{story_id}`) 4. Инжектит статические инструкции из `services/langgraph/src/prompts/developer_worker/INSTRUCTIONS.md` → agent-specific file (`CLAUDE.md` / `AGENTS.md`) 5. Инжектит динамический `TASK.md` в `/workspace/TASK.md` с project-specific задачей. Previous tasks archived in `.story/old_tasks/` -6. Запускает coding agent (Droid или Claude Code) с одной строкой: `claude -p "Read TASK.md"` +6. Запускает coding agent (Droid, Claude Code или Codex) в non-interactive режиме 7. Агент коммитит и пушит на feature branch. Worker-wrapper pulls from current branch (not hardcoded `main`) 8. Агент сообщает результат через HTTP: `curl -X POST localhost:9090/result -d '{"success":true,"commit":"","summary":"..."}'` 9. При невозможности выполнения: `curl -X POST localhost:9090/result -d '{"success":false,"reason":"..."}'` @@ -66,6 +106,6 @@ Developer node в Engineering Subgraph использует coding agents чер | Узел | Инструмент | Статус | |------|------------|--------| | **Scaffolder** | Copier template | ✅ Реализовано | -| **Developer** | Factory.ai Droid / Claude Code | ✅ Реализовано (Native execution, Flat Dev Environment) | +| **Developer** | Factory.ai Droid / Claude Code / OpenAI Codex | ✅ Реализовано (Native execution, Flat Dev Environment) | | **Tester** | Функциональный узел (запуск тестов) | ⚠️ Заглушка (временно Developer сам запускает тесты через `make`) | | **DevOps** | GitHub Actions (deploy.yml) | ✅ Реализовано | diff --git a/docs/reports/codegen-orchestrator-642-codex-worker-runtime.md b/docs/reports/codegen-orchestrator-642-codex-worker-runtime.md new file mode 100644 index 00000000..0719a5a5 --- /dev/null +++ b/docs/reports/codegen-orchestrator-642-codex-worker-runtime.md @@ -0,0 +1,73 @@ +# Codex developer-worker report + +Task: `codegen_orchestrator-642` +Date: 2026-07-18 + +## Result + +Codex is a strict `AgentType` value carried through project config, LangGraph, +the existing worker command DTO, worker-manager, the agent-specific image, and +worker-wrapper. The queue envelope is unchanged. Unknown agent types fail at +the typed or explicit routing boundary. + +`auth_mode=host_session` requires a separate `HOST_CODEX_HOME` with directory +mode `0700`, `auth.json` and `config.toml` mode `0600`, a non-empty JSON session, +refresh-capable access and refresh tokens, and +`cli_auth_credentials_store = "file"`. The profile is mounted read-write +only into Codex workers at `/home/worker/.codex`; `CODEX_HOME` points there and +no API key is required. + +## Verification + +```text +make test-unit +Passed: 8 +Failed: 0 +All unit tests passed! + +make lint +All checks passed! + +uv run pytest packages/worker-wrapper/tests/component -q +7 passed + +docker compose config --quiet --no-env-resolution +exit 0 + +make rebuild-worker-images +worker-base-common:latest built +worker-base-claude:latest built +worker-base-factory:latest built +worker-base-codex:latest built +Worker images rebuilt + +make check-worker-images +Worker base images up to date + +docker run --rm --entrypoint sh worker-base-codex:latest -c \ + 'set -eu; test "$(id -un)" = worker; test "$CODEX_HOME" = /home/worker/.codex; \ + test -d "$CODEX_HOME"; test -w "$CODEX_HOME"; codex --version' +codex-cli 0.144.6 +user=worker codex_home=/home/worker/.codex writable=yes + +docker run --rm worker-base-codex:latest healthcheck +Healthcheck passed + +docker network create codex-smoke-642 +docker run -d --name codex-redis-smoke-642 --network codex-smoke-642 redis:7-alpine +docker run -d --name codex-worker-smoke-642 --network codex-smoke-642 \ + -e WORKER_REDIS_URL=redis://codex-redis-smoke-642:6379/0 \ + -e WORKER_INPUT_STREAM=codex:smoke:input \ + -e WORKER_OUTPUT_STREAM=codex:smoke:output \ + -e WORKER_CONSUMER_GROUP=codex-smoke \ + -e WORKER_CONSUMER_NAME=codex-smoke-642 \ + -e WORKER_AGENT_TYPE=codex worker-base-codex:latest +docker logs codex-worker-smoke-642 | rg worker_wrapper_starting +docker exec codex-worker-smoke-642 worker-wrapper healthcheck +Healthcheck passed +wrapper=running agent=codex user=worker codex_home=/home/worker/.codex +``` + +The wrapper smoke used an isolated Redis container, no real session material, +and printed no credentials. Both temporary containers and their Docker network +were removed after the check. diff --git a/packages/worker-wrapper/src/worker_wrapper/runners/codex.py b/packages/worker-wrapper/src/worker_wrapper/runners/codex.py new file mode 100644 index 00000000..2e2d293c --- /dev/null +++ b/packages/worker-wrapper/src/worker_wrapper/runners/codex.py @@ -0,0 +1,11 @@ +from dataclasses import dataclass + +from .base import AgentRunner + + +@dataclass +class CodexRunner(AgentRunner): + """Runner for Codex CLI non-interactive developer work.""" + + def build_command(self, prompt: str) -> list[str]: + return ["codex", "exec", "--sandbox", "workspace-write", prompt] diff --git a/packages/worker-wrapper/src/worker_wrapper/wrapper.py b/packages/worker-wrapper/src/worker_wrapper/wrapper.py index d8be83db..d5171292 100644 --- a/packages/worker-wrapper/src/worker_wrapper/wrapper.py +++ b/packages/worker-wrapper/src/worker_wrapper/wrapper.py @@ -591,6 +591,7 @@ async def execute_agent(self, data: dict[str, Any]) -> None: # Select Runner from .runners.claude import ClaudeRunner + from .runners.codex import CodexRunner from .runners.factory import FactoryRunner from .runners.noop import NoopRunner @@ -598,6 +599,8 @@ async def execute_agent(self, data: dict[str, Any]) -> None: runner = ClaudeRunner(session_id=session_id) elif self.config.agent_type == AgentType.FACTORY: runner = FactoryRunner() + elif self.config.agent_type == AgentType.CODEX: + runner = CodexRunner() elif self.config.agent_type == AgentType.NOOP: runner = NoopRunner() else: @@ -605,7 +608,11 @@ async def execute_agent(self, data: dict[str, Any]) -> None: prompt = self._resolve_prompt(data) cmd = runner.build_command(prompt=prompt) - logger.info("executing_agent_command", cmd=cmd) + logger.info( + "executing_agent_command", + executable=cmd[0], + agent_type=self.config.agent_type, + ) # Build subprocess env: remove /app from PYTHONPATH. # The worker image sets PYTHONPATH=/app so the wrapper itself can @@ -650,17 +657,22 @@ async def execute_agent(self, data: dict[str, Any]) -> None: stdout = stdout_bytes.decode().strip() stderr = stderr_bytes.decode().strip() - # Capture stdout tail for analytics/debugging (last ~10KB) - max_tail = 10_000 - combined = stdout - if stderr: - combined = f"{stdout}\n--- stderr ---\n{stderr}" if stdout else stderr - self._agent_stdout_tail = combined[-max_tail:] if combined else None + # Codex stdout/stderr are transport diagnostics, never business output. + # Do not persist or log them because CLI diagnostics can include data + # from the mounted session or repository. + if self.config.agent_type == AgentType.CODEX: + self._agent_stdout_tail = None + else: + max_tail = 10_000 + combined = stdout + if stderr: + combined = f"{stdout}\n--- stderr ---\n{stderr}" if stdout else stderr + self._agent_stdout_tail = combined[-max_tail:] if combined else None if proc.returncode != 0: - logger.error( - "agent_process_failed", stderr=stderr, stdout=stdout, exit_code=proc.returncode - ) + logger.error("agent_process_failed", exit_code=proc.returncode) + if self.config.agent_type == AgentType.CODEX: + raise RuntimeError(f"Codex agent process failed with code {proc.returncode}") detail = stderr or stdout or "no output" raise RuntimeError(f"Agent process failed with code {proc.returncode}: {detail}") @@ -763,7 +775,7 @@ def _resolve_prompt(self, data: dict[str, Any]) -> str: if not raw: raise ValueError("Task data missing 'content' or 'prompt'") - if self.config.agent_type == AgentType.CLAUDE: + if self.config.agent_type in {AgentType.CLAUDE, AgentType.CODEX}: return "Read TASK.md and AGENTS.md, then complete the task described in TASK.md." return raw diff --git a/packages/worker-wrapper/tests/component/test_full_cycle.py b/packages/worker-wrapper/tests/component/test_full_cycle.py index e0b7b33b..00c35684 100644 --- a/packages/worker-wrapper/tests/component/test_full_cycle.py +++ b/packages/worker-wrapper/tests/component/test_full_cycle.py @@ -5,6 +5,8 @@ import pytest from worker_wrapper.wrapper import WorkerWrapper, WorkerWrapperConfig +from shared.contracts.vocab import AgentType + @pytest.fixture def wrapper_config(): @@ -85,3 +87,35 @@ async def test_handles_execution_failure(self, wrapper_config, fake_redis): await wrapper.execute_agent({"content": "fail"}) assert "Agent process failed" in str(exc.value) + + @pytest.mark.asyncio + async def test_codex_exec_uses_workspace_sandbox_without_output_bridge( + self, wrapper_config, fake_redis + ): + mock_redis_client = MagicMock() + mock_redis_client.redis = fake_redis + codex_config = WorkerWrapperConfig( + **(wrapper_config.model_dump() | {"agent_type": AgentType.CODEX}) + ) + wrapper = WorkerWrapper(config=codex_config, redis_client=mock_redis_client) + mock_process = MockProcess( + stdout=b"transport output must not become a result", + stderr=b"transport diagnostics must not become a result", + returncode=0, + ) + + with patch("asyncio.create_subprocess_exec", new_callable=AsyncMock) as mock_exec: + mock_exec.return_value = mock_process + + result = await wrapper.execute_agent({"content": "not part of the command"}) + + assert result is None + assert mock_exec.call_args.args[:4] == ( + "codex", + "exec", + "--sandbox", + "workspace-write", + ) + assert "TASK.md" in mock_exec.call_args.args[4] + assert "not part of the command" not in mock_exec.call_args.args[4] + assert wrapper._agent_stdout_tail is None diff --git a/packages/worker-wrapper/tests/unit/test_http_integration.py b/packages/worker-wrapper/tests/unit/test_http_integration.py index 250f9926..e792db40 100644 --- a/packages/worker-wrapper/tests/unit/test_http_integration.py +++ b/packages/worker-wrapper/tests/unit/test_http_integration.py @@ -10,6 +10,7 @@ import json from unittest.mock import AsyncMock, MagicMock, patch +import pytest from worker_wrapper.config import WorkerWrapperConfig from worker_wrapper.wrapper import WorkerWrapper @@ -187,6 +188,31 @@ async def crashing_agent(data): assert result.status == WorkerResultStatus.FAILED assert result.agent_stdout_tail == "Partial output before crash" + async def test_codex_diagnostics_are_not_persisted_or_returned(self): + raw_diagnostic = "refresh-token-must-not-leak" + config = _make_config(agent_type="codex") + wrapper = WorkerWrapper(config=config, redis_client=_make_redis_mock()) + process = MagicMock(returncode=1) + process.communicate = AsyncMock( + return_value=( + f"stdout {raw_diagnostic}".encode(), + f"stderr {raw_diagnostic}".encode(), + ) + ) + + with patch( + "worker_wrapper.session.SessionManager.get_or_create_session", + new_callable=AsyncMock, + return_value="unused", + ): + with patch("asyncio.create_subprocess_exec", new_callable=AsyncMock) as spawn: + spawn.return_value = process + with pytest.raises(RuntimeError) as exc_info: + await wrapper.execute_agent({"prompt": "do stuff"}) + + assert raw_diagnostic not in str(exc_info.value) + assert wrapper._agent_stdout_tail is None + class TestWatchdog: """When agent exits without HTTP result, auto-resume or fail.""" diff --git a/packages/worker-wrapper/tests/unit/test_runners.py b/packages/worker-wrapper/tests/unit/test_runners.py index dcf62818..82bae15e 100644 --- a/packages/worker-wrapper/tests/unit/test_runners.py +++ b/packages/worker-wrapper/tests/unit/test_runners.py @@ -5,6 +5,7 @@ import pytest from worker_wrapper.runners.claude import ClaudeRunner +from worker_wrapper.runners.codex import CodexRunner from worker_wrapper.runners.factory import FactoryRunner from worker_wrapper.runners.noop import NoopRunner @@ -59,6 +60,19 @@ def test_build_command_uses_droid_exec(self): assert "Fix bug" in cmd +class TestCodexRunner: + def test_build_command_uses_non_interactive_workspace_sandbox(self): + cmd = CodexRunner().build_command(prompt="Read TASK.md and AGENTS.md") + + assert cmd == [ + "codex", + "exec", + "--sandbox", + "workspace-write", + "Read TASK.md and AGENTS.md", + ] + + class TestNoopRunner: def test_pushes_checked_out_branch_and_reports_success_over_http(self): command = " ".join(NoopRunner().build_command(prompt="ignored")) diff --git a/services/langgraph/src/config/settings.py b/services/langgraph/src/config/settings.py index ed0098f3..4f47ac81 100644 --- a/services/langgraph/src/config/settings.py +++ b/services/langgraph/src/config/settings.py @@ -12,6 +12,7 @@ default_agent_type_field, redis_url_field, ) +from shared.contracts.vocab import AgentType class Settings(BaseSettings): @@ -22,7 +23,7 @@ class Settings(BaseSettings): api_base_url: str = api_base_url_field(required=True) # Worker configuration - default_agent_type: str = default_agent_type_field() + default_agent_type: AgentType = default_agent_type_field() # Optional: Mount host Claude session for dev agents (avoids API key need) mount_claude_session: bool = True diff --git a/services/langgraph/src/nodes/developer.py b/services/langgraph/src/nodes/developer.py index 482cee26..48ba6318 100644 --- a/services/langgraph/src/nodes/developer.py +++ b/services/langgraph/src/nodes/developer.py @@ -92,7 +92,13 @@ async def run(self, state: dict) -> dict: try: agent_type = AgentType(agent_type_str) except ValueError: - agent_type = AgentType.CLAUDE + error = f"Unknown developer agent_type: {agent_type_str!r}" + logger.error("unknown_developer_agent_type", agent_type=agent_type_str) + return { + "messages": [AIMessage(content=error)], + "engineering_status": EngineeringStatus.FAILED, + "errors": state.get("errors", []) + [error], + } action = state.get("action", "create") feature_description = state.get("description") diff --git a/services/langgraph/tests/unit/test_developer_node.py b/services/langgraph/tests/unit/test_developer_node.py index c67b6eb3..7b205694 100644 --- a/services/langgraph/tests/unit/test_developer_node.py +++ b/services/langgraph/tests/unit/test_developer_node.py @@ -15,6 +15,7 @@ from shared.contracts.dto.engineering import EngineeringStatus from shared.contracts.dto.project import ProjectDTO, ProjectStatus from shared.contracts.dto.repository import RepositoryDTO +from shared.contracts.vocab import AgentType from src.clients.worker_spawner import SpawnResult _NOW = datetime(2025, 1, 1, tzinfo=UTC) @@ -70,6 +71,48 @@ def _make_state(*, action="create", status=ProjectStatus.ACTIVE.value, modules=N } +class TestDeveloperAgentRouting: + @pytest.mark.asyncio + @patch("src.nodes.developer.request_spawn", new_callable=AsyncMock) + @patch("src.nodes.developer.api_client") + @patch("src.nodes.developer.GitHubAppClient") + async def test_codex_project_config_reaches_worker_spawn( + self, mock_github_cls, mock_api, mock_spawn + ): + mock_github_cls.return_value.get_token = AsyncMock(return_value="ghs_fake") + mock_api.get_project = AsyncMock(return_value=None) + mock_api.get_primary_repository = AsyncMock(return_value=_repo()) + mock_spawn.return_value = SpawnResult( + request_id="req-1", + success=True, + exit_code=0, + output="Done", + commit_sha="abc123", + ) + state = _make_state() + state["project_spec"]["config"]["agent_type"] = "codex" + + from src.nodes.developer import DeveloperNode + + await DeveloperNode().run(state) + + assert mock_spawn.await_args.kwargs["agent_type"] is AgentType.CODEX + + @pytest.mark.asyncio + @patch("src.nodes.developer.request_spawn", new_callable=AsyncMock) + async def test_unknown_project_agent_fails_without_spawning(self, mock_spawn): + state = _make_state() + state["project_spec"]["config"]["agent_type"] = "mystery" + + from src.nodes.developer import DeveloperNode + + result = await DeveloperNode().run(state) + + assert result["engineering_status"] == EngineeringStatus.FAILED + assert any("mystery" in error for error in result["errors"]) + mock_spawn.assert_not_awaited() + + class TestDeveloperNodeCommitValidation: @pytest.mark.asyncio @patch("src.nodes.developer.request_spawn", new_callable=AsyncMock) diff --git a/services/telegram_bot/src/config.py b/services/telegram_bot/src/config.py index c6996409..fcdee98b 100644 --- a/services/telegram_bot/src/config.py +++ b/services/telegram_bot/src/config.py @@ -14,6 +14,7 @@ redis_url_field, telegram_token_field, ) +from shared.contracts.vocab import AgentType class Settings(BaseSettings): @@ -25,7 +26,7 @@ class Settings(BaseSettings): telegram_bot_token: str = telegram_token_field(required=True) # Worker configuration - default_agent_type: str = default_agent_type_field() + default_agent_type: AgentType = default_agent_type_field() # LK (dashboard) lk_domain: str = Field(alias="LK_DOMAIN") diff --git a/services/worker-manager/images/worker-base-codex/Dockerfile b/services/worker-manager/images/worker-base-codex/Dockerfile new file mode 100644 index 00000000..ca509c4d --- /dev/null +++ b/services/worker-manager/images/worker-base-codex/Dockerfile @@ -0,0 +1,23 @@ +# Worker base image with a pinned OpenAI Codex CLI. +# Inherits the shared wrapper, workspace, and non-root worker user. + +ARG BASE_IMAGE=worker-base-common:latest +FROM ${BASE_IMAGE} + +ARG CODEX_CLI_VERSION=0.144.6 + +USER root + +RUN apt-get update && apt-get install -y --no-install-recommends \ + nodejs \ + npm \ + ripgrep \ + && npm install --global "@openai/codex@${CODEX_CLI_VERSION}" \ + && npm cache clean --force \ + && rm -rf /var/lib/apt/lists/* + +RUN mkdir -p /home/worker/.codex && chown worker:worker /home/worker/.codex + +ENV CODEX_HOME=/home/worker/.codex + +USER worker diff --git a/services/worker-manager/src/agents/__init__.py b/services/worker-manager/src/agents/__init__.py index 6eb5f234..bc963511 100644 --- a/services/worker-manager/src/agents/__init__.py +++ b/services/worker-manager/src/agents/__init__.py @@ -1,5 +1,6 @@ from .base import AgentConfig from .claude_code import ClaudeCodeAgent +from .codex import CodexAgent from .factory_droid import FactoryDroidAgent -__all__ = ["AgentConfig", "ClaudeCodeAgent", "FactoryDroidAgent"] +__all__ = ["AgentConfig", "ClaudeCodeAgent", "CodexAgent", "FactoryDroidAgent"] diff --git a/services/worker-manager/src/agents/codex.py b/services/worker-manager/src/agents/codex.py new file mode 100644 index 00000000..8b0e5a9c --- /dev/null +++ b/services/worker-manager/src/agents/codex.py @@ -0,0 +1,16 @@ +from typing import List + +from .base import AgentConfig + + +class CodexAgent(AgentConfig): + """Configuration for the OpenAI Codex CLI developer worker.""" + + def get_install_commands(self) -> List[str]: + return [] + + def get_instruction_path(self) -> str: + return "/workspace/AGENTS.md" + + def get_agent_command(self) -> str: + return "codex exec" diff --git a/services/worker-manager/src/codex_auth.py b/services/worker-manager/src/codex_auth.py new file mode 100644 index 00000000..81c924df --- /dev/null +++ b/services/worker-manager/src/codex_auth.py @@ -0,0 +1,54 @@ +"""Fail-fast validation for the dedicated Codex host-session profile.""" + +import json +from pathlib import Path +import stat +import tomllib + + +def _mode(path: Path) -> int: + return stat.S_IMODE(path.stat().st_mode) + + +def validate_codex_host_session(profile_path: str | None) -> None: + """Validate the file-backed ChatGPT session without exposing its contents.""" + if not profile_path: + raise RuntimeError( + "HOST_CODEX_HOME is required for Codex auth_mode=host_session; " + "configure a dedicated profile created with codex login --device-auth" + ) + + profile = Path(profile_path) + if not profile.is_dir(): + raise RuntimeError(f"HOST_CODEX_HOME is not an existing directory: {profile}") + if _mode(profile) != 0o700: + raise RuntimeError(f"HOST_CODEX_HOME must have mode 0700: {profile}") + + auth_path = profile / "auth.json" + if not auth_path.is_file() or auth_path.stat().st_size == 0: + raise RuntimeError(f"Codex host session is missing a non-empty auth.json: {profile}") + if _mode(auth_path) != 0o600: + raise RuntimeError(f"Codex auth.json must have mode 0600: {profile}") + try: + auth_data = json.loads(auth_path.read_text()) + except (OSError, json.JSONDecodeError) as exc: + raise RuntimeError(f"Codex auth.json is unreadable or invalid JSON: {profile}") from exc + if not isinstance(auth_data, dict) or not auth_data: + raise RuntimeError(f"Codex auth.json does not contain a cached session: {profile}") + tokens = auth_data.get("tokens") + if not isinstance(tokens, dict) or not all( + isinstance(tokens.get(name), str) and tokens[name] for name in ("access_token", "refresh_token") + ): + raise RuntimeError(f"Codex auth.json does not contain a refresh-capable ChatGPT session: {profile}") + + config_path = profile / "config.toml" + if not config_path.is_file(): + raise RuntimeError(f"Codex host session is missing config.toml: {profile}") + if _mode(config_path) != 0o600: + raise RuntimeError(f"Codex config.toml must have mode 0600: {profile}") + try: + config = tomllib.loads(config_path.read_text()) + except (OSError, tomllib.TOMLDecodeError) as exc: + raise RuntimeError(f"Codex config.toml is unreadable or invalid TOML: {profile}") from exc + if config.get("cli_auth_credentials_store") != "file": + raise RuntimeError('Codex config.toml must set cli_auth_credentials_store = "file"') diff --git a/services/worker-manager/src/config.py b/services/worker-manager/src/config.py index 9cec9bac..d79da218 100644 --- a/services/worker-manager/src/config.py +++ b/services/worker-manager/src/config.py @@ -24,6 +24,12 @@ class WorkerManagerSettings(BaseSettings): # Host path to .claude directory (for mounting into workers) HOST_CLAUDE_DIR: str | None = None + # Dedicated host Codex profile. It must not point at the operator's live + # ~/.codex directory. The validation path is the same profile mounted + # read-only into worker-manager by Compose. + HOST_CODEX_HOME: str | None = None + HOST_CODEX_VALIDATION_PATH: str | None = None + # Worker subprocess timeout (seconds). Live LLM agents (Claude/Factory) need # well over the noop budget to write and iterate on real code; keep within the # harness LLM_ENGINEERING_TIMEOUT. The noop runner uses its own short timeout. diff --git a/services/worker-manager/src/consumer.py b/services/worker-manager/src/consumer.py index 0f56b308..118d673a 100644 --- a/services/worker-manager/src/consumer.py +++ b/services/worker-manager/src/consumer.py @@ -125,6 +125,7 @@ async def _handle_create(self, cmd: CreateWorkerCommand) -> CreateWorkerResponse env_vars=env_vars, auth_mode=cmd.config.auth_mode, host_claude_dir=cmd.config.host_claude_dir or settings.HOST_CLAUDE_DIR, + host_codex_home=cmd.config.host_codex_home or settings.HOST_CODEX_HOME, api_key=cmd.config.api_key, worker_type=cmd.config.worker_type, project_id=cmd.config.project_id, diff --git a/services/worker-manager/src/container_config.py b/services/worker-manager/src/container_config.py index 290cda07..b5061a31 100644 --- a/services/worker-manager/src/container_config.py +++ b/services/worker-manager/src/container_config.py @@ -14,6 +14,7 @@ class WorkerContainerConfig: capabilities: List[str] auth_mode: str = "host_session" # "host_session" or "api_key" host_claude_dir: Optional[str] = None + host_codex_home: Optional[str] = None api_key: Optional[str] = None workspace_host_path: Optional[str] = None @@ -46,9 +47,14 @@ def to_env_vars( "WORKER_CONSUMER_NAME": self.worker_id, } + if self.agent_type == AgentType.CODEX: + env["CODEX_HOME"] = "/home/worker/.codex" + if self.auth_mode == "api_key" and self.api_key: if self.agent_type == AgentType.FACTORY: env["FACTORY_API_KEY"] = self.api_key + elif self.agent_type == AgentType.CODEX: + env["OPENAI_API_KEY"] = self.api_key else: env["ANTHROPIC_API_KEY"] = self.api_key @@ -62,10 +68,16 @@ def to_volume_mounts(self) -> Dict[str, Dict[str, str]]: volumes = {} # Mount host session directory if in host_session mode - if self.auth_mode == "host_session" and self.host_claude_dir: + if self.auth_mode == "host_session" and self.agent_type == AgentType.CLAUDE and self.host_claude_dir: # Mount to /home/worker/.claude inside container volumes[self.host_claude_dir] = {"bind": "/home/worker/.claude", "mode": "rw"} + if self.auth_mode == "host_session" and self.agent_type == AgentType.CODEX and self.host_codex_home: + volumes[self.host_codex_home] = { + "bind": "/home/worker/.codex", + "mode": "rw", + } + # Mount workspace directory if provided if self.workspace_host_path: volumes[self.workspace_host_path] = {"bind": "/workspace", "mode": "rw"} @@ -104,6 +116,6 @@ def to_docker_run_kwargs(self, network_name: Optional[str] = None) -> Dict[str, def _mem_limit(self) -> str: """Return the Docker memory limit for this worker agent.""" - if self.agent_type in {AgentType.CLAUDE, AgentType.FACTORY}: + if self.agent_type in {AgentType.CLAUDE, AgentType.FACTORY, AgentType.CODEX}: return "4g" return "2g" diff --git a/services/worker-manager/src/image_builder.py b/services/worker-manager/src/image_builder.py index d8f50f0e..81f0da9c 100644 --- a/services/worker-manager/src/image_builder.py +++ b/services/worker-manager/src/image_builder.py @@ -8,8 +8,7 @@ - Compute deterministic hashes for image caching - Provide image tags for cache lookup -Agent CLIs (Claude Code, Factory Droid) are pre-installed in agent-specific -base images (worker-base-claude, worker-base-factory) for faster builds. +Agent CLIs are pre-installed in agent-specific base images for faster builds. """ import hashlib @@ -19,15 +18,18 @@ AGENT_BASE_IMAGES = { "claude": "worker-base-claude:latest", "factory": "worker-base-factory:latest", + "codex": "worker-base-codex:latest", "noop": "worker-base-claude:latest", # Reuse claude base (has git+bash) } -DEFAULT_BASE_IMAGE = "worker-base-claude:latest" - def get_base_image(agent_type: str) -> str: """Get the appropriate base image for the agent type.""" - return AGENT_BASE_IMAGES.get(agent_type.lower(), DEFAULT_BASE_IMAGE) + normalized = agent_type.value if hasattr(agent_type, "value") else str(agent_type) + try: + return AGENT_BASE_IMAGES[normalized.lower()] + except KeyError: + raise ValueError(f"Unknown agent type: {normalized}") from None # Capability to installation commands mapping @@ -109,12 +111,12 @@ def generate_dockerfile(self, capabilities: list[str], agent_type: str = "claude """ Generate Dockerfile content for given capabilities. - Agent CLI is already pre-installed in the base image (worker-base-claude - or worker-base-factory), so we only add capability-specific packages. + Agent CLI is already pre-installed in its agent-specific base image, so + we only add capability-specific packages. Args: capabilities: List of capabilities to install (e.g., ["GIT", "CURL"]) - agent_type: Type of agent ("claude" or "factory") + agent_type: Type of agent ("claude", "factory", "codex", or "noop") Returns: Complete Dockerfile content as string @@ -179,7 +181,7 @@ def get_image_tag(self, capabilities: list[str], prefix: str, agent_type: str = Args: capabilities: List of capabilities prefix: Image name prefix (e.g., "worker" or "worker-test") - agent_type: Type of agent ("claude" or "factory") + agent_type: Type of agent ("claude", "factory", "codex", or "noop") Returns: Full image tag (e.g., "worker:a1b2c3d4e5f6") diff --git a/services/worker-manager/src/manager.py b/services/worker-manager/src/manager.py index 4b8d65b1..c06369ce 100644 --- a/services/worker-manager/src/manager.py +++ b/services/worker-manager/src/manager.py @@ -291,11 +291,15 @@ async def ensure_or_build_image( def _get_agent(self, agent_type: AgentType): """Get agent instance by type.""" - from .agents import ClaudeCodeAgent, FactoryDroidAgent + from .agents import ClaudeCodeAgent, CodexAgent, FactoryDroidAgent if agent_type == AgentType.FACTORY: return FactoryDroidAgent() - return ClaudeCodeAgent() + if agent_type == AgentType.CODEX: + return CodexAgent() + if agent_type in {AgentType.CLAUDE, AgentType.NOOP}: + return ClaudeCodeAgent() + raise ValueError(f"Unknown agent type: {agent_type}") # Statuses that indicate the worker is no longer alive and can be cleaned up _TERMINAL_STATUSES = frozenset({WorkerStatus.DEAD, WorkerStatus.FAILED, WorkerStatus.STOPPED}) @@ -342,6 +346,7 @@ async def create_worker_with_capabilities( task_content: str | None = None, auth_mode: str = "host_session", host_claude_dir: str | None = None, + host_codex_home: str | None = None, api_key: str | None = None, env_vars: Dict[str, str] = None, worker_type: str = "developer", @@ -360,6 +365,12 @@ async def create_worker_with_capabilities( project_id=project_id, ) + if agent_type == AgentType.CODEX and auth_mode == "host_session": + from .codex_auth import validate_codex_host_session + + validation_path = settings.HOST_CODEX_VALIDATION_PATH or host_codex_home + validate_codex_host_session(validation_path) + if project_id: existing_worker = await self._check_project_lock(project_id) if existing_worker: @@ -396,6 +407,7 @@ async def create_worker_with_capabilities( capabilities=capabilities, auth_mode=auth_mode, host_claude_dir=host_claude_dir, + host_codex_home=host_codex_home, api_key=api_key, ) diff --git a/services/worker-manager/tests/unit/test_agent_factories.py b/services/worker-manager/tests/unit/test_agent_factories.py index e737114a..1106bca9 100644 --- a/services/worker-manager/tests/unit/test_agent_factories.py +++ b/services/worker-manager/tests/unit/test_agent_factories.py @@ -1,4 +1,5 @@ from src.agents.claude_code import ClaudeCodeAgent +from src.agents.codex import CodexAgent from src.agents.factory_droid import FactoryDroidAgent @@ -28,3 +29,10 @@ def test_get_install_commands_returns_empty(self): def test_get_instruction_path_returns_agents_md(self): """Factory uses AGENTS.md for instructions.""" assert FactoryDroidAgent().get_instruction_path() == "/workspace/AGENTS.md" + + +class TestCodexAgent: + def test_uses_agents_md_and_codex_exec(self): + agent = CodexAgent() + assert agent.get_instruction_path() == "/workspace/AGENTS.md" + assert agent.get_agent_command() == "codex exec" diff --git a/services/worker-manager/tests/unit/test_codex_auth.py b/services/worker-manager/tests/unit/test_codex_auth.py new file mode 100644 index 00000000..24f0a93a --- /dev/null +++ b/services/worker-manager/tests/unit/test_codex_auth.py @@ -0,0 +1,83 @@ +import json +import os +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from src.codex_auth import validate_codex_host_session +from src.manager import WorkerManager +from shared.contracts.vocab import AgentType + + +def _write_profile(path, *, auth_mode=0o600, config_mode=0o600): + path.mkdir(mode=0o700) + auth_path = path / "auth.json" + auth_path.write_text( + json.dumps( + { + "tokens": { + "access_token": "test-access", + "refresh_token": "test-refresh", + } + } + ) + ) + auth_path.chmod(auth_mode) + config_path = path / "config.toml" + config_path.write_text('cli_auth_credentials_store = "file"\n') + config_path.chmod(config_mode) + + +def test_valid_codex_host_session(tmp_path): + profile = tmp_path / "codex-worker" + _write_profile(profile) + + validate_codex_host_session(str(profile)) + + +def test_missing_codex_home_fails_fast(tmp_path): + with pytest.raises(RuntimeError, match="HOST_CODEX_HOME"): + validate_codex_host_session(str(tmp_path / "missing")) + + +@pytest.mark.asyncio +async def test_manager_rejects_missing_codex_session_before_image_resolution(tmp_path): + docker = MagicMock() + docker.image_exists = AsyncMock() + manager = WorkerManager(redis=MagicMock(), docker_client=docker) + + with pytest.raises(RuntimeError, match="HOST_CODEX_HOME"): + await manager.create_worker_with_capabilities( + worker_id="worker-codex", + capabilities=[], + base_image="worker-base-codex:latest", + agent_type=AgentType.CODEX, + host_codex_home=str(tmp_path / "missing"), + ) + + docker.image_exists.assert_not_awaited() + + +@pytest.mark.parametrize( + ("mutate", "message"), + [ + (lambda profile: os.chmod(profile, 0o755), "0700"), + (lambda profile: os.chmod(profile / "auth.json", 0o644), "0600"), + (lambda profile: (profile / "auth.json").write_text(""), "auth.json"), + ( + lambda profile: (profile / "auth.json").write_text(json.dumps({"tokens": {"access_token": "test-access"}})), + "refresh-capable", + ), + ( + lambda profile: (profile / "config.toml").write_text('cli_auth_credentials_store = "keyring"\n'), + "cli_auth_credentials_store", + ), + ], +) +def test_unsuitable_codex_session_fails_fast(tmp_path, mutate, message): + profile = tmp_path / "codex-worker" + _write_profile(profile) + mutate(profile) + + with pytest.raises(RuntimeError, match=message): + validate_codex_host_session(str(profile)) diff --git a/services/worker-manager/tests/unit/test_container_config.py b/services/worker-manager/tests/unit/test_container_config.py index 941dc9d3..67c0a6f1 100644 --- a/services/worker-manager/tests/unit/test_container_config.py +++ b/services/worker-manager/tests/unit/test_container_config.py @@ -35,6 +35,53 @@ def test_host_session_mode_adds_volume_mount(self): assert "/home/user/.claude" in volumes assert volumes["/home/user/.claude"]["bind"] == "/home/worker/.claude" + def test_codex_host_session_uses_dedicated_rw_mount(self): + config = WorkerContainerConfig( + worker_id="test-1", + worker_type="developer", + agent_type="codex", + capabilities=["GIT"], + auth_mode="host_session", + host_codex_home="/home/user/.codex-worker", + ) + + volumes = config.to_volume_mounts() + + assert volumes["/home/user/.codex-worker"] == { + "bind": "/home/worker/.codex", + "mode": "rw", + } + assert all(source != "/home/user/.codex" for source in volumes) + + def test_codex_host_session_exports_container_codex_home(self): + config = WorkerContainerConfig( + worker_id="test-1", + worker_type="developer", + agent_type="codex", + capabilities=["GIT"], + auth_mode="host_session", + host_codex_home="/home/user/.codex-worker", + ) + + env = config.to_env_vars(redis_url="redis://r", api_url="http://api") + + assert env["CODEX_HOME"] == "/home/worker/.codex" + assert "OPENAI_API_KEY" not in env + + def test_codex_api_key_mode_uses_openai_variable(self): + config = WorkerContainerConfig( + worker_id="test-1", + worker_type="developer", + agent_type="codex", + capabilities=["GIT"], + auth_mode="api_key", + api_key="sk-openai-test", + ) + + env = config.to_env_vars(redis_url="redis://r", api_url="http://api") + + assert env["OPENAI_API_KEY"] == "sk-openai-test" + def test_api_key_mode_adds_env_var(self): """API key auth mode should add ANTHROPIC_API_KEY.""" config = WorkerContainerConfig( @@ -83,6 +130,15 @@ def test_factory_worker_gets_real_agent_memory_limit(self): kwargs = config.to_docker_run_kwargs() assert kwargs["mem_limit"] == "4g" + def test_codex_worker_gets_real_agent_memory_limit(self): + config = WorkerContainerConfig( + worker_id="test-1", + worker_type="developer", + agent_type="codex", + capabilities=[], + ) + assert config.to_docker_run_kwargs()["mem_limit"] == "4g" + def test_to_docker_run_kwargs_with_network_name(self): """With network_name, should attach to that network.""" config = WorkerContainerConfig( diff --git a/services/worker-manager/tests/unit/test_image_builder.py b/services/worker-manager/tests/unit/test_image_builder.py index 31f09728..023e4ec4 100644 --- a/services/worker-manager/tests/unit/test_image_builder.py +++ b/services/worker-manager/tests/unit/test_image_builder.py @@ -10,7 +10,7 @@ import pytest # This import will fail initially (RED phase) - module doesn't exist yet -from src.image_builder import ImageBuilder, compute_image_hash +from src.image_builder import ImageBuilder, compute_image_hash, get_base_image class TestComputeImageHash: @@ -45,6 +45,13 @@ def test_same_capabilities_different_agent_produce_different_hash(self): hash_factory = compute_image_hash(["GIT"], agent_type="factory") assert hash_claude != hash_factory + def test_codex_has_distinct_hash(self): + hash_codex = compute_image_hash(["GIT"], agent_type="codex") + assert hash_codex not in { + compute_image_hash(["GIT"], agent_type="claude"), + compute_image_hash(["GIT"], agent_type="factory"), + } + def test_hash_deterministic_with_agent_type(self): """Agent type should be part of deterministic hash.""" h1 = compute_image_hash(["GIT", "CURL"], agent_type="claude") @@ -118,6 +125,14 @@ def test_dockerfile_agent_type_creates_unique_layer(self, builder): assert "factory" in dockerfile_factory assert dockerfile_claude != dockerfile_factory + def test_codex_uses_dedicated_base_image(self, builder): + dockerfile = builder.generate_dockerfile(capabilities=["GIT"], agent_type="codex") + assert dockerfile.startswith("FROM worker-base-codex:latest") + + def test_unknown_agent_type_is_rejected(self): + with pytest.raises(ValueError, match="Unknown agent type"): + get_base_image("unknown") + class TestImageBuilderImageTag: """Test image tag generation.""" diff --git a/shared/config.py b/shared/config.py index 7ee2da80..e05c946d 100644 --- a/shared/config.py +++ b/shared/config.py @@ -138,13 +138,13 @@ def default_agent_type_field(): """Default agent type field definition. Can be set via `DEFAULT_AGENT_TYPE` environment variable. - Valid values: "claude" or "factory" + Valid values: "claude", "factory", or "codex" Defaults to "claude". """ return Field( default="claude", alias="DEFAULT_AGENT_TYPE", - description="Default AI agent to use (claude or factory)", + description="Default AI agent to use (claude, factory, or codex)", ) diff --git a/shared/contracts/queues/worker.py b/shared/contracts/queues/worker.py index d8dd77e3..9ceeb904 100644 --- a/shared/contracts/queues/worker.py +++ b/shared/contracts/queues/worker.py @@ -64,6 +64,7 @@ class WorkerConfig(BaseModel): env_vars: dict[str, str] = {} auth_mode: Literal["host_session", "api_key"] = "host_session" host_claude_dir: str | None = None + host_codex_home: str | None = None api_key: str | None = None project_id: str | None = None # Project ID for workspace persistence repo_id: str | None = None # Repository ID — mount pre-scaffolded workspace diff --git a/shared/contracts/vocab.py b/shared/contracts/vocab.py index f2f47980..22465a9c 100644 --- a/shared/contracts/vocab.py +++ b/shared/contracts/vocab.py @@ -14,6 +14,7 @@ class AgentType(StrEnum): CLAUDE = "claude" # Claude Code FACTORY = "factory" # Factory.ai Droid + CODEX = "codex" # OpenAI Codex CLI NOOP = "noop" # No-op runner for E2E testing (empty commit + push) @@ -21,9 +22,10 @@ class WorkerCliKind(StrEnum): """CLI-agent wire identity reported on `worker:events`. Deliberately distinct from :class:`AgentType`: these are the historical - `worker_type` values a running CLI reports about itself, and they do not map - one-to-one onto the agent we ask for (claude/factory/noop). Kept separate on - purpose, not merged. + `worker_type` values a running CLI reports about itself. The Codex spelling + overlaps with :class:`AgentType`, while the Claude and Factory spellings do + not. The concepts stay separate because this field reports CLI identity, + not the requested developer-worker type. """ DROID = "droid" diff --git a/shared/tests/test_worker_contracts.py b/shared/tests/test_worker_contracts.py index 64318531..3f8bf304 100644 --- a/shared/tests/test_worker_contracts.py +++ b/shared/tests/test_worker_contracts.py @@ -10,6 +10,22 @@ class TestScaffoldConfig: + def test_codex_worker_config_roundtrip_keeps_auth_profile(self): + config = WorkerConfig( + name="dev-codex", + worker_type="developer", + agent_type="codex", + instructions="Read AGENTS.md", + allowed_commands=["*"], + capabilities=[WorkerCapability.GIT], + host_codex_home="/srv/codex-worker", + ) + + restored = WorkerConfig.model_validate_json(config.model_dump_json()) + + assert restored.agent_type is AgentType.CODEX + assert restored.host_codex_home == "/srv/codex-worker" + def test_roundtrip_serialization(self): """ScaffoldConfig survives JSON round-trip through CreateWorkerCommand.""" scaffold = ScaffoldConfig( diff --git a/shared/tests/unit/test_vocab.py b/shared/tests/unit/test_vocab.py index 1a653047..af57258c 100644 --- a/shared/tests/unit/test_vocab.py +++ b/shared/tests/unit/test_vocab.py @@ -27,7 +27,7 @@ class TestCanonicalValues: def test_agent_type_values(self): - assert {a.value for a in AgentType} == {"claude", "factory", "noop"} + assert {a.value for a in AgentType} == {"claude", "factory", "codex", "noop"} def test_action_type_values(self): assert {a.value for a in ActionType} == {"create", "feature", "fix"} @@ -63,6 +63,7 @@ class _M(BaseModel): type: AgentType assert _M(type="factory").type is AgentType.FACTORY + assert _M(type="codex").type is AgentType.CODEX def test_dto_rejects_unknown(self): class _M(BaseModel): @@ -151,8 +152,9 @@ def test_values_are_the_historical_wire_set(self): assert {k.value for k in WorkerCliKind} == {"droid", "claude_code", "codex"} def test_distinct_from_agent_type(self): - # Explicitly NOT merged with AgentType (claude/factory/noop). - assert {k.value for k in WorkerCliKind}.isdisjoint({a.value for a in AgentType}) + assert WorkerCliKind is not AgentType + assert WorkerCliKind.CODEX.value == AgentType.CODEX.value + assert WorkerCliKind.CLAUDE_CODE.value != AgentType.CLAUDE.value def test_worker_event_parses_cli_kind(self): ev = parse_worker_event( From eaacc03df46a0077f6fbc6ba25302db1610bcbad Mon Sep 17 00:00:00 2001 From: vladmesh <16962535+vladmesh@users.noreply.github.com> Date: Sat, 18 Jul 2026 19:37:32 +0300 Subject: [PATCH 2/3] fix: complete Codex worker deployment path --- .github/workflows/deploy.yml | 1 + docker-compose.prod.yml | 3 ++- docs/CHANGELOG.md | 6 +++++- docs/DEPLOY.md | 1 + docs/coding-agents.md | 4 ++++ ...codegen-orchestrator-642-codex-worker-runtime.md | 12 ++++++++++++ infra/scripts/pull-worker-images.sh | 1 + .../src/worker_wrapper/runners/codex.py | 10 +++++++++- .../tests/component/test_full_cycle.py | 8 ++++++-- packages/worker-wrapper/tests/unit/test_runners.py | 2 ++ services/worker-manager/src/container_config.py | 2 +- .../tests/unit/test_container_config.py | 5 +++-- shared/tests/unit/test_codex_deployment_contract.py | 13 +++++++++++++ 13 files changed, 60 insertions(+), 8 deletions(-) create mode 100644 shared/tests/unit/test_codex_deployment_contract.py diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 962aff87..07cfd37c 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -74,6 +74,7 @@ jobs: # Worker agents FACTORY_API_KEY=${{ secrets.FACTORY_API_KEY }} HOST_CLAUDE_DIR=${{ secrets.HOST_CLAUDE_DIR }} + HOST_CODEX_HOME=${{ secrets.HOST_CODEX_HOME }} # Observability LANGCHAIN_API_KEY=${{ secrets.LANGCHAIN_API_KEY }} diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 7ff16c16..2d477ef9 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -7,7 +7,8 @@ # - DB credentials have no defaults (fail-fast if .env is misconfigured) # - Redis AOF persistence enabled # -# Secret paths (SSH key, GitHub App PEM, Telethon session, HOST_CLAUDE_DIR) +# Secret paths (SSH key, GitHub App PEM, Telethon session, HOST_CLAUDE_DIR, +# HOST_CODEX_HOME) # are controlled via env vars in .env — see docs/DEPLOY.md for required values. services: diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 1656c613..2ac1561a 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -9,8 +9,12 @@ non-root worker with the same workspace and resource limits. Host-session mode validates and mounts an isolated file-backed `HOST_CODEX_HOME` read-write for token refresh, without requiring an API key or defaulting to `~/.codex`. + Production deployment persists that host profile path and pulls the Codex + base image alongside the common, Claude, and Factory images. Codex runs non-interactively with workspace-only writes and reports through - the existing HTTP bridge; its stdout and stderr are not persisted as results. + the existing HTTP bridge; sandboxed network commands are enabled for the + bridge, dependency access, and Git push. Its stdout and stderr are not + persisted as results. Optional API-key mode uses `CODEX_API_KEY`. - Make live harness API reads fail loud before body parsing. Parsed `tests/live` API responses now call `raise_for_status()` first, including scaffold/project polling and auth-gated server, ssh-key and port-allocation checks, with a mock-transport regression for a rejected polling call. diff --git a/docs/DEPLOY.md b/docs/DEPLOY.md index d0d503d6..2bc30a9a 100644 --- a/docs/DEPLOY.md +++ b/docs/DEPLOY.md @@ -102,6 +102,7 @@ All secrets must be configured in the repository's **production** environment. |--------|-------------| | `FACTORY_API_KEY` | Factory.ai API key | | `HOST_CLAUDE_DIR` | Path to `.claude` directory on prod server | +| `HOST_CODEX_HOME` | Path to the dedicated file-backed Codex profile described in `docs/coding-agents.md` | ### Admin UI diff --git a/docs/coding-agents.md b/docs/coding-agents.md index 339b1d1e..d41b4e1c 100644 --- a/docs/coding-agents.md +++ b/docs/coding-agents.md @@ -45,6 +45,7 @@ Codex is available only for developer workers. The image pins Codex CLI ```bash codex exec --sandbox workspace-write \ + --config sandbox_workspace_write.network_access=true \ "Read TASK.md and AGENTS.md, then complete the task described in TASK.md." ``` @@ -52,6 +53,9 @@ The task is in `/workspace/TASK.md`, and the shared developer instructions are in `/workspace/AGENTS.md`. The agent must report success or failure through `POST http://localhost:9090/result`. CLI stdout and stderr are diagnostics and are neither accepted as the business result nor persisted for Codex workers. +The per-run network override is required because `workspace-write` otherwise +blocks the agent's localhost result call, dependency access, and Git push. The +Docker worker network remains the outer isolation boundary. ### Dedicated ChatGPT session profile diff --git a/docs/reports/codegen-orchestrator-642-codex-worker-runtime.md b/docs/reports/codegen-orchestrator-642-codex-worker-runtime.md index 0719a5a5..abf38209 100644 --- a/docs/reports/codegen-orchestrator-642-codex-worker-runtime.md +++ b/docs/reports/codegen-orchestrator-642-codex-worker-runtime.md @@ -10,6 +10,11 @@ the existing worker command DTO, worker-manager, the agent-specific image, and worker-wrapper. The queue envelope is unchanged. Unknown agent types fail at the typed or explicit routing boundary. +The runner keeps filesystem writes in the `workspace-write` sandbox and +enables sandboxed network commands so the agent can reach the localhost result +bridge, dependencies, and Git remote. Optional API-key mode uses the CLI's +`CODEX_API_KEY`; host-session mode sets neither API-key variable. + `auth_mode=host_session` requires a separate `HOST_CODEX_HOME` with directory mode `0700`, `auth.json` and `config.toml` mode `0600`, a non-empty JSON session, refresh-capable access and refresh tokens, and @@ -34,6 +39,13 @@ uv run pytest packages/worker-wrapper/tests/component -q docker compose config --quiet --no-env-resolution exit 0 +uv run pytest shared/tests/unit/test_codex_deployment_contract.py -q +1 passed + +Production deploy writes `HOST_CODEX_HOME` and pulls +`worker-base-common`, `worker-base-claude`, `worker-base-factory`, and +`worker-base-codex` from GHCR before starting the stack. + make rebuild-worker-images worker-base-common:latest built worker-base-claude:latest built diff --git a/infra/scripts/pull-worker-images.sh b/infra/scripts/pull-worker-images.sh index 475109eb..2e452ef1 100755 --- a/infra/scripts/pull-worker-images.sh +++ b/infra/scripts/pull-worker-images.sh @@ -21,6 +21,7 @@ IMAGES=( "worker-base-common" "worker-base-claude" "worker-base-factory" + "worker-base-codex" ) echo "Logging in to GHCR..." diff --git a/packages/worker-wrapper/src/worker_wrapper/runners/codex.py b/packages/worker-wrapper/src/worker_wrapper/runners/codex.py index 2e2d293c..ed442c38 100644 --- a/packages/worker-wrapper/src/worker_wrapper/runners/codex.py +++ b/packages/worker-wrapper/src/worker_wrapper/runners/codex.py @@ -8,4 +8,12 @@ class CodexRunner(AgentRunner): """Runner for Codex CLI non-interactive developer work.""" def build_command(self, prompt: str) -> list[str]: - return ["codex", "exec", "--sandbox", "workspace-write", prompt] + return [ + "codex", + "exec", + "--sandbox", + "workspace-write", + "--config", + "sandbox_workspace_write.network_access=true", + prompt, + ] diff --git a/packages/worker-wrapper/tests/component/test_full_cycle.py b/packages/worker-wrapper/tests/component/test_full_cycle.py index 00c35684..3273dbea 100644 --- a/packages/worker-wrapper/tests/component/test_full_cycle.py +++ b/packages/worker-wrapper/tests/component/test_full_cycle.py @@ -116,6 +116,10 @@ async def test_codex_exec_uses_workspace_sandbox_without_output_bridge( "--sandbox", "workspace-write", ) - assert "TASK.md" in mock_exec.call_args.args[4] - assert "not part of the command" not in mock_exec.call_args.args[4] + assert mock_exec.call_args.args[4:6] == ( + "--config", + "sandbox_workspace_write.network_access=true", + ) + assert "TASK.md" in mock_exec.call_args.args[6] + assert "not part of the command" not in mock_exec.call_args.args[6] assert wrapper._agent_stdout_tail is None diff --git a/packages/worker-wrapper/tests/unit/test_runners.py b/packages/worker-wrapper/tests/unit/test_runners.py index 82bae15e..8a27bfaf 100644 --- a/packages/worker-wrapper/tests/unit/test_runners.py +++ b/packages/worker-wrapper/tests/unit/test_runners.py @@ -69,6 +69,8 @@ def test_build_command_uses_non_interactive_workspace_sandbox(self): "exec", "--sandbox", "workspace-write", + "--config", + "sandbox_workspace_write.network_access=true", "Read TASK.md and AGENTS.md", ] diff --git a/services/worker-manager/src/container_config.py b/services/worker-manager/src/container_config.py index b5061a31..dcb93b0d 100644 --- a/services/worker-manager/src/container_config.py +++ b/services/worker-manager/src/container_config.py @@ -54,7 +54,7 @@ def to_env_vars( if self.agent_type == AgentType.FACTORY: env["FACTORY_API_KEY"] = self.api_key elif self.agent_type == AgentType.CODEX: - env["OPENAI_API_KEY"] = self.api_key + env["CODEX_API_KEY"] = self.api_key else: env["ANTHROPIC_API_KEY"] = self.api_key diff --git a/services/worker-manager/tests/unit/test_container_config.py b/services/worker-manager/tests/unit/test_container_config.py index 67c0a6f1..3f0046f9 100644 --- a/services/worker-manager/tests/unit/test_container_config.py +++ b/services/worker-manager/tests/unit/test_container_config.py @@ -68,7 +68,7 @@ def test_codex_host_session_exports_container_codex_home(self): assert env["CODEX_HOME"] == "/home/worker/.codex" assert "OPENAI_API_KEY" not in env - def test_codex_api_key_mode_uses_openai_variable(self): + def test_codex_api_key_mode_uses_exec_scoped_variable(self): config = WorkerContainerConfig( worker_id="test-1", worker_type="developer", @@ -80,7 +80,8 @@ def test_codex_api_key_mode_uses_openai_variable(self): env = config.to_env_vars(redis_url="redis://r", api_url="http://api") - assert env["OPENAI_API_KEY"] == "sk-openai-test" + assert env["CODEX_API_KEY"] == "sk-openai-test" + assert "OPENAI_API_KEY" not in env def test_api_key_mode_adds_env_var(self): """API key auth mode should add ANTHROPIC_API_KEY.""" diff --git a/shared/tests/unit/test_codex_deployment_contract.py b/shared/tests/unit/test_codex_deployment_contract.py new file mode 100644 index 00000000..e3f8ff76 --- /dev/null +++ b/shared/tests/unit/test_codex_deployment_contract.py @@ -0,0 +1,13 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[3] + + +def test_production_deploy_provisions_codex_worker_runtime(): + pull_script = (ROOT / "infra/scripts/pull-worker-images.sh").read_text() + deploy_workflow = (ROOT / ".github/workflows/deploy.yml").read_text() + deploy_runbook = (ROOT / "docs/DEPLOY.md").read_text() + + assert '"worker-base-codex"' in pull_script + assert "HOST_CODEX_HOME=${{ secrets.HOST_CODEX_HOME }}" in deploy_workflow + assert "`HOST_CODEX_HOME`" in deploy_runbook From cab6e2f21e9c2dc1242537765bef91e3aee6c7cc Mon Sep 17 00:00:00 2001 From: vladmesh <16962535+vladmesh@users.noreply.github.com> Date: Sat, 18 Jul 2026 19:45:26 +0300 Subject: [PATCH 3/3] docs: document Codex worker across active guides --- README.md | 2 +- docs/GLOSSARY.md | 4 ++-- docs/NODES.md | 2 +- docs/ci-gate-baseline.md | 2 +- docs/playbooks/line2-engineering.md | 4 ++-- infra/scripts/pull-worker-images.sh | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index a0a95a83..05e673a0 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ - **Автономность**: Человек заходит раз в несколько дней, смотрит отчёты, докидывает деньги - **Агенты как узлы графа**: Product Owner — это агент LangGraph, управляющий процессом. -- **Worker Manager**: Запускает изолированные контейнеры для Engineering/DevOps задач (Claude Code, Factory.ai). +- **Worker Manager**: Запускает изолированные контейнеры для Engineering/DevOps задач (Claude Code, Factory.ai, OpenAI Codex). - **Нелинейность**: Агенты могут вызывать друг друга в любом порядке - **Spec-first**: Используем [service-template](https://github.com/vladmesh/service-template) для генерации кода diff --git a/docs/GLOSSARY.md b/docs/GLOSSARY.md index 050db2ed..0a300d30 100644 --- a/docs/GLOSSARY.md +++ b/docs/GLOSSARY.md @@ -30,7 +30,7 @@ Docker-контейнер с CLI coding agent внутри. Использует **Developer Worker** — Контейнер с coding agent. Для задач внутри Story — переиспользуется между задачами (worker_id хранится в Redis hash `story:workers`). Для standalone задач — эфемерный, удаляется после завершения. Stateless — контекст это код в репо + ошибки. **Управляется:** `worker-manager` -**Конфигурация:** Промпты хранятся в `services/langgraph/src/prompts/developer_worker/INSTRUCTIONS.md`. Worker-manager маппит их в agent-specific файлы через `get_instruction_path()`: Claude → `CLAUDE.md`, Factory → `AGENTS.md`. Также инжектится `TASK.md` с конкретной задачей. +**Конфигурация:** Промпты хранятся в `services/langgraph/src/prompts/developer_worker/INSTRUCTIONS.md`. Worker-manager маппит их в agent-specific файлы через `get_instruction_path()`: Claude → `CLAUDE.md`, Factory и Codex → `AGENTS.md`. Также инжектится `TASK.md` с конкретной задачей. ### Project Status (Статус проекта) Жизненный цикл проекта. Минимальный набор: `draft` → `active` → `paused` / `archived`. Не содержит процессных статусов (scaffolding, deploying) — активность определяется дочерними сущностями (Story, Run). @@ -71,7 +71,7 @@ LangGraph ReactAgent, живущий внутри сервиса langgraph, вы ### CLI-Agent (CLI-Агент) AI который работает внутри Developer Worker контейнера. -**Реализации:** Claude Code, Factory.ai Droid. +**Реализации:** Claude Code, Factory.ai Droid, OpenAI Codex CLI. **Отличие от Service Agent:** CLI-Agent — это "личность" в эфемерном контейнере с доступом к bash и ФС, а Service Agent — нода в графе langgraph-сервиса, общающаяся через @tool. diff --git a/docs/NODES.md b/docs/NODES.md index 2fa1fcdc..39017e70 100644 --- a/docs/NODES.md +++ b/docs/NODES.md @@ -44,7 +44,7 @@ 2. Architect Consumer (langgraph) ждёт завершения scaffold (poll project.status != draft, до 5 мин), затем декомпозирует story в tasks (видит tree, specs summary: модели, домены, события) 3. Task Dispatcher находит разблокированные tasks, создаёт Runs, публикует в `engineering:queue` с `branch=story/{story_id}` 4. Engineering worker создает GitHub-репозиторий и устанавливает registry secrets -5. Спавнит контейнер через `worker-manager` (Claude Code / Factory.ai) +5. Спавнит контейнер через `worker-manager` (Claude Code / Factory.ai / OpenAI Codex) 6. Worker-manager creates/checks out `story/{story_id}` branch, инжектит инструкции из `services/langgraph/src/prompts/developer_worker/INSTRUCTIONS.md` и `TASK.md` (в `/workspace/TASK.md`) 7. Агент работает на feature branch и пушит туда diff --git a/docs/ci-gate-baseline.md b/docs/ci-gate-baseline.md index 4b8ebfae..0109469c 100644 --- a/docs/ci-gate-baseline.md +++ b/docs/ci-gate-baseline.md @@ -38,7 +38,7 @@ Path-routed checks: - Service matrix covers `api`, `langgraph`, `scheduler`, `telegram_bot`, `worker-manager`, and `infra`. - Integration matrix covers `backend`, `template`, `frontend`, `infra`, and `po-tools`. -- `backend` integration stays `workflow_dispatch` only because it exercises DIND worker containers and Claude/factory worker boundaries. It is preserved for manual full-matrix runs, but it is not a deterministic PR merge-gate suite. +- `backend` integration stays `workflow_dispatch` only because it exercises DIND worker containers and coding-agent worker boundaries. It is preserved for manual full-matrix runs, but it is not a deterministic PR merge-gate suite. - Shared code, packages, workflow edits, Makefile edits, dependency root changes, Docker test infrastructure, and integration test changes trigger the broad affected matrices instead of a single service subset. Skipped-command guard: diff --git a/docs/playbooks/line2-engineering.md b/docs/playbooks/line2-engineering.md index 8e69e479..7a6ea77c 100644 --- a/docs/playbooks/line2-engineering.md +++ b/docs/playbooks/line2-engineering.md @@ -1,7 +1,7 @@ # Line 2: Engineering Flow Playbook Manual test playbook: submit engineering tasks for every valid module combination, -verify that Claude Code (or Factory.ai) builds a working project end-to-end. +verify that Claude Code, Factory.ai, or OpenAI Codex builds a working project end-to-end. **Executor**: Human or Claude Code, step by step. **Not a script** — adapt commands as needed. @@ -479,4 +479,4 @@ For running all 7 tests at a given level: - **Telegram bot token**: For Levels A/B, not needed (code generation doesn't require real tokens). For Level C with tg_bot, provide a real token from @BotFather before triggering. - **Parallel runs**: Possible but not recommended. Worker-manager handles one container at a time. Multiple engineering tasks will queue up. -- **Cost**: Each test spawns a Claude Code (or Factory.ai) worker session. Budget accordingly. +- **Cost**: Each test spawns a paid coding-agent worker session. Budget accordingly. diff --git a/infra/scripts/pull-worker-images.sh b/infra/scripts/pull-worker-images.sh index 2e452ef1..8163c6a5 100755 --- a/infra/scripts/pull-worker-images.sh +++ b/infra/scripts/pull-worker-images.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Pull worker base images from GHCR and retag to local names -# expected by worker-manager (worker-base-common:latest, worker-base-claude:latest, etc.) +# expected by worker-manager (common, Claude, Factory, and Codex images) # # Required env vars: # GHCR_TOKEN — GitHub token with packages:read scope