diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f2f0dc1e1..31982a975 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,6 +36,7 @@ jobs: bash -n crates/paw-codex-worker/scripts/webhook-intake-smoke.sh bash -n crates/paw-codex-worker/scripts/production-readiness.sh bash -n crates/paw-codex-worker/scripts/production-preflight.sh + bash -n crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh bash -n crates/paw-codex-worker/scripts/production-git-ancestry-guard.sh bash -n crates/paw-codex-worker/scripts/production-observe-only.sh bash -n crates/paw-codex-worker/scripts/production-preflight-github-smoke.sh @@ -47,6 +48,28 @@ jobs: bash -n crates/paw-codex-worker/scripts/paw-patrol-acceptance.sh crates/paw-codex-worker/scripts/ci-actions-runtime-smoke.sh crates/paw-codex-worker/scripts/production-git-ancestry-guard-smoke.sh + - name: Agent Answers proof precheck + run: | + set -euo pipefail + PROOF_DIR="${RUNNER_TEMP}/agent-answers-live-proof-precheck" + rm -rf "$PROOF_DIR" + PRECHECK_ONLY=1 \ + PROOF_DIR="$PROOF_DIR" \ + ALLOW_PRODUCTION_WRITE=1 \ + CONFIRM_AGENT_ANSWERS_LIVE_PROOF=1 \ + CONFIRM_PRODUCTION_DATADOG_QUERY_SECRETS=1 \ + TEMPER_URL=https://temperpaw-ci.example.invalid \ + WORKER_TOKEN=ci-proof-token \ + DIRECTED_EVOLUTION_ORGANISM_ID=agent-answers-ci-organism \ + DIRECTED_EVOLUTION_DIRECTION_ID=agent-answers-ci-direction \ + START_LOCAL_WORKER=0 \ + crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh + jq -e '.status == "ready" and .no_mutation == true and .would_create_live_episode == true and .evidence_requirements.mandatory_datadog_observer_evidence == true and .evidence_requirements.mandatory_datadog_telemetry_evaluator_evidence == true' "$PROOF_DIR/summary.json" + grep -F '"no_mutation": true' "$PROOF_DIR/summary.json" + grep -F '"mandatory_datadog_observer_evidence": true' "$PROOF_DIR/summary.json" + grep -F '"mandatory_datadog_telemetry_evaluator_evidence": true' "$PROOF_DIR/summary.json" + test ! -e "$PROOF_DIR/episode-start.json" + test ! -e "$PROOF_DIR/agent-answers-episode-contract.json" - name: WASM build (os-apps) run: | set -e diff --git a/.proofs/081-directed-evolution-local-feedback-closure.md b/.proofs/081-directed-evolution-local-feedback-closure.md new file mode 100644 index 000000000..08035272d --- /dev/null +++ b/.proofs/081-directed-evolution-local-feedback-closure.md @@ -0,0 +1,131 @@ +# Directed Evolution Local Feedback Closure Proof + +Date: 2026-06-03 + +This proof records the strongest proof cycle available without production +Temper credentials. It is not a substitute for the required live Agent Answers +Datadog proof, but it does exercise the current Genesis-authored Directed +Evolution app, shared `paw-orchestration` worker entities, and the local +TemperPaw worker loop against a fresh local Temper runtime. + +## Runtime + +- Genesis worktree: + `/Users/seshendranalla/Development/genesis-worktrees/directed-evolution-live-backend` +- TemperPaw worktree: + `/Users/seshendranalla/Development/temperpaw-worktrees/directed-evolution-hotload-variants` +- Local proof bundle: + `/tmp/directed-evolution-local-proof-foreground.udcnRR` +- Local Temper URL: `http://127.0.0.1:3201` +- App catalog: temp symlink catalog containing: + - `directed-evolution` + - `paw-orchestration` + - `agent-answers` + - `agent-answers-evaluation` + +Startup installed the relevant app surfaces: + +```text +App 'agent-answers' installed for 'default': Answer, Question +App 'agent-answers-evaluation' installed for 'default': TrialMetricDefinition, TrialSuite, ValidatorRun +App 'directed-evolution' installed for 'default': AdaptationGoal, AutonomyPolicy, Direction, EliminationRule, Episode, EpisodeStartRequest, EvaluationStage, EvidenceArtifact, Generation, LineageEdge, Measurement, MetricDefinition, Mutation, Organism, OrganismVersion, Pressure, Promotion, ScoringRule, SelectionPressure, SelectionProtocol, Signal, SimulatedUserPlan, StageResult, Trial, Variant, ViabilityConstraint, WorkItemReceipt +App 'paw-orchestration' installed for 'default': BudgetLedger, HeartbeatRun, Organization, WorkItem, WorkerAgent, WorkerProvider, WorkerRun +``` + +Metadata checks confirmed these live entity/action surfaces: + +```text +EntitySet Name="Organisms" True +EntitySet Name="EpisodeStartRequests" True +EntitySet Name="WorkItems" True +Action Name="SubmitEpisodeStartRequest" True +``` + +## Episode Start + +The proof seeded a baseline Agent Answers organism and direction through real +OData create/action calls, then submitted `EpisodeStartRequest`: + +```text +Organisms org-agent-answers create -> 201 +OrganismVersions ov-agent-answers-local-proof create -> 201 +OrganismVersions ov-agent-answers-local-proof MarkOrganismVersionParent -> 200 +Organisms org-agent-answers ActivateOrganism -> 200 +Directions dir-agent-answers-local-proof create -> 201 +Directions dir-agent-answers-local-proof ProposeDirection -> 200 +EpisodeStartRequests episode-start-local-proof create -> 201 +EpisodeStartRequests episode-start-local-proof SubmitEpisodeStartRequest -> 200 +``` + +The inline `episode_start_requestor` path materialized: + +```text +EpisodeStartRequests: count=1 statuses={'Started': 1} +Directions: count=1 statuses={'Selected': 1} +Episodes: count=1 statuses={'Running': 1} +Generations: count=1 statuses={'Generating': 1} +WorkItems: count=3 statuses={'Queued': 3} +``` + +The started episode was +`en-019e8af2-b5b2-7200-b291-1916d528457c`, with generation +`en-019e8af2-b97d-7b22-8232-40d7c50a5dec`. + +## Worker Loop + +The local worker was started with execution disabled: + +```sh +TEMPER_URL=http://127.0.0.1:3201 \ +TEMPER_TENANT=default \ +WORKER_ID=local-proof-worker \ +PAW_CODEX_ENABLE_EXECUTION=0 \ +PAW_CODEX_POLL_ON_START=1 \ +./target/debug/paw-codex-worker run +``` + +It registered, connected to the local event stream, claimed Directed Evolution +work items, created shared `WorkerRun` rows, and routed receipts back into +Directed Evolution. Final captured state: + +```text +WorkItems: count=13 statuses={'Succeeded': 9, 'Cancelled': 4} +WorkerRuns: count=9 statuses={'Succeeded': 9} +EvidenceArtifacts: count=9 statuses={'Linked': 9} +Variants: count=3 statuses={'Eliminated': 3} +Trials: count=3 statuses={'Succeeded': 3} +StageResults: count=9 statuses={'Eliminated': 3, 'Pending': 3, 'Running': 3} +Episodes: count=1 statuses={'Failed': 1} +Generations: count=1 statuses={'Failed': 1} +``` + +The episode failed closed because dry-run worker output is not a viable real +variant/evaluation proof: + +```text +All variants were eliminated before selection. +``` + +That failure is expected for this local proof. The proof still demonstrates the +fresh feedback-closure mechanics through episode materialization, shared work +item creation, worker claiming, worker-run provenance, evidence artifact +linking, trial routing, and fail-closed elimination. + +## Remaining Live Proof Blocker + +The required production Agent Answers proof remains blocked. Production +preflight bundle +`/tmp/paw-patrol-production-preflight-20260603T001306Z-903` reported missing: + +- `TEMPER_URL` +- `WORKER_TOKEN` +- `PATROL_OPERATOR_TOKEN` +- signed webhook secrets +- launchd plist/load state +- production local Codex worker-id confirmation +- checkout ancestry against current main + +Datadog MCP access was available, and recent TemperPaw telemetry could be read, +but no fresh Directed Evolution / Agent Answers Datadog proof trace was found. +The live proof can only be closed after those production credentials and worker +activation gates are supplied. diff --git a/Cargo.lock b/Cargo.lock index 95ce1b278..fee71959f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -665,7 +665,7 @@ dependencies = [ "cap-primitives", "cap-std", "io-lifetimes", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -694,7 +694,7 @@ dependencies = [ "maybe-owned", "rustix 1.1.4", "rustix-linux-procfs", - "windows-sys 0.59.0", + "windows-sys 0.52.0", "winx", ] @@ -971,9 +971,9 @@ dependencies = [ [[package]] name = "cmov" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f88a43d011fc4a6876cb7344703e297c71dda42494fee094d5f7c76bf13f746" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" [[package]] name = "cobs" @@ -1352,9 +1352,9 @@ dependencies = [ [[package]] name = "crypto-common" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77727bb15fa921304124b128af125e7e3b968275d1b108b379190264f4423710" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" dependencies = [ "hybrid-array", ] @@ -1576,7 +1576,7 @@ checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ "block-buffer 0.12.0", "const-oid 0.10.2", - "crypto-common 0.2.1", + "crypto-common 0.2.2", "ctutils", ] @@ -1855,7 +1855,7 @@ checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" dependencies = [ "cfg-if", "rustix 1.1.4", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -1996,7 +1996,7 @@ checksum = "94e7099f6313ecacbe1256e8ff9d617b75d1bcb16a6fddef94866d225a01a14a" dependencies = [ "io-lifetimes", "rustix 1.1.4", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -2840,7 +2840,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2285ddfe3054097ef4b2fe909ef8c3bcd1ea52a8f0d274416caebeef39f04a65" dependencies = [ "io-lifetimes", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -3951,6 +3951,7 @@ name = "paw-codex-worker" version = "0.1.0" dependencies = [ "anyhow", + "base64 0.22.1", "futures-util", "libc", "reqwest", @@ -4348,12 +4349,12 @@ dependencies = [ [[package]] name = "prost" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" dependencies = [ "bytes", - "prost-derive 0.14.3", + "prost-derive 0.14.4", ] [[package]] @@ -4384,12 +4385,12 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.12.1", "proc-macro2", "quote", "syn", @@ -4397,11 +4398,11 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" dependencies = [ - "prost 0.14.3", + "prost 0.14.4", ] [[package]] @@ -5833,9 +5834,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "symbolic-common" -version = "12.18.2" +version = "12.18.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bbc8b4f883265fb2207a0d6ce67095f4bd6cf13e5f9183eb8b3e73fa1b2466a" +checksum = "332615d90111d8eeaf86a84dc9bbe9f65d0d8c5cf11b4caccedc37754eb0dcfd" dependencies = [ "debugid", "memmap2", @@ -5845,9 +5846,9 @@ dependencies = [ [[package]] name = "symbolic-demangle" -version = "12.18.2" +version = "12.18.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eda470a26ea40eb5cafc35487718e23a2f19153d3a589affb9e8f7fa1aa73b3" +checksum = "912017718eb4d21930546245af9a3475c9dccf15675a5c215664e76621afc471" dependencies = [ "cpp_demangle", "rustc-demangle", @@ -5924,7 +5925,7 @@ dependencies = [ "fd-lock", "io-lifetimes", "rustix 0.38.44", - "windows-sys 0.59.0", + "windows-sys 0.52.0", "winx", ] @@ -5937,13 +5938,13 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" [[package]] name = "temper-actor-runtime" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "anyhow", "async-trait", "chrono", "deadpool-postgres", - "prost 0.14.3", + "prost 0.14.4", "prost-types", "serde", "serde_json", @@ -5960,7 +5961,7 @@ dependencies = [ [[package]] name = "temper-authz" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "cedar-policy", "opentelemetry", @@ -5974,7 +5975,7 @@ dependencies = [ [[package]] name = "temper-evolution" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "chrono", "serde", @@ -5991,7 +5992,7 @@ dependencies = [ [[package]] name = "temper-jit" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", @@ -6003,7 +6004,7 @@ dependencies = [ [[package]] name = "temper-observe" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "chrono", "opentelemetry", @@ -6026,7 +6027,7 @@ dependencies = [ [[package]] name = "temper-odata" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "axum 0.8.9", "chrono", @@ -6040,7 +6041,7 @@ dependencies = [ [[package]] name = "temper-optimize" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", @@ -6054,7 +6055,7 @@ dependencies = [ [[package]] name = "temper-platform" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "anyhow", "async-trait", @@ -6087,7 +6088,7 @@ dependencies = [ [[package]] name = "temper-runtime" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "anyhow", "chrono", @@ -6103,7 +6104,7 @@ dependencies = [ [[package]] name = "temper-sandbox" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "anyhow", "monty", @@ -6117,7 +6118,7 @@ dependencies = [ [[package]] name = "temper-server" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "aes-gcm", "async-stream", @@ -6170,7 +6171,7 @@ dependencies = [ [[package]] name = "temper-spec" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "quick-xml", "serde", @@ -6183,7 +6184,7 @@ dependencies = [ [[package]] name = "temper-store-postgres" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "chrono", "opentelemetry", @@ -6201,7 +6202,7 @@ dependencies = [ [[package]] name = "temper-store-redis" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "chrono", "fred", @@ -6218,7 +6219,7 @@ dependencies = [ [[package]] name = "temper-store-turso" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "libsql", "opentelemetry", @@ -6234,7 +6235,7 @@ dependencies = [ [[package]] name = "temper-verify" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "proptest", "serde", @@ -6248,7 +6249,7 @@ dependencies = [ [[package]] name = "temper-wasm" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "async-stream", "async-trait", @@ -8202,7 +8203,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f3fd376f71958b862e7afb20cfe5a22830e1963462f3a17f49d82a6c1d1f42d" dependencies = [ "bitflags 2.11.1", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] diff --git a/Dockerfile b/Dockerfile index 6d1f0deb5..e4c4e960a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,7 +24,7 @@ COPY os-apps ./os-apps # catalog discovers them. Pin the default ref to the exact Katagami commit baked # into this image; callers may still override KATAGAMI_REF with a branch or tag. ARG KATAGAMI_REF=00c4db68c8ca119bc22df98ca6417d6ecf0f8a79 -ARG TEMPER_OBSERVABILITY_REV=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c +ARG TEMPER_OBSERVABILITY_REV=9d1b39f5d92b8783f6572a20d1cf62caf4724b23 RUN apt-get update && apt-get install -y --no-install-recommends git ca-certificates && rm -rf /var/lib/apt/lists/* \ && rm -rf os-apps/katagami-curation os-apps/katagami-commons \ && git init /tmp/katagami \ diff --git a/crates/paw-codex-worker/Cargo.toml b/crates/paw-codex-worker/Cargo.toml index 4504e8e04..969b4edf7 100644 --- a/crates/paw-codex-worker/Cargo.toml +++ b/crates/paw-codex-worker/Cargo.toml @@ -12,12 +12,13 @@ path = "src/main.rs" [dependencies] anyhow = { workspace = true } +base64 = "0.22" futures-util = "0.3" libc = "0.2" reqwest = { version = "0.12", features = ["json", "stream"] } serde = { workspace = true } serde_json = { workspace = true } -temper-observe = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-observe = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } tokio = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } diff --git a/crates/paw-codex-worker/README.md b/crates/paw-codex-worker/README.md index 2f7a7ce01..be33f0278 100644 --- a/crates/paw-codex-worker/README.md +++ b/crates/paw-codex-worker/README.md @@ -78,6 +78,10 @@ Script inventory: SecurityFinding, ProofPacket, and DailyBrief loop. - `production-preflight.sh`: read-only production blocker inventory for tokens, Railway, GitHub PRs, launchd, and webhook secrets. +- `directed-evolution-agent-answers-live-proof.sh`: guarded live Agent Answers + Directed Evolution proof driver that starts the semantic episode, waits for + worker/evidence closure, and requires Datadog-measured evidence plus a + successful terminal episode status. - `production-readiness.sh`: guarded doctor/build/plist path before loading the Mac mini daemon. - `production-observe-only.sh`: guarded production proof with execution off. @@ -352,6 +356,70 @@ crates/paw-codex-worker/scripts/production-preflight-diff.sh \ /tmp/current-preflight/summary.json ``` +After preflight and readiness gates are clear, run the guarded Agent Answers +Directed Evolution proof driver. It refuses to mutate production unless +`ALLOW_PRODUCTION_WRITE=1` and `CONFIRM_AGENT_ANSWERS_LIVE_PROOF=1` are set, +then calls `paw-codex-worker directed-evolution-start-episode`, waits for +`WorkItems`, `WorkerRuns`, `EvidenceArtifacts`, terminal episode state, and at +least one `datadog-measured` artifact with a Datadog URL. It also requires +distinct `observer` and `telemetry_evaluator` `datadog-measured` +EvidenceArtifacts with structured query, time window, result count, +interpretation, and zero-result meaning fields, so the proof closes the +Datadog observer/evaluator loop instead of relying on aggregate telemetry. The +script treats attempts to disable Datadog evidence or terminal-success +requirements as blockers, not proof configuration. When `START_LOCAL_WORKER=0`, +also set `CONFIRM_PRODUCTION_DATADOG_QUERY_SECRETS=1` after verifying the +production worker/vault has `dd_api_key` and `dd_app_key`; when +`START_LOCAL_WORKER=1`, export `DD_API_KEY` or `DATADOG_API_KEY` plus +`DD_APP_KEY` for the local worker: + +```sh +ALLOW_PRODUCTION_WRITE=1 \ +CONFIRM_AGENT_ANSWERS_LIVE_PROOF=1 \ +CONFIRM_PRODUCTION_DATADOG_QUERY_SECRETS=1 \ +TEMPER_URL=https://your-railway-temperpaw.example \ +WORKER_TOKEN="$TEMPER_WORKER_TOKEN" \ +DIRECTED_EVOLUTION_ORGANISM_ID="$AGENT_ANSWERS_ORGANISM_ID" \ +DIRECTED_EVOLUTION_DIRECTION_ID="$AGENT_ANSWERS_DIRECTION_ID" \ +START_LOCAL_WORKER=0 \ +crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh +``` + +Use `PRECHECK_ONLY=1` with the same confirmations and identifiers to generate a +ready proof bundle without mutating production. The precheck writes +`summary.json` and `proof.md`, exits before the worker build, metadata read, +episode start call, worker start, episode polling, and Datadog polling, and +marks the bundle with `no_mutation: true`. Its `evidence_requirements` section +declares `mandatory_datadog_observer_evidence` and +`mandatory_datadog_telemetry_evaluator_evidence`, making the no-mutation proof +explicit about the role-specific Datadog gates the next live run must satisfy: + +```sh +PRECHECK_ONLY=1 \ +ALLOW_PRODUCTION_WRITE=1 \ +CONFIRM_AGENT_ANSWERS_LIVE_PROOF=1 \ +CONFIRM_PRODUCTION_DATADOG_QUERY_SECRETS=1 \ +TEMPER_URL=https://your-railway-temperpaw.example \ +WORKER_TOKEN="$TEMPER_WORKER_TOKEN" \ +DIRECTED_EVOLUTION_ORGANISM_ID="$AGENT_ANSWERS_ORGANISM_ID" \ +DIRECTED_EVOLUTION_DIRECTION_ID="$AGENT_ANSWERS_DIRECTION_ID" \ +START_LOCAL_WORKER=0 \ +crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh +``` + +Set `START_LOCAL_WORKER=1` on the production Mac mini when launchd has not +already started the worker pool. In that mode, the proof driver starts a local +worker process pool and defaults `WORKER_SLOT_COUNT` and `MAX_CONCURRENT_RUNS` +to `3`, with each local slot running `MAX_CONCURRENT_RUNS=1`. Slot 1 keeps +`WORKER_ID`, later slots use identities such as `-slot-02`, and each +slot writes its own log under the proof bundle's `local-worker-logs/` +directory. Local worker processes inherit the Datadog query environment and the +production-secret confirmation is not accepted as a substitute for +`DD_API_KEY`/`DD_APP_KEY`. The proof bundle is written under +`/tmp/directed-evolution-agent-answers-live-proof-*` and includes +`summary.json`, `proof.md`, `human-blockers.json` when blocked, the episode +contract, and terminal episode state. + Use `STRICT=1` when blocked gates should make the command fail, for example in a final cutover checklist: @@ -381,11 +449,12 @@ crates/paw-codex-worker/scripts/production-readiness-smoke.sh ``` On success, the smoke writes `summary.json`, `proof.md`, -`production-readiness.log`, and a rendered launchd plist under +`production-readiness.log`, and a rendered launchd plist pool under `/tmp/paw-patrol-production-readiness-proof-*`. It checks OData, the event stream, fake Codex availability, a guarded `codex exec` doctor smoke, -`PAW_CODEX_ENABLE_EXECUTION=0`, plist rendering, and that the worker token was -not printed to the readiness log. +`PAW_CODEX_ENABLE_EXECUTION=0`, `WORKER_SLOT_COUNT=3` process-pool rendering, +isolated worker slot identities, and that the worker token was not printed to +the readiness log. On the production Mac mini, use the guarded bootstrap wrapper to avoid manually copying production tokens. It links the known Railway `temperpaw` production diff --git a/crates/paw-codex-worker/fixtures/fake-codex.sh b/crates/paw-codex-worker/fixtures/fake-codex.sh index ff44ef894..6f63ed055 100755 --- a/crates/paw-codex-worker/fixtures/fake-codex.sh +++ b/crates/paw-codex-worker/fixtures/fake-codex.sh @@ -25,6 +25,22 @@ while [ "$#" -gt 0 ]; do fi shift ;; + --model) + shift + if [ "$#" -eq 0 ]; then + echo "--model requires a value" >&2 + exit 2 + fi + shift + ;; + -c) + shift + if [ "$#" -eq 0 ]; then + echo "-c requires a value" >&2 + exit 2 + fi + shift + ;; --cd) shift if [ "$#" -eq 0 ]; then diff --git a/crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh b/crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh new file mode 100755 index 000000000..73370d62f --- /dev/null +++ b/crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh @@ -0,0 +1,891 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(git rev-parse --show-toplevel)" +STAMP="$(date -u +%Y%m%dT%H%M%SZ)" +PROOF_DIR="${PROOF_DIR:-/tmp/directed-evolution-agent-answers-live-proof-${STAMP}-$$}" +SUMMARY_JSON="${PROOF_DIR}/summary.json" +PROOF_MD="${PROOF_DIR}/proof.md" +HUMAN_BLOCKERS_JSON="${PROOF_DIR}/human-blockers.json" +BLOCKERS_TSV="${PROOF_DIR}/human-blockers.tsv" +CONTRACT_JSON="${PROOF_DIR}/agent-answers-episode-contract.json" +START_JSON="${PROOF_DIR}/episode-start.json" +WORKER_LOG_DIR="${PROOF_DIR}/local-worker-logs" + +TEMPER_TENANT="${TEMPER_TENANT:-default}" +WORKER_ID="${WORKER_ID:-mac-mini-codex-prod}" +WORKER_BIN="${WORKER_BIN:-${ROOT}/target/release/paw-codex-worker}" +REPO_ROOT="${REPO_ROOT:-$ROOT}" +WORKSPACE_ROOT="${WORKSPACE_ROOT:-$(dirname "$ROOT")}" +CODEX_BIN="${CODEX_BIN:-codex}" +BUILD_WORKER="${BUILD_WORKER:-1}" +START_LOCAL_WORKER="${START_LOCAL_WORKER:-0}" +PRECHECK_ONLY="${PRECHECK_ONLY:-0}" +PAW_CODEX_ENABLE_EXECUTION="${PAW_CODEX_ENABLE_EXECUTION:-1}" +PAW_CODEX_POLL_ON_START="${PAW_CODEX_POLL_ON_START:-1}" +MAX_CONCURRENT_RUNS="${MAX_CONCURRENT_RUNS:-3}" +WORKER_SLOT_COUNT="${WORKER_SLOT_COUNT:-$MAX_CONCURRENT_RUNS}" +READY_ATTEMPTS="${READY_ATTEMPTS:-900}" +POLL_SECONDS="${POLL_SECONDS:-2}" +REQUIRE_DATADOG_EVIDENCE="${REQUIRE_DATADOG_EVIDENCE:-1}" +REQUIRE_EPISODE_SUCCESS="${REQUIRE_EPISODE_SUCCESS:-1}" +DIRECTED_EVOLUTION_PROOF_ACTOR="${DIRECTED_EVOLUTION_PROOF_ACTOR:-codex-agent-answers-live-proof}" + +WORKER_PIDS=() + +log() { + printf '[directed-evolution-agent-answers-proof] %s\n' "$*" +} + +fail() { + log "$*" + exit 1 +} + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || fail "missing required command: $1" +} + +add_blocker() { + local gate="$1" + local detail="$2" + local evidence="$3" + printf '%s\t%s\t%s\n' "$gate" "$detail" "$evidence" >>"$BLOCKERS_TSV" +} + +cleanup() { + local pid + for pid in "${WORKER_PIDS[@]:-}"; do + if [[ -n "$pid" ]] && kill -0 "$pid" >/dev/null 2>&1; then + kill "$pid" >/dev/null 2>&1 || true + wait "$pid" >/dev/null 2>&1 || true + fi + done +} +trap cleanup EXIT + +slot_worker_id() { + local slot="$1" + if [[ "$slot" == "1" ]]; then + printf '%s\n' "$WORKER_ID" + else + printf '%s-slot-%02d\n' "$WORKER_ID" "$slot" + fi +} + +slot_worker_log() { + local slot="$1" + printf '%s/slot-%02d.log\n' "$WORKER_LOG_DIR" "$slot" +} + +worker_slot_ids_json() { + local ids_json="[]" + local slot + for slot in $(seq 1 "$WORKER_SLOT_COUNT"); do + ids_json="$(jq -c --arg worker "$(slot_worker_id "$slot")" '. + [$worker]' <<<"$ids_json")" + done + printf '%s\n' "$ids_json" +} + +worker_slot_logs_json() { + local logs_json="[]" + local slot + for slot in $(seq 1 "$WORKER_SLOT_COUNT"); do + logs_json="$(jq -c --arg log_path "$(slot_worker_log "$slot")" '. + [$log_path]' <<<"$logs_json")" + done + printf '%s\n' "$logs_json" +} + +mkdir -p "$PROOF_DIR" +: >"$BLOCKERS_TSV" + +require_cmd curl +require_cmd git +require_cmd jq + +record_precondition_blockers() { + if [[ "${ALLOW_PRODUCTION_WRITE:-0}" != "1" ]]; then + add_blocker "confirm:allow_production_write" \ + "ALLOW_PRODUCTION_WRITE=1 is required because this script creates a live Directed Evolution Episode." \ + "operator confirmation required" + fi + + if [[ "${CONFIRM_AGENT_ANSWERS_LIVE_PROOF:-0}" != "1" ]]; then + add_blocker "confirm:agent_answers_live_proof" \ + "CONFIRM_AGENT_ANSWERS_LIVE_PROOF=1 is required to run a live Agent Answers Directed Evolution proof cycle." \ + "operator confirmation required" + fi + + if [[ -z "${TEMPER_URL:-}" ]]; then + add_blocker "env:temper_url" \ + "TEMPER_URL is required for the production TemperPaw control plane." \ + "human input required" + fi + + if [[ -z "${WORKER_TOKEN:-}" ]]; then + add_blocker "env:worker_token" \ + "WORKER_TOKEN is required for the episode starter and optional local worker." \ + "mint or provide a Temper worker credential" + fi + + if [[ -z "${DIRECTED_EVOLUTION_DIRECTION_ID:-}" ]]; then + add_blocker "env:directed_evolution_direction_id" \ + "DIRECTED_EVOLUTION_DIRECTION_ID is required and must point at the fresh Agent Answers direction under review." \ + "create or select the production Direction first" + fi + + if [[ -z "${DIRECTED_EVOLUTION_ORGANISM_ID:-}" ]]; then + add_blocker "env:directed_evolution_organism_id" \ + "DIRECTED_EVOLUTION_ORGANISM_ID is required and must point at the Agent Answers organism." \ + "create or select the production Organism first" + fi + + if [[ "$START_LOCAL_WORKER" == "1" && "$PAW_CODEX_ENABLE_EXECUTION" != "1" ]]; then + add_blocker "env:paw_codex_enable_execution" \ + "START_LOCAL_WORKER=1 requires PAW_CODEX_ENABLE_EXECUTION=1 so Codex can execute variant/reviewer/evaluator work." \ + "set PAW_CODEX_ENABLE_EXECUTION=1" + fi + + if ! [[ "$MAX_CONCURRENT_RUNS" =~ ^[0-9]+$ ]] || [[ "$MAX_CONCURRENT_RUNS" -lt 1 ]]; then + add_blocker "env:max_concurrent_runs" \ + "MAX_CONCURRENT_RUNS must be a positive integer for the Directed Evolution proof contract." \ + "set MAX_CONCURRENT_RUNS=3 or another positive concurrency budget" + fi + + if ! [[ "$WORKER_SLOT_COUNT" =~ ^[0-9]+$ ]] || [[ "$WORKER_SLOT_COUNT" -lt 1 ]]; then + add_blocker "env:worker_slot_count" \ + "WORKER_SLOT_COUNT must be a positive integer for the Directed Evolution proof worker pool." \ + "set WORKER_SLOT_COUNT=3 or another positive pool size" + elif [[ "$START_LOCAL_WORKER" == "1" && "$WORKER_SLOT_COUNT" -lt 2 ]]; then + add_blocker "env:worker_slot_count" \ + "START_LOCAL_WORKER=1 requires WORKER_SLOT_COUNT >= 2 so the live proof exercises a local Codex worker process pool, not a single worker." \ + "set WORKER_SLOT_COUNT=3" + fi + + if [[ "$REQUIRE_DATADOG_EVIDENCE" != "1" ]]; then + add_blocker "env:require_datadog_evidence" \ + "REQUIRE_DATADOG_EVIDENCE must remain 1; Datadog-measured evidence is mandatory for this proof." \ + "do not disable the Datadog observer/evaluator gate" + fi + + if [[ "$REQUIRE_DATADOG_EVIDENCE" == "1" ]]; then + if [[ "$START_LOCAL_WORKER" == "1" ]]; then + if [[ -z "${DD_API_KEY:-${DATADOG_API_KEY:-}}" ]]; then + add_blocker "env:dd_api_key" \ + "START_LOCAL_WORKER=1 requires DD_API_KEY or DATADOG_API_KEY so Datadog observer/evaluator WorkItems can query live telemetry." \ + "export the Datadog API key before starting the local proof worker" + fi + + if [[ -z "${DD_APP_KEY:-}" ]]; then + add_blocker "env:dd_app_key" \ + "START_LOCAL_WORKER=1 requires DD_APP_KEY so Datadog observer/evaluator WorkItems can query live telemetry." \ + "export the Datadog application key before starting the local proof worker" + fi + elif [[ "${CONFIRM_PRODUCTION_DATADOG_QUERY_SECRETS:-0}" != "1" ]]; then + add_blocker "confirm:production_datadog_query_secrets" \ + "START_LOCAL_WORKER=0 expects an already-running production worker pool; confirm it has dd_api_key and dd_app_key secrets before starting the live proof." \ + "set CONFIRM_PRODUCTION_DATADOG_QUERY_SECRETS=1 after verifying the production worker/vault Datadog query credentials" + fi + fi + + if [[ "$REQUIRE_EPISODE_SUCCESS" != "1" ]]; then + add_blocker "env:require_episode_success" \ + "REQUIRE_EPISODE_SUCCESS must remain 1; failed/cancelled/abandoned episodes cannot be reported as a passed proof." \ + "do not disable the terminal-success gate" + fi +} + +write_blocked_summary_and_exit() { + if [[ ! -s "$BLOCKERS_TSV" ]]; then + return 0 + fi + + jq -Rn ' + [inputs | split("\t") | { + gate: .[0], + detail: .[1], + evidence: .[2] + }] + ' <"$BLOCKERS_TSV" >"$HUMAN_BLOCKERS_JSON" + jq -n \ + --arg status "blocked" \ + --arg proof_dir "$PROOF_DIR" \ + --argjson blockers "$(cat "$HUMAN_BLOCKERS_JSON")" \ + '{ + status: $status, + proof_dir: $proof_dir, + proof_kind: "live Agent Answers Directed Evolution proof", + human_blockers: $blockers + }' >"$SUMMARY_JSON" + cat >"$PROOF_MD" <"$SUMMARY_JSON" + cat >"$PROOF_MD" </dev/null 2>&1; then + return 0 + fi + sleep "$POLL_SECONDS" + done + fail "control plane did not become readable at ${TEMPER_URL}" +} + +field() { + local key="$1" + jq -r --arg key "$key" ' + def norm: ascii_downcase | gsub("_"; ""); + . as $root + | ($root[$key] + // $root[($key | ascii_downcase)] + // ($root | to_entries[]? | select(.key | norm == ($key | norm)) | .value) + // .fields[$key] + // .fields[($key | ascii_downcase)] + // (.fields // {} | to_entries[]? | select(.key | norm == ($key | norm)) | .value) + // "") + ' +} + +list_set() { + local set="$1" + if [[ "${SCRIPT_SELF_TEST:-0}" == "1" ]]; then + cat "${SCRIPT_SELF_TEST_FIXTURE_DIR}/${set}.json" + return 0 + fi + curl_json "${TEMPER_URL}/tdata/${set}" +} + +count_for_episode() { + local set="$1" + local episode_id="$2" + list_set "$set" | jq -r --arg episode_id "$episode_id" ' + def rows: + if type == "array" then . + elif has("value") then .value + elif has("entities") then .entities + elif has("items") then .items + else [] + end; + def f($name): + .[$name] + // .[($name | ascii_downcase)] + // .fields[$name] + // .fields[($name | ascii_downcase)] + // ""; + def present($value): + $value != null and ($value | tostring | length > 0); + def corr: + (f("CorrelationJson") | fromjson? // {}); + [rows[]? | select( + f("EpisodeId") == $episode_id + or f("episode_id") == $episode_id + or corr.episode_id == $episode_id + or corr.datadog.join_fields.episode_id == $episode_id + or corr.output.episode_id == $episode_id + )] | length + ' +} + +datadog_evidence_count() { + local episode_id="$1" + local role="${2:-}" + list_set EvidenceArtifacts | jq -r --arg episode_id "$episode_id" --arg role "$role" ' + def rows: + if type == "array" then . + elif has("value") then .value + elif has("entities") then .entities + elif has("items") then .items + else [] + end; + def f($name): + .[$name] + // .[($name | ascii_downcase)] + // .fields[$name] + // .fields[($name | ascii_downcase)] + // ""; + def present($value): + $value != null and ($value | tostring | length > 0); + def corr: + (f("CorrelationJson") | fromjson? // {}); + def evidence_scope: + (corr.output.evidence_scope // corr.output.evidenceScope // []) + | if type == "array" then . + elif type == "object" then [.] + else [] + end; + def belongs_to_episode: + f("EpisodeId") == $episode_id + or f("episode_id") == $episode_id + or corr.episode_id == $episode_id + or corr.datadog.join_fields.episode_id == $episode_id + or corr.output.episode_id == $episode_id; + def datadog_url: + if (f("Uri") | tostring | test("^https://app\\.")) + then (f("Uri") | tostring) + else (evidence_scope + | map(.datadog_url // .datadogUrl // "") + | map(select(test("^https://app\\."))) + | .[0]) + end + // ""; + def role_matches: + $role == "" + or f("Role") == $role + or corr.role == $role + or corr.datadog.role == $role + or corr.datadog.join_fields.role == $role + or corr.output.role == $role + or corr.output.evaluator_role == $role + or corr.output.evaluatorRole == $role + or (evidence_scope | any((.role // .Role // .evaluator_role // .evaluatorRole // "") == $role)); + [rows[]? | select( + belongs_to_episode + and + f("EvidenceProvenance") == "datadog-measured" + and present(f("Query")) + and present(f("TimeWindow")) + and present(f("ResultCount")) + and present(f("Interpretation")) + and present(f("ZeroResultMeaning")) + and (datadog_url | test("^https://app\\.")) + and role_matches + )] | length + ' +} + +datadog_role_evidence_count() { + local episode_id="$1" + local role="$2" + datadog_evidence_count "$episode_id" "$role" +} + +wait_for_episode_terminal() { + local episode_id="$1" + local body status + for _ in $(seq 1 "$READY_ATTEMPTS"); do + body="$(curl_json "$(entity_url Episodes "$episode_id")")" + status="$(field Status <<<"$body")" + case "$status" in + Completed|Complete|Succeeded|Promoted|NoPromotion|Failed|Cancelled|Abandoned) + printf '%s' "$body" + return 0 + ;; + esac + sleep "$POLL_SECONDS" + done + log "Episodes('${episode_id}') did not reach a terminal status; latest state follows" + curl_json "$(entity_url Episodes "$episode_id")" | tee "${PROOF_DIR}/episode-timeout.json" | jq . + exit 1 +} + +wait_for_min_count() { + local label="$1" + local set="$2" + local episode_id="$3" + local minimum="$4" + local count + for _ in $(seq 1 "$READY_ATTEMPTS"); do + count="$(count_for_episode "$set" "$episode_id")" + if [[ "$count" -ge "$minimum" ]]; then + printf '%s\n' "$count" + return 0 + fi + sleep "$POLL_SECONDS" + done + fail "${label} did not reach count >= ${minimum} for episode ${episode_id}" +} + +wait_for_datadog_evidence() { + local episode_id="$1" + local count + for _ in $(seq 1 "$READY_ATTEMPTS"); do + count="$(datadog_evidence_count "$episode_id")" + if [[ "$count" -ge 1 ]]; then + printf '%s\n' "$count" + return 0 + fi + sleep "$POLL_SECONDS" + done + fail "no datadog-measured EvidenceArtifacts with datadog_url/query/window/result_count/interpretation/zero_result_meaning were observed" +} + +wait_for_datadog_role_evidence() { + local episode_id="$1" + local role="$2" + local count + for _ in $(seq 1 "$READY_ATTEMPTS"); do + count="$(datadog_role_evidence_count "$episode_id" "$role")" + if [[ "$count" -ge 1 ]]; then + printf '%s\n' "$count" + return 0 + fi + sleep "$POLL_SECONDS" + done + fail "no datadog-measured EvidenceArtifacts for ${role} with datadog_url/query/window/result_count/interpretation/zero_result_meaning were observed" +} + +run_self_test() { + SCRIPT_SELF_TEST_FIXTURE_DIR="${PROOF_DIR}/self-test-fixtures" + export SCRIPT_SELF_TEST_FIXTURE_DIR + mkdir -p "$SCRIPT_SELF_TEST_FIXTURE_DIR" + + cat >"${SCRIPT_SELF_TEST_FIXTURE_DIR}/EvidenceArtifacts.json" <<'JSON' +{ + "value": [ + { + "fields": { + "EpisodeId": "episode-self-test", + "EvidenceProvenance": "datadog-measured", + "Uri": "https://app.datadoghq.com/logs?query=directed-evolution", + "Query": "service:temper-platform directed_evolution.episode_id:episode-self-test", + "TimeWindow": "now-15m to now", + "ResultCount": 0, + "Interpretation": "No runtime request logs were observed for the fixture.", + "ZeroResultMeaning": "failure", + "CorrelationJson": "{\"episode_id\":\"episode-self-test\",\"role\":\"observer\",\"datadog\":{\"role\":\"observer\",\"join_fields\":{\"episode_id\":\"episode-self-test\",\"role\":\"observer\"}},\"output\":{\"evidence_scope\":[{\"datadog_url\":\"https://app.datadoghq.com/logs?query=directed-evolution\",\"role\":\"observer\"}]}}" + } + }, + { + "fields": { + "EpisodeId": "episode-self-test", + "EvidenceProvenance": "datadog-measured", + "Uri": "https://app.datadoghq.com/logs?query=telemetry-evaluator", + "Query": "service:temper-platform directed_evolution.episode_id:episode-self-test role:telemetry_evaluator", + "TimeWindow": "now-15m to now", + "ResultCount": 3, + "Interpretation": "Telemetry evaluator observed proof-run events.", + "ZeroResultMeaning": "failure", + "CorrelationJson": "{\"episode_id\":\"episode-self-test\",\"datadog\":{\"role\":\"telemetry_evaluator\",\"join_fields\":{\"episode_id\":\"episode-self-test\",\"role\":\"telemetry_evaluator\"}},\"output\":{\"evaluator_role\":\"telemetry_evaluator\",\"evidence_scope\":[{\"datadog_url\":\"https://app.datadoghq.com/logs?query=telemetry-evaluator\",\"role\":\"telemetry_evaluator\"}]}}" + } + }, + { + "fields": { + "EpisodeId": "other-episode", + "EvidenceProvenance": "datadog-measured", + "Uri": "https://app.datadoghq.com/logs?query=other", + "Query": "other", + "TimeWindow": "now-15m to now", + "ResultCount": 7, + "Interpretation": "Different episode.", + "ZeroResultMeaning": "neutral" + } + } + ] +} +JSON + + local count + count="$(datadog_evidence_count "episode-self-test")" + if [[ "$count" != "2" ]]; then + fail "self-test expected two structured Datadog evidence artifacts, got ${count}" + fi + + local observer_count + observer_count="$(datadog_role_evidence_count "episode-self-test" observer)" + if [[ "$observer_count" != "1" ]]; then + fail "self-test expected one observer Datadog evidence artifact, got ${observer_count}" + fi + + local telemetry_count + telemetry_count="$(datadog_role_evidence_count "episode-self-test" telemetry_evaluator)" + if [[ "$telemetry_count" != "1" ]]; then + fail "self-test expected one telemetry_evaluator Datadog evidence artifact, got ${telemetry_count}" + fi + + local failed_status + failed_status="$(field Status <<'JSON' +{"fields":{"Status":"Failed"}} +JSON +)" + case "$failed_status" in + Completed|Complete|Succeeded|Promoted|NoPromotion) + fail "self-test misclassified Failed as a successful terminal episode" + ;; + Failed) + ;; + *) + fail "self-test could not read failed terminal status" + ;; + esac + + log "self-test passed" +} + +if [[ "${SCRIPT_SELF_TEST:-0}" == "1" ]]; then + run_self_test + exit 0 +fi + +record_precondition_blockers +write_blocked_summary_and_exit +write_precheck_ready_summary_and_exit + +if [[ "$BUILD_WORKER" == "1" ]]; then + log "building paw-codex-worker release binary" + cargo build -p paw-codex-worker --release +fi + +if [[ ! -x "$WORKER_BIN" ]]; then + fail "worker binary is not executable: ${WORKER_BIN}" +fi + +wait_for_metadata + +jq -n \ + --arg direction_id "$DIRECTED_EVOLUTION_DIRECTION_ID" \ + --arg organism_id "$DIRECTED_EVOLUTION_ORGANISM_ID" \ + --arg parent_version_id "${DIRECTED_EVOLUTION_PARENT_VERSION_ID:-}" \ + --arg actor "$DIRECTED_EVOLUTION_PROOF_ACTOR" \ + '{ + direction_id: $direction_id, + organism_id: $organism_id, + parent_version_id: $parent_version_id, + adaptation_goal: "Improve Agent Answers while preserving question, answer, acceptance, citation, and reviewer workflows.", + human_notes: "Fresh live Agent Answers Directed Evolution proof. Requires worker provenance, simulated-user evidence, state verification, and Datadog observer/evaluator evidence before proof closure.", + created_by_worker_run_id: ("manual:" + $actor), + evaluator_ref: "genesis://nerdsane/agent-answers-evaluation@production", + selected_by: $actor, + selection_notes: "Live proof selection is bounded to the Agent Answers organism and pinned Genesis evaluator.", + started_by: $actor, + start_reason: "Operator-confirmed live Agent Answers Directed Evolution proof cycle." + }' >"$CONTRACT_JSON" + +log "starting semantic Agent Answers Directed Evolution episode" +TEMPER_URL="$TEMPER_URL" \ +TEMPER_TENANT="$TEMPER_TENANT" \ +WORKER_ID="$WORKER_ID" \ +WORKER_TOKEN="$WORKER_TOKEN" \ +REPO_ROOT="$REPO_ROOT" \ +WORKSPACE_ROOT="$WORKSPACE_ROOT" \ +CODEX_BIN="$CODEX_BIN" \ +PAW_CODEX_ENABLE_EXECUTION="$PAW_CODEX_ENABLE_EXECUTION" \ +PAW_CODEX_POLL_ON_START="$PAW_CODEX_POLL_ON_START" \ +"$WORKER_BIN" directed-evolution-start-episode "$CONTRACT_JSON" >"$START_JSON" + +episode_id="$(jq -r '.episode_id // ""' "$START_JSON")" +if [[ -z "$episode_id" ]]; then + fail "directed-evolution-start-episode did not return episode_id" +fi + +if [[ "$START_LOCAL_WORKER" == "1" ]]; then + log "starting ${WORKER_SLOT_COUNT} local worker slots for live proof" + mkdir -p "$WORKER_LOG_DIR" + for slot in $(seq 1 "$WORKER_SLOT_COUNT"); do + slot_worker="$(slot_worker_id "$slot")" + slot_log="$(slot_worker_log "$slot")" + log "starting local worker slot ${slot} as ${slot_worker}" + TEMPER_URL="$TEMPER_URL" \ + TEMPER_TENANT="$TEMPER_TENANT" \ + WORKER_ID="$slot_worker" \ + WORKER_TOKEN="$WORKER_TOKEN" \ + REPO_ROOT="$REPO_ROOT" \ + WORKSPACE_ROOT="$WORKSPACE_ROOT" \ + CODEX_BIN="$CODEX_BIN" \ + PAW_CODEX_ENABLE_EXECUTION="$PAW_CODEX_ENABLE_EXECUTION" \ + PAW_CODEX_POLL_ON_START="$PAW_CODEX_POLL_ON_START" \ + MAX_CONCURRENT_RUNS="1" \ + "$WORKER_BIN" run >"$slot_log" 2>&1 & + WORKER_PIDS+=("$!") + done +else + log "START_LOCAL_WORKER=0; expecting an already-running production worker pool" +fi + +work_item_count="$(wait_for_min_count WorkItems WorkItems "$episode_id" 1)" +worker_run_count="$(wait_for_min_count WorkerRuns WorkerRuns "$episode_id" 1)" +evidence_count="$(wait_for_min_count EvidenceArtifacts EvidenceArtifacts "$episode_id" 1)" +datadog_count="$(wait_for_datadog_evidence "$episode_id")" +datadog_observer_count="$(wait_for_datadog_role_evidence "$episode_id" observer)" +datadog_telemetry_evaluator_count="$(wait_for_datadog_role_evidence "$episode_id" telemetry_evaluator)" + +episode_body="$(wait_for_episode_terminal "$episode_id")" +episode_status="$(field Status <<<"$episode_body")" +jq . <<<"$episode_body" >"${PROOF_DIR}/episode-terminal.json" + +case "$episode_status" in + Completed|Complete|Succeeded|Promoted|NoPromotion) + ;; + *) + jq -n \ + --arg status "failed" \ + --arg proof_dir "$PROOF_DIR" \ + --arg episode_id "$episode_id" \ + --arg episode_status "$episode_status" \ + --arg worker_id "$WORKER_ID" \ + --arg start_local_worker "$START_LOCAL_WORKER" \ + --arg execution_enabled "$PAW_CODEX_ENABLE_EXECUTION" \ + --arg worker_slot_count "$WORKER_SLOT_COUNT" \ + --argjson worker_slot_ids "$(worker_slot_ids_json)" \ + --argjson worker_slot_logs "$(worker_slot_logs_json)" \ + --argjson work_item_count "$work_item_count" \ + --argjson worker_run_count "$worker_run_count" \ + --argjson evidence_count "$evidence_count" \ + --argjson datadog_evidence_count "$datadog_count" \ + --argjson datadog_observer_evidence_count "$datadog_observer_count" \ + --argjson datadog_telemetry_evaluator_evidence_count "$datadog_telemetry_evaluator_count" \ + '{ + status: $status, + proof_kind: "live Agent Answers Directed Evolution proof", + proof_dir: $proof_dir, + agent_answers: { + episode_id: $episode_id, + episode_status: $episode_status + }, + worker: { + worker_id: $worker_id, + start_local_worker: $start_local_worker, + execution_enabled: $execution_enabled, + worker_slot_count: ($worker_slot_count | tonumber), + worker_slot_ids: $worker_slot_ids, + local_worker_logs: $worker_slot_logs, + process_pool_model: (if $start_local_worker == "1" then "local worker process pool" else "already-running production worker pool" end) + }, + evidence: { + work_items: $work_item_count, + worker_runs: $worker_run_count, + evidence_artifacts: $evidence_count, + datadog_measured_evidence_artifacts: $datadog_evidence_count, + datadog_observer_evidence_artifacts: $datadog_observer_evidence_count, + datadog_telemetry_evaluator_evidence_artifacts: $datadog_telemetry_evaluator_evidence_count + }, + failure_reason: "terminal episode status was not successful" + }' >"$SUMMARY_JSON" + fail "episode ${episode_id} reached ${episode_status}; live proof requires a successful terminal status" + ;; +esac + +summary_json="$(jq -n \ + --arg status "passed" \ + --arg proof_dir "$PROOF_DIR" \ + --arg temper_url "$TEMPER_URL" \ + --arg tenant "$TEMPER_TENANT" \ + --arg episode_id "$episode_id" \ + --arg episode_status "$episode_status" \ + --arg direction_id "$DIRECTED_EVOLUTION_DIRECTION_ID" \ + --arg organism_id "$DIRECTED_EVOLUTION_ORGANISM_ID" \ + --arg worker_id "$WORKER_ID" \ + --arg start_local_worker "$START_LOCAL_WORKER" \ + --arg execution_enabled "$PAW_CODEX_ENABLE_EXECUTION" \ + --arg worker_slot_count "$WORKER_SLOT_COUNT" \ + --argjson worker_slot_ids "$(worker_slot_ids_json)" \ + --argjson worker_slot_logs "$(worker_slot_logs_json)" \ + --arg mandatory_datadog_evidence "$REQUIRE_DATADOG_EVIDENCE" \ + --argjson work_item_count "$work_item_count" \ + --argjson worker_run_count "$worker_run_count" \ + --argjson evidence_count "$evidence_count" \ + --argjson datadog_evidence_count "$datadog_count" \ + --argjson datadog_observer_evidence_count "$datadog_observer_count" \ + --argjson datadog_telemetry_evaluator_evidence_count "$datadog_telemetry_evaluator_count" \ + '{ + status: $status, + proof_kind: "live Agent Answers Directed Evolution proof", + proof_dir: $proof_dir, + control_plane: { + temper_url: $temper_url, + tenant: $tenant + }, + agent_answers: { + organism_id: $organism_id, + direction_id: $direction_id, + episode_id: $episode_id, + episode_status: $episode_status + }, + worker: { + worker_id: $worker_id, + start_local_worker: $start_local_worker, + execution_enabled: $execution_enabled, + worker_slot_count: ($worker_slot_count | tonumber), + worker_slot_ids: $worker_slot_ids, + local_worker_logs: $worker_slot_logs, + process_pool_model: (if $start_local_worker == "1" then "local worker process pool" else "already-running production worker pool" end) + }, + evidence: { + work_items: $work_item_count, + worker_runs: $worker_run_count, + evidence_artifacts: $evidence_count, + datadog_measured_evidence_artifacts: $datadog_evidence_count, + datadog_observer_evidence_artifacts: $datadog_observer_evidence_count, + datadog_telemetry_evaluator_evidence_artifacts: $datadog_telemetry_evaluator_evidence_count, + mandatory_datadog_evidence: ($mandatory_datadog_evidence == "1") + } + }')" + +printf '%s\n' "$summary_json" >"$SUMMARY_JSON" + +cat >"$PROOF_MD" <0" -assert_contains "$LAUNCHD_PLIST" "PAW_CODEX_DOCTOR_EXEC_SMOKE" -assert_contains "$LAUNCHD_PLIST" "1" -assert_contains "$LAUNCHD_PLIST" "PAW_CODEX_EXEC_TIMEOUT_SECS" -assert_contains "$LAUNCHD_PLIST" "1200" +slot_plists_json="[]" +slot_worker_ids_json="[]" +for slot in $(seq 1 "$WORKER_SLOT_COUNT"); do + slot_suffix="$(printf '%02d' "$slot")" + slot_plist="$(dirname "$LAUNCHD_PLIST")/com.temperpaw.paw-codex-worker.slot-${slot_suffix}.plist" + if [[ "$slot" == "1" ]]; then + slot_worker_id="$WORKER_ID" + else + slot_worker_id="${WORKER_ID}-slot-${slot_suffix}" + fi + + test -s "$slot_plist" + assert_contains "$slot_plist" "com.temperpaw.paw-codex-worker.slot-${slot_suffix}" + assert_contains "$slot_plist" "$TEMPER_URL" + assert_contains "$slot_plist" "$slot_worker_id" + assert_contains "$slot_plist" "PAW_CODEX_ENABLE_EXECUTION" + assert_contains "$slot_plist" "0" + assert_contains "$slot_plist" "PAW_CODEX_DOCTOR_EXEC_SMOKE" + assert_contains "$slot_plist" "1" + assert_contains "$slot_plist" "PAW_CODEX_EXEC_TIMEOUT_SECS" + assert_contains "$slot_plist" "1200" + assert_contains "$slot_plist" "MAX_CONCURRENT_RUNS" + + slot_plists_json="$(jq -c --arg path "$slot_plist" '. + [$path]' <<<"$slot_plists_json")" + slot_worker_ids_json="$(jq -c --arg worker "$slot_worker_id" '. + [$worker]' <<<"$slot_worker_ids_json")" +done summary_json="$(jq -n \ --arg temper_url "$TEMPER_URL" \ --arg worker_id "$WORKER_ID" \ + --argjson worker_slot_count "$WORKER_SLOT_COUNT" \ + --argjson worker_slot_ids "$slot_worker_ids_json" \ --arg readiness_log "$READINESS_LOG" \ - --arg launchd_plist "$LAUNCHD_PLIST" \ + --argjson launchd_plists "$slot_plists_json" \ --arg server_log "$SERVER_LOG" \ '{ status: "passed", @@ -218,6 +245,8 @@ summary_json="$(jq -n \ }, worker: { worker_id: $worker_id, + worker_slot_count: $worker_slot_count, + worker_slot_ids: $worker_slot_ids, execution_enabled: false, codex_binary: "fake-codex fixture", codex_exec_smoke: "doctor pass" @@ -225,7 +254,8 @@ summary_json="$(jq -n \ launchd: { rendered: true, installed: false, - plist: $launchd_plist + process_pool_model: "launchd worker slot per local Codex process", + plists: $launchd_plists }, logs: { production_readiness: $readiness_log, diff --git a/crates/paw-codex-worker/scripts/production-readiness.sh b/crates/paw-codex-worker/scripts/production-readiness.sh index 4fa617e09..0e987f0e7 100755 --- a/crates/paw-codex-worker/scripts/production-readiness.sh +++ b/crates/paw-codex-worker/scripts/production-readiness.sh @@ -15,6 +15,8 @@ PAW_CODEX_POLL_ON_START="${PAW_CODEX_POLL_ON_START:-1}" PAW_CODEX_DOCTOR_EXEC_SMOKE="${PAW_CODEX_DOCTOR_EXEC_SMOKE:-0}" PAW_CODEX_EXEC_TIMEOUT_SECS="${PAW_CODEX_EXEC_TIMEOUT_SECS:-1200}" MAX_CONCURRENT_RUNS="${MAX_CONCURRENT_RUNS:-1}" +WORKER_SLOT_COUNT="${WORKER_SLOT_COUNT:-$MAX_CONCURRENT_RUNS}" +LAUNCHD_STALE_SLOT_SCAN_MAX="${LAUNCHD_STALE_SLOT_SCAN_MAX:-64}" WRITE_LAUNCHD_PLIST="${WRITE_LAUNCHD_PLIST:-0}" INSTALL_LAUNCHD="${INSTALL_LAUNCHD:-0}" LAUNCHD_PLIST="${LAUNCHD_PLIST:-$HOME/Library/LaunchAgents/com.temperpaw.paw-codex-worker.plist}" @@ -41,10 +43,13 @@ require_env() { } worker_env() { + local worker_id="$1" + local launchd_label="$2" + shift 2 env \ TEMPER_URL="$TEMPER_URL" \ TEMPER_TENANT="$TEMPER_TENANT" \ - WORKER_ID="$WORKER_ID" \ + WORKER_ID="$worker_id" \ WORKER_TOKEN="$WORKER_TOKEN" \ REPO_ROOT="$REPO_ROOT" \ WORKSPACE_ROOT="$WORKSPACE_ROOT" \ @@ -53,15 +58,91 @@ worker_env() { PAW_CODEX_POLL_ON_START="$PAW_CODEX_POLL_ON_START" \ PAW_CODEX_DOCTOR_EXEC_SMOKE="$PAW_CODEX_DOCTOR_EXEC_SMOKE" \ PAW_CODEX_EXEC_TIMEOUT_SECS="$PAW_CODEX_EXEC_TIMEOUT_SECS" \ - MAX_CONCURRENT_RUNS="$MAX_CONCURRENT_RUNS" \ + MAX_CONCURRENT_RUNS="1" \ + PAW_CODEX_LAUNCHD_LABEL="$launchd_label" \ "$@" } +slot_worker_id() { + local slot="$1" + if [[ "$slot" == "1" ]]; then + printf '%s\n' "$WORKER_ID" + else + printf '%s-slot-%02d\n' "$WORKER_ID" "$slot" + fi +} + +slot_launchd_label() { + local slot="$1" + if [[ "$WORKER_SLOT_COUNT" == "1" ]]; then + printf '%s\n' "com.temperpaw.paw-codex-worker" + else + slot_launchd_label_numbered "$slot" + fi +} + +slot_launchd_label_numbered() { + local slot="$1" + printf 'com.temperpaw.paw-codex-worker.slot-%02d\n' "$slot" +} + +slot_launchd_plist() { + local slot="$1" + if [[ "$WORKER_SLOT_COUNT" == "1" ]]; then + printf '%s\n' "$LAUNCHD_PLIST" + else + slot_launchd_plist_numbered "$slot" + fi +} + +slot_launchd_plist_numbered() { + local slot="$1" + local dir + dir="$(dirname "$LAUNCHD_PLIST")" + printf '%s/com.temperpaw.paw-codex-worker.slot-%02d.plist\n' "$dir" "$slot" +} + +launchd_bootout_agent() { + local label="$1" + local plist="$2" + launchctl bootout "gui/$(id -u)/${label}" >/dev/null 2>&1 || true + if [[ -f "$plist" ]]; then + launchctl bootout "gui/$(id -u)" "$plist" >/dev/null 2>&1 || true + fi +} + +unload_stale_launchd_agents() { + local max_slot="$LAUNCHD_STALE_SLOT_SCAN_MAX" + if [[ "$WORKER_SLOT_COUNT" -gt "$max_slot" ]]; then + max_slot="$WORKER_SLOT_COUNT" + fi + + log "unloading stale paw-codex-worker launchd agents" + launchd_bootout_agent "com.temperpaw.paw-codex-worker" "$LAUNCHD_PLIST" + for slot in $(seq 1 "$max_slot"); do + launchd_bootout_agent "$(slot_launchd_label_numbered "$slot")" "$(slot_launchd_plist_numbered "$slot")" + done + + local launchd_dir + launchd_dir="$(dirname "$LAUNCHD_PLIST")" + for plist in "${launchd_dir}"/com.temperpaw.paw-codex-worker.slot-*.plist; do + [[ -e "$plist" ]] || continue + launchctl bootout "gui/$(id -u)" "$plist" >/dev/null 2>&1 || true + done +} + require_cmd cargo require_cmd git require_env TEMPER_URL require_env WORKER_TOKEN +if ! [[ "$WORKER_SLOT_COUNT" =~ ^[0-9]+$ ]] || [[ "$WORKER_SLOT_COUNT" -lt 1 ]]; then + fail "WORKER_SLOT_COUNT must be a positive integer" +fi +if ! [[ "$LAUNCHD_STALE_SLOT_SCAN_MAX" =~ ^[0-9]+$ ]] || [[ "$LAUNCHD_STALE_SLOT_SCAN_MAX" -lt 1 ]]; then + fail "LAUNCHD_STALE_SLOT_SCAN_MAX must be a positive integer" +fi + if [[ "$REQUIRE_MAIN_ANCESTRY" == "1" ]]; then bash "${ROOT}/crates/paw-codex-worker/scripts/production-git-ancestry-guard.sh" fi @@ -70,6 +151,7 @@ log "repo root: ${ROOT}" log "temper url: ${TEMPER_URL}" log "tenant: ${TEMPER_TENANT}" log "worker id: ${WORKER_ID}" +log "worker slot count: ${WORKER_SLOT_COUNT}" log "repo checkout: ${REPO_ROOT}" log "worktree root: ${WORKSPACE_ROOT}" log "codex binary: ${CODEX_BIN}" @@ -80,23 +162,39 @@ log "building paw-codex-worker release binary" cargo build -p paw-codex-worker --release log "running paw-codex-worker doctor" -worker_env "$WORKER_BIN" doctor +worker_env "$(slot_worker_id 1)" "$(slot_launchd_label 1)" "$WORKER_BIN" doctor + +log "registering worker slot identities" +for slot in $(seq 1 "$WORKER_SLOT_COUNT"); do + slot_worker="$(slot_worker_id "$slot")" + slot_label="$(slot_launchd_label "$slot")" + worker_env "$slot_worker" "$slot_label" "$WORKER_BIN" register-worker-agent +done if [[ "$WRITE_LAUNCHD_PLIST" == "1" ]]; then - log "rendering launchd-plist to ${LAUNCHD_PLIST}" - mkdir -p "$(dirname "$LAUNCHD_PLIST")" - worker_env "$WORKER_BIN" launchd-plist >"$LAUNCHD_PLIST" - chmod 600 "$LAUNCHD_PLIST" + for slot in $(seq 1 "$WORKER_SLOT_COUNT"); do + slot_worker="$(slot_worker_id "$slot")" + slot_label="$(slot_launchd_label "$slot")" + slot_plist="$(slot_launchd_plist "$slot")" + log "rendering launchd plist ${slot} for ${slot_worker} to ${slot_plist}" + mkdir -p "$(dirname "$slot_plist")" + worker_env "$slot_worker" "$slot_label" "$WORKER_BIN" launchd-plist >"$slot_plist" + chmod 600 "$slot_plist" + done elif [[ "$INSTALL_LAUNCHD" == "1" ]]; then fail "INSTALL_LAUNCHD=1 requires WRITE_LAUNCHD_PLIST=1 so the exact plist is rendered first" fi if [[ "$INSTALL_LAUNCHD" == "1" ]]; then require_cmd launchctl - log "loading launchd agent ${LAUNCHD_PLIST}" - launchctl bootout "gui/$(id -u)" "$LAUNCHD_PLIST" >/dev/null 2>&1 || true - launchctl bootstrap "gui/$(id -u)" "$LAUNCHD_PLIST" - launchctl kickstart -k "gui/$(id -u)/com.temperpaw.paw-codex-worker" + unload_stale_launchd_agents + for slot in $(seq 1 "$WORKER_SLOT_COUNT"); do + slot_label="$(slot_launchd_label "$slot")" + slot_plist="$(slot_launchd_plist "$slot")" + log "loading launchd agent ${slot_label} from ${slot_plist}" + launchctl bootstrap "gui/$(id -u)" "$slot_plist" + launchctl kickstart -k "gui/$(id -u)/${slot_label}" + done fi log "production readiness check passed" diff --git a/crates/paw-codex-worker/src/boot_watch.rs b/crates/paw-codex-worker/src/boot_watch.rs index a81a1b105..3a2aa00e9 100644 --- a/crates/paw-codex-worker/src/boot_watch.rs +++ b/crates/paw-codex-worker/src/boot_watch.rs @@ -76,10 +76,50 @@ async fn claim_boot_queued_directed_evolution_work_items( client: &reqwest::Client, config: &Config, ) -> Result<()> { + if let Err(error) = recover_pending_directed_evolution_stage_results(client, config).await { + warn!(%error, "Directed Evolution stage-result recovery pass failed"); + } let ids = query_boot_entity_ids(client, config, "WorkItems", "Queued").await?; - for work_item_id in ids { - handle_queued_directed_evolution_work_item(client, config, &work_item_id).await?; + if ids.is_empty() { + return Ok(()); } + let concurrency = config.max_concurrent_runs.max(1); + info!( + queued_work_item_count = ids.len(), + max_concurrent_runs = concurrency, + "claiming queued Directed Evolution WorkItems" + ); + let results = futures_util::stream::iter(ids.into_iter().map(|work_item_id| { + let client = client.clone(); + let config = config.clone(); + async move { + let result = + handle_queued_directed_evolution_work_item(&client, &config, &work_item_id).await; + (work_item_id, result) + } + })) + .buffer_unordered(concurrency) + .collect::>() + .await; + + let mut failure_count = 0usize; + for (work_item_id, result) in results { + if let Err(error) = result { + failure_count += 1; + warn!( + work_item_id = %work_item_id, + %error, + "Directed Evolution WorkItem pickup failed" + ); + } + } + if failure_count > 0 { + bail!( + "{} Directed Evolution WorkItem pickup task(s) failed", + failure_count + ); + } + Ok(()) } @@ -100,7 +140,16 @@ async fn query_boot_entity_ids( .await .with_context(|| format!("query {entity_set} with status {status} on boot"))?; if !response.status().is_success() { - warn!(entity_set, status, http_status = %response.status(), "boot query failed"); + if response.status() == reqwest::StatusCode::NOT_FOUND { + debug!( + entity_set, + status, + http_status = %response.status(), + "optional boot query entity set is not installed" + ); + } else { + warn!(entity_set, status, http_status = %response.status(), "boot query failed"); + } return Ok(Vec::new()); } diff --git a/crates/paw-codex-worker/src/cli.rs b/crates/paw-codex-worker/src/cli.rs index 01b0ffe0d..1c738aaa5 100644 --- a/crates/paw-codex-worker/src/cli.rs +++ b/crates/paw-codex-worker/src/cli.rs @@ -6,6 +6,9 @@ fn parse_worker_command(args: impl IntoIterator) -> WorkerCommand "launchd-plist" | "plist" | "--launchd-plist" => { return WorkerCommand::LaunchdPlist; } + "register-worker-agent" | "--register-worker-agent" => { + return WorkerCommand::RegisterWorkerAgent; + } "directed-evolution-start-episode" | "de-start-episode" => { return WorkerCommand::DirectedEvolutionStartEpisode { contract_path: args.next(), diff --git a/crates/paw-codex-worker/src/datadog_patrol.rs b/crates/paw-codex-worker/src/datadog_patrol.rs index 82917e75d..0aff7b122 100644 --- a/crates/paw-codex-worker/src/datadog_patrol.rs +++ b/crates/paw-codex-worker/src/datadog_patrol.rs @@ -99,8 +99,9 @@ async fn investigate_datadog_with_codex( let workdir = ensure_worktree(config, worker_run).await?; let prompt = datadog_mcp_patrol_prompt(patrol_run_id, worker_run); - let output = run_codex_exec_command(config, &workdir, prompt, "run Codex Datadog MCP Patrol") - .await?; + let output = + run_codex_exec_command_with_datadog_mcp(config, &workdir, prompt, "run Codex Datadog MCP Patrol") + .await?; let stdout = String::from_utf8_lossy(&output.stdout); let stderr = String::from_utf8_lossy(&output.stderr); diff --git a/crates/paw-codex-worker/src/directed_evolution.rs b/crates/paw-codex-worker/src/directed_evolution.rs index 0b2698221..595046d14 100644 --- a/crates/paw-codex-worker/src/directed_evolution.rs +++ b/crates/paw-codex-worker/src/directed_evolution.rs @@ -1,514 +1,13 @@ -async fn handle_queued_directed_evolution_work_item( - client: &reqwest::Client, - config: &Config, - work_item_id: &str, -) -> Result<()> { - info!(work_item_id, "saw queued Directed Evolution WorkItem"); - let work_item = fetch_directed_evolution_work_item(client, config, work_item_id).await?; - if work_item.status != "Queued" { - debug!( - work_item_id, - status = %work_item.status, - "Directed Evolution WorkItem no longer queued" - ); - return Ok(()); - } - if work_item.role.trim().is_empty() { - debug!(work_item_id, "Directed Evolution WorkItem has no brain role yet"); - return Ok(()); - } - if let Some(reason) = - stale_directed_evolution_work_item_reason(client, config, &work_item).await? - { - eliminate_stale_directed_evolution_stage_result(client, config, &work_item, &reason) - .await?; - post_directed_evolution_action( - client, - config, - "WorkItems", - &work_item.id, - "CancelWorkItem", - json!({ "Reason": reason }), - ) - .await?; - info!( - work_item_id = %work_item.id, - role = %work_item.role, - "cancelled stale Directed Evolution WorkItem" - ); - return Ok(()); - } - - let brain_run_id = create_entity(client, config, "BrainRuns", json!({})).await?; - post_directed_evolution_action( - client, - config, - "WorkItems", - &work_item.id, - "ClaimWorkItem", - json!({ - "WorkerId": config.worker_id, - "ClaimedBy": config.worker_id, - }), - ) - .await?; - post_directed_evolution_action( - client, - config, - "BrainRuns", - &brain_run_id, - "StartBrainRun", - json!({ - "Role": work_item.role, - "WorkItemId": work_item.id, - "AgentKind": directed_evolution_agent_kind_for_role(&work_item.role), - "Model": directed_evolution_model_for_role(&work_item.role), - "ParentSessionId": env::var("CODEX_SESSION_ID").unwrap_or_default(), - "CorrelationJson": work_item.correlation_json, - }), - ) - .await?; - post_directed_evolution_action( - client, - config, - "WorkItems", - &work_item.id, - "StartWorkItem", - json!({ "BrainRunId": brain_run_id }), - ) - .await?; - info!( - work_item_id = %work_item.id, - brain_run_id = %brain_run_id, - role = %work_item.role, - target_entity_type = %work_item.target_entity_type, - target_entity_id = %work_item.target_entity_id, - "started Directed Evolution worker run" - ); - - match run_directed_evolution_codex_role(client, config, &work_item).await { - Ok(output_json) => { - let summary = directed_evolution_summary(&work_item, &output_json); - let evidence_artifact_id = record_directed_evolution_brain_evidence( - client, - config, - &work_item, - &brain_run_id, - "codex_brain_run", - &output_json, - &summary, - ) - .await?; - post_directed_evolution_action( - client, - config, - "BrainRuns", - &brain_run_id, - "SucceedBrainRun", - json!({ - "OutputJson": output_json, - "EvidenceArtifactId": evidence_artifact_id, - "Summary": summary, - }), - ) - .await?; - post_directed_evolution_action( - client, - config, - "WorkItems", - &work_item.id, - "SucceedWorkItem", - json!({ - "ResultJson": output_json, - "EvidenceArtifactId": evidence_artifact_id, - "Summary": summary, - }), - ) - .await?; - info!( - work_item_id = %work_item.id, - brain_run_id = %brain_run_id, - role = %work_item.role, - evidence_artifact_id = %evidence_artifact_id, - "completed Directed Evolution Codex brain run" - ); - Ok(()) - } - Err(error) => { - let failure_reason = format!("Directed Evolution Codex role failed: {error}"); - let evidence_artifact_id = match record_directed_evolution_brain_evidence( - client, - config, - &work_item, - &brain_run_id, - "codex_brain_run_failure", - &serde_json::to_string(&json!({ - "status": "failed", - "failure_reason": failure_reason, - }))?, - &failure_reason, - ) - .await - { - Ok(id) => id, - Err(report_error) => { - warn!(%report_error, work_item_id, brain_run_id, "failed to record Directed Evolution failure evidence"); - String::new() - } - }; - if let Err(report_error) = post_directed_evolution_action( - client, - config, - "BrainRuns", - &brain_run_id, - "FailBrainRun", - json!({ - "FailureReason": failure_reason, - "EvidenceArtifactId": evidence_artifact_id, - }), - ) - .await - { - warn!(%report_error, work_item_id, brain_run_id, "failed to report BrainRun failure"); - } - post_directed_evolution_action( - client, - config, - "WorkItems", - &work_item.id, - "FailWorkItem", - json!({ - "FailureReason": failure_reason, - "EvidenceArtifactId": evidence_artifact_id, - }), - ) - .await?; - warn!( - work_item_id = %work_item.id, - brain_run_id = %brain_run_id, - role = %work_item.role, - evidence_artifact_id = %evidence_artifact_id, - "failed Directed Evolution Codex brain run" - ); - Ok(()) - } - } -} - -async fn stale_directed_evolution_work_item_reason( - client: &reqwest::Client, - config: &Config, - work_item: &DirectedEvolutionWorkItemState, -) -> Result> { - if !directed_evolution_stage_evaluator_role(&work_item.role) - || work_item.target_entity_type != "StageResult" - { - return Ok(None); - } - - let stage_fields = - fetch_directed_evolution_entity_fields(client, config, "StageResults", &work_item.target_entity_id) - .await?; - let variant_fields = { - let variant_id = value_field_string(&stage_fields, &["VariantId", "variant_id"]); - if variant_id.trim().is_empty() { - json!({}) - } else { - fetch_directed_evolution_entity_fields(client, config, "Variants", &variant_id).await? - } - }; - Ok(stale_directed_evolution_stage_work_reason( - work_item, - &stage_fields, - &variant_fields, - )) -} - -async fn eliminate_stale_directed_evolution_stage_result( - client: &reqwest::Client, - config: &Config, - work_item: &DirectedEvolutionWorkItemState, - reason: &str, -) -> Result<()> { - if !stale_stage_work_targets_stage_result(work_item) { - return Ok(()); - } - - let stage_fields = - fetch_directed_evolution_entity_fields(client, config, "StageResults", &work_item.target_entity_id) - .await?; - if !stale_stage_result_should_eliminate(&stage_fields) { - return Ok(()); - } - - post_directed_evolution_action( - client, - config, - "StageResults", - &work_item.target_entity_id, - "EliminateStageResult", - json!({ - "EliminationRuleId": "stale-after-variant-terminal", - "EvidenceArtifactId": value_field_string(&stage_fields, &["EvidenceArtifactId", "evidence_artifact_id"]), - "Reason": reason, - }), - ) - .await -} - -fn stale_directed_evolution_stage_work_reason( - work_item: &DirectedEvolutionWorkItemState, - stage_fields: &Value, - variant_fields: &Value, -) -> Option { - if !stale_stage_work_targets_stage_result(work_item) { - return None; - } - let stage_status = value_field_string(stage_fields, &["Status", "status"]); - if !stage_status.trim().is_empty() && stage_status != "Running" { - return Some(format!( - "Target StageResult {} is already {}; skipping stale {} work", - work_item.target_entity_id, stage_status, work_item.role - )); - } - let variant_status = value_field_string(variant_fields, &["Status", "status"]); - if matches!( - variant_status.as_str(), - "Eliminated" | "Promoted" | "Superseded" | "Failed" - ) { - return Some(format!( - "Target variant is already {}; skipping stale {} work for StageResult {}", - variant_status, work_item.role, work_item.target_entity_id - )); - } - None -} - -fn stale_stage_work_targets_stage_result(work_item: &DirectedEvolutionWorkItemState) -> bool { - directed_evolution_stage_evaluator_role(&work_item.role) - && work_item.target_entity_type == "StageResult" -} - -fn directed_evolution_stage_evaluator_role(role: &str) -> bool { - matches!( - role, - "reviewer" - | "viability_evaluator" - | "state_verifier" - | "telemetry_evaluator" - | "wasm_evaluator" - ) -} - -fn stale_stage_result_should_eliminate(stage_fields: &Value) -> bool { - matches!( - value_field_string(stage_fields, &["Status", "status"]).as_str(), - "Running" | "Failed" - ) -} - - +include!("directed_evolution/work_item_handler.rs"); +include!("directed_evolution/work_item_guards.rs"); include!("directed_evolution/evidence.rs"); include!("directed_evolution/human_episode_defaults.rs"); include!("directed_evolution/human_episode_plan.rs"); include!("directed_evolution/human_episode.rs"); - -async fn post_directed_evolution_action( - client: &reqwest::Client, - config: &Config, - entity_set: &str, - entity_id: &str, - action: &str, - body: Value, -) -> Result<()> { - post_entity_action_with_namespace( - client, - config, - entity_set, - entity_id, - DIRECTED_EVOLUTION_NAMESPACE, - action, - body, - ) - .await -} - -async fn run_directed_evolution_codex_role( - client: &reqwest::Client, - config: &Config, - work_item: &DirectedEvolutionWorkItemState, -) -> Result { - let prompt = directed_evolution_prompt(work_item); - info!( - work_item_id = %work_item.id, - role = %work_item.role, - target_entity_type = %work_item.target_entity_type, - target_entity_id = %work_item.target_entity_id, - execution_enabled = config.enable_execution, - "executing Directed Evolution Codex role" - ); - if !config.enable_execution { - return serde_json::to_string(&json!({ - "status": "dry_run", - "role": work_item.role, - "work_item_id": work_item.id, - "target": { - "entity_type": work_item.target_entity_type, - "entity_id": work_item.target_entity_id, - }, - "prompt_preview": truncate_middle(&prompt, 1200), - })) - .context("serialize Directed Evolution dry-run output"); - } - if directed_evolution_mechanical_evaluator_role(&work_item.role) { - let payload = run_directed_evolution_mechanical_evaluator(client, config, work_item).await?; - return serde_json::to_string(&payload) - .context("serialize Directed Evolution mechanical evaluator output"); - } - if work_item.role == "promoter" { - let materialization = - materialize_directed_evolution_promotion(client, config, work_item).await?; - return serde_json::to_string(&directed_evolution_promotion_output(&materialization)) - .context("serialize Directed Evolution promoter output"); - } - - let workdir = resolve_directed_evolution_workdir(client, config, work_item).await?; - let readonly_status_before = if directed_evolution_role_may_write_repo(&work_item.role) { - None - } else { - directed_evolution_git_status_snapshot(&workdir.path).await? - }; - let output = match run_codex_exec_command( - config, - &workdir.path, - prompt, - "run Directed Evolution Codex role", - ) - .await - { - Ok(output) => output, - Err(error) => { - return recover_directed_evolution_variant_output( - client, config, work_item, &workdir, error, - ) - .await; - } - }; - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - if !output.status.success() { - return recover_directed_evolution_variant_output( - client, - config, - work_item, - &workdir, - anyhow!( - "codex role {} failed with status {:?}: {}", - work_item.role, - output.status.code(), - truncate_middle(&format!("{stdout}\n{stderr}"), 4_000) - ), - ) - .await; - } - let mut payload = parse_codex_jsonish(&stdout).unwrap_or_else(|| { - json!({ - "status": "succeeded", - "summary": truncate_middle(&stdout, 4_000), - }) - }); - if let Some(status_before) = readonly_status_before { - ensure_directed_evolution_readonly_workdir_unchanged(&workdir.path, &status_before).await?; - } - payload = finalize_directed_evolution_output(client, config, work_item, &workdir, payload).await?; - if let Some(object) = payload.as_object_mut() { - object - .entry("role".to_string()) - .or_insert_with(|| json!(work_item.role)); - object - .entry("work_item_id".to_string()) - .or_insert_with(|| json!(work_item.id)); - object.entry("target".to_string()).or_insert_with(|| { - json!({ - "entity_type": work_item.target_entity_type, - "entity_id": work_item.target_entity_id, - }) - }); - object.entry("execution".to_string()).or_insert_with(|| { - json!({ - "workdir": workdir.path.display().to_string(), - "stdout_bytes": stdout.len(), - "stderr_bytes": stderr.len(), - }) - }); - } - serde_json::to_string(&payload) - .context("serialize Directed Evolution Codex output") -} - -fn directed_evolution_agent_kind_for_role(role: &str) -> &'static str { - if matches!(role, "promoter" | "state_verifier" | "wasm_evaluator") { - "temperpaw-worker" - } else { - "codex" - } -} - -fn directed_evolution_model_for_role(role: &str) -> &'static str { - if matches!(role, "promoter" | "state_verifier" | "wasm_evaluator") { - "deterministic-worker" - } else { - "codex-cli" - } -} - -async fn recover_directed_evolution_variant_output( - client: &reqwest::Client, - config: &Config, - work_item: &DirectedEvolutionWorkItemState, - workdir: &DirectedEvolutionWorkdir, - error: anyhow::Error, -) -> Result { - if work_item.role != "variant_generator" - || env_flag("PAW_DE_RECOVER_VARIANT_CHANGES_ON_CODEX_ERROR") == Some(false) - { - return Err(error); - } - let changed_files = match git_changed_files(&workdir.path).await { - Ok(files) if !files.is_empty() => files, - _ => return Err(error), - }; - let error_summary = truncate_middle(&error.to_string(), 1_200); - warn!( - work_item_id = %work_item.id, - changed_file_count = changed_files.len(), - error = %error_summary, - "recovering Directed Evolution variant from git changes after Codex execution error" - ); - let payload = json!({ - "status": "recovered_after_codex_exec_error", - "summary": format!( - "Recovered a Directed Evolution variant from {} changed file(s) after Codex exited before returning JSON.", - changed_files.len() - ), - "changed_files": changed_files, - "verification_notes": format!( - "Codex produced repository changes but did not finish the structured response: {error_summary}. Later evaluation stages must validate the hot-loaded runtime before selection." - ), - "reasoning_summary": "The worker recovered concrete file mutations left by the variant-generator brain and will publish them as a candidate for downstream evaluation.", - "recovery": { - "reason": "codex_exec_error_after_file_changes", - "error": error_summary, - "worker_id": config.worker_id, - } - }); - let payload = finalize_directed_evolution_output(client, config, work_item, workdir, payload) - .await?; - serde_json::to_string(&payload).context("serialize recovered Directed Evolution variant output") -} - - +include!("directed_evolution/actions.rs"); +include!("directed_evolution/codex_role.rs"); include!("directed_evolution/workdir.rs"); +include!("directed_evolution/recovery.rs"); include!("directed_evolution/mechanical_evaluator.rs"); include!("directed_evolution/prompt.rs"); include!("directed_evolution/tests.rs"); diff --git a/crates/paw-codex-worker/src/directed_evolution/actions.rs b/crates/paw-codex-worker/src/directed_evolution/actions.rs new file mode 100644 index 000000000..3ade9fb6a --- /dev/null +++ b/crates/paw-codex-worker/src/directed_evolution/actions.rs @@ -0,0 +1,129 @@ +async fn post_directed_evolution_action( + client: &reqwest::Client, + config: &Config, + entity_set: &str, + entity_id: &str, + action: &str, + body: Value, +) -> Result<()> { + post_entity_action_with_namespace( + client, + config, + entity_set, + entity_id, + DIRECTED_EVOLUTION_NAMESPACE, + action, + body, + ) + .await +} + +async fn post_paw_orchestration_action( + client: &reqwest::Client, + config: &Config, + entity_set: &str, + entity_id: &str, + action: &str, + body: Value, +) -> Result<()> { + post_entity_action_with_namespace( + client, + config, + entity_set, + entity_id, + PAW_ORCHESTRATION_NAMESPACE, + action, + body, + ) + .await +} + +async fn post_paw_orchestration_action_with_headers( + client: &reqwest::Client, + config: &Config, + request_headers: &HeaderMap, + entity_set: &str, + entity_id: &str, + action: &str, + body: Value, +) -> Result<()> { + let url = config.entity_action_url_with_namespace( + entity_set, + entity_id, + PAW_ORCHESTRATION_NAMESPACE, + action, + ); + let response = client + .post(url) + .headers(request_headers.clone()) + .json(&body) + .send() + .await + .with_context(|| format!("dispatch PawOrchestration {entity_set}.{action}"))?; + if !response.status().is_success() { + let status = response.status(); + let text = response.text().await.unwrap_or_default(); + bail!("PawOrchestration {entity_set}.{action} returned {status}: {text}"); + } + Ok(()) +} + +async fn route_directed_evolution_work_item_success( + client: &reqwest::Client, + config: &Config, + work_item: &DirectedEvolutionWorkItemState, + worker_run_id: &str, + output_json: &str, + evidence_artifact_id: &str, + summary: &str, +) -> Result<()> { + let receipt_id = create_entity(client, config, "WorkItemReceipts", json!({})).await?; + post_directed_evolution_action( + client, + config, + "WorkItemReceipts", + &receipt_id, + "RouteSucceededWorkItem", + json!({ + "WorkItemId": work_item.id, + "Role": work_item.role, + "TargetEntityType": work_item.target_entity_type, + "TargetEntityId": work_item.target_entity_id, + "WorkerRunId": worker_run_id, + "ResultJson": output_json, + "EvidenceArtifactId": evidence_artifact_id, + "Summary": summary, + "CorrelationJson": work_item.correlation_json, + }), + ) + .await +} + +async fn route_directed_evolution_work_item_failure( + client: &reqwest::Client, + config: &Config, + work_item: &DirectedEvolutionWorkItemState, + worker_run_id: &str, + failure_reason: &str, + evidence_artifact_id: &str, +) -> Result<()> { + let receipt_id = create_entity(client, config, "WorkItemReceipts", json!({})).await?; + post_directed_evolution_action( + client, + config, + "WorkItemReceipts", + &receipt_id, + "RouteFailedWorkItem", + json!({ + "WorkItemId": work_item.id, + "Role": work_item.role, + "TargetEntityType": work_item.target_entity_type, + "TargetEntityId": work_item.target_entity_id, + "WorkerRunId": worker_run_id, + "FailureReason": failure_reason, + "EvidenceArtifactId": evidence_artifact_id, + "CorrelationJson": work_item.correlation_json, + }), + ) + .await +} diff --git a/crates/paw-codex-worker/src/directed_evolution/codex_role.rs b/crates/paw-codex-worker/src/directed_evolution/codex_role.rs new file mode 100644 index 000000000..907fb8940 --- /dev/null +++ b/crates/paw-codex-worker/src/directed_evolution/codex_role.rs @@ -0,0 +1,215 @@ +async fn run_directed_evolution_codex_role( + client: &reqwest::Client, + config: &Config, + work_item: &DirectedEvolutionWorkItemState, +) -> Result { + let prompt = directed_evolution_prompt(work_item); + info!( + work_item_id = %work_item.id, + role = %work_item.role, + target_entity_type = %work_item.target_entity_type, + target_entity_id = %work_item.target_entity_id, + execution_enabled = config.enable_execution, + "executing Directed Evolution Codex role" + ); + if !config.enable_execution { + if directed_evolution_work_item_requires_datadog_evidence(work_item) { + bail!( + "Directed Evolution role {} requires live Codex execution and Datadog evidence", + work_item.role + ); + } + return serde_json::to_string(&json!({ + "status": "dry_run", + "role": work_item.role, + "work_item_id": work_item.id, + "target": { + "entity_type": work_item.target_entity_type, + "entity_id": work_item.target_entity_id, + }, + "prompt_preview": truncate_middle(&prompt, 1200), + })) + .context("serialize Directed Evolution dry-run output"); + } + if directed_evolution_mechanical_evaluator_role(&work_item.role) { + let payload = run_directed_evolution_mechanical_evaluator(client, config, work_item).await?; + return serde_json::to_string(&payload) + .context("serialize Directed Evolution mechanical evaluator output"); + } + if work_item.role == "promoter" { + let materialization = + materialize_directed_evolution_promotion(client, config, work_item).await?; + return serde_json::to_string(&directed_evolution_promotion_output(&materialization)) + .context("serialize Directed Evolution promoter output"); + } + + let workdir = resolve_directed_evolution_workdir(client, config, work_item).await?; + let readonly_status_before = if directed_evolution_role_may_write_repo(&work_item.role) { + None + } else { + directed_evolution_git_status_snapshot(&workdir.path).await? + }; + let output = match if directed_evolution_work_item_requires_datadog_evidence(work_item) { + run_codex_exec_command_with_datadog_mcp( + config, + &workdir.path, + prompt, + "run Directed Evolution Datadog-backed role", + ) + .await + } else { + run_codex_exec_command(config, &workdir.path, prompt, "run Directed Evolution Codex role") + .await + } { + Ok(output) => output, + Err(error) => { + return recover_directed_evolution_variant_output( + client, config, work_item, &workdir, error, + ) + .await; + } + }; + let stdout = String::from_utf8_lossy(&output.stdout).to_string(); + let stderr = String::from_utf8_lossy(&output.stderr).to_string(); + if !output.status.success() { + return recover_directed_evolution_variant_output( + client, + config, + work_item, + &workdir, + anyhow!( + "codex role {} failed with status {:?}: {}", + work_item.role, + output.status.code(), + truncate_middle(&format!("{stdout}\n{stderr}"), 4_000) + ), + ) + .await; + } + let mut payload = parse_codex_jsonish(&stdout).unwrap_or_else(|| { + json!({ + "status": "succeeded", + "summary": truncate_middle(&stdout, 4_000), + }) + }); + if let Some(status_before) = readonly_status_before { + ensure_directed_evolution_readonly_workdir_unchanged(&workdir.path, &status_before).await?; + } + payload = finalize_directed_evolution_output(client, config, work_item, &workdir, payload).await?; + if let Some(object) = payload.as_object_mut() { + object + .entry("role".to_string()) + .or_insert_with(|| json!(work_item.role)); + object + .entry("work_item_id".to_string()) + .or_insert_with(|| json!(work_item.id)); + object.entry("target".to_string()).or_insert_with(|| { + json!({ + "entity_type": work_item.target_entity_type, + "entity_id": work_item.target_entity_id, + }) + }); + object.entry("execution".to_string()).or_insert_with(|| { + json!({ + "workdir": workdir.path.display().to_string(), + "stdout_bytes": stdout.len(), + "stderr_bytes": stderr.len(), + }) + }); + } + ensure_directed_evolution_required_datadog_evidence(work_item, &payload)?; + serde_json::to_string(&payload).context("serialize Directed Evolution Codex output") +} + +fn directed_evolution_agent_kind_for_role(role: &str) -> &'static str { + if matches!(role, "promoter" | "state_verifier" | "wasm_evaluator") { + "temperpaw-worker" + } else { + "codex" + } +} + +fn directed_evolution_role_requires_datadog_mcp(role: &str) -> bool { + matches!(role, "observer" | "telemetry_evaluator") +} + +fn directed_evolution_work_item_requires_datadog_evidence( + work_item: &DirectedEvolutionWorkItemState, +) -> bool { + directed_evolution_role_requires_datadog_mcp(&work_item.role) + || directed_evolution_text_requires_datadog_evidence(&work_item.prompt_ref) + || directed_evolution_text_requires_datadog_evidence(&work_item.context_ref) + || directed_evolution_text_requires_datadog_evidence(&work_item.output_schema_ref) + || directed_evolution_text_requires_datadog_evidence(&work_item.correlation_json) +} + +fn directed_evolution_text_requires_datadog_evidence(value: &str) -> bool { + value + .to_ascii_lowercase() + .contains("datadog_evidence_scope") +} + +fn directed_evolution_worker_provider_id() -> String { + worker_provider_id() +} + +fn directed_evolution_model_for_role(role: &str) -> String { + if matches!(role, "promoter" | "state_verifier" | "wasm_evaluator") { + "deterministic-worker".to_string() + } else { + directed_evolution_codex_model_label() + } +} + +fn directed_evolution_codex_model_label() -> String { + env::var("PAW_CODEX_MODEL") + .ok() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| "codex-cli".to_string()) +} + +async fn recover_directed_evolution_variant_output( + client: &reqwest::Client, + config: &Config, + work_item: &DirectedEvolutionWorkItemState, + workdir: &DirectedEvolutionWorkdir, + error: anyhow::Error, +) -> Result { + if work_item.role != "variant_generator" + || env_flag("PAW_DE_RECOVER_VARIANT_CHANGES_ON_CODEX_ERROR") == Some(false) + { + return Err(error); + } + let changed_files = match git_changed_files(&workdir.path).await { + Ok(files) if !files.is_empty() => files, + _ => return Err(error), + }; + let error_summary = truncate_middle(&error.to_string(), 1_200); + warn!( + work_item_id = %work_item.id, + changed_file_count = changed_files.len(), + error = %error_summary, + "recovering Directed Evolution variant from git changes after Codex execution error" + ); + let payload = json!({ + "status": "recovered_after_codex_exec_error", + "summary": format!( + "Recovered a Directed Evolution variant from {} changed file(s) after Codex exited before returning JSON.", + changed_files.len() + ), + "changed_files": changed_files, + "verification_notes": format!( + "Codex produced repository changes but did not finish the structured response: {error_summary}. Later evaluation stages must validate the hot-loaded runtime before selection." + ), + "reasoning_summary": "The worker recovered concrete file mutations left by the variant-generator worker run and will publish them as a candidate for downstream evaluation.", + "recovery": { + "reason": "codex_exec_error_after_file_changes", + "error": error_summary, + "worker_id": config.worker_id, + } + }); + let payload = + finalize_directed_evolution_output(client, config, work_item, workdir, payload).await?; + serde_json::to_string(&payload).context("serialize recovered Directed Evolution variant output") +} diff --git a/crates/paw-codex-worker/src/directed_evolution/evidence.rs b/crates/paw-codex-worker/src/directed_evolution/evidence.rs index b2ee68d2a..14604d158 100644 --- a/crates/paw-codex-worker/src/directed_evolution/evidence.rs +++ b/crates/paw-codex-worker/src/directed_evolution/evidence.rs @@ -1,8 +1,8 @@ -async fn record_directed_evolution_brain_evidence( +async fn record_directed_evolution_worker_evidence( client: &reqwest::Client, config: &Config, work_item: &DirectedEvolutionWorkItemState, - brain_run_id: &str, + worker_run_id: &str, artifact_kind: &str, output_json: &str, summary: &str, @@ -16,7 +16,7 @@ async fn record_directed_evolution_brain_evidence( let uri = directed_evolution_evidence_uri(work_item, &output_value); let correlation = json!({ "work_item_id": work_item.id, - "brain_run_id": brain_run_id, + "worker_run_id": worker_run_id, "role": work_item.role, "target_entity_type": work_item.target_entity_type, "target_entity_id": work_item.target_entity_id, @@ -54,8 +54,8 @@ async fn record_directed_evolution_brain_evidence( &evidence_id, "LinkEvidenceArtifact", json!({ - "TargetEntityType": "BrainRun", - "TargetEntityId": brain_run_id, + "TargetEntityType": "WorkerRun", + "TargetEntityId": worker_run_id, }), ) .await?; @@ -81,7 +81,11 @@ fn directed_evolution_first_evidence_scope_summary( else { return DirectedEvolutionEvidenceScopeSummary::default(); }; - let Some(first) = items.first() else { + let Some(first) = items + .iter() + .find(|item| directed_evolution_scope_satisfies_datadog_contract(item)) + .or_else(|| items.first()) + else { return DirectedEvolutionEvidenceScopeSummary::default(); }; DirectedEvolutionEvidenceScopeSummary { @@ -108,7 +112,91 @@ fn directed_evolution_first_evidence_scope_summary( } } +fn ensure_directed_evolution_required_datadog_evidence( + work_item: &DirectedEvolutionWorkItemState, + output: &Value, +) -> Result<()> { + if !directed_evolution_work_item_requires_datadog_evidence(work_item) { + return Ok(()); + } + let role = work_item.role.as_str(); + let Some(items) = output + .get("evidence_scope") + .or_else(|| output.get("evidenceScope")) + .and_then(Value::as_array) + else { + bail!("{role} output missing mandatory Datadog evidence_scope"); + }; + if items + .iter() + .any(directed_evolution_scope_satisfies_datadog_contract) + { + return Ok(()); + } + bail!("{role} output missing complete Datadog evidence_scope entry") +} + +fn directed_evolution_scope_satisfies_datadog_contract(scope: &Value) -> bool { + let required = [ + ("query", &["query", "Query"][..]), + ("time_window", &["time_window", "timeWindow", "TimeWindow"][..]), + ( + "result_count", + &["result_count", "resultCount", "ResultCount", "count"][..], + ), + ("interpretation", &["interpretation", "Interpretation"][..]), + ( + "zero_result_meaning", + &[ + "zero_result_meaning", + "zeroResultMeaning", + "ZeroResultMeaning", + ][..], + ), + ]; + for (_, aliases) in required { + if value_field_string(scope, aliases).trim().is_empty() { + return false; + } + } + let Some(url) = scope + .get("datadog_url") + .or_else(|| scope.get("datadogUrl")) + .and_then(Value::as_str) + .map(str::trim) + .filter(|value| !value.is_empty()) + else { + return false; + }; + is_datadog_app_url(url) +} + fn directed_evolution_evidence_provenance(role: &str, output: &Value) -> String { + let explicit = value_field_string( + output, + &[ + "provenance_kind", + "provenanceKind", + "EvidenceProvenance", + "evidence_provenance", + ], + ); + if !explicit.trim().is_empty() { + return explicit; + } + if role == "observer" + && output + .get("evidence_scope") + .or_else(|| output.get("evidenceScope")) + .and_then(Value::as_array) + .is_some_and(|items| { + items + .iter() + .any(directed_evolution_scope_satisfies_datadog_contract) + }) + { + return "datadog-measured".to_string(); + } let fallback = match role { "simulated_user" => "agent-observed", "state_verifier" => "state-verified", @@ -117,18 +205,7 @@ fn directed_evolution_evidence_provenance(role: &str, output: &Value) -> String "reviewer" | "viability_evaluator" => "brain-judged", _ => "agent-observed", }; - nonempty( - value_field_string( - output, - &[ - "provenance_kind", - "provenanceKind", - "EvidenceProvenance", - "evidence_provenance", - ], - ), - fallback.to_string(), - ) + fallback.to_string() } fn directed_evolution_evidence_uri( @@ -214,7 +291,24 @@ fn directed_evolution_datadog_context(work_item: &DirectedEvolutionWorkItemState let service = env::var("DD_SERVICE").unwrap_or_else(|_| "temperpaw".to_string()); let env_name = env::var("DD_ENV").unwrap_or_else(|_| "local".to_string()); let site = env::var("DD_SITE").unwrap_or_else(|_| "datadoghq.com".to_string()); - let query = format!("service:{service} env:{env_name} @work_item_id:{}", work_item.id); + let correlation = serde_json::from_str::(&work_item.correlation_json) + .unwrap_or_else(|_| json!({})); + let join_fields = directed_evolution_datadog_join_fields(work_item, &correlation); + let mut query_parts = vec![ + format!("service:{service}"), + format!("env:{env_name}"), + format!("@work_item_id:{}", work_item.id), + ]; + for (key, value) in &join_fields { + if !value.trim().is_empty() { + query_parts.push(format!("@{key}:{value}")); + } + } + let query = query_parts.join(" "); + let runtime_observation_query = format!( + "service:{service} env:{env_name} \"app usage:\" @observation_metadata:*{}*", + work_item.id + ); json!({ "service": service, "env": env_name, @@ -223,14 +317,54 @@ fn directed_evolution_datadog_context(work_item: &DirectedEvolutionWorkItemState "target_entity_type": work_item.target_entity_type, "target_entity_id": work_item.target_entity_id, "control_tenant": config_tenant_label(), + "join_fields": join_fields, "query": query, "logs_url": format!( "https://app.{site}/logs?query={}", encode_url_component(&query) ), + "runtime_observation_query": runtime_observation_query, + "runtime_observation_logs_url": format!( + "https://app.{site}/logs?query={}", + encode_url_component(&runtime_observation_query) + ), }) } +fn directed_evolution_datadog_join_fields( + work_item: &DirectedEvolutionWorkItemState, + correlation: &Value, +) -> BTreeMap { + let mut fields = BTreeMap::new(); + for key in [ + "episode_id", + "direction_id", + "generation_id", + "variant_id", + "evaluation_stage_id", + "stage_result_id", + "trial_id", + "simulated_user_plan_id", + "persona_index", + "run_index", + "runtime_ref", + "app_ref", + ] { + let value = correlation + .get(key) + .and_then(Value::as_str) + .map(str::to_string) + .or_else(|| correlation.get(key).map(Value::to_string)) + .unwrap_or_default(); + if !value.trim().is_empty() { + fields.insert(key.to_string(), value); + } + } + fields.insert("work_item_id".to_string(), work_item.id.clone()); + fields.insert("role".to_string(), work_item.role.clone()); + fields +} + fn config_tenant_label() -> String { env::var("TEMPER_TENANT").unwrap_or_else(|_| "default".to_string()) } diff --git a/crates/paw-codex-worker/src/directed_evolution/human_episode.rs b/crates/paw-codex-worker/src/directed_evolution/human_episode.rs index 3dcba9b72..165fce7cb 100644 --- a/crates/paw-codex-worker/src/directed_evolution/human_episode.rs +++ b/crates/paw-codex-worker/src/directed_evolution/human_episode.rs @@ -78,7 +78,7 @@ async fn start_directed_evolution_human_episode( json!({ "EpisodeId": episode_id, "GoalStatement": plan.adaptation_goal, - "CreatedByBrainRunId": plan.created_by_brain_run_id, + "CreatedByWorkerRunId": plan.created_by_worker_run_id, "HumanNotes": plan.human_notes, }), ) @@ -140,7 +140,7 @@ async fn start_directed_evolution_human_episode( "EpisodeId": episode_id, "ConstraintStatement": constraint.statement, "ConstraintKind": constraint.kind, - "CreatedByBrainRunId": plan.created_by_brain_run_id, + "CreatedByWorkerRunId": plan.created_by_worker_run_id, }), ) .await?; @@ -202,7 +202,7 @@ async fn start_directed_evolution_human_episode( "RuleStatement": rule.statement, "MetricIdsJson": json!(metric_ids_for_rule).to_string(), "ThresholdJson": rule.threshold.to_string(), - "CreatedByBrainRunId": plan.created_by_brain_run_id, + "CreatedByWorkerRunId": plan.created_by_worker_run_id, }), ) .await?; @@ -231,7 +231,7 @@ async fn start_directed_evolution_human_episode( "RuleStatement": rule.statement, "MetricIdsJson": json!(metric_ids_for_rule).to_string(), "Weight": rule.weight, - "CreatedByBrainRunId": plan.created_by_brain_run_id, + "CreatedByWorkerRunId": plan.created_by_worker_run_id, }), ) .await?; @@ -299,7 +299,7 @@ async fn start_directed_evolution_human_episode( "ScoringRuleIdsJson": json!(scoring_rule_ids).to_string(), "EvaluatorRef": plan.evaluator_ref, "DecisionPolicy": plan.selection_notes, - "CreatedByBrainRunId": plan.created_by_brain_run_id, + "CreatedByWorkerRunId": plan.created_by_worker_run_id, }), ) .await?; diff --git a/crates/paw-codex-worker/src/directed_evolution/human_episode_defaults.rs b/crates/paw-codex-worker/src/directed_evolution/human_episode_defaults.rs index 5aa40341c..9eb071254 100644 --- a/crates/paw-codex-worker/src/directed_evolution/human_episode_defaults.rs +++ b/crates/paw-codex-worker/src/directed_evolution/human_episode_defaults.rs @@ -113,6 +113,18 @@ fn default_human_episode_metrics() -> Vec { interpretation: "Simulated users record observations; evaluator decides whether friction is acceptable.".to_string(), hard_constraint: false, }, + DirectedEvolutionMetricPlan { + name: "simulated_user_app_blocker_count".to_string(), + kind: "journey".to_string(), + unit: "trials".to_string(), + higher_is_better: false, + description: "Simulated-user blockers classified as app behavior rather than runtime access.".to_string(), + provenance_kind: "state-verified".to_string(), + evaluator_ref: String::new(), + evaluator_module: "trial-state-verifier".to_string(), + interpretation: "The router counts terminal Trial entities and only hard-eliminates app-behavior blockers; runtime-access blockers remain visible evidence.".to_string(), + hard_constraint: true, + }, DirectedEvolutionMetricPlan { name: "runtime_error_count".to_string(), kind: "telemetry".to_string(), @@ -162,6 +174,8 @@ fn default_human_episode_elimination_rules() -> Vec, - #[serde(default, alias = "Description")] + #[serde(default, alias = "Description", alias = "description")] description: String, - #[serde(default, alias = "ProvenanceKind", alias = "provenanceKind")] + #[serde( + default, + alias = "ProvenanceKind", + alias = "provenanceKind", + alias = "provenance_kind" + )] provenance_kind: String, - #[serde(default, alias = "EvaluatorRef", alias = "evaluatorRef")] + #[serde(default, alias = "EvaluatorRef", alias = "evaluatorRef", alias = "evaluator_ref")] evaluator_ref: String, - #[serde(default, alias = "EvaluatorModule", alias = "evaluatorModule")] + #[serde( + default, + alias = "EvaluatorModule", + alias = "evaluatorModule", + alias = "evaluator_module" + )] evaluator_module: String, - #[serde(default, alias = "Interpretation")] + #[serde(default, alias = "Interpretation", alias = "interpretation")] interpretation: String, - #[serde(default, alias = "HardConstraint", alias = "hardConstraint")] + #[serde( + default, + alias = "HardConstraint", + alias = "hardConstraint", + alias = "hard_constraint" + )] hard_constraint: Option, } #[derive(Clone, Debug, Deserialize)] struct DirectedEvolutionEliminationRuleInput { - #[serde(default, alias = "RuleStatement", alias = "ruleStatement")] + #[serde(default, alias = "RuleStatement", alias = "ruleStatement", alias = "rule_statement")] statement: String, - #[serde(default, alias = "MetricNames", alias = "metricNames")] + #[serde(default, alias = "MetricNames", alias = "metricNames", alias = "metric_names")] metric_names: Vec, - #[serde(default, alias = "MetricIds", alias = "metricIds")] + #[serde(default, alias = "MetricIds", alias = "metricIds", alias = "metric_ids")] metric_ids: Vec, #[serde( default = "empty_directed_evolution_json_object", @@ -100,33 +134,53 @@ struct DirectedEvolutionEliminationRuleInput { #[derive(Clone, Debug, Deserialize)] struct DirectedEvolutionScoringRuleInput { - #[serde(default, alias = "RuleStatement", alias = "ruleStatement")] + #[serde(default, alias = "RuleStatement", alias = "ruleStatement", alias = "rule_statement")] statement: String, - #[serde(default, alias = "MetricNames", alias = "metricNames")] + #[serde(default, alias = "MetricNames", alias = "metricNames", alias = "metric_names")] metric_names: Vec, - #[serde(default, alias = "MetricIds", alias = "metricIds")] + #[serde(default, alias = "MetricIds", alias = "metricIds", alias = "metric_ids")] metric_ids: Vec, - #[serde(default, alias = "Weight")] + #[serde(default, alias = "Weight", alias = "weight")] weight: String, } #[derive(Clone, Debug, Deserialize)] struct DirectedEvolutionEvaluationStageInput { - #[serde(default, alias = "StageName", alias = "stageName")] + #[serde(default, alias = "StageName", alias = "stageName", alias = "stage_name")] name: String, - #[serde(default, alias = "StageKind", alias = "stageKind")] + #[serde(default, alias = "StageKind", alias = "stageKind", alias = "stage_kind")] kind: String, - #[serde(default, alias = "ExecutorKind", alias = "executorKind")] + #[serde(default, alias = "ExecutorKind", alias = "executorKind", alias = "executor_kind")] executor: String, - #[serde(default, alias = "RequiredEvidence", alias = "requiredEvidence")] + #[serde( + default, + alias = "RequiredEvidence", + alias = "requiredEvidence", + alias = "required_evidence" + )] required_evidence: Vec, - #[serde(default, alias = "MeasurementProvenance", alias = "measurementProvenance")] + #[serde( + default, + alias = "MeasurementProvenance", + alias = "measurementProvenance", + alias = "measurement_provenance" + )] measurement_provenance: String, - #[serde(default, alias = "EvaluatorRef", alias = "evaluatorRef")] + #[serde(default, alias = "EvaluatorRef", alias = "evaluatorRef", alias = "evaluator_ref")] evaluator_ref: String, - #[serde(default, alias = "EvaluatorModule", alias = "evaluatorModule")] + #[serde( + default, + alias = "EvaluatorModule", + alias = "evaluatorModule", + alias = "evaluator_module" + )] evaluator_module: String, - #[serde(default, alias = "DecisionAuthority", alias = "decisionAuthority")] + #[serde( + default, + alias = "DecisionAuthority", + alias = "decisionAuthority", + alias = "decision_authority" + )] decision_authority: String, } @@ -138,7 +192,7 @@ struct DirectedEvolutionEpisodePlan { autonomy_lane: String, adaptation_goal: String, human_notes: String, - created_by_brain_run_id: String, + created_by_worker_run_id: String, evaluator_ref: String, metrics: Vec, viability_constraints: Vec, @@ -327,8 +381,8 @@ fn directed_evolution_episode_plan_from_input( input.human_notes, "Human and Codex agreed to this episode contract in chat.".to_string(), ), - created_by_brain_run_id: nonempty( - input.created_by_brain_run_id.clone(), + created_by_worker_run_id: nonempty( + input.created_by_worker_run_id.clone(), env::var("CODEX_SESSION_ID").unwrap_or_else(|_| "chat-codex".to_string()), ), evaluator_ref: nonempty( @@ -362,7 +416,7 @@ fn directed_evolution_episode_plan_from_input( selected_by: nonempty(input.selected_by, "human+codex-chat".to_string()), selection_notes: nonempty( input.selection_notes, - "Selected after human-brain negotiation in Codex chat.".to_string(), + "Selected after human-agent negotiation in Codex chat.".to_string(), ), started_by: nonempty(input.started_by, "codex".to_string()), start_reason: nonempty( diff --git a/crates/paw-codex-worker/src/directed_evolution/mechanical_evaluator.rs b/crates/paw-codex-worker/src/directed_evolution/mechanical_evaluator.rs index bb325086d..24975a027 100644 --- a/crates/paw-codex-worker/src/directed_evolution/mechanical_evaluator.rs +++ b/crates/paw-codex-worker/src/directed_evolution/mechanical_evaluator.rs @@ -137,6 +137,15 @@ fn directed_evolution_state_verifier_output( )); } + let missing_required_changed_files = + directed_evolution_missing_required_changed_files(evaluation_stage, &changed_files); + if !missing_required_changed_files.is_empty() { + findings.push(format!( + "Variant did not change required file(s): {}", + missing_required_changed_files.join(", ") + )); + } + let passed = findings.is_empty(); let viability_regression_count = if passed { 0 } else { findings.len() }; let summary = if passed { @@ -181,6 +190,12 @@ fn directed_evolution_state_verifier_output( "unit": "files", "provenance_kind": provenance, "interpretation": "Number of changed files under the pinned evaluator bundle." + }, + "required_changed_file_missing_count": { + "value": missing_required_changed_files.len(), + "unit": "files", + "provenance_kind": provenance, + "interpretation": "Number of stage RequiredEvidence changed_file: requirements not present on the Mutation." } }, "decision_basis": { @@ -247,6 +262,32 @@ fn directed_evolution_changed_files(fields: &Value) -> Option> { ) } +fn directed_evolution_missing_required_changed_files( + evaluation_stage: &Value, + changed_files: &[String], +) -> Vec { + parse_json_string_array(&value_field_string( + evaluation_stage, + &["RequiredEvidenceJson", "required_evidence_json"], + )) + .into_iter() + .filter_map(|requirement| { + requirement + .strip_prefix("changed_file:") + .or_else(|| requirement.strip_prefix("changed-path:")) + .map(str::trim) + .filter(|path| !path.is_empty()) + .map(str::to_string) + }) + .filter(|required_path| { + !changed_files.iter().any(|changed_path| { + let normalized = changed_path.trim_start_matches("./"); + normalized == required_path || normalized.ends_with(&format!("/{required_path}")) + }) + }) + .collect() +} + fn directed_evolution_evaluator_path(path: &str) -> bool { let normalized = path.trim_start_matches("./"); normalized.starts_with("apps/agent-answers-evaluation/") diff --git a/crates/paw-codex-worker/src/directed_evolution/prompt.rs b/crates/paw-codex-worker/src/directed_evolution/prompt.rs index afb46c62c..eb291d009 100644 --- a/crates/paw-codex-worker/src/directed_evolution/prompt.rs +++ b/crates/paw-codex-worker/src/directed_evolution/prompt.rs @@ -1,7 +1,7 @@ fn directed_evolution_prompt(work_item: &DirectedEvolutionWorkItemState) -> String { let role_contract = match work_item.role.as_str() { "observer" => { - "Observe real signals and infer pressures. If the signal source or correlation mentions Datadog, use authenticated Datadog MCP tools to inspect the relevant logs, traces, metrics, monitors, or dashboards. Do not treat the signal summary as proof. Return candidate pressures, rejected interpretations, structured evidence_scope entries with datadog_url values when available, and confidence." + "Observe real signals and infer pressures. Datadog inspection is mandatory: use authenticated Datadog MCP tools to inspect relevant logs, traces, metrics, monitors, or dashboards alongside Temper runtime evidence and Genesis/app history. Do not treat the signal summary as proof. Return multiple candidate pressures or directions when evidence supports them, rejected interpretations, structured evidence_scope entries with query, time_window, result_count, interpretation, zero_result_meaning, datadog_url, and confidence. If usable Datadog evidence is unavailable, return actionable=false and explain the missing evidence." } "direction_framer" => { "Frame human-legible directions from pressures. Return direction title, rationale, source signals, required human gate, and likely adaptation goals." @@ -10,16 +10,16 @@ fn directed_evolution_prompt(work_item: &DirectedEvolutionWorkItemState) -> Stri "Generate one bounded candidate variant for the target organism. Make the concrete app-bundle changes, avoid long exploratory verification, and return the concise JSON object immediately after the mutation is complete." } "simulated_user" => { - "Act as an AI simulated user against the target organism. Exercise the live runtime when a RuntimeRef is provided. Do not judge viability, do not pass/fail the variant, and do not select a winner. Return journeys, observations, friction, unmet intents, user-visible measurements, and evidence_scope entries from the app interaction only." + "Act as an AI simulated user against the target organism. Exercise the live runtime when a RuntimeRef is provided. Start with /tdata and /tdata/$metadata for the parsed runtime tenant; a 404 from / or decorative app routes is not a blocker when OData works. Do not judge viability, do not pass/fail the variant, and do not select a winner. Return journeys, observations, friction, unmet intents, user-visible measurements, blocker_kind, and evidence_scope entries from the app interaction only." } "reviewer" | "viability_evaluator" => { - "Evaluate a variant against the Adaptation Goal, Viability Constraints, Selection Protocol, and recorded evidence. Use authenticated Datadog MCP tools only when the stage requires telemetry evidence. Return pass/fail reasoning, explicit provenance_kind, metrics, structured evidence_scope entries, and risk notes." + "Evaluate a variant against the Adaptation Goal, Viability Constraints, Selection Protocol, and recorded evidence. This is not a telemetry role by default: do not fail because Datadog runtime-request logs are absent unless the literal stage prompt explicitly says the stage is Datadog-measured. Return pass/fail reasoning, explicit non-telemetry provenance_kind, metrics, structured evidence_scope entries, and risk notes." } "state_verifier" => { "Run deterministic state/spec verification against the variant. Use specs, CSDL, Cedar, runtime state, and transition checks rather than subjective judgment. Return pass/fail, provenance_kind=state-verified, metrics, decision_basis, and concrete evidence." } "telemetry_evaluator" => { - "Evaluate Datadog or runtime telemetry evidence for the variant. Query logs, traces, metrics, or monitors when available. Return pass/fail, provenance_kind=datadog-measured or runtime-measured, metrics, query summaries, result counts, zero-result meaning, and Datadog URLs when available." + "Evaluate Datadog telemetry evidence for the variant. Datadog is the primary judging surface: query runtime app-usage logs and traces for the runtime service/tenant parsed from RuntimeRef. Scope by the generic Temper observation metadata emitted by the runtime, for example observation_metadata containing de.episode_id/de.variant_id in logs or temper.observation.de.* attributes in traces. Do not require a producer-specific field unless Datadog field discovery proves it is indexed. Return top-level provenance_kind=datadog-measured. The first evidence_scope item must include surface=logs|traces, the exact Datadog query, time_window, result_count, interpretation, zero_result_meaning, and a Datadog URL. Zero matching runtime app-usage telemetry is failure for this stage; runtime OData probes may be supporting evidence but must not replace Datadog." } "wasm_evaluator" => { "Run or inspect the pinned deterministic evaluator bundle for this stage. Do not let the variant alter the evaluator judging it. Return pass/fail, provenance_kind=wasm-computed, metrics, evaluator inputs, and evidence." @@ -34,14 +34,14 @@ fn directed_evolution_prompt(work_item: &DirectedEvolutionWorkItemState) -> Stri "Explain the episode outcome for Mission Control. Return concise human-facing narrative, lineage impact, and evidence links." } _ => { - "Execute the Directed Evolution brain role described by this WorkItem. Return structured evidence and next-state recommendations." + "Execute the Directed Evolution worker role described by this WorkItem. Return structured evidence and next-state recommendations." } }; let output_contract = directed_evolution_output_contract(&work_item.role); let prompt_body = directed_evolution_worker_prompt_body(&work_item.role, &literal_prompt_ref(&work_item.prompt_ref)); format!( - r#"You are a Codex brain run executing a Directed Evolution WorkItem. + r#"You are a Codex WorkerRun executing a Directed Evolution WorkItem. Role: {role} WorkItemId: {work_item_id} @@ -79,14 +79,28 @@ fn directed_evolution_output_contract(role: &str) -> &'static str { "observer" => { r#"{ "actionable": true, - "pressure_class": "growth|repair|performance|policy|ux", - "pressure_summary": "...", - "title": "...", - "direction_summary": "...", - "autonomy_lane": "human-approval|repair-auto", - "proposed_adaptation_goal": "...", - "proposed_viability_constraints": ["..."], - "evidence_scope": [{"surface":"logs|traces|metrics|monitors|runtime","query":"...","result_summary":"...","datadog_url":"https://app.datadoghq.com/..."}], + "directions": [ + { + "pressure_class": "growth|repair|performance|policy|ux", + "pressure_summary": "...", + "title": "...", + "direction_summary": "...", + "autonomy_lane": "human-approval|repair-auto", + "proposed_adaptation_goal": "...", + "proposed_viability_constraints": ["..."] + }, + { + "pressure_class": "growth|repair|performance|policy|ux", + "pressure_summary": "...", + "title": "...", + "direction_summary": "...", + "autonomy_lane": "human-approval|repair-auto", + "proposed_adaptation_goal": "...", + "proposed_viability_constraints": ["..."] + } + ], + "rejected_interpretations": [{"interpretation":"...","why_rejected":"..."}], + "evidence_scope": [{"surface":"logs|traces|metrics|monitors|runtime|genesis","query":"...","time_window":"...","result_count":0,"interpretation":"...","zero_result_meaning":"success|failure|neutral","datadog_url":"https://app.datadoghq.com/..."}], "evidence_refs": ["..."], "reasoning_summary": "..." }"# @@ -99,6 +113,7 @@ fn directed_evolution_output_contract(role: &str) -> &'static str { "runtime_ref": "runnable variant URL or local ref", "changed_files": ["..."], "diff_ref": "...", + "diff_patch": "unified diff patch if available", "verification_notes": "...", "reasoning_summary": "..." }"# @@ -115,17 +130,34 @@ fn directed_evolution_output_contract(role: &str) -> &'static str { "evidence_scope": [{"surface":"runtime|app","query":"...","result_summary":"..."}], "evidence_refs": ["..."], "blocker": "", + "blocker_kind": "none|runtime-access|app-behavior|ambiguous", + "reasoning_summary": "..." +}"# + } + "telemetry_evaluator" => { + r#"{ + "passed": true, + "status": "passed|failed", + "summary": "...", + "failure_reason": "", + "evaluator_role": "telemetry_evaluator", + "provenance_kind": "datadog-measured", + "metrics": {"runtime_request_log_count":{"value":0,"unit":"logs","provenance_kind":"datadog-measured","interpretation":"..."}}, + "decision_basis": {"why":"Datadog query result is the pass/fail source; runtime OData probes are supporting only.","tradeoffs":["Do not over-scope by runtime_ref unless field discovery shows it is indexed."]}, + "inputs": {"telemetry":["Datadog logs/traces query scoped by Temper observation metadata and runtime tenant"],"observations":[],"state":[]}, + "evidence_scope": [{"surface":"logs|traces","query":"service:temperpaw @tenant: \"app usage:\" @observation_metadata:*de.variant_id* ...","time_window":"...","result_count":1,"interpretation":"...","zero_result_meaning":"failure","datadog_url":"https://app.datadoghq.com/logs?query=..."}], + "evidence_refs": ["..."], "reasoning_summary": "..." }"# } - "reviewer" | "viability_evaluator" | "state_verifier" | "telemetry_evaluator" | "wasm_evaluator" => { + "reviewer" | "viability_evaluator" | "state_verifier" | "wasm_evaluator" => { r#"{ "passed": true, "status": "passed|failed", "summary": "...", "failure_reason": "", "evaluator_role": "reviewer|viability_evaluator|state_verifier|telemetry_evaluator|wasm_evaluator", - "provenance_kind": "brain-judged|state-verified|wasm-computed|runtime-measured|datadog-measured", + "provenance_kind": "brain-judged|state-verified|wasm-computed|runtime-measured", "metrics": {"metric_name":{"value":0,"unit":"score","provenance_kind":"...","interpretation":"..."}}, "decision_basis": {"why":"...","tradeoffs":["..."]}, "inputs": {"observations":["..."],"telemetry":["..."],"state":["..."]}, @@ -215,7 +247,8 @@ fn directed_evolution_worker_prompt_body_with_public_api_url( - Do not start a foreground long-lived server. If a local server is absolutely required, start it in the background, stop it before returning, and include the cleanup in evidence_scope.\n\ - If runtime execution is unavailable, fail the stage with clear evidence instead of hanging.\n\ - Simulated-user roles must only report observations and must not include passed, failed, viable, score, winner, or selection fields.\n\ -- Evaluator roles may pass/fail only from recorded observations, deterministic checks, runtime measurements, or telemetry evidence with explicit provenance_kind.\n", +- Evaluator roles may pass/fail only from recorded observations, deterministic checks, runtime measurements, or telemetry evidence with explicit provenance_kind.\n\ +- For reviewer, viability_evaluator, state_verifier, and wasm_evaluator roles, Datadog absence is not a failure unless the literal stage prompt names Datadog as required evidence.\n", ); body } diff --git a/crates/paw-codex-worker/src/directed_evolution/recovery.rs b/crates/paw-codex-worker/src/directed_evolution/recovery.rs new file mode 100644 index 000000000..8804cff7e --- /dev/null +++ b/crates/paw-codex-worker/src/directed_evolution/recovery.rs @@ -0,0 +1,400 @@ +async fn recover_pending_directed_evolution_stage_results( + client: &reqwest::Client, + config: &Config, +) -> Result<()> { + let stage_results = + query_directed_evolution_rows(client, config, "StageResults", "Status eq 'Pending'", 300) + .await?; + let trials = query_directed_evolution_rows(client, config, "Trials", "", 1000).await?; + let director_headers = directed_evolution_director_headers(config)?; + + for stage_result in stage_results { + let stage_result_id = directed_evolution_row_id(&stage_result); + let fields = directed_evolution_row_fields(&stage_result); + let episode_id = value_field_string(&fields, &["EpisodeId", "episode_id"]); + let generation_id = value_field_string(&fields, &["GenerationId", "generation_id"]); + let variant_id = value_field_string(&fields, &["VariantId", "variant_id"]); + let stage_id = value_field_string(&fields, &["EvaluationStageId", "evaluation_stage_id"]); + if stage_result_id.is_empty() + || episode_id.is_empty() + || generation_id.is_empty() + || variant_id.is_empty() + || stage_id.is_empty() + { + continue; + } + + let stage_fields = + fetch_directed_evolution_entity_fields(client, config, "EvaluationStages", &stage_id) + .await?; + let stage_kind = value_field_string(&stage_fields, &["StageKind", "stage_kind"]) + .to_ascii_lowercase(); + let role = recovered_stage_role(&stage_fields); + let ready = if stage_kind.contains("simulated") { + directed_evolution_trials_terminal_for_stage(&trials, &stage_result_id) + } else if role == "telemetry_evaluator" { + directed_evolution_trials_terminal_for_variant(&trials, &variant_id) + } else { + false + }; + if !ready { + continue; + } + if recovered_stage_work_item_exists(client, config, &stage_result_id, &role).await? { + continue; + } + + let variant_fields = + fetch_directed_evolution_entity_fields(client, config, "Variants", &variant_id) + .await?; + if value_field_string(&variant_fields, &["Status", "status"]) != "Active" { + continue; + } + + let work_item_id = create_directed_evolution_entity_with_headers( + client, + config, + &director_headers, + "WorkItems", + ) + .await?; + let prompt = recovered_stage_prompt(RecoveredStagePrompt { + role: &role, + episode_id: &episode_id, + generation_id: &generation_id, + variant_id: &variant_id, + stage_id: &stage_id, + stage_result_id: &stage_result_id, + work_item_id: &work_item_id, + stage_fields: &stage_fields, + variant_fields: &variant_fields, + trials: &trials, + }); + if let Err(error) = post_directed_evolution_action_with_headers( + client, + config, + &director_headers, + "StageResults", + &stage_result_id, + "StartStageResult", + json!({ + "EpisodeId": episode_id, + "GenerationId": generation_id, + "VariantId": variant_id, + "EvaluationStageId": stage_id, + "WorkItemId": work_item_id, + }), + ) + .await + { + let cancel_reason = + format!("recovery skipped StageResult start because it changed state: {error}"); + if let Err(cancel_error) = post_paw_orchestration_action_with_headers( + client, + config, + &director_headers, + "WorkItems", + &work_item_id, + "CancelWorkItem", + json!({ "Reason": cancel_reason }), + ) + .await + { + warn!( + stage_result_id, + work_item_id, + %cancel_error, + "could not cancel placeholder Directed Evolution recovery WorkItem" + ); + } + info!( + stage_result_id, + work_item_id, + role, + %error, + "skipped Directed Evolution stage-result recovery because another actor changed the stage" + ); + continue; + } + post_paw_orchestration_action_with_headers( + client, + config, + &director_headers, + "WorkItems", + &work_item_id, + "QueueWorkItem", + json!({ + "Role": role, + "TargetEntityType": "StageResult", + "TargetEntityId": stage_result_id, + "PromptRef": format!("literal:{prompt}"), + "ContextRef": format!("stage_result:{stage_result_id}"), + "OutputSchemaRef": "directed-evolution.stage-evaluation.v1", + "CorrelationJson": json!({ + "episode_id": episode_id, + "generation_id": generation_id, + "variant_id": variant_id, + "evaluation_stage_id": stage_id, + "stage_result_id": stage_result_id, + "role": role, + "recovered_after_terminal_trials": true, + }) + .to_string(), + }), + ) + .await?; + info!( + stage_result_id, + work_item_id, + role, + "recovered pending Directed Evolution stage result after terminal trials" + ); + } + + Ok(()) +} + +fn recovered_stage_role(stage_fields: &Value) -> String { + let executor = value_field_string(stage_fields, &["ExecutorKind", "executor_kind"]); + if directed_evolution_stage_evaluator_role(&executor) { + return executor; + } + let kind = value_field_string(stage_fields, &["StageKind", "stage_kind"]).to_ascii_lowercase(); + let provenance = + value_field_string(stage_fields, &["MeasurementProvenance", "measurement_provenance"]) + .to_ascii_lowercase(); + if kind.contains("telemetry") || kind.contains("datadog") || provenance.contains("datadog") { + "telemetry_evaluator".to_string() + } else if kind.contains("simulated") { + "viability_evaluator".to_string() + } else { + "reviewer".to_string() + } +} + +fn directed_evolution_trials_terminal_for_stage(trials: &[Value], stage_result_id: &str) -> bool { + let matching = trials + .iter() + .filter(|trial| { + value_field_string( + &directed_evolution_row_fields(trial), + &["StageResultId", "stage_result_id"], + ) == stage_result_id + }) + .collect::>(); + !matching.is_empty() && matching.iter().all(directed_evolution_trial_terminal) +} + +fn directed_evolution_trials_terminal_for_variant(trials: &[Value], variant_id: &str) -> bool { + let matching = trials + .iter() + .filter(|trial| { + value_field_string( + &directed_evolution_row_fields(trial), + &["VariantId", "variant_id"], + ) == variant_id + }) + .collect::>(); + !matching.is_empty() && matching.iter().all(directed_evolution_trial_terminal) +} + +fn directed_evolution_trial_terminal(trial: &&Value) -> bool { + matches!( + value_field_string( + &directed_evolution_row_fields(trial), + &["Status", "status"] + ) + .as_str(), + "Succeeded" | "Failed" | "Archived" + ) +} + +async fn recovered_stage_work_item_exists( + client: &reqwest::Client, + config: &Config, + stage_result_id: &str, + role: &str, +) -> Result { + let work_items = query_directed_evolution_rows(client, config, "WorkItems", "", 1000).await?; + Ok(work_items.iter().any(|item| { + let fields = directed_evolution_row_fields(item); + value_field_string(&fields, &["TargetEntityType", "target_entity_type"]) == "StageResult" + && value_field_string(&fields, &["TargetEntityId", "target_entity_id"]) == stage_result_id + && value_field_string(&fields, &["Role", "role"]) == role + && matches!( + value_field_string(&fields, &["Status", "status"]).as_str(), + "Queued" | "Claimed" | "Running" | "Succeeded" + ) + })) +} + +struct RecoveredStagePrompt<'a> { + role: &'a str, + episode_id: &'a str, + generation_id: &'a str, + variant_id: &'a str, + stage_id: &'a str, + stage_result_id: &'a str, + work_item_id: &'a str, + stage_fields: &'a Value, + variant_fields: &'a Value, + trials: &'a [Value], +} + +fn recovered_stage_prompt(args: RecoveredStagePrompt<'_>) -> String { + let stage_name = value_field_string(args.stage_fields, &["StageName", "stage_name"]); + let stage_kind = value_field_string(args.stage_fields, &["StageKind", "stage_kind"]); + let app_ref = value_field_string(args.variant_fields, &["AppRef", "app_ref"]); + let runtime_ref = value_field_string(args.variant_fields, &["RuntimeRef", "runtime_ref"]); + let variant_summary = value_field_string(args.variant_fields, &["Summary", "summary"]); + let trial_context = recovered_trial_context(args.trials, args.variant_id); + let header_metadata = serde_json::json!({ + "de.episode_id": args.episode_id, + "de.generation_id": args.generation_id, + "de.variant_id": args.variant_id, + "de.stage_id": args.stage_id, + "de.stage_result_id": args.stage_result_id, + "de.work_item_id": args.work_item_id, + "de.runtime_ref": runtime_ref, + "de.app_ref": app_ref, + }); + let header_block = format!("X-Temper-Observe-Metadata: {header_metadata}"); + + if args.role == "telemetry_evaluator" { + return format!( + concat!( + "Evaluate Directed Evolution Datadog telemetry after simulated users completed.\n", + "EpisodeId: {episode_id}\n", + "GenerationId: {generation_id}\n", + "VariantId: {variant_id}\n", + "EvaluationStageId: {stage_id}\n", + "StageResultId: {stage_result_id}\n", + "StageName: {stage_name}\n", + "StageKind: {stage_kind}\n", + "TemperApiBase: {base}\n", + "AppRef: {app_ref}\n", + "RuntimeRef: {runtime_ref}\n", + "VariantSummary: {variant_summary}\n\n", + "RecordedTrialEvidence:\n{trial_context}\n\n", + "DirectedEvolutionHeaders:\n{header_block}\n\n", + "This is a Datadog-measured telemetry stage. Use Datadog MCP aggregate/SQL evidence over brittle log-explorer field syntax: filter for \"directed evolution runtime request\", select directed_evolution.episode_id, directed_evolution.variant_id, and tenant, and count rows for this exact episode, variant, and runtime tenant parsed from RuntimeRef. Return provenance_kind=datadog-measured. The first evidence_scope item must include query, time_window, result_count, interpretation, zero_result_meaning, and datadog_url. Zero matching runtime-request logs is failure; runtime probes may support but must not replace Datadog. Return exactly one concise JSON object with passed/status/summary/failure_reason/evaluator_role/provenance_kind/metrics/decision_basis/inputs/evidence_scope/evidence_refs/reasoning_summary.", + ), + base = configless_public_temper_api_base(), + episode_id = args.episode_id, + generation_id = args.generation_id, + variant_id = args.variant_id, + stage_id = args.stage_id, + stage_result_id = args.stage_result_id, + stage_name = stage_name, + stage_kind = stage_kind, + app_ref = app_ref, + runtime_ref = runtime_ref, + variant_summary = variant_summary, + trial_context = trial_context, + header_block = header_block + ); + } + + format!( + concat!( + "Evaluate the completed simulated-user trial observations for a Directed Evolution variant.\n", + "EpisodeId: {episode_id}\n", + "GenerationId: {generation_id}\n", + "VariantId: {variant_id}\n", + "EvaluationStageId: {stage_id}\n", + "StageResultId: {stage_result_id}\n", + "StageName: {stage_name}\n", + "StageKind: {stage_kind}\n", + "TemperApiBase: {base}\n", + "AppRef: {app_ref}\n", + "RuntimeRef: {runtime_ref}\n", + "VariantSummary: {variant_summary}\n\n", + "RecordedTrialEvidence:\n{trial_context}\n\n", + "Use only recorded simulated-user observations, blockers, friction, and live-runtime evidence. Do not query Datadog for this non-telemetry stage, do not select a winner, and do not modify files or evaluators. Return exactly one concise JSON object with passed/status/summary/failure_reason/evaluator_role=viability_evaluator/provenance_kind=brain-judged/metrics/decision_basis/inputs/evidence_scope/evidence_refs/reasoning_summary.", + ), + base = configless_public_temper_api_base(), + episode_id = args.episode_id, + generation_id = args.generation_id, + variant_id = args.variant_id, + stage_id = args.stage_id, + stage_result_id = args.stage_result_id, + stage_name = stage_name, + stage_kind = stage_kind, + app_ref = app_ref, + runtime_ref = runtime_ref, + variant_summary = variant_summary, + trial_context = trial_context + ) +} + +fn recovered_trial_context(trials: &[Value], variant_id: &str) -> String { + let mut rows = Vec::new(); + for trial in trials { + let fields = directed_evolution_row_fields(trial); + if value_field_string(&fields, &["VariantId", "variant_id"]) != variant_id { + continue; + } + rows.push(format!( + "- {} {} run={} summary={} blocker={} intent={}", + directed_evolution_row_id(trial), + value_field_string(&fields, &["Status", "status"]), + value_field_string(&fields, &["RunIndex", "run_index"]), + value_field_string(&fields, &["Summary", "summary", "FailureReason", "failure_reason"]), + value_field_string(&fields, &["Blocker", "blocker"]), + value_field_string(&fields, &["IntentSatisfied", "intent_satisfied"]), + )); + } + if rows.is_empty() { + "No trial evidence found for this variant.".to_string() + } else { + rows.join("\n") + } +} + +fn configless_public_temper_api_base() -> String { + env::var("DIRECTED_EVOLUTION_PUBLIC_API_URL") + .ok() + .or_else(|| env::var("DIRECTED_EVOLUTION_GENESIS_URL").ok()) + .or_else(|| env::var("TEMPER_URL").ok()) + .unwrap_or_else(|| "https://genesis-production-164d.up.railway.app".to_string()) +} + +async fn query_directed_evolution_rows( + client: &reqwest::Client, + config: &Config, + entity_set: &str, + filter: &str, + top: usize, +) -> Result> { + let mut url = format!("{}/tdata/{entity_set}?$top={top}", config.temper_url); + if !filter.trim().is_empty() { + url.push_str("&$filter="); + url.push_str(&filter.replace(' ', "%20").replace('\'', "%27")); + } + let response = client + .get(url) + .headers(headers(config)?) + .send() + .await + .with_context(|| format!("query Directed Evolution {entity_set}"))?; + if !response.status().is_success() { + let status = response.status(); + let text = response.text().await.unwrap_or_default(); + bail!("query Directed Evolution {entity_set} returned {status}: {text}"); + } + let body: Value = response.json().await.context("parse Directed Evolution query")?; + Ok(body + .get("value") + .and_then(Value::as_array) + .cloned() + .unwrap_or_default()) +} + +fn directed_evolution_row_fields(row: &Value) -> Value { + row.get("fields").cloned().unwrap_or_else(|| json!({})) +} + +fn directed_evolution_row_id(row: &Value) -> String { + let fields = directed_evolution_row_fields(row); + first_string(row, &fields, &["entity_id", "id", "Id"], &["Id", "id"]) +} diff --git a/crates/paw-codex-worker/src/directed_evolution/tests.rs b/crates/paw-codex-worker/src/directed_evolution/tests.rs index 94bf8c112..4991c2356 100644 --- a/crates/paw-codex-worker/src/directed_evolution/tests.rs +++ b/crates/paw-codex-worker/src/directed_evolution/tests.rs @@ -92,7 +92,10 @@ mod directed_evolution_tests { let prompt = directed_evolution_prompt(&work_item); - assert!(prompt.contains("Query logs, traces, metrics, or monitors")); + assert!(prompt.contains("Datadog is the primary judging surface")); + assert!(prompt.contains("observation_metadata")); + assert!(prompt.contains("temper.observation.de.*")); + assert!(prompt.contains("de.variant_id")); assert!(prompt.contains("provenance_kind=datadog-measured")); assert!(prompt.contains("result_count")); assert!(prompt.contains("zero_result_meaning")); @@ -319,7 +322,46 @@ mod directed_evolution_tests { } #[test] - fn mechanical_evaluator_roles_are_not_codex_brains() { + fn mechanical_state_verifier_rejects_missing_required_changed_file() { + let work_item = DirectedEvolutionWorkItemState { + id: "wi-state".to_string(), + status: "Running".to_string(), + role: "state_verifier".to_string(), + target_entity_type: "StageResult".to_string(), + target_entity_id: "sr-required-file".to_string(), + prompt_ref: String::new(), + context_ref: String::new(), + output_schema_ref: String::new(), + correlation_json: "{}".to_string(), + }; + + let output = directed_evolution_state_verifier_output( + &work_item, + &json!({ "VariantId": "var-required-file", "EvaluationStageId": "stage-required-file" }), + &json!({ "MutationId": "mut-required-file" }), + &json!({ + "ChangedFilesJson": "[\"APP.md\",\"specs/question.ioa.toml\",\"specs/model.csdl.xml\"]" + }), + &json!({ + "RequiredEvidenceJson": "[\"changed_file:specs/answer.ioa.toml\",\"no_evaluator_mutation\"]" + }), + ); + + assert_eq!(output["passed"], false); + assert_eq!( + output["metrics"]["required_changed_file_missing_count"]["value"], + 1 + ); + assert!( + output["failure_reason"] + .as_str() + .unwrap() + .contains("specs/answer.ioa.toml") + ); + } + + #[test] + fn mechanical_evaluator_roles_do_not_spawn_codex_agents() { assert!(directed_evolution_mechanical_evaluator_role("state_verifier")); assert!(directed_evolution_mechanical_evaluator_role("wasm_evaluator")); assert_eq!( @@ -359,6 +401,30 @@ mod directed_evolution_tests { } } + #[test] + fn directed_evolution_repo_mapping_accepts_object_path() { + let previous = env::var_os("DIRECTED_EVOLUTION_ORGANISM_REPOS_JSON"); + unsafe { + env::set_var( + "DIRECTED_EVOLUTION_ORGANISM_REPOS_JSON", + r#"{"org-agent-answers":{"path":"/tmp/agent-answers","owner":"arni-labs","app":"agent-answers"}}"#, + ); + } + + let path = + directed_evolution_repo_from_mapping("org-agent-answers", "arni-labs/agent-answers@abc123") + .expect("repo should resolve from object path"); + + assert_eq!(path, PathBuf::from("/tmp/agent-answers")); + unsafe { + if let Some(value) = previous { + env::set_var("DIRECTED_EVOLUTION_ORGANISM_REPOS_JSON", value); + } else { + env::remove_var("DIRECTED_EVOLUTION_ORGANISM_REPOS_JSON"); + } + } + } + #[test] fn directed_evolution_app_ref_maps_to_genesis_app_id() { let app = directed_evolution_genesis_app_from_ref("nerdsane/agent-answers@abc123") @@ -371,6 +437,108 @@ mod directed_evolution_tests { assert_eq!(app.pinned_ref(), "nerdsane/agent-answers@abc123"); } + #[test] + fn directed_evolution_genesis_bundle_url_targets_pinned_version() { + let app = directed_evolution_genesis_app_from_ref("nerdsane/agent-answers@abc123") + .expect("app ref should parse"); + + let url = directed_evolution_genesis_bundle_url( + "https://genesis-production-164d.up.railway.app/", + &app, + ); + + assert_eq!( + url, + "https://genesis-production-164d.up.railway.app/api/genesis/apps/nerdsane/agent-answers/versions/abc123/bundle" + ); + } + + #[test] + fn directed_evolution_bundle_path_rejects_traversal() { + assert!(safe_directed_evolution_bundle_path("specs/answer.ioa.toml").is_ok()); + assert!(safe_directed_evolution_bundle_path("../answer.ioa.toml").is_err()); + assert!(safe_directed_evolution_bundle_path("/tmp/answer.ioa.toml").is_err()); + assert!(safe_directed_evolution_bundle_path(".git/config").is_err()); + assert!(safe_directed_evolution_bundle_path(".Git/config").is_err()); + assert!(safe_directed_evolution_bundle_path("specs/.git/config").is_err()); + assert!(safe_directed_evolution_bundle_path("specs/.GIT/config").is_err()); + } + + #[tokio::test] + async fn directed_evolution_bundle_materializes_git_repo() { + let repo = env::temp_dir().join(format!( + "paw-codex-worker-bundle-test-{}-{}", + std::process::id(), + SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("time") + .as_nanos() + )); + let app = directed_evolution_genesis_app_from_ref("nerdsane/agent-answers@abc123") + .expect("app ref should parse"); + let bundle = json!({ + "apps": [{ + "files": [ + { + "path": "APP.md", + "content_base64": "IyBBZ2VudCBBbnN3ZXJzCg==" + }, + { + "path": "specs/answer.ioa.toml", + "content_base64": "W2F1dG9tYXRvbl0KbmFtZSA9ICJBbnN3ZXIiCg==" + }, + { + "path": ".gitignore", + "content_base64": "ZGlzdC8K" + }, + { + "path": "dist/app.wasm", + "content_base64": "AAECAw==" + } + ] + }] + }); + + materialize_directed_evolution_bundle_repo(&repo, &app, &bundle) + .await + .expect("bundle materializes"); + + assert!(repo.join(".git").exists()); + assert_eq!( + fs::read_to_string(repo.join("APP.md")).expect("read APP.md"), + "# Agent Answers\n" + ); + let head = std::process::Command::new("git") + .arg("-C") + .arg(&repo) + .args(["rev-parse", "HEAD"]) + .output() + .expect("run git"); + assert!( + head.status.success(), + "git rev-parse failed: {}", + String::from_utf8_lossy(&head.stderr) + ); + assert!(!String::from_utf8_lossy(&head.stdout).trim().is_empty()); + let listed = std::process::Command::new("git") + .arg("-C") + .arg(&repo) + .args(["ls-files", "--", "dist/app.wasm"]) + .output() + .expect("run git ls-files"); + assert!( + listed.status.success(), + "git ls-files failed: {}", + String::from_utf8_lossy(&listed.stderr) + ); + assert_eq!( + String::from_utf8_lossy(&listed.stdout).trim(), + "dist/app.wasm", + "forced bundle materialization should commit files ignored by bundled .gitignore" + ); + fs::remove_dir_all(repo).ok(); + } + #[test] fn directed_evolution_variant_tenant_is_stable_and_safe() { let work_item = DirectedEvolutionWorkItemState { @@ -466,26 +634,34 @@ mod directed_evolution_tests { } #[test] - fn directed_evolution_canonical_push_targets_main_with_registry_tenant() { - let app = directed_evolution_genesis_app_from_ref("nerdsane/agent-answers@abc123") - .expect("app ref should parse"); + fn directed_evolution_registry_tenant_defaults_to_default() { + let previous = env::var_os("DIRECTED_EVOLUTION_REGISTRY_TENANT"); + unsafe { + env::remove_var("DIRECTED_EVOLUTION_REGISTRY_TENANT"); + } - let args = directed_evolution_canonical_push_args( - "https://genesis-production-164d.up.railway.app/", - &app, - "default", - ); + let config = Config { + temper_url: "http://temper.test".to_string(), + tenant: "de-control".to_string(), + worker_id: "worker".to_string(), + worker_token: None, + workspace_root: PathBuf::from("/tmp/workspaces"), + repo_root: PathBuf::from("/tmp/repo"), + codex_bin: "codex".to_string(), + max_concurrent_runs: 1, + enable_execution: true, + poll_on_start: true, + codex_exec_smoke: false, + codex_exec_timeout: Duration::from_secs(60), + }; - assert_eq!( - args, - vec![ - "-c", - "http.extraHeader=x-tenant-id: default", - "push", - "https://genesis-production-164d.up.railway.app/nerdsane/agent-answers.git", - "HEAD:refs/heads/main", - ] - ); + assert_eq!(directed_evolution_registry_tenant(&config), "default"); + + unsafe { + if let Some(value) = previous { + env::set_var("DIRECTED_EVOLUTION_REGISTRY_TENANT", value); + } + } } #[test] @@ -576,6 +752,190 @@ mod directed_evolution_tests { assert!(!is_datadog_app_url("https://example.com/logs")); } + #[test] + fn directed_evolution_evidence_summary_preserves_numeric_result_count() { + let summary = directed_evolution_first_evidence_scope_summary(&json!({ + "evidence_scope": [ + { + "surface": "logs", + "query": "service:temper-platform", + "time_window": "2026-05-28T00:00:00Z to 2026-05-29T23:59:59Z", + "result_count": 78, + "interpretation": "Datadog returned matching runtime-request logs.", + "zero_result_meaning": "failure" + } + ] + })); + + assert_eq!(summary.query, "service:temper-platform"); + assert_eq!(summary.result_count, "78"); + assert_eq!(summary.zero_result_meaning, "failure"); + } + + #[test] + fn directed_evolution_evidence_summary_prefers_complete_datadog_scope() { + let summary = directed_evolution_first_evidence_scope_summary(&json!({ + "evidence_scope": [ + { + "surface": "genesis", + "query": "app history", + "result_summary": "Found prior changes." + }, + { + "surface": "logs", + "query": "service:temper-platform @directed_evolution.variant_id:v3", + "time_window": "2026-06-02T14:00:00Z/2026-06-02T15:00:00Z", + "result_count": 31, + "interpretation": "Runtime logs were present for v3.", + "zero_result_meaning": "failure", + "datadog_url": "https://app.datadoghq.com/logs?query=v3" + } + ] + })); + + assert_eq!( + summary.query, + "service:temper-platform @directed_evolution.variant_id:v3" + ); + assert_eq!(summary.result_count, "31"); + assert_eq!(summary.interpretation, "Runtime logs were present for v3."); + } + + #[test] + fn directed_evolution_datadog_roles_require_structured_evidence() { + fn work_item(role: &str, prompt_ref: &str) -> DirectedEvolutionWorkItemState { + DirectedEvolutionWorkItemState { + id: format!("wi-{role}"), + status: "Queued".to_string(), + role: role.to_string(), + target_entity_type: "StageResult".to_string(), + target_entity_id: "sr-1".to_string(), + prompt_ref: prompt_ref.to_string(), + context_ref: "stage-result:sr-1".to_string(), + output_schema_ref: "schema-1".to_string(), + correlation_json: "{}".to_string(), + } + } + let observer = work_item("observer", "literal:observe"); + let telemetry = work_item("telemetry_evaluator", "literal:evaluate telemetry"); + let plain_reviewer = work_item("reviewer", "literal:review without Datadog gate"); + let datadog_reviewer = work_item( + "reviewer", + "literal:RequiredEvidence: [\"datadog_evidence_scope\"]", + ); + let datadog_viability = work_item( + "viability_evaluator", + "literal:RequiredEvidence: [\"datadog_evidence_scope\"]", + ); + + assert!(directed_evolution_work_item_requires_datadog_evidence( + &observer + )); + assert!(directed_evolution_work_item_requires_datadog_evidence( + &telemetry + )); + assert!(!directed_evolution_work_item_requires_datadog_evidence( + &plain_reviewer + )); + assert!(directed_evolution_work_item_requires_datadog_evidence( + &datadog_reviewer + )); + assert!(directed_evolution_work_item_requires_datadog_evidence( + &datadog_viability + )); + + assert!( + ensure_directed_evolution_required_datadog_evidence( + &observer, + &json!({ + "evidence_scope": [ + { + "surface": "genesis", + "query": "app history", + "result_summary": "Found prior answer changes." + }, + { + "surface": "logs", + "query": "service:temper-platform", + "time_window": "2026-06-02T14:00:00Z/2026-06-02T15:00:00Z", + "result_count": 12, + "interpretation": "Runtime request logs were present.", + "zero_result_meaning": "failure", + "datadog_url": "https://app.datadoghq.com/logs?query=service%3Atemper-platform" + } + ] + }) + ) + .is_ok() + ); + assert!( + ensure_directed_evolution_required_datadog_evidence( + &telemetry, + &json!({ + "evidence_scope": [{ + "query": "service:temper-platform", + "time_window": "2026-06-02T14:00:00Z/2026-06-02T15:00:00Z", + "result_count": 0, + "interpretation": "No runtime logs were present.", + "zero_result_meaning": "failure" + }] + }) + ) + .is_err() + ); + assert!( + ensure_directed_evolution_required_datadog_evidence( + &plain_reviewer, + &json!({"summary": "Reviewer evidence can be non-Datadog."}) + ) + .is_ok() + ); + assert!( + ensure_directed_evolution_required_datadog_evidence( + &datadog_reviewer, + &json!({"summary": "Reviewer prompt required Datadog but omitted it."}) + ) + .is_err() + ); + assert!( + ensure_directed_evolution_required_datadog_evidence( + &datadog_viability, + &json!({ + "evidence_scope": [{ + "query": "service:temper-platform @directed_evolution.variant_id:v2", + "time_window": "2026-06-02T14:00:00Z/2026-06-02T15:00:00Z", + "result_count": 4, + "interpretation": "Viability Datadog check returned runtime logs.", + "zero_result_meaning": "failure", + "datadog_url": "https://app.datadoghq.com/logs?query=v2" + }] + }) + ) + .is_ok() + ); + } + + #[test] + fn directed_evolution_observer_datadog_scope_records_datadog_provenance() { + let output = json!({ + "actionable": true, + "evidence_scope": [{ + "surface": "logs", + "query": "@directed_evolution.episode_id:episode-1", + "time_window": "now-24h to now", + "result_count": 206, + "interpretation": "Episode-scoped runtime logs were present.", + "zero_result_meaning": "failure", + "datadog_url": "https://app.datadoghq.com/logs?query=episode-1" + }] + }); + + assert_eq!( + directed_evolution_evidence_provenance("observer", &output), + "datadog-measured" + ); + } + #[test] fn directed_evolution_summary_uses_agent_summary_when_available() { let work_item = DirectedEvolutionWorkItemState { @@ -632,6 +992,89 @@ mod directed_evolution_tests { assert!(context["query"].as_str().unwrap().contains("@work_item_id:wi-dd")); } + #[test] + fn directed_evolution_claim_conflict_is_benign_race_signature() { + let message = + "WorkItems.ClaimWorkItem returned 409 Conflict: Action 'ClaimWorkItem' not valid from state 'Running'"; + + assert!(directed_evolution_claim_conflict_message(message)); + assert!(!directed_evolution_claim_conflict_message( + "WorkItems.StartWorkItem returned 409 Conflict: Action 'StartWorkItem' not valid from state 'Queued'" + )); + } + + #[test] + fn directed_evolution_exclusive_key_conflict_detects_active_peer() { + let rows = vec![ + json!({ + "entity_id": "wi-selector-current", + "fields": {"Status": "Queued"} + }), + json!({ + "entity_id": "wi-selector-active", + "fields": {"Status": "Running"} + }), + json!({ + "entity_id": "wi-selector-done", + "fields": {"Status": "Succeeded"} + }), + ]; + + assert_eq!( + directed_evolution_exclusive_key_conflict_from_rows( + "wi-selector-current", + &rows, + ExclusiveKeyConflictPolicy::AnyActivePeer, + ) + .as_deref(), + Some("wi-selector-active") + ); + assert_eq!( + directed_evolution_exclusive_key_conflict_from_rows( + "wi-selector-active", + &rows, + ExclusiveKeyConflictPolicy::AnyActivePeer, + ), + None + ); + } + + #[test] + fn directed_evolution_post_claim_conflict_lets_lowest_active_work_item_win() { + let rows = vec![ + json!({ + "entity_id": "wi-selector-003", + "fields": {"Status": "Claimed"} + }), + json!({ + "entity_id": "wi-selector-001", + "fields": {"Status": "Claimed"} + }), + json!({ + "entity_id": "wi-selector-004", + "fields": {"Status": "Running"} + }), + ]; + + assert_eq!( + directed_evolution_exclusive_key_conflict_from_rows( + "wi-selector-003", + &rows, + ExclusiveKeyConflictPolicy::LowerActivePeer, + ) + .as_deref(), + Some("wi-selector-001") + ); + assert_eq!( + directed_evolution_exclusive_key_conflict_from_rows( + "wi-selector-001", + &rows, + ExclusiveKeyConflictPolicy::LowerActivePeer, + ), + None + ); + } + #[test] fn human_episode_contract_uses_direction_and_organism_defaults() { let input: DirectedEvolutionHumanEpisodeInput = serde_json::from_value(json!({ @@ -666,7 +1109,7 @@ mod directed_evolution_tests { "Humans compare candidate answers before acceptance." ); assert_eq!(plan.viability_constraints[0].statement, "Answer creation still works."); - assert_eq!(plan.metrics.len(), 4); + assert_eq!(plan.metrics.len(), 5); assert_eq!(plan.simulated_user_plan.users_per_variant, 3); assert_eq!(plan.simulated_user_plan.runs_per_persona, 2); assert!( @@ -738,6 +1181,77 @@ mod directed_evolution_tests { assert!(!plan.evaluator_ref.trim().is_empty()); } + #[test] + fn human_episode_contract_accepts_snake_case_semantics() { + let input: DirectedEvolutionHumanEpisodeInput = serde_json::from_value(json!({ + "direction_id": "direction-growth", + "adaptation_goal": "Improve answer comparison.", + "metrics": [ + { + "metric_name": "required_changed_file_missing_count", + "metric_kind": "state", + "unit": "files", + "higher_is_better": false, + "provenance_kind": "state-verified", + "evaluator_ref": "genesis://nerdsane/agent-answers-evaluation@frozen", + "evaluator_module": "state-verifier", + "hard_constraint": true + } + ], + "viability_constraints": [ + { + "constraint_statement": "Answer spec must change.", + "constraint_kind": "state-verified" + } + ], + "elimination_rules": [ + { + "rule_statement": "Eliminate missing answer spec mutation.", + "metric_names": ["required_changed_file_missing_count"], + "threshold": { "max": 0 } + } + ], + "scoring_rules": [ + { + "rule_statement": "Prefer exact answer-spec fit.", + "metric_names": ["required_changed_file_missing_count"], + "weight": "0.20" + } + ], + "evaluation_stages": [ + { + "stage_name": "Deterministic state verification", + "stage_kind": "state_verification", + "executor_kind": "state_verifier", + "required_evidence": ["changed_file:specs/answer.ioa.toml"], + "measurement_provenance": "state-verified", + "evaluator_ref": "genesis://nerdsane/agent-answers-evaluation@frozen", + "evaluator_module": "state-verifier", + "decision_authority": "mechanical" + } + ] + })) + .expect("contract input should parse"); + + let plan = directed_evolution_episode_plan_from_input( + input, + &json!({ + "OrganismId": "org-agent-answers", + "ProposedAdaptationGoal": "Direction goal", + }), + &json!({ "ParentVersionId": "ov-parent" }), + ) + .expect("episode plan should resolve"); + + assert_eq!(plan.metrics[0].name, "required_changed_file_missing_count"); + assert!(!plan.metrics[0].higher_is_better); + assert!(plan.metrics[0].hard_constraint); + assert_eq!(plan.viability_constraints[0].statement, "Answer spec must change."); + assert_eq!(plan.elimination_rules[0].metric_names[0], "required_changed_file_missing_count"); + assert_eq!(plan.scoring_rules[0].weight, "0.20"); + assert_eq!(plan.evaluation_stages[0].required_evidence[0], "changed_file:specs/answer.ioa.toml"); + } + #[test] fn directed_evolution_start_episode_command_accepts_contract_path() { let command = parse_worker_command([ diff --git a/crates/paw-codex-worker/src/directed_evolution/work_item_guards.rs b/crates/paw-codex-worker/src/directed_evolution/work_item_guards.rs new file mode 100644 index 000000000..ac6952ca3 --- /dev/null +++ b/crates/paw-codex-worker/src/directed_evolution/work_item_guards.rs @@ -0,0 +1,223 @@ +fn directed_evolution_claim_conflict_message(message: &str) -> bool { + message.contains("WorkItems.ClaimWorkItem returned 409") + && message.contains("not valid from state") +} + +struct DirectedEvolutionExclusiveKeyConflict { + exclusive_key: String, + active_work_item_id: String, +} + +async fn directed_evolution_exclusive_key_conflict( + client: &reqwest::Client, + config: &Config, + work_item: &DirectedEvolutionWorkItemState, +) -> Result> { + directed_evolution_exclusive_key_conflict_with_policy( + client, + config, + work_item, + ExclusiveKeyConflictPolicy::AnyActivePeer, + ) + .await +} + +async fn directed_evolution_exclusive_key_post_claim_conflict( + client: &reqwest::Client, + config: &Config, + work_item: &DirectedEvolutionWorkItemState, +) -> Result> { + directed_evolution_exclusive_key_conflict_with_policy( + client, + config, + work_item, + ExclusiveKeyConflictPolicy::LowerActivePeer, + ) + .await +} + +#[derive(Clone, Copy)] +enum ExclusiveKeyConflictPolicy { + AnyActivePeer, + LowerActivePeer, +} + +async fn directed_evolution_exclusive_key_conflict_with_policy( + client: &reqwest::Client, + config: &Config, + work_item: &DirectedEvolutionWorkItemState, + policy: ExclusiveKeyConflictPolicy, +) -> Result> { + let fields = + fetch_directed_evolution_entity_fields(client, config, "WorkItems", &work_item.id).await?; + let exclusive_key = value_field_string(&fields, &["ExclusiveKey", "exclusive_key"]); + if exclusive_key.trim().is_empty() { + return Ok(None); + } + + let escaped_key = exclusive_key.replace('\'', "''"); + let rows = query_directed_evolution_rows( + client, + config, + "WorkItems", + &format!("ExclusiveKey eq '{escaped_key}'"), + 100, + ) + .await?; + if let Some(active_work_item_id) = + directed_evolution_exclusive_key_conflict_from_rows(&work_item.id, &rows, policy) + { + return Ok(Some(DirectedEvolutionExclusiveKeyConflict { + exclusive_key, + active_work_item_id, + })); + } + + Ok(None) +} + +fn directed_evolution_exclusive_key_conflict_from_rows( + work_item_id: &str, + rows: &[Value], + policy: ExclusiveKeyConflictPolicy, +) -> Option { + let mut active_ids = Vec::new(); + for row in rows { + let row_fields = directed_evolution_row_fields(row); + let row_id = directed_evolution_row_id(row); + if row_id == work_item_id { + continue; + } + let status = value_field_string(&row_fields, &["Status", "status"]); + if matches!(status.as_str(), "Claimed" | "Running") { + active_ids.push(row_id); + } + } + active_ids.sort(); + match policy { + ExclusiveKeyConflictPolicy::AnyActivePeer => active_ids.into_iter().next(), + ExclusiveKeyConflictPolicy::LowerActivePeer => active_ids + .into_iter() + .find(|active_id| active_id.as_str() < work_item_id), + } +} + +async fn stale_directed_evolution_work_item_reason( + client: &reqwest::Client, + config: &Config, + work_item: &DirectedEvolutionWorkItemState, +) -> Result> { + if !directed_evolution_stage_evaluator_role(&work_item.role) + || work_item.target_entity_type != "StageResult" + { + return Ok(None); + } + + let stage_fields = fetch_directed_evolution_entity_fields( + client, + config, + "StageResults", + &work_item.target_entity_id, + ) + .await?; + let variant_fields = { + let variant_id = value_field_string(&stage_fields, &["VariantId", "variant_id"]); + if variant_id.trim().is_empty() { + json!({}) + } else { + fetch_directed_evolution_entity_fields(client, config, "Variants", &variant_id).await? + } + }; + Ok(stale_directed_evolution_stage_work_reason( + work_item, + &stage_fields, + &variant_fields, + )) +} + +async fn eliminate_stale_directed_evolution_stage_result( + client: &reqwest::Client, + config: &Config, + work_item: &DirectedEvolutionWorkItemState, + reason: &str, +) -> Result<()> { + if !stale_stage_work_targets_stage_result(work_item) { + return Ok(()); + } + + let stage_fields = fetch_directed_evolution_entity_fields( + client, + config, + "StageResults", + &work_item.target_entity_id, + ) + .await?; + if !stale_stage_result_should_eliminate(&stage_fields) { + return Ok(()); + } + + post_directed_evolution_action( + client, + config, + "StageResults", + &work_item.target_entity_id, + "EliminateStageResult", + json!({ + "EliminationRuleId": "stale-after-variant-terminal", + "EvidenceArtifactId": value_field_string(&stage_fields, &["EvidenceArtifactId", "evidence_artifact_id"]), + "Reason": reason, + }), + ) + .await +} + +fn stale_directed_evolution_stage_work_reason( + work_item: &DirectedEvolutionWorkItemState, + stage_fields: &Value, + variant_fields: &Value, +) -> Option { + if !stale_stage_work_targets_stage_result(work_item) { + return None; + } + let stage_status = value_field_string(stage_fields, &["Status", "status"]); + if !stage_status.trim().is_empty() && stage_status != "Running" { + return Some(format!( + "Target StageResult {} is already {}; skipping stale {} work", + work_item.target_entity_id, stage_status, work_item.role + )); + } + let variant_status = value_field_string(variant_fields, &["Status", "status"]); + if matches!( + variant_status.as_str(), + "Eliminated" | "Promoted" | "Superseded" | "Failed" + ) { + return Some(format!( + "Target variant is already {}; skipping stale {} work for StageResult {}", + variant_status, work_item.role, work_item.target_entity_id + )); + } + None +} + +fn stale_stage_work_targets_stage_result(work_item: &DirectedEvolutionWorkItemState) -> bool { + directed_evolution_stage_evaluator_role(&work_item.role) + && work_item.target_entity_type == "StageResult" +} + +fn directed_evolution_stage_evaluator_role(role: &str) -> bool { + matches!( + role, + "reviewer" + | "viability_evaluator" + | "state_verifier" + | "telemetry_evaluator" + | "wasm_evaluator" + ) +} + +fn stale_stage_result_should_eliminate(stage_fields: &Value) -> bool { + matches!( + value_field_string(stage_fields, &["Status", "status"]).as_str(), + "Running" | "Failed" + ) +} diff --git a/crates/paw-codex-worker/src/directed_evolution/work_item_handler.rs b/crates/paw-codex-worker/src/directed_evolution/work_item_handler.rs new file mode 100644 index 000000000..d314e5672 --- /dev/null +++ b/crates/paw-codex-worker/src/directed_evolution/work_item_handler.rs @@ -0,0 +1,337 @@ +async fn handle_queued_directed_evolution_work_item( + client: &reqwest::Client, + config: &Config, + work_item_id: &str, +) -> Result<()> { + info!(work_item_id, "saw queued Directed Evolution WorkItem"); + let work_item = fetch_directed_evolution_work_item(client, config, work_item_id).await?; + if work_item.status != "Queued" { + debug!( + work_item_id, + status = %work_item.status, + "Directed Evolution WorkItem no longer queued" + ); + return Ok(()); + } + if work_item.role.trim().is_empty() { + debug!(work_item_id, "Directed Evolution WorkItem has no worker role yet"); + return Ok(()); + } + if let Some(conflict) = + directed_evolution_exclusive_key_conflict(client, config, &work_item).await? + { + let reason = format!( + "Exclusive key {} is already active on WorkItem {}; cancelling duplicate {} work", + conflict.exclusive_key, conflict.active_work_item_id, work_item.role + ); + post_paw_orchestration_action( + client, + config, + "WorkItems", + &work_item.id, + "CancelWorkItem", + json!({ "Reason": reason }), + ) + .await?; + info!( + work_item_id = %work_item.id, + active_work_item_id = %conflict.active_work_item_id, + exclusive_key = %conflict.exclusive_key, + "cancelled duplicate exclusive Directed Evolution WorkItem" + ); + return Ok(()); + } + if let Some(reason) = + stale_directed_evolution_work_item_reason(client, config, &work_item).await? + { + if let Err(error) = + eliminate_stale_directed_evolution_stage_result(client, config, &work_item, &reason) + .await + { + warn!( + work_item_id = %work_item.id, + role = %work_item.role, + %error, + "stale Directed Evolution WorkItem cleanup could not mutate StageResult; cancelling WorkItem only" + ); + } + if let Err(error) = post_paw_orchestration_action( + client, + config, + "WorkItems", + &work_item.id, + "CancelWorkItem", + json!({ "Reason": reason }), + ) + .await + && !error.to_string().contains("not valid from state 'Cancelled'") + { + return Err(error); + } + info!( + work_item_id = %work_item.id, + role = %work_item.role, + "cancelled stale Directed Evolution WorkItem" + ); + return Ok(()); + } + + if let Err(error) = post_paw_orchestration_action( + client, + config, + "WorkItems", + &work_item.id, + "ClaimWorkItem", + json!({ + "WorkerId": config.worker_id, + "ClaimedBy": config.worker_id, + }), + ) + .await + { + let message = error.to_string(); + if directed_evolution_claim_conflict_message(&message) { + let current = fetch_directed_evolution_work_item(client, config, &work_item.id).await?; + info!( + work_item_id = %work_item.id, + role = %work_item.role, + status = %current.status, + "Directed Evolution WorkItem claim lost to another worker" + ); + if current.status != "Queued" { + return Ok(()); + } + } + return Err(error).with_context(|| { + format!( + "claim Directed Evolution WorkItem {} as {}", + work_item.id, config.worker_id + ) + }); + } + if let Some(conflict) = + directed_evolution_exclusive_key_post_claim_conflict(client, config, &work_item).await? + { + let reason = format!( + "Exclusive key {} is already claimed by lower-priority WorkItem {}; cancelling duplicate {} work before WorkerRun start", + conflict.exclusive_key, conflict.active_work_item_id, work_item.role + ); + post_paw_orchestration_action( + client, + config, + "WorkItems", + &work_item.id, + "CancelWorkItem", + json!({ "Reason": reason }), + ) + .await?; + info!( + work_item_id = %work_item.id, + active_work_item_id = %conflict.active_work_item_id, + exclusive_key = %conflict.exclusive_key, + "cancelled duplicate exclusive Directed Evolution WorkItem after claim" + ); + return Ok(()); + } + + let worker_run_id = create_entity(client, config, "WorkerRuns", json!({})).await?; + post_paw_orchestration_action( + client, + config, + "WorkerRuns", + &worker_run_id, + "StartWorkerRun", + json!({ + "Role": work_item.role, + "WorkItemId": work_item.id, + "WorkerId": config.worker_id, + "ProviderId": directed_evolution_worker_provider_id(), + "AgentKind": directed_evolution_agent_kind_for_role(&work_item.role), + "Model": directed_evolution_model_for_role(&work_item.role), + "SessionId": "", + "ParentSessionId": env::var("CODEX_SESSION_ID").unwrap_or_default(), + "CorrelationJson": work_item.correlation_json, + }), + ) + .await?; + post_paw_orchestration_action( + client, + config, + "WorkItems", + &work_item.id, + "StartWorkItem", + json!({ "WorkerRunId": worker_run_id }), + ) + .await?; + info!( + work_item_id = %work_item.id, + worker_run_id = %worker_run_id, + role = %work_item.role, + target_entity_type = %work_item.target_entity_type, + target_entity_id = %work_item.target_entity_id, + "started Directed Evolution worker run" + ); + + match run_directed_evolution_codex_role(client, config, &work_item).await { + Ok(output_json) => { + let summary = directed_evolution_summary(&work_item, &output_json); + let evidence_artifact_id = record_directed_evolution_worker_evidence( + client, + config, + &work_item, + &worker_run_id, + "codex_worker_run", + &output_json, + &summary, + ) + .await?; + post_paw_orchestration_action( + client, + config, + "WorkerRuns", + &worker_run_id, + "SucceedWorkerRun", + json!({ + "OutputJson": output_json, + "EvidenceArtifactId": evidence_artifact_id, + "Summary": summary, + }), + ) + .await?; + if let Err(route_error) = route_directed_evolution_work_item_success( + client, + config, + &work_item, + &worker_run_id, + &output_json, + &evidence_artifact_id, + &summary, + ) + .await + { + let failure_reason = format!( + "Directed Evolution receipt routing failed after WorkerRun success: {route_error}" + ); + warn!( + work_item_id = %work_item.id, + worker_run_id = %worker_run_id, + role = %work_item.role, + evidence_artifact_id = %evidence_artifact_id, + %route_error, + "Directed Evolution WorkItem receipt routing failed after WorkerRun success" + ); + post_paw_orchestration_action( + client, + config, + "WorkItems", + &work_item.id, + "FailWorkItem", + json!({ + "FailureReason": failure_reason, + "EvidenceArtifactId": evidence_artifact_id, + }), + ) + .await?; + return Ok(()); + } + post_paw_orchestration_action( + client, + config, + "WorkItems", + &work_item.id, + "SucceedWorkItem", + json!({ + "ResultJson": output_json, + "EvidenceArtifactId": evidence_artifact_id, + "Summary": summary, + }), + ) + .await?; + info!( + work_item_id = %work_item.id, + worker_run_id = %worker_run_id, + role = %work_item.role, + evidence_artifact_id = %evidence_artifact_id, + "completed Directed Evolution Codex worker run" + ); + Ok(()) + } + Err(error) => { + let failure_reason = format!("Directed Evolution Codex role failed: {error}"); + let evidence_artifact_id = match record_directed_evolution_worker_evidence( + client, + config, + &work_item, + &worker_run_id, + "codex_worker_run_failure", + &serde_json::to_string(&json!({ + "status": "failed", + "failure_reason": failure_reason, + }))?, + &failure_reason, + ) + .await + { + Ok(id) => id, + Err(report_error) => { + warn!(%report_error, work_item_id, worker_run_id, "failed to record Directed Evolution failure evidence"); + String::new() + } + }; + if let Err(report_error) = post_paw_orchestration_action( + client, + config, + "WorkerRuns", + &worker_run_id, + "FailWorkerRun", + json!({ + "FailureReason": failure_reason, + "EvidenceArtifactId": evidence_artifact_id, + }), + ) + .await + { + warn!(%report_error, work_item_id, worker_run_id, "failed to report WorkerRun failure"); + } + if let Err(route_error) = route_directed_evolution_work_item_failure( + client, + config, + &work_item, + &worker_run_id, + &failure_reason, + &evidence_artifact_id, + ) + .await + { + warn!( + work_item_id = %work_item.id, + worker_run_id = %worker_run_id, + role = %work_item.role, + evidence_artifact_id = %evidence_artifact_id, + %route_error, + "Directed Evolution failure receipt routing failed; failing WorkItem anyway" + ); + } + post_paw_orchestration_action( + client, + config, + "WorkItems", + &work_item.id, + "FailWorkItem", + json!({ + "FailureReason": failure_reason, + "EvidenceArtifactId": evidence_artifact_id, + }), + ) + .await?; + warn!( + work_item_id = %work_item.id, + worker_run_id = %worker_run_id, + role = %work_item.role, + evidence_artifact_id = %evidence_artifact_id, + "failed Directed Evolution Codex worker run" + ); + Ok(()) + } + } +} diff --git a/crates/paw-codex-worker/src/directed_evolution/workdir.rs b/crates/paw-codex-worker/src/directed_evolution/workdir.rs index f16197963..037d1a731 100644 --- a/crates/paw-codex-worker/src/directed_evolution/workdir.rs +++ b/crates/paw-codex-worker/src/directed_evolution/workdir.rs @@ -10,6 +10,7 @@ async fn resolve_directed_evolution_workdir( path: config.repo_root.clone(), app_ref: String::new(), branch_ref: String::new(), + canonical_genesis_bundle: false, }); }; if let Some(path) = @@ -23,6 +24,7 @@ async fn resolve_directed_evolution_workdir( path: worktree, app_ref: context.app_ref, branch_ref, + canonical_genesis_bundle: false, }); } if !context.branch_ref.trim().is_empty() { @@ -33,19 +35,54 @@ async fn resolve_directed_evolution_workdir( path: worktree, app_ref: context.app_ref, branch_ref: context.branch_ref, + canonical_genesis_bundle: false, }); } return Ok(DirectedEvolutionWorkdir { path, app_ref: context.app_ref, branch_ref: context.branch_ref, + canonical_genesis_bundle: false, + }); + } + if !context.branch_ref.trim().is_empty() + && let Some(worktree) = + directed_evolution_existing_workspace_worktree(config, &context.branch_ref).await? + { + return Ok(DirectedEvolutionWorkdir { + path: worktree, + app_ref: context.app_ref, + branch_ref: context.branch_ref, + canonical_genesis_bundle: false, + }); + } + if let Some(path) = + directed_evolution_repo_from_genesis_bundle(client, config, &context.app_ref, &work_item.id) + .await? + { + if directed_evolution_role_may_write_repo(&work_item.role) { + let branch_ref = directed_evolution_variant_branch(work_item); + let worktree = + ensure_directed_evolution_worktree(config, &path, &branch_ref, true).await?; + return Ok(DirectedEvolutionWorkdir { + path: worktree, + app_ref: context.app_ref, + branch_ref, + canonical_genesis_bundle: false, + }); + } + return Ok(DirectedEvolutionWorkdir { + path, + app_ref: context.app_ref, + branch_ref: context.branch_ref, + canonical_genesis_bundle: true, }); } if directed_evolution_role_may_write_repo(&work_item.role) && env_flag("PAW_DE_ALLOW_GLOBAL_REPO_FOR_VARIANTS") != Some(true) { bail!( - "Directed Evolution {} WorkItem {} resolved app_ref '{}' for organism '{}' but no repository mapping was configured. Set DIRECTED_EVOLUTION_ORGANISM_REPOS_JSON or PAW_DE_ALLOW_GLOBAL_REPO_FOR_VARIANTS=true.", + "Directed Evolution {} WorkItem {} resolved app_ref '{}' for organism '{}' but no repository mapping or canonical Genesis bundle worktree was available. Set DIRECTED_EVOLUTION_ORGANISM_REPOS_JSON, DIRECTED_EVOLUTION_GENESIS_URL, or PAW_DE_ALLOW_GLOBAL_REPO_FOR_VARIANTS=true.", work_item.role, work_item.id, context.app_ref, @@ -56,6 +93,7 @@ async fn resolve_directed_evolution_workdir( path: config.repo_root.clone(), app_ref: context.app_ref, branch_ref: context.branch_ref, + canonical_genesis_bundle: false, }) } @@ -64,6 +102,7 @@ struct DirectedEvolutionWorkdir { path: PathBuf, app_ref: String, branch_ref: String, + canonical_genesis_bundle: bool, } #[derive(Clone, Debug)] @@ -260,6 +299,13 @@ fn directed_evolution_repo_from_mapping(organism_id: &str, app_ref: &str) -> Opt if let Some(path) = mapping.get(key).and_then(Value::as_str) { return Some(PathBuf::from(path)); } + if let Some(path) = mapping + .get(key) + .and_then(|value| value.get("path")) + .and_then(Value::as_str) + { + return Some(PathBuf::from(path)); + } } None } @@ -272,6 +318,169 @@ fn directed_evolution_variant_branch(work_item: &DirectedEvolutionWorkItemState) format!("directed-evolution/{}", sanitize_git_ref_component(&work_item.id)) } +async fn directed_evolution_repo_from_genesis_bundle( + client: &reqwest::Client, + config: &Config, + app_ref: &str, + work_item_id: &str, +) -> Result> { + if app_ref.trim().is_empty() { + return Ok(None); + } + let Some(genesis_url) = directed_evolution_genesis_url() else { + return Ok(None); + }; + let app = directed_evolution_genesis_app_from_ref(app_ref)?; + let repo = directed_evolution_materialized_bundle_repo_path(config, &app, work_item_id); + if repo.join(".git").exists() { + return Ok(Some(repo)); + } + + let url = directed_evolution_genesis_bundle_url(&genesis_url, &app); + let registry_tenant = directed_evolution_registry_tenant(config); + let response = client + .get(&url) + .headers(headers_for_tenant(config, ®istry_tenant)?) + .send() + .await + .with_context(|| format!("fetch canonical Genesis bundle {}", app.pinned_ref()))?; + let status = response.status(); + let body = response + .text() + .await + .context("read canonical Genesis bundle response")?; + if !status.is_success() { + bail!( + "fetch canonical Genesis bundle {} failed: HTTP {} body={}", + app.pinned_ref(), + status, + body + ); + } + let bundle: Value = serde_json::from_str(&body) + .with_context(|| format!("parse canonical Genesis bundle {}", app.pinned_ref()))?; + materialize_directed_evolution_bundle_repo(&repo, &app, &bundle).await?; + Ok(Some(repo)) +} + +fn directed_evolution_materialized_bundle_repo_path( + config: &Config, + app: &DirectedEvolutionGenesisApp, + work_item_id: &str, +) -> PathBuf { + config + .workspace_root + .join("directed-evolution") + .join("canonical-bundles") + .join(app.app_id()) + .join(sanitize_path_component(&app.hash)) + .join(sanitize_path_component(work_item_id)) +} + +fn directed_evolution_genesis_bundle_url(genesis_url: &str, app: &DirectedEvolutionGenesisApp) -> String { + format!( + "{}/api/genesis/apps/{}/{}/versions/{}/bundle", + genesis_url.trim_end_matches('/'), + app.owner, + app.name, + app.hash + ) +} + +async fn materialize_directed_evolution_bundle_repo( + repo: &Path, + app: &DirectedEvolutionGenesisApp, + bundle: &Value, +) -> Result<()> { + if repo.exists() { + fs::remove_dir_all(repo) + .with_context(|| format!("remove stale canonical bundle repo {}", repo.display()))?; + } + fs::create_dir_all(repo) + .with_context(|| format!("create canonical bundle repo {}", repo.display()))?; + + for (path, content) in directed_evolution_bundle_files(bundle)? { + let file_path = repo.join(&path); + if let Some(parent) = file_path.parent() { + fs::create_dir_all(parent) + .with_context(|| format!("create bundle directory {}", parent.display()))?; + } + fs::write(&file_path, content) + .with_context(|| format!("write bundle file {}", file_path.display()))?; + } + + git_capture(repo, &["init"]).await?; + ensure_git_identity(repo).await?; + git_capture(repo, &["add", "-f", "-A"]).await?; + git_capture_owned( + repo, + vec![ + "commit".to_string(), + "-m".to_string(), + format!("Materialize canonical Genesis bundle {}", app.pinned_ref()), + ], + ) + .await?; + Ok(()) +} + +fn directed_evolution_bundle_files(bundle: &Value) -> Result)>> { + let apps = bundle + .get("apps") + .and_then(Value::as_array) + .context("canonical Genesis bundle missing apps array")?; + let app = apps + .first() + .context("canonical Genesis bundle has no apps")?; + let files = app + .get("files") + .and_then(Value::as_array) + .context("canonical Genesis bundle app missing files array")?; + let mut out = Vec::new(); + for file in files { + let path = file + .get("path") + .and_then(Value::as_str) + .context("canonical Genesis bundle file missing path")?; + let path = safe_directed_evolution_bundle_path(path)?; + let content_base64 = file + .get("content_base64") + .and_then(Value::as_str) + .context("canonical Genesis bundle file missing content_base64")?; + let content = base64::engine::general_purpose::STANDARD + .decode(content_base64) + .context("decode canonical Genesis bundle file")?; + out.push((path, content)); + } + Ok(out) +} + +fn safe_directed_evolution_bundle_path(path: &str) -> Result { + let path = Path::new(path); + if path.as_os_str().is_empty() { + bail!("canonical Genesis bundle file path is empty"); + } + let mut safe = PathBuf::new(); + for component in path.components() { + match component { + std::path::Component::Normal(part) => { + if part.to_string_lossy().eq_ignore_ascii_case(".git") { + bail!( + "canonical Genesis bundle file path may not contain .git: {}", + path.display() + ); + } + safe.push(part) + } + _ => bail!("canonical Genesis bundle file path is unsafe: {}", path.display()), + } + } + if safe.as_os_str().is_empty() { + bail!("canonical Genesis bundle file path is empty"); + } + Ok(safe) +} + async fn ensure_directed_evolution_worktree( config: &Config, base_repo: &Path, @@ -286,6 +495,9 @@ async fn ensure_directed_evolution_worktree( if worktree.join(".git").exists() { return Ok(worktree); } + if let Some(existing) = existing_directed_evolution_worktree_for_branch(base_repo, &branch).await? { + return Ok(existing); + } fs::create_dir_all(worktree.parent().unwrap_or(&config.workspace_root)) .with_context(|| format!("create {}", worktree.display()))?; let inside = git_capture(base_repo, &["rev-parse", "--is-inside-work-tree"]).await?; @@ -307,6 +519,51 @@ async fn ensure_directed_evolution_worktree( Ok(worktree) } +async fn existing_directed_evolution_worktree_for_branch( + base_repo: &Path, + branch: &str, +) -> Result> { + let list = git_capture(base_repo, &["worktree", "list", "--porcelain"]).await?; + let expected_ref = format!("refs/heads/{branch}"); + let mut current_path: Option = None; + + for line in list.lines().chain(std::iter::once("")) { + if let Some(path) = line.strip_prefix("worktree ") { + current_path = Some(PathBuf::from(path)); + continue; + } + if let Some(active_branch) = line.strip_prefix("branch ") + && active_branch == expected_ref + { + return Ok(current_path); + } + if line.trim().is_empty() { + current_path = None; + } + } + + Ok(None) +} + +async fn directed_evolution_existing_workspace_worktree( + config: &Config, + branch_ref: &str, +) -> Result> { + let branch = sanitize_git_branch(branch_ref); + let worktree = config + .workspace_root + .join("directed-evolution") + .join(sanitize_path_component(&branch)); + if !worktree.join(".git").exists() { + return Ok(None); + } + let current_branch = git_capture(&worktree, &["rev-parse", "--abbrev-ref", "HEAD"]).await?; + if current_branch.trim() == branch { + return Ok(Some(worktree)); + } + Ok(None) +} + async fn finalize_directed_evolution_output( client: &reqwest::Client, config: &Config, @@ -326,6 +583,7 @@ async fn finalize_directed_evolution_output( ); } git_capture(&workdir.path, &["add", "-A"]).await?; + let diff_patch = git_capture(&workdir.path, &["diff", "--cached", "--binary"]).await?; git_capture_owned( &workdir.path, vec![ @@ -355,6 +613,7 @@ async fn finalize_directed_evolution_output( "diff_ref".to_string(), json!(format!("git:{}:{commit}", workdir.branch_ref)), ); + object.insert("diff_patch".to_string(), json!(diff_patch)); } if let Some(runtime) = hotload_directed_evolution_variant(client, config, work_item, workdir, app_name, &commit) @@ -452,6 +711,8 @@ async fn hotload_directed_evolution_variant( let registry_tenant = directed_evolution_registry_tenant(config); push_directed_evolution_variant_commit(&workdir.path, &genesis_url, &app, &branch, ®istry_tenant) .await?; + publish_directed_evolution_app_version(client, config, &genesis_url, &app, ®istry_tenant, &branch) + .await?; let tenant_prefix = env::var("DIRECTED_EVOLUTION_VARIANT_TENANT_PREFIX") .unwrap_or_else(|_| "de-variant".to_string()); let tenant = directed_evolution_variant_tenant(&tenant_prefix, work_item); @@ -493,7 +754,7 @@ async fn materialize_directed_evolution_promotion( let workdir = resolve_directed_evolution_workdir(client, config, work_item).await?; let head = git_capture(&workdir.path, &["rev-parse", "HEAD"]).await?; let head = head.trim(); - if head != app.hash { + if head != app.hash && !workdir.canonical_genesis_bundle { bail!( "Directed Evolution promoter expected worktree {} at {}, found {}", workdir.path.display(), @@ -503,10 +764,22 @@ async fn materialize_directed_evolution_promotion( } let registry_tenant = directed_evolution_registry_tenant(config); - push_directed_evolution_canonical_commit(&workdir.path, &genesis_url, &app, ®istry_tenant) - .await?; - publish_directed_evolution_app_version(client, config, &genesis_url, &app, ®istry_tenant) + let ref_name = if workdir.branch_ref.trim().is_empty() { + directed_evolution_variant_branch(work_item) + } else { + workdir.branch_ref.clone() + }; + if !workdir.canonical_genesis_bundle { + publish_directed_evolution_app_version( + client, + config, + &genesis_url, + &app, + ®istry_tenant, + &ref_name, + ) .await?; + } let production_tenant = directed_evolution_production_tenant(); install_directed_evolution_variant( client, @@ -519,12 +792,20 @@ async fn materialize_directed_evolution_promotion( .await?; let canonical_app_ref = app.pinned_ref(); + let summary = if workdir.canonical_genesis_bundle { + format!( + "Hot-loaded existing canonical Genesis app ref {} into tenant {}.", + canonical_app_ref, production_tenant + ) + } else { + format!( + "Published {} from winner ref {} and hot-loaded it into tenant {}.", + canonical_app_ref, ref_name, production_tenant + ) + }; Ok(DirectedEvolutionPromotionMaterialization { runtime_ref: format!("temper://tenant/{production_tenant}/app/{canonical_app_ref}"), - summary: format!( - "Published {} to Genesis main and hot-loaded it into tenant {}.", - canonical_app_ref, production_tenant - ), + summary, digest: app.hash.clone(), canonical_app_ref, production_tenant, @@ -570,28 +851,6 @@ async fn push_directed_evolution_variant_commit( Ok(()) } -async fn push_directed_evolution_canonical_commit( - workdir: &Path, - genesis_url: &str, - app: &DirectedEvolutionGenesisApp, - registry_tenant: &str, -) -> Result<()> { - git_capture_owned( - workdir, - directed_evolution_canonical_push_args(genesis_url, app, registry_tenant), - ) - .await?; - Ok(()) -} - -fn directed_evolution_canonical_push_args( - genesis_url: &str, - app: &DirectedEvolutionGenesisApp, - registry_tenant: &str, -) -> Vec { - directed_evolution_variant_push_args(genesis_url, app, "main", registry_tenant) -} - fn directed_evolution_variant_push_args( genesis_url: &str, app: &DirectedEvolutionGenesisApp, @@ -643,6 +902,14 @@ async fn install_directed_evolution_variant( let status = response.status(); let body = response.text().await.unwrap_or_default(); if !status.is_success() { + if status == reqwest::StatusCode::CONFLICT && body.contains("state 'Installed'") { + tracing::info!( + app_ref = %app.pinned_ref(), + target_tenant, + "Directed Evolution install already materialized; treating Installed conflict as success" + ); + return Ok(()); + } bail!( "Directed Evolution variant install failed for {} into tenant {}: HTTP {} body={}", app.pinned_ref(), @@ -660,6 +927,7 @@ async fn publish_directed_evolution_app_version( genesis_url: &str, app: &DirectedEvolutionGenesisApp, registry_tenant: &str, + ref_name: &str, ) -> Result<()> { let url = format!( "{}/tdata/Apps('{}')/App.PublishNewVersion?await_integration=true", @@ -671,7 +939,7 @@ async fn publish_directed_evolution_app_version( .headers(headers_for_tenant(config, registry_tenant)?) .json(&json!({ "NewHash": app.hash, - "RefName": "main", + "RefName": ref_name, })) .send() .await @@ -689,8 +957,8 @@ async fn publish_directed_evolution_app_version( Ok(()) } -fn directed_evolution_registry_tenant(config: &Config) -> String { - env::var("DIRECTED_EVOLUTION_REGISTRY_TENANT").unwrap_or_else(|_| config.tenant.clone()) +fn directed_evolution_registry_tenant(_config: &Config) -> String { + env::var("DIRECTED_EVOLUTION_REGISTRY_TENANT").unwrap_or_else(|_| "default".to_string()) } fn directed_evolution_production_tenant() -> String { @@ -740,7 +1008,7 @@ async fn ensure_directed_evolution_readonly_workdir_unchanged( let status_after = git_capture(workdir, &["status", "--porcelain"]).await?; if status_after != status_before { bail!( - "Directed Evolution read-only brain role modified {}. Before status: {:?}. After status: {:?}", + "Directed Evolution read-only worker role modified {}. Before status: {:?}. After status: {:?}", workdir.display(), status_before, status_after @@ -817,9 +1085,15 @@ fn sanitize_id_component(value: &str) -> String { fn value_field_string(fields: &Value, keys: &[&str]) -> String { keys.iter() - .find_map(|key| fields.get(*key).and_then(Value::as_str)) - .unwrap_or("") - .to_string() + .find_map(|key| { + fields.get(*key).and_then(|value| match value { + Value::String(value) => Some(value.clone()), + Value::Number(value) => Some(value.to_string()), + Value::Bool(value) => Some(value.to_string()), + _ => None, + }) + }) + .unwrap_or_default() } fn env_flag(key: &str) -> Option { diff --git a/crates/paw-codex-worker/src/event_loop.rs b/crates/paw-codex-worker/src/event_loop.rs index 7e442d7e8..01397b388 100644 --- a/crates/paw-codex-worker/src/event_loop.rs +++ b/crates/paw-codex-worker/src/event_loop.rs @@ -197,7 +197,7 @@ async fn handle_requested_review_run( worker_run_id = %worker_run.id, "claiming ReviewRun for local Codex reviewer" ); - post_entity_action( + if let Err(error) = post_entity_action( client, config, "ReviewRuns", @@ -205,7 +205,20 @@ async fn handle_requested_review_run( "Claim", json!({ "reviewer_id": config.worker_id }), ) - .await?; + .await + { + let message = error.to_string(); + if action_conflict_message(&message) { + info!( + review_run_id, + worker_run_id = %worker_run.id, + worker_id = %config.worker_id, + "ReviewRun claim lost to another worker" + ); + return Ok(()); + } + return Err(error); + } post_entity_action( client, config, @@ -348,7 +361,18 @@ async fn handle_queued_evaluation_run( return Ok(()); } - claim_evaluation_run(client, config, evaluation_run_id).await?; + if let Err(error) = claim_evaluation_run(client, config, evaluation_run_id).await { + let message = error.to_string(); + if action_conflict_message(&message) { + info!( + evaluation_run_id, + worker_id = %config.worker_id, + "EvaluationRun claim lost to another worker" + ); + return Ok(()); + } + return Err(error); + } if !is_repo_sweep { return run_code_change_evaluation(client, config, evaluation_run_id, &worker_run).await; @@ -416,7 +440,21 @@ async fn fail_terminal_queued_evaluation_if_needed( }; if evaluation_run_is_claimable_by_worker(evaluation_run, &config.worker_id) { - fail_queued_evaluation_run(client, config, evaluation_run_id, &blocker).await?; + if let Err(error) = + fail_queued_evaluation_run(client, config, evaluation_run_id, &blocker).await + { + let message = error.to_string(); + if action_conflict_message(&message) { + info!( + evaluation_run_id, + worker_id = %config.worker_id, + failure_classification = %blocker.failure_classification, + "EvaluationRun terminal claim lost to another worker" + ); + return Ok(true); + } + return Err(error); + } } else { warn!( evaluation_run_id, diff --git a/crates/paw-codex-worker/src/execution.rs b/crates/paw-codex-worker/src/execution.rs index 00289cc88..ee25cba3c 100644 --- a/crates/paw-codex-worker/src/execution.rs +++ b/crates/paw-codex-worker/src/execution.rs @@ -378,6 +378,21 @@ async fn run_codex_exec_command( .await } +async fn run_codex_exec_command_with_datadog_mcp( + config: &Config, + workdir: &Path, + prompt: String, + context_label: &str, +) -> Result { + run_codex_exec_command_with_args( + config, + workdir, + codex_exec_args_with_datadog_mcp(workdir, &prompt), + context_label, + ) + .await +} + async fn run_codex_exec_command_with_args( config: &Config, workdir: &Path, @@ -488,8 +503,23 @@ fn signal_process_group(pid: u32, signal: libc::c_int, label: &str, context: &st async fn terminate_process_group(_pid: u32, _context: &str) {} fn codex_exec_args(workdir: &Path, prompt: &str) -> Vec { + codex_exec_args_with_datadog_option(workdir, prompt, false) +} + +fn codex_exec_args_with_datadog_mcp(workdir: &Path, prompt: &str) -> Vec { + codex_exec_args_with_datadog_option(workdir, prompt, true) +} + +fn codex_exec_args_with_datadog_option( + workdir: &Path, + prompt: &str, + force_datadog_mcp: bool, +) -> Vec { let mut args = vec!["exec".into(), "--ignore-user-config".into()]; - if let Some(datadog_mcp_url) = codex_datadog_mcp_url() { + if let Some(model) = codex_exec_model() { + args.extend(["--model".into(), model.into()]); + } + if let Some(datadog_mcp_url) = codex_datadog_mcp_url(force_datadog_mcp) { args.extend([ "-c".into(), format!( @@ -510,11 +540,18 @@ fn codex_exec_args(workdir: &Path, prompt: &str) -> Vec { args } -fn codex_datadog_mcp_url() -> Option { +fn codex_exec_model() -> Option { + env::var("PAW_CODEX_MODEL") + .ok() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) +} + +fn codex_datadog_mcp_url(force_enabled: bool) -> Option { let enabled = env::var("PAW_CODEX_ENABLE_DATADOG_MCP") .map(|value| value == "1" || value.eq_ignore_ascii_case("true")) .unwrap_or(false); - if !enabled { + if !force_enabled && !enabled { return None; } Some( diff --git a/crates/paw-codex-worker/src/fake_codex_tests.rs b/crates/paw-codex-worker/src/fake_codex_tests.rs index 60bdcbb04..b4d4fe757 100644 --- a/crates/paw-codex-worker/src/fake_codex_tests.rs +++ b/crates/paw-codex-worker/src/fake_codex_tests.rs @@ -21,6 +21,20 @@ fn fake_codex_fixture_only_uses_reviewer_mode_for_reviewer_prompt() { "implementation prompt should write the marker even if task text mentions a reviewer" ); + let implementation_with_config = StdCommand::new(&fixture) + .arg("exec") + .arg("-c") + .arg("mcp_servers.datadog.url=\"https://mcp.datadoghq.test/mcp\"") + .arg("--cd") + .arg(&root) + .arg("Implement a task with a Codex config override.") + .output() + .expect("run fake implementation with config override"); + assert!( + implementation_with_config.status.success(), + "fake implementation should accept Codex -c config overrides" + ); + let review = StdCommand::new(&fixture) .arg("exec") .arg("You are the independent reviewer for a TemperPaw paw-patrol WorkerRun.") diff --git a/crates/paw-codex-worker/src/http_headers.rs b/crates/paw-codex-worker/src/http_headers.rs index d38d4416b..c6f9443d6 100644 --- a/crates/paw-codex-worker/src/http_headers.rs +++ b/crates/paw-codex-worker/src/http_headers.rs @@ -12,6 +12,9 @@ fn extract_sse_data(frame: &str) -> Option { fn headers(config: &Config) -> Result { let mut headers = HeaderMap::new(); + let principal_kind = + env::var("PAW_CODEX_PRINCIPAL_KIND").unwrap_or_else(|_| "agent".to_string()); + let agent_type = env::var("PAW_CODEX_AGENT_TYPE").unwrap_or_else(|_| "worker".to_string()); headers.insert( "x-tenant-id", HeaderValue::from_str(&config.tenant).context("invalid TEMPER_TENANT")?, @@ -20,8 +23,14 @@ fn headers(config: &Config) -> Result { "x-temper-principal-id", HeaderValue::from_str(&config.worker_id).context("invalid WORKER_ID")?, ); - headers.insert("x-temper-principal-kind", HeaderValue::from_static("agent")); - headers.insert("x-temper-agent-type", HeaderValue::from_static("worker")); + headers.insert( + "x-temper-principal-kind", + HeaderValue::from_str(&principal_kind).context("invalid PAW_CODEX_PRINCIPAL_KIND")?, + ); + headers.insert( + "x-temper-agent-type", + HeaderValue::from_str(&agent_type).context("invalid PAW_CODEX_AGENT_TYPE")?, + ); headers.insert( "x-agent-id", HeaderValue::from_str(&config.worker_id).context("invalid WORKER_ID")?, @@ -41,6 +50,24 @@ fn event_stream_headers(config: &Config) -> Result { "x-tenant-id", HeaderValue::from_str(&config.tenant).context("invalid TEMPER_TENANT")?, ); + if let Ok(kind) = env::var("PAW_CODEX_EVENT_STREAM_PRINCIPAL_KIND") + && !kind.trim().is_empty() + { + let principal_id = env::var("PAW_CODEX_EVENT_STREAM_PRINCIPAL_ID") + .ok() + .filter(|value| !value.trim().is_empty()) + .unwrap_or_else(|| config.worker_id.clone()); + headers.insert( + "x-temper-principal-kind", + HeaderValue::from_str(kind.trim()) + .context("invalid PAW_CODEX_EVENT_STREAM_PRINCIPAL_KIND")?, + ); + headers.insert( + "x-temper-principal-id", + HeaderValue::from_str(principal_id.trim()) + .context("invalid PAW_CODEX_EVENT_STREAM_PRINCIPAL_ID")?, + ); + } if let Some(token) = &config.worker_token { headers.insert( AUTHORIZATION, @@ -59,4 +86,3 @@ fn event_stream_headers(config: &Config) -> Result { fn required_env(key: &str) -> Result { env::var(key).with_context(|| format!("{key} is required")) } - diff --git a/crates/paw-codex-worker/src/launchd.rs b/crates/paw-codex-worker/src/launchd.rs index 899d79f79..8eacbe1e8 100644 --- a/crates/paw-codex-worker/src/launchd.rs +++ b/crates/paw-codex-worker/src/launchd.rs @@ -10,6 +10,11 @@ fn launchd_worker_binary_path() -> PathBuf { } fn render_launchd_plist(config: &Config, worker_bin: &Path, eval_commands: Option<&str>) -> String { + let launchd_label = env::var("PAW_CODEX_LAUNCHD_LABEL") + .ok() + .filter(|value| !value.trim().is_empty()) + .unwrap_or_else(|| "com.temperpaw.paw-codex-worker".to_string()); + let max_concurrent_runs = config.max_concurrent_runs.to_string(); let mut env_vars = vec![ ("TEMPER_URL", config.temper_url.as_str()), ("TEMPER_TENANT", config.tenant.as_str()), @@ -18,7 +23,7 @@ fn render_launchd_plist(config: &Config, worker_bin: &Path, eval_commands: Optio ("REPO_ROOT", path_str(&config.repo_root)), ("CODEX_BIN", config.codex_bin.as_str()), ("PATH", launchd_path().leak()), - ("MAX_CONCURRENT_RUNS", "1"), + ("MAX_CONCURRENT_RUNS", max_concurrent_runs.as_str()), ( "PAW_CODEX_ENABLE_EXECUTION", if config.enable_execution { "1" } else { "0" }, @@ -86,6 +91,11 @@ fn render_launchd_plist(config: &Config, worker_bin: &Path, eval_commands: Optio if let Some(commands) = eval_commands.filter(|value| !value.trim().is_empty()) { env_vars.push(("PAW_CODEX_EVAL_COMMANDS", commands)); } + if let Ok(model) = env::var("PAW_CODEX_MODEL") + && !model.trim().is_empty() + { + env_vars.push(("PAW_CODEX_MODEL", model.leak())); + } let environment = env_vars .into_iter() @@ -105,7 +115,7 @@ fn render_launchd_plist(config: &Config, worker_bin: &Path, eval_commands: Optio Label - com.temperpaw.paw-codex-worker + {} ProgramArguments {} @@ -120,13 +130,16 @@ fn render_launchd_plist(config: &Config, worker_bin: &Path, eval_commands: Optio {} StandardOutPath - /tmp/paw-codex-worker.out.log + /tmp/{}.out.log StandardErrorPath - /tmp/paw-codex-worker.err.log + /tmp/{}.err.log "#, + escape_plist(&launchd_label), escape_plist(&worker_bin.display().to_string()), - environment + environment, + escape_plist(&launchd_label), + escape_plist(&launchd_label) ) } diff --git a/crates/paw-codex-worker/src/main.rs b/crates/paw-codex-worker/src/main.rs index 8ce231f5c..ba9b60dd8 100644 --- a/crates/paw-codex-worker/src/main.rs +++ b/crates/paw-codex-worker/src/main.rs @@ -1,4 +1,5 @@ use anyhow::{Context, Result, anyhow, bail}; +use base64::Engine; use futures_util::StreamExt; use repo_health::{ extract_repo_sweep_snapshot_id, parse_repo_health_agent_output, repo_health_agent_prompt, @@ -7,6 +8,7 @@ use repo_health::{ use reqwest::header::{ACCEPT, AUTHORIZATION, CONTENT_TYPE, HeaderMap, HeaderValue}; use serde::Deserialize; use serde_json::{Value, json}; +use std::collections::BTreeMap; use std::env; use std::fs; use std::io::{self, Read}; @@ -43,6 +45,11 @@ async fn main() -> Result<()> { ); return Ok(()); } + if command == WorkerCommand::RegisterWorkerAgent { + ensure_worker_agent_registered(&client, &config).await?; + report_worker_heartbeat(&client, &config).await?; + return Ok(()); + } if let WorkerCommand::DirectedEvolutionStartEpisode { contract_path } = &command { let result = run_directed_evolution_human_episode_command( &client, @@ -63,6 +70,9 @@ async fn main() -> Result<()> { worker_capabilities = %worker_capabilities().join(","), "paw-codex-worker starting" ); + if let Err(error) = ensure_worker_agent_registered(&client, &config).await { + warn!(%error, "failed to ensure WorkerAgent registration"); + } if let Err(error) = report_worker_heartbeat(&client, &config).await { warn!(%error, "failed to report WorkerAgent heartbeat"); } diff --git a/crates/paw-codex-worker/src/temper_api.rs b/crates/paw-codex-worker/src/temper_api.rs index 81d12e40a..9bcba3c3c 100644 --- a/crates/paw-codex-worker/src/temper_api.rs +++ b/crates/paw-codex-worker/src/temper_api.rs @@ -157,19 +157,129 @@ async fn claim_worker_run( } async fn report_worker_heartbeat(client: &reqwest::Client, config: &Config) -> Result<()> { - post_entity_action( + let body = json!({ + "last_seen_at": generated_at_label(), + "status_summary": "paw-codex-worker running under openclaw launchd", + "capabilities": worker_capabilities().join(","), + }); + match post_entity_action_with_namespace( client, config, "WorkerAgents", &config.worker_id, + PAW_ORCHESTRATION_NAMESPACE, "ReportHeartbeat", - json!({ - "last_seen_at": generated_at_label(), - "status_summary": "paw-codex-worker running under openclaw launchd", - "capabilities": worker_capabilities().join(","), - }), + body.clone(), ) .await + { + Ok(()) => Ok(()), + Err(error) => { + warn!( + %error, + worker_id = %config.worker_id, + "WorkerAgent heartbeat via PawOrchestration failed; falling back to installed Patrol namespace" + ); + post_entity_action( + client, + config, + "WorkerAgents", + &config.worker_id, + "ReportHeartbeat", + body, + ) + .await + } + } +} + +async fn ensure_worker_agent_registered(client: &reqwest::Client, config: &Config) -> Result<()> { + if worker_agent_is_registered(client, config).await? { + return Ok(()); + } + let id = match create_entity( + client, + config, + "WorkerAgents", + worker_agent_registration_body(config), + ) + .await + { + Ok(id) => id, + Err(error) => { + if action_conflict_message(&error.to_string()) + && worker_agent_is_registered(client, config).await? + { + return Ok(()); + } + return Err(error); + } + }; + if id != config.worker_id { + bail!( + "WorkerAgent registration returned unexpected id {id}; expected {}", + config.worker_id + ); + } + Ok(()) +} + +async fn worker_agent_is_registered(client: &reqwest::Client, config: &Config) -> Result { + let response = client + .get(config.entity_url("WorkerAgents", &config.worker_id)) + .headers(headers(config)?) + .send() + .await + .with_context(|| format!("fetch WorkerAgent {}", config.worker_id))?; + if response.status().is_success() { + return Ok(true); + } + if response.status() == reqwest::StatusCode::NOT_FOUND { + return Ok(false); + } + let status = response.status(); + let text = response.text().await.unwrap_or_default(); + bail!( + "fetch WorkerAgent {} returned {}: {}", + config.worker_id, + status, + text + ) +} + +fn worker_agent_registration_body(config: &Config) -> Value { + json!({ + "id": config.worker_id, + "Id": config.worker_id, + "worker_id": config.worker_id, + "WorkerId": config.worker_id, + "provider_id": worker_provider_id(), + "ProviderId": worker_provider_id(), + "display_name": config.worker_id, + "DisplayName": config.worker_id, + "capabilities": worker_capabilities().join(","), + "Capabilities": worker_capabilities().join(","), + "host": worker_host_label(), + "Host": worker_host_label(), + "worktree_root": path_str(&config.workspace_root), + "WorktreeRoot": path_str(&config.workspace_root), + }) +} + +fn worker_provider_id() -> String { + env::var("WORKER_PROVIDER_ID") + .ok() + .filter(|value| !value.trim().is_empty()) + .unwrap_or_else(|| "local-codex".to_string()) +} + +fn worker_host_label() -> String { + env::var("HOSTNAME") + .or_else(|_| env::var("COMPUTERNAME")) + .ok() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| "local".to_string()) } async fn start_local_worker_run( diff --git a/crates/paw-codex-worker/src/tests.rs b/crates/paw-codex-worker/src/tests.rs index b2a912b39..0168c5400 100644 --- a/crates/paw-codex-worker/src/tests.rs +++ b/crates/paw-codex-worker/src/tests.rs @@ -1,893 +1 @@ -#[cfg(test)] -mod tests { - use super::*; - use std::ffi::OsString; - use std::fs; - #[cfg(unix)] - use std::os::unix::fs::PermissionsExt; - use std::time::{SystemTime, UNIX_EPOCH}; - use tokio::sync::Mutex; - - include!("datadog_patrol_tests.rs"); - include!("github_patrol_tests.rs"); - include!("evaluation_tests.rs"); - include!("event_stream_tests.rs"); - include!("fake_codex_tests.rs"); - include!("codex_plan_tests.rs"); - include!("pull_request_tests.rs"); - include!("repo_health_parser_tests.rs"); - include!("worker_http_tests.rs"); - - static ENV_LOCK: Mutex<()> = Mutex::const_new(()); - - #[test] - fn worker_run_state_reads_temper_odata_fields() { - let value = json!({ - "entity_id": "wr-1", - "status": "Queued", - "fields": { - "task": "Do useful work", - "work_cycle_id": "wc-1", - "worktree_path": "/tmp/worktree", - "branch_name": "codex/test", - "runner_kind": "local_codex", - "allowed_worker_id": "mac-mini-codex-1", - "worker_id": "mac-mini-codex-1", - "provider_id": "local-codex", - "required_capabilities": "local_codex,repo_write" - } - }); - - let worker_run = worker_run_from_odata_value(value).expect("worker run should parse"); - - assert_eq!(worker_run.id, "wr-1"); - assert_eq!(worker_run.status, "Queued"); - assert_eq!(worker_run.task, "Do useful work"); - assert_eq!(worker_run.work_cycle_id, "wc-1"); - assert_eq!(worker_run.worktree_path, "/tmp/worktree"); - assert_eq!(worker_run.branch_name, "codex/test"); - assert_eq!(worker_run.runner_kind, "local_codex"); - assert_eq!(worker_run.allowed_worker_id, "mac-mini-codex-1"); - assert_eq!(worker_run.worker_id, "mac-mini-codex-1"); - assert_eq!(worker_run.provider_id, "local-codex"); - assert_eq!(worker_run.required_capabilities, "local_codex,repo_write"); - } - - #[test] - fn local_worker_claims_only_configured_local_codex_runs() { - let configured = WorkerRunState { - id: "wr-1".to_string(), - status: "Queued".to_string(), - task: "Do useful work".to_string(), - work_cycle_id: "wc-1".to_string(), - worktree_path: "/tmp/worktree".to_string(), - branch_name: "codex/test".to_string(), - runner_kind: "local_codex".to_string(), - allowed_worker_id: "mac-mini-codex-1".to_string(), - worker_id: String::new(), - provider_id: "local-codex".to_string(), - required_capabilities: "local_codex,repo_write".to_string(), - }; - let unconfigured = WorkerRunState { - id: "wr-2".to_string(), - status: "Queued".to_string(), - task: String::new(), - work_cycle_id: String::new(), - worktree_path: String::new(), - branch_name: String::new(), - runner_kind: String::new(), - allowed_worker_id: String::new(), - worker_id: String::new(), - provider_id: String::new(), - required_capabilities: String::new(), - }; - let cloud = WorkerRunState { - id: "wr-3".to_string(), - status: "Queued".to_string(), - task: "Cloud overflow".to_string(), - work_cycle_id: "wc-cloud".to_string(), - worktree_path: String::new(), - branch_name: String::new(), - runner_kind: "codex_cloud".to_string(), - allowed_worker_id: "mac-mini-codex-1".to_string(), - worker_id: String::new(), - provider_id: "local-codex".to_string(), - required_capabilities: "local_codex,repo_write".to_string(), - }; - let no_worktree_assignment = WorkerRunState { - id: "wr-4".to_string(), - status: "Queued".to_string(), - task: "Fix a Discord trace leak".to_string(), - work_cycle_id: "wc-no-worktree".to_string(), - worktree_path: String::new(), - branch_name: String::new(), - runner_kind: "local_codex".to_string(), - allowed_worker_id: "mac-mini-codex-1".to_string(), - worker_id: String::new(), - provider_id: "local-codex".to_string(), - required_capabilities: "local_codex,repo_write".to_string(), - }; - - assert!(worker_run_is_claimable_by_local_codex( - &configured, - "mac-mini-codex-1" - )); - assert!(!worker_run_is_claimable_by_local_codex( - &configured, - "other-worker" - )); - assert!(!worker_run_is_claimable_by_local_codex( - &unconfigured, - "mac-mini-codex-1" - )); - assert!(!worker_run_is_claimable_by_local_codex( - &cloud, - "mac-mini-codex-1" - )); - assert!(!worker_run_is_claimable_by_local_codex( - &no_worktree_assignment, - "mac-mini-codex-1" - )); - } - - #[test] - fn local_worker_recovers_only_own_running_codex_runs() { - let recoverable = WorkerRunState { - id: "wr-1".to_string(), - status: "Running".to_string(), - task: "Resume this work".to_string(), - work_cycle_id: "wc-1".to_string(), - worktree_path: "/tmp/worktree".to_string(), - branch_name: "codex/test".to_string(), - runner_kind: "local_codex".to_string(), - allowed_worker_id: "mac-mini-codex-1".to_string(), - worker_id: "mac-mini-codex-1".to_string(), - provider_id: "local-codex".to_string(), - required_capabilities: "local_codex,repo_write".to_string(), - }; - let other_worker = WorkerRunState { - worker_id: "other-worker".to_string(), - ..recoverable.clone() - }; - let queued = WorkerRunState { - status: "Queued".to_string(), - ..recoverable.clone() - }; - - assert!(worker_run_is_recoverable_by_local_codex( - &recoverable, - "mac-mini-codex-1" - )); - assert!(!worker_run_is_recoverable_by_local_codex( - &other_worker, - "mac-mini-codex-1" - )); - assert!(!worker_run_is_recoverable_by_local_codex( - &queued, - "mac-mini-codex-1" - )); - } - - #[test] - fn repo_sweep_worker_runs_are_detected_for_review_and_evaluation() { - let worker_run = WorkerRunState { - id: "wr-1".to_string(), - status: "Done".to_string(), - task: "RepoGraphSnapshot: snap-1\nWorkCycle: wc-1".to_string(), - work_cycle_id: "wc-1".to_string(), - worktree_path: "/tmp/worktree".to_string(), - branch_name: "codex/repo-sweep".to_string(), - runner_kind: "local_codex".to_string(), - allowed_worker_id: "mac-mini-codex-1".to_string(), - worker_id: String::new(), - provider_id: "local-codex".to_string(), - required_capabilities: "local_codex,repo_write".to_string(), - }; - let normal_worker_run = WorkerRunState { - id: "wr-2".to_string(), - status: "Done".to_string(), - task: "Fix a Discord reply bug".to_string(), - work_cycle_id: "wc-2".to_string(), - worktree_path: "/tmp/worktree".to_string(), - branch_name: "codex/bugfix".to_string(), - runner_kind: "local_codex".to_string(), - allowed_worker_id: "mac-mini-codex-1".to_string(), - worker_id: String::new(), - provider_id: "local-codex".to_string(), - required_capabilities: "local_codex,repo_write".to_string(), - }; - - assert!(worker_run_is_repo_sweep(&worker_run)); - assert!(!worker_run_is_repo_sweep(&normal_worker_run)); - } - - #[test] - fn worker_command_defaults_to_run_and_accepts_doctor() { - assert_eq!( - parse_worker_command(Vec::::new()), - WorkerCommand::Run - ); - assert_eq!( - parse_worker_command(vec!["doctor".to_string()]), - WorkerCommand::Doctor - ); - assert_eq!( - parse_worker_command(vec!["--doctor".to_string()]), - WorkerCommand::Doctor - ); - assert_eq!( - parse_worker_command(vec!["launchd-plist".to_string()]), - WorkerCommand::LaunchdPlist - ); - assert_eq!( - parse_worker_command(vec!["--launchd-plist".to_string()]), - WorkerCommand::LaunchdPlist - ); - assert_eq!( - parse_worker_command(vec!["run".to_string()]), - WorkerCommand::Run - ); - } - - #[test] - fn launchd_plist_renders_concrete_worker_environment() { - let config = Config { - temper_url: "https://temperpaw.example.test".to_string(), - tenant: "prod".to_string(), - worker_id: "mac-mini-codex-1".to_string(), - worker_token: Some("secret&token".to_string()), - workspace_root: PathBuf::from("/Users/me/Development/temperpaw-worktrees"), - repo_root: PathBuf::from("/Users/me/Development/temperpaw"), - codex_bin: "/Users/me/.local/bin/codex".to_string(), - max_concurrent_runs: 1, - enable_execution: true, - poll_on_start: false, - codex_exec_smoke: true, - codex_exec_timeout: Duration::from_secs(90), - }; - - let plist = render_launchd_plist( - &config, - Path::new("/Users/me/.local/bin/paw-codex-worker"), - Some("cargo test -p temperpaw --test paw_patrol_foundation"), - ); - - for needle in [ - "Label", - "com.temperpaw.paw-codex-worker", - "/Users/me/.local/bin/paw-codex-worker", - "TEMPER_URL", - "https://temperpaw.example.test", - "WORKER_ID", - "mac-mini-codex-1", - "PATH", - "/opt/homebrew/bin:/opt/homebrew/sbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Users/openclaw/.cargo/bin", - "PAW_CODEX_ENABLE_EXECUTION", - "1", - "PAW_CODEX_POLL_ON_START", - "0", - "PAW_CODEX_DOCTOR_EXEC_SMOKE", - "1", - "PAW_CODEX_EXEC_TIMEOUT_SECS", - "90", - "PAW_CODEX_WORKER_CAPABILITIES", - "local_codex,repo_write,review,evaluation,datadog_query,github_query", - "PAW_CODEX_WORKER_ENV_FILE", - "/Users/openclaw/.config/temperpaw/paw-codex-worker.env", - "PAW_CODEX_FORBIDDEN_DONE_PATHS", - "PAW_CODEX_EVAL_COMMANDS", - "cargo test -p temperpaw --test paw_patrol_foundation", - ] { - assert!(plist.contains(needle), "plist should contain {needle}"); - } - assert!( - !plist.contains("WORKER_TOKEN"), - "launchd plist should keep WORKER_TOKEN in the 0600 env file, not in launchctl-visible environment" - ); - assert!( - !plist.contains("PAW_CODEX_INHERIT_USER_CONFIG"), - "tool profile opt-in should stay in the worker env file so launchd does not freeze a default" - ); - } - - #[test] - fn doctor_status_fails_only_on_failures() { - let pass_and_warn = vec![ - DoctorCheck::pass("repo", "ok".to_string()), - DoctorCheck::warn("token", "missing token".to_string()), - ]; - let fail = vec![DoctorCheck::fail("odata", "not reachable".to_string())]; - - assert!(!doctor_has_failures(&pass_and_warn)); - assert!(doctor_has_failures(&fail)); - assert_eq!(doctor_status_label(DoctorStatus::Pass), "pass"); - assert_eq!(doctor_status_label(DoctorStatus::Warn), "warn"); - assert_eq!(doctor_status_label(DoctorStatus::Fail), "fail"); - } - - #[tokio::test] - async fn doctor_codex_exec_smoke_runs_fixture_when_enabled() { - let fixture = Path::new(env!("CARGO_MANIFEST_DIR")).join("fixtures/fake-codex.sh"); - let root = unique_temp_dir(); - fs::create_dir_all(&root).expect("temp dir"); - let config = Config { - temper_url: "http://127.0.0.1:3497".to_string(), - tenant: "default".to_string(), - worker_id: "mac-mini-codex-1".to_string(), - worker_token: Some("secret".to_string()), - workspace_root: root.clone(), - repo_root: root.clone(), - codex_bin: fixture.display().to_string(), - max_concurrent_runs: 1, - enable_execution: false, - poll_on_start: true, - codex_exec_smoke: true, - codex_exec_timeout: Duration::from_secs(30), - }; - - let check = check_codex_exec_smoke(&config).await; - - assert_eq!(check.name, "codex_exec_smoke"); - assert_eq!(check.status, DoctorStatus::Pass); - assert!( - check.detail.contains("PAW_CODEX_DOCTOR_EXEC_OK"), - "unexpected detail: {}", - check.detail - ); - fs::remove_dir_all(root).ok(); - } - - #[tokio::test] - async fn run_codex_times_out_stuck_local_exec() { - let fixture = Path::new(env!("CARGO_MANIFEST_DIR")).join("fixtures/fake-codex.sh"); - let root = unique_temp_dir(); - fs::create_dir_all(&root).expect("temp dir"); - let config = Config { - temper_url: "http://127.0.0.1:3497".to_string(), - tenant: "default".to_string(), - worker_id: "mac-mini-codex-1".to_string(), - worker_token: Some("secret".to_string()), - workspace_root: root.clone(), - repo_root: root.clone(), - codex_bin: fixture.display().to_string(), - max_concurrent_runs: 1, - enable_execution: true, - poll_on_start: true, - codex_exec_smoke: false, - codex_exec_timeout: Duration::from_millis(100), - }; - let worker_run = WorkerRunState { - id: "wr-timeout".to_string(), - status: "Running".to_string(), - task: "PAW_FAKE_CODEX_HANG: simulate a stuck Codex child".to_string(), - work_cycle_id: "wc-timeout".to_string(), - worktree_path: root.display().to_string(), - branch_name: "codex/timeout".to_string(), - runner_kind: "local_codex".to_string(), - allowed_worker_id: "mac-mini-codex-1".to_string(), - worker_id: "mac-mini-codex-1".to_string(), - provider_id: "local-codex".to_string(), - required_capabilities: "local_codex,repo_write".to_string(), - }; - - let result = run_codex(&config, &worker_run).await; - - let error = result.expect_err("stuck codex exec should time out"); - let message = format!("{error:#}"); - assert!( - message.contains("codex exec timed out after"), - "unexpected timeout error: {message}" - ); - fs::remove_dir_all(root).ok(); - } - - #[tokio::test] - async fn run_codex_timeout_cleans_child_process_group() { - let fixture = Path::new(env!("CARGO_MANIFEST_DIR")).join("fixtures/fake-codex.sh"); - let root = unique_temp_dir(); - fs::create_dir_all(&root).expect("temp dir"); - let marker = root.join("orphan-survived"); - let config = Config { - temper_url: "http://127.0.0.1:3497".to_string(), - tenant: "default".to_string(), - worker_id: "mac-mini-codex-1".to_string(), - worker_token: Some("secret".to_string()), - workspace_root: root.clone(), - repo_root: root.clone(), - codex_bin: fixture.display().to_string(), - max_concurrent_runs: 1, - enable_execution: true, - poll_on_start: true, - codex_exec_smoke: false, - codex_exec_timeout: Duration::from_millis(100), - }; - let worker_run = WorkerRunState { - id: "wr-timeout-tree".to_string(), - status: "Running".to_string(), - task: format!("PAW_FAKE_CODEX_ORPHAN:{}", marker.display()), - work_cycle_id: "wc-timeout-tree".to_string(), - worktree_path: root.display().to_string(), - branch_name: "codex/timeout-tree".to_string(), - runner_kind: "local_codex".to_string(), - allowed_worker_id: "mac-mini-codex-1".to_string(), - worker_id: "mac-mini-codex-1".to_string(), - provider_id: "local-codex".to_string(), - required_capabilities: "local_codex,repo_write".to_string(), - }; - - let result = run_codex(&config, &worker_run).await; - - let error = result.expect_err("stuck codex exec should time out"); - assert!( - format!("{error:#}").contains("codex exec timed out after"), - "unexpected timeout error: {error:#}" - ); - sleep(Duration::from_millis(1_500)).await; - assert!( - !marker.exists(), - "timed-out codex exec must not leave descendant processes running" - ); - fs::remove_dir_all(root).ok(); - } - - #[test] - fn codex_review_verdict_requires_explicit_marker() { - let approved = - parse_codex_review_verdict("SUMMARY: Looks good\nVERDICT: approve\nLIVE_E2E: passed"); - let changes = - parse_codex_review_verdict("VERDICT: request_changes\nSUMMARY: Missing E2E proof"); - let escalated = - parse_codex_review_verdict("VERDICT: escalate\nSUMMARY: Cedar risk needs human"); - let unknown = parse_codex_review_verdict("Looks fine to me"); - - assert_eq!(approved.action, ReviewDecisionAction::Approve); - assert_eq!(changes.action, ReviewDecisionAction::RequestChanges); - assert_eq!(escalated.action, ReviewDecisionAction::Escalate); - assert_eq!(unknown.action, ReviewDecisionAction::Escalate); - assert!(unknown.summary.contains("explicit VERDICT")); - } - - #[test] - fn forbidden_done_path_scan_catches_deployment_sensitive_edits() { - let status = "\ - M dd-dashboards/temperpaw-overview.json - M os-apps/paw-channels/wasm/transport_reconcile/Cargo.lock -?? .proofs/patrol.md -"; - - let violation = forbidden_done_path_violations( - status, - "os-apps/paw-agent/,os-apps/paw-channels/,crates/paw-triggers/", - ); - - assert_eq!( - violation, - vec!["os-apps/paw-channels/wasm/transport_reconcile/Cargo.lock"] - ); - } - - #[test] - fn evaluation_commands_default_to_temperpaw_foundation_check() { - let commands = evaluation_commands(None); - assert_eq!( - commands, - vec!["cargo test -p temperpaw --test paw_patrol_foundation -- --nocapture"] - ); - - let configured = - evaluation_commands(Some("cargo check -p paw-codex-worker\n git diff --check ")); - assert_eq!( - configured, - vec!["cargo check -p paw-codex-worker", "git diff --check"] - ); - } - - #[test] - fn codex_success_summary_includes_git_evidence_for_proof_packets() { - let worker_run = WorkerRunState { - id: "wr-proof".to_string(), - status: "Running".to_string(), - task: "Fix a Discord trace leak".to_string(), - work_cycle_id: "wc-proof".to_string(), - worktree_path: "/tmp/paw-worktree".to_string(), - branch_name: "codex/trace-leak".to_string(), - runner_kind: "local_codex".to_string(), - allowed_worker_id: "mac-mini-codex-1".to_string(), - worker_id: "mac-mini-codex-1".to_string(), - provider_id: "local-codex".to_string(), - required_capabilities: "local_codex,repo_write".to_string(), - }; - let evidence = WorktreeEvidence { - status_short: " M crates/temperpaw/src/discord.rs\n?? docs/proofs/trace.md\n" - .to_string(), - diff_stat: "crates/temperpaw/src/discord.rs | 12 ++++++------".to_string(), - }; - - let summary = format_codex_success_summary( - &worker_run, - Path::new("/tmp/paw-worktree"), - "implemented the fix", - &evidence, - None, - ); - - assert!(summary.contains("codex exec completed for WorkerRun wr-proof")); - assert!(summary.contains("Worktree: /tmp/paw-worktree")); - assert!(summary.contains("```git-status")); - assert!(summary.contains(" M crates/temperpaw/src/discord.rs")); - assert!(summary.contains("?? docs/proofs/trace.md")); - assert!(summary.contains("```git-diff-stat")); - assert!(summary.contains("Codex stdout")); - } - - #[tokio::test] - async fn codex_exec_bypasses_sandbox_for_assigned_worktree() { - let _guard = ENV_LOCK.lock().await; - let _datadog_mcp = - EnvOverride::set("PAW_CODEX_ENABLE_DATADOG_MCP", OsString::from("0")); - let args = codex_exec_args(Path::new("/tmp/paw-worktree"), "Create the proof file"); - let args = args - .iter() - .map(|arg| arg.to_string_lossy().to_string()) - .collect::>(); - - assert_eq!( - args, - vec![ - "exec", - "--ignore-user-config", - "--ephemeral", - "--dangerously-bypass-approvals-and-sandbox", - "--cd", - "/tmp/paw-worktree", - "--skip-git-repo-check", - "Create the proof file" - ] - ); - } - - #[tokio::test] - async fn codex_exec_keeps_datadog_mcp_disabled_when_env_unset() { - let _guard = ENV_LOCK.lock().await; - let _datadog_mcp = EnvOverride::remove("PAW_CODEX_ENABLE_DATADOG_MCP"); - let _datadog_url = EnvOverride::remove("PAW_CODEX_DATADOG_MCP_URL"); - let args = codex_exec_args(Path::new("/tmp/paw-worktree"), "Create the proof file"); - let args = args - .iter() - .map(|arg| arg.to_string_lossy().to_string()) - .collect::>(); - - assert_eq!(args[0], "exec"); - assert_eq!(args[1], "--ignore-user-config"); - assert!(!args.contains(&"-c".to_string())); - assert!( - !args - .iter() - .any(|arg| arg.contains("mcp_servers.datadog.url")) - ); - } - - #[tokio::test] - async fn codex_exec_can_enable_datadog_mcp_without_inheriting_user_config() { - let _guard = ENV_LOCK.lock().await; - let _datadog_mcp = - EnvOverride::set("PAW_CODEX_ENABLE_DATADOG_MCP", OsString::from("1")); - let _datadog_url = EnvOverride::set( - "PAW_CODEX_DATADOG_MCP_URL", - OsString::from("https://mcp.datadoghq.test/mcp?toolsets=logs"), - ); - let args = codex_exec_args( - Path::new("/tmp/paw-worktree"), - "Inspect Datadog evidence for the evolution direction", - ); - let args = args - .iter() - .map(|arg| arg.to_string_lossy().to_string()) - .collect::>(); - - assert_eq!( - &args[0..4], - &[ - "exec".to_string(), - "--ignore-user-config".to_string(), - "-c".to_string(), - "mcp_servers.datadog.url=\"https://mcp.datadoghq.test/mcp?toolsets=logs\"" - .to_string(), - ] - ); - assert!(args.contains(&"--ephemeral".to_string())); - assert!(args.contains(&"--dangerously-bypass-approvals-and-sandbox".to_string())); - assert!(args.contains(&"--skip-git-repo-check".to_string())); - assert_eq!( - args.last().map(String::as_str), - Some("Inspect Datadog evidence for the evolution direction") - ); - } - - #[tokio::test] - async fn codex_exec_uses_default_datadog_mcp_url_when_enabled() { - let _guard = ENV_LOCK.lock().await; - let _datadog_mcp = - EnvOverride::set("PAW_CODEX_ENABLE_DATADOG_MCP", OsString::from("1")); - let _datadog_url = EnvOverride::remove("PAW_CODEX_DATADOG_MCP_URL"); - let args = codex_exec_args( - Path::new("/tmp/paw-worktree"), - "Inspect Datadog evidence for the evolution direction", - ); - let args = args - .iter() - .map(|arg| arg.to_string_lossy().to_string()) - .collect::>(); - - assert!(args.contains( - &"mcp_servers.datadog.url=\"https://mcp.datadoghq.com/api/unstable/mcp-server/mcp?toolsets=all\"" - .to_string() - )); - } - - #[test] - fn toml_basic_string_escapes_codex_config_values() { - assert_eq!( - toml_basic_string("https://mcp.example.test/a\"b\\c\n"), - "\"https://mcp.example.test/a\\\"b\\\\c\\n\"" - ); - } - - #[test] - fn worker_proof_text_does_not_call_assigned_worktree_current_checkout() { - let worker_run = WorkerRunState { - id: "wr-existing-worktree".to_string(), - status: "Running".to_string(), - task: "Inspect an existing assigned worktree".to_string(), - work_cycle_id: "wc-existing-worktree".to_string(), - worktree_path: "/tmp/paw-existing-worktree".to_string(), - branch_name: String::new(), - runner_kind: "local_codex".to_string(), - allowed_worker_id: "mac-mini-codex-1".to_string(), - worker_id: "mac-mini-codex-1".to_string(), - provider_id: "local-codex".to_string(), - required_capabilities: "local_codex,repo_write".to_string(), - }; - let evidence = WorktreeEvidence { - status_short: String::new(), - diff_stat: String::new(), - }; - - let summary = format_codex_success_summary( - &worker_run, - Path::new("/tmp/paw-existing-worktree"), - "inspected", - &evidence, - None, - ); - let review = ReviewRunState { - status: "Requested".to_string(), - worker_run_id: worker_run.id.clone(), - proof_packet_id: "proof-1".to_string(), - }; - let prompt = codex_review_prompt(&worker_run, &review); - - assert!(summary.contains("Branch: (assigned worktree without branch)")); - assert!(prompt.contains("Branch: (assigned worktree without branch)")); - assert!(!summary.contains("Branch: (current checkout)")); - assert!(!prompt.contains("Branch: (current checkout)")); - } - - #[test] - fn review_prompt_allows_pr_ready_changes_with_deployment_pending() { - let worker_run = WorkerRunState { - id: "wr-dashboard".to_string(), - status: "Done".to_string(), - task: "Update Datadog dashboard JSON but do not deploy production.".to_string(), - work_cycle_id: "wc-dashboard".to_string(), - worktree_path: "/tmp/paw-dashboard".to_string(), - branch_name: "codex/dashboard".to_string(), - runner_kind: "local_codex".to_string(), - allowed_worker_id: "mac-mini-codex-1".to_string(), - worker_id: "mac-mini-codex-1".to_string(), - provider_id: "local-codex".to_string(), - required_capabilities: "local_codex,repo_write,review".to_string(), - }; - let review = ReviewRunState { - status: "Requested".to_string(), - worker_run_id: worker_run.id.clone(), - proof_packet_id: "proof-dashboard".to_string(), - }; - - let prompt = codex_review_prompt(&worker_run, &review); - - assert!(prompt.contains("deployment-pending residual risk")); - assert!(prompt.contains("Do not require production deployment")); - } - - #[test] - fn repo_health_review_prompt_reviews_scan_contract_not_patch_contract() { - let worker_run = WorkerRunState { - id: "wr-repo-scan".to_string(), - status: "Done".to_string(), - task: "RepoGraphSnapshot: snap-1\nWorkCycle: wc-1\nRun an agent-led repo health patrol.".to_string(), - work_cycle_id: "wc-1".to_string(), - worktree_path: "/tmp/paw-repo-scan".to_string(), - branch_name: "codex/paw-repo-sweep-snap-1".to_string(), - runner_kind: "local_codex".to_string(), - allowed_worker_id: "mac-mini-codex-1".to_string(), - worker_id: "mac-mini-codex-1".to_string(), - provider_id: "local-codex".to_string(), - required_capabilities: "local_codex,repo_write,evaluation".to_string(), - }; - let review = ReviewRunState { - status: "Requested".to_string(), - worker_run_id: worker_run.id.clone(), - proof_packet_id: "proof-1".to_string(), - }; - - let prompt = codex_review_prompt(&worker_run, &review); - - assert!(prompt.contains("repo-health Patrol scan reviewer")); - assert!(prompt.contains("RepoGraphSnapshot.ScanComplete")); - assert!(prompt.contains("do not require an implementation patch")); - assert!(!prompt.contains("Inspect the git diff, changed files, tests/proofs")); - } - - #[test] - fn review_evaluation_and_work_cycle_state_read_temper_odata_fields() { - let review = review_run_from_odata_value(json!({ - "entity_id": "rev-1", - "status": "Requested", - "fields": { - "work_cycle_id": "wc-1", - "worker_run_id": "wr-1", - "proof_packet_id": "proof-1" - } - })) - .expect("review run"); - let evaluation = evaluation_run_from_odata_value(json!({ - "entity_id": "eval-1", - "status": "Queued", - "fields": { - "work_cycle_id": "wc-1", - "required_checks": "[\"repo-sweep\"]" - } - })) - .expect("evaluation run"); - let work_cycle = work_cycle_from_odata_value(json!({ - "entity_id": "wc-1", - "status": "Reviewing", - "fields": { - "implementer_worker_run_id": "wr-1", - "reviewer_run_id": "rev-1", - "evaluation_run_id": "eval-1", - "review_passed": "true" - } - })) - .expect("work cycle"); - - assert_eq!(review.worker_run_id, "wr-1"); - assert_eq!(evaluation.work_cycle_id, "wc-1"); - assert_eq!(work_cycle.status, "Reviewing"); - assert_eq!(work_cycle.implementer_worker_run_id, "wr-1"); - assert!(work_cycle.review_passed); - } - - fn unique_temp_dir() -> PathBuf { - let nanos = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("time") - .as_nanos(); - env::temp_dir().join(format!( - "paw-codex-worker-test-{}-{nanos}", - std::process::id() - )) - } - - struct EnvOverride { - key: &'static str, - previous: Option, - } - - impl EnvOverride { - fn set(key: &'static str, value: OsString) -> Self { - let previous = env::var_os(key); - unsafe { - env::set_var(key, value); - } - Self { key, previous } - } - - fn remove(key: &'static str) -> Self { - let previous = env::var_os(key); - unsafe { - env::remove_var(key); - } - Self { key, previous } - } - } - - impl Drop for EnvOverride { - fn drop(&mut self) { - unsafe { - if let Some(value) = &self.previous { - env::set_var(self.key, value); - } else { - env::remove_var(self.key); - } - } - } - } - - fn run_raw_git_for_test(args: &[&str]) { - let output = std::process::Command::new("git") - .args(args) - .output() - .expect("run git"); - assert!( - output.status.success(), - "git {} failed: {}{}{}", - args.join(" "), - String::from_utf8_lossy(&output.stdout), - if output.stdout.is_empty() || output.stderr.is_empty() { - "" - } else { - "\n" - }, - String::from_utf8_lossy(&output.stderr) - ); - } - - fn run_git_for_test(workdir: &Path, args: &[&str]) { - let output = std::process::Command::new("git") - .arg("-C") - .arg(workdir) - .args(args) - .output() - .expect("run git"); - assert!( - output.status.success(), - "git -C {} {} failed: {}{}{}", - workdir.display(), - args.join(" "), - String::from_utf8_lossy(&output.stdout), - if output.stdout.is_empty() || output.stderr.is_empty() { - "" - } else { - "\n" - }, - String::from_utf8_lossy(&output.stderr) - ); - } - - fn run_git_capture_for_test(workdir: &Path, args: &[&str]) -> String { - let output = std::process::Command::new("git") - .arg("-C") - .arg(workdir) - .args(args) - .output() - .expect("run git"); - assert!( - output.status.success(), - "git -C {} {} failed: {}{}{}", - workdir.display(), - args.join(" "), - String::from_utf8_lossy(&output.stdout), - if output.stdout.is_empty() || output.stderr.is_empty() { - "" - } else { - "\n" - }, - String::from_utf8_lossy(&output.stderr) - ); - String::from_utf8_lossy(&output.stdout) - .trim_end() - .to_string() - } - - #[cfg(unix)] - fn make_executable(path: &Path) { - let mut permissions = fs::metadata(path).expect("metadata").permissions(); - permissions.set_mode(0o755); - fs::set_permissions(path, permissions).expect("chmod"); - } - - #[cfg(not(unix))] - fn make_executable(_path: &Path) {} -} +include!("tests/core.rs"); diff --git a/crates/paw-codex-worker/src/tests/core.rs b/crates/paw-codex-worker/src/tests/core.rs new file mode 100644 index 000000000..4faf2211f --- /dev/null +++ b/crates/paw-codex-worker/src/tests/core.rs @@ -0,0 +1,1157 @@ +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::OsString; + use std::fs; + #[cfg(unix)] + use std::os::unix::fs::PermissionsExt; + use std::time::{SystemTime, UNIX_EPOCH}; + use tokio::sync::Mutex; + + include!("../datadog_patrol_tests.rs"); + include!("../github_patrol_tests.rs"); + include!("../evaluation_tests.rs"); + include!("../event_stream_tests.rs"); + include!("../fake_codex_tests.rs"); + include!("../codex_plan_tests.rs"); + include!("../pull_request_tests.rs"); + include!("../repo_health_parser_tests.rs"); + include!("../worker_http_tests.rs"); + + static ENV_LOCK: Mutex<()> = Mutex::const_new(()); + + #[test] + fn worker_run_state_reads_temper_odata_fields() { + let value = json!({ + "entity_id": "wr-1", + "status": "Queued", + "fields": { + "task": "Do useful work", + "work_cycle_id": "wc-1", + "worktree_path": "/tmp/worktree", + "branch_name": "codex/test", + "runner_kind": "local_codex", + "allowed_worker_id": "mac-mini-codex-1", + "worker_id": "mac-mini-codex-1", + "provider_id": "local-codex", + "required_capabilities": "local_codex,repo_write" + } + }); + + let worker_run = worker_run_from_odata_value(value).expect("worker run should parse"); + + assert_eq!(worker_run.id, "wr-1"); + assert_eq!(worker_run.status, "Queued"); + assert_eq!(worker_run.task, "Do useful work"); + assert_eq!(worker_run.work_cycle_id, "wc-1"); + assert_eq!(worker_run.worktree_path, "/tmp/worktree"); + assert_eq!(worker_run.branch_name, "codex/test"); + assert_eq!(worker_run.runner_kind, "local_codex"); + assert_eq!(worker_run.allowed_worker_id, "mac-mini-codex-1"); + assert_eq!(worker_run.worker_id, "mac-mini-codex-1"); + assert_eq!(worker_run.provider_id, "local-codex"); + assert_eq!(worker_run.required_capabilities, "local_codex,repo_write"); + } + + #[test] + fn local_worker_claims_only_configured_local_codex_runs() { + let configured = WorkerRunState { + id: "wr-1".to_string(), + status: "Queued".to_string(), + task: "Do useful work".to_string(), + work_cycle_id: "wc-1".to_string(), + worktree_path: "/tmp/worktree".to_string(), + branch_name: "codex/test".to_string(), + runner_kind: "local_codex".to_string(), + allowed_worker_id: "mac-mini-codex-1".to_string(), + worker_id: String::new(), + provider_id: "local-codex".to_string(), + required_capabilities: "local_codex,repo_write".to_string(), + }; + let unconfigured = WorkerRunState { + id: "wr-2".to_string(), + status: "Queued".to_string(), + task: String::new(), + work_cycle_id: String::new(), + worktree_path: String::new(), + branch_name: String::new(), + runner_kind: String::new(), + allowed_worker_id: String::new(), + worker_id: String::new(), + provider_id: String::new(), + required_capabilities: String::new(), + }; + let cloud = WorkerRunState { + id: "wr-3".to_string(), + status: "Queued".to_string(), + task: "Cloud overflow".to_string(), + work_cycle_id: "wc-cloud".to_string(), + worktree_path: String::new(), + branch_name: String::new(), + runner_kind: "codex_cloud".to_string(), + allowed_worker_id: "mac-mini-codex-1".to_string(), + worker_id: String::new(), + provider_id: "local-codex".to_string(), + required_capabilities: "local_codex,repo_write".to_string(), + }; + let no_worktree_assignment = WorkerRunState { + id: "wr-4".to_string(), + status: "Queued".to_string(), + task: "Fix a Discord trace leak".to_string(), + work_cycle_id: "wc-no-worktree".to_string(), + worktree_path: String::new(), + branch_name: String::new(), + runner_kind: "local_codex".to_string(), + allowed_worker_id: "mac-mini-codex-1".to_string(), + worker_id: String::new(), + provider_id: "local-codex".to_string(), + required_capabilities: "local_codex,repo_write".to_string(), + }; + + assert!(worker_run_is_claimable_by_local_codex( + &configured, + "mac-mini-codex-1" + )); + assert!(!worker_run_is_claimable_by_local_codex( + &configured, + "other-worker" + )); + assert!(!worker_run_is_claimable_by_local_codex( + &unconfigured, + "mac-mini-codex-1" + )); + assert!(!worker_run_is_claimable_by_local_codex( + &cloud, + "mac-mini-codex-1" + )); + assert!(!worker_run_is_claimable_by_local_codex( + &no_worktree_assignment, + "mac-mini-codex-1" + )); + } + + #[test] + fn local_worker_recovers_only_own_running_codex_runs() { + let recoverable = WorkerRunState { + id: "wr-1".to_string(), + status: "Running".to_string(), + task: "Resume this work".to_string(), + work_cycle_id: "wc-1".to_string(), + worktree_path: "/tmp/worktree".to_string(), + branch_name: "codex/test".to_string(), + runner_kind: "local_codex".to_string(), + allowed_worker_id: "mac-mini-codex-1".to_string(), + worker_id: "mac-mini-codex-1".to_string(), + provider_id: "local-codex".to_string(), + required_capabilities: "local_codex,repo_write".to_string(), + }; + let other_worker = WorkerRunState { + worker_id: "other-worker".to_string(), + ..recoverable.clone() + }; + let queued = WorkerRunState { + status: "Queued".to_string(), + ..recoverable.clone() + }; + + assert!(worker_run_is_recoverable_by_local_codex( + &recoverable, + "mac-mini-codex-1" + )); + assert!(!worker_run_is_recoverable_by_local_codex( + &other_worker, + "mac-mini-codex-1" + )); + assert!(!worker_run_is_recoverable_by_local_codex( + &queued, + "mac-mini-codex-1" + )); + } + + #[test] + fn repo_sweep_worker_runs_are_detected_for_review_and_evaluation() { + let worker_run = WorkerRunState { + id: "wr-1".to_string(), + status: "Done".to_string(), + task: "RepoGraphSnapshot: snap-1\nWorkCycle: wc-1".to_string(), + work_cycle_id: "wc-1".to_string(), + worktree_path: "/tmp/worktree".to_string(), + branch_name: "codex/repo-sweep".to_string(), + runner_kind: "local_codex".to_string(), + allowed_worker_id: "mac-mini-codex-1".to_string(), + worker_id: String::new(), + provider_id: "local-codex".to_string(), + required_capabilities: "local_codex,repo_write".to_string(), + }; + let normal_worker_run = WorkerRunState { + id: "wr-2".to_string(), + status: "Done".to_string(), + task: "Fix a Discord reply bug".to_string(), + work_cycle_id: "wc-2".to_string(), + worktree_path: "/tmp/worktree".to_string(), + branch_name: "codex/bugfix".to_string(), + runner_kind: "local_codex".to_string(), + allowed_worker_id: "mac-mini-codex-1".to_string(), + worker_id: String::new(), + provider_id: "local-codex".to_string(), + required_capabilities: "local_codex,repo_write".to_string(), + }; + + assert!(worker_run_is_repo_sweep(&worker_run)); + assert!(!worker_run_is_repo_sweep(&normal_worker_run)); + } + + #[test] + fn worker_command_defaults_to_run_and_accepts_doctor() { + assert_eq!( + parse_worker_command(Vec::::new()), + WorkerCommand::Run + ); + assert_eq!( + parse_worker_command(vec!["doctor".to_string()]), + WorkerCommand::Doctor + ); + assert_eq!( + parse_worker_command(vec!["--doctor".to_string()]), + WorkerCommand::Doctor + ); + assert_eq!( + parse_worker_command(vec!["launchd-plist".to_string()]), + WorkerCommand::LaunchdPlist + ); + assert_eq!( + parse_worker_command(vec!["--launchd-plist".to_string()]), + WorkerCommand::LaunchdPlist + ); + assert_eq!( + parse_worker_command(vec!["register-worker-agent".to_string()]), + WorkerCommand::RegisterWorkerAgent + ); + assert_eq!( + parse_worker_command(vec!["run".to_string()]), + WorkerCommand::Run + ); + } + + #[tokio::test] + async fn launchd_plist_renders_concrete_worker_environment() { + let _guard = ENV_LOCK.lock().await; + let _model = EnvOverride::set("PAW_CODEX_MODEL", OsString::from("gpt-5-codex")); + let config = Config { + temper_url: "https://temperpaw.example.test".to_string(), + tenant: "prod".to_string(), + worker_id: "mac-mini-codex-1".to_string(), + worker_token: Some("secret&token".to_string()), + workspace_root: PathBuf::from("/Users/me/Development/temperpaw-worktrees"), + repo_root: PathBuf::from("/Users/me/Development/temperpaw"), + codex_bin: "/Users/me/.local/bin/codex".to_string(), + max_concurrent_runs: 1, + enable_execution: true, + poll_on_start: false, + codex_exec_smoke: true, + codex_exec_timeout: Duration::from_secs(90), + }; + + let plist = render_launchd_plist( + &config, + Path::new("/Users/me/.local/bin/paw-codex-worker"), + Some("cargo test -p temperpaw --test paw_patrol_foundation"), + ); + + for needle in [ + "Label", + "com.temperpaw.paw-codex-worker", + "/Users/me/.local/bin/paw-codex-worker", + "TEMPER_URL", + "https://temperpaw.example.test", + "WORKER_ID", + "mac-mini-codex-1", + "PATH", + "/opt/homebrew/bin:/opt/homebrew/sbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Users/openclaw/.cargo/bin", + "PAW_CODEX_ENABLE_EXECUTION", + "1", + "PAW_CODEX_POLL_ON_START", + "0", + "PAW_CODEX_DOCTOR_EXEC_SMOKE", + "1", + "PAW_CODEX_EXEC_TIMEOUT_SECS", + "90", + "PAW_CODEX_WORKER_CAPABILITIES", + "local_codex,repo_write,review,evaluation,datadog_query,github_query", + "PAW_CODEX_WORKER_ENV_FILE", + "/Users/openclaw/.config/temperpaw/paw-codex-worker.env", + "PAW_CODEX_FORBIDDEN_DONE_PATHS", + "PAW_CODEX_EVAL_COMMANDS", + "cargo test -p temperpaw --test paw_patrol_foundation", + "PAW_CODEX_MODEL", + "gpt-5-codex", + ] { + assert!(plist.contains(needle), "plist should contain {needle}"); + } + assert!( + !plist.contains("WORKER_TOKEN"), + "launchd plist should keep WORKER_TOKEN in the 0600 env file, not in launchctl-visible environment" + ); + assert!( + !plist.contains("PAW_CODEX_INHERIT_USER_CONFIG"), + "tool profile opt-in should stay in the worker env file so launchd does not freeze a default" + ); + } + + #[test] + fn doctor_status_fails_only_on_failures() { + let pass_and_warn = vec![ + DoctorCheck::pass("repo", "ok".to_string()), + DoctorCheck::warn("token", "missing token".to_string()), + ]; + let fail = vec![DoctorCheck::fail("odata", "not reachable".to_string())]; + + assert!(!doctor_has_failures(&pass_and_warn)); + assert!(doctor_has_failures(&fail)); + assert_eq!(doctor_status_label(DoctorStatus::Pass), "pass"); + assert_eq!(doctor_status_label(DoctorStatus::Warn), "warn"); + assert_eq!(doctor_status_label(DoctorStatus::Fail), "fail"); + } + + #[tokio::test] + async fn doctor_codex_exec_smoke_runs_fixture_when_enabled() { + let fixture = Path::new(env!("CARGO_MANIFEST_DIR")).join("fixtures/fake-codex.sh"); + let root = unique_temp_dir(); + fs::create_dir_all(&root).expect("temp dir"); + let config = Config { + temper_url: "http://127.0.0.1:3497".to_string(), + tenant: "default".to_string(), + worker_id: "mac-mini-codex-1".to_string(), + worker_token: Some("secret".to_string()), + workspace_root: root.clone(), + repo_root: root.clone(), + codex_bin: fixture.display().to_string(), + max_concurrent_runs: 1, + enable_execution: false, + poll_on_start: true, + codex_exec_smoke: true, + codex_exec_timeout: Duration::from_secs(30), + }; + + let check = check_codex_exec_smoke(&config).await; + + assert_eq!(check.name, "codex_exec_smoke"); + assert_eq!(check.status, DoctorStatus::Pass); + assert!( + check.detail.contains("PAW_CODEX_DOCTOR_EXEC_OK"), + "unexpected detail: {}", + check.detail + ); + fs::remove_dir_all(root).ok(); + } + + #[tokio::test] + async fn run_codex_times_out_stuck_local_exec() { + let fixture = Path::new(env!("CARGO_MANIFEST_DIR")).join("fixtures/fake-codex.sh"); + let root = unique_temp_dir(); + fs::create_dir_all(&root).expect("temp dir"); + let config = Config { + temper_url: "http://127.0.0.1:3497".to_string(), + tenant: "default".to_string(), + worker_id: "mac-mini-codex-1".to_string(), + worker_token: Some("secret".to_string()), + workspace_root: root.clone(), + repo_root: root.clone(), + codex_bin: fixture.display().to_string(), + max_concurrent_runs: 1, + enable_execution: true, + poll_on_start: true, + codex_exec_smoke: false, + codex_exec_timeout: Duration::from_millis(100), + }; + let worker_run = WorkerRunState { + id: "wr-timeout".to_string(), + status: "Running".to_string(), + task: "PAW_FAKE_CODEX_HANG: simulate a stuck Codex child".to_string(), + work_cycle_id: "wc-timeout".to_string(), + worktree_path: root.display().to_string(), + branch_name: "codex/timeout".to_string(), + runner_kind: "local_codex".to_string(), + allowed_worker_id: "mac-mini-codex-1".to_string(), + worker_id: "mac-mini-codex-1".to_string(), + provider_id: "local-codex".to_string(), + required_capabilities: "local_codex,repo_write".to_string(), + }; + + let result = run_codex(&config, &worker_run).await; + + let error = result.expect_err("stuck codex exec should time out"); + let message = format!("{error:#}"); + assert!( + message.contains("codex exec timed out after"), + "unexpected timeout error: {message}" + ); + fs::remove_dir_all(root).ok(); + } + + #[tokio::test] + async fn run_codex_timeout_cleans_child_process_group() { + let fixture = Path::new(env!("CARGO_MANIFEST_DIR")).join("fixtures/fake-codex.sh"); + let root = unique_temp_dir(); + fs::create_dir_all(&root).expect("temp dir"); + let marker = root.join("orphan-survived"); + let config = Config { + temper_url: "http://127.0.0.1:3497".to_string(), + tenant: "default".to_string(), + worker_id: "mac-mini-codex-1".to_string(), + worker_token: Some("secret".to_string()), + workspace_root: root.clone(), + repo_root: root.clone(), + codex_bin: fixture.display().to_string(), + max_concurrent_runs: 1, + enable_execution: true, + poll_on_start: true, + codex_exec_smoke: false, + codex_exec_timeout: Duration::from_millis(100), + }; + let worker_run = WorkerRunState { + id: "wr-timeout-tree".to_string(), + status: "Running".to_string(), + task: format!("PAW_FAKE_CODEX_ORPHAN:{}", marker.display()), + work_cycle_id: "wc-timeout-tree".to_string(), + worktree_path: root.display().to_string(), + branch_name: "codex/timeout-tree".to_string(), + runner_kind: "local_codex".to_string(), + allowed_worker_id: "mac-mini-codex-1".to_string(), + worker_id: "mac-mini-codex-1".to_string(), + provider_id: "local-codex".to_string(), + required_capabilities: "local_codex,repo_write".to_string(), + }; + + let result = run_codex(&config, &worker_run).await; + + let error = result.expect_err("stuck codex exec should time out"); + assert!( + format!("{error:#}").contains("codex exec timed out after"), + "unexpected timeout error: {error:#}" + ); + sleep(Duration::from_millis(1_500)).await; + assert!( + !marker.exists(), + "timed-out codex exec must not leave descendant processes running" + ); + fs::remove_dir_all(root).ok(); + } + + #[test] + fn codex_review_verdict_requires_explicit_marker() { + let approved = + parse_codex_review_verdict("SUMMARY: Looks good\nVERDICT: approve\nLIVE_E2E: passed"); + let changes = + parse_codex_review_verdict("VERDICT: request_changes\nSUMMARY: Missing E2E proof"); + let escalated = + parse_codex_review_verdict("VERDICT: escalate\nSUMMARY: Cedar risk needs human"); + let unknown = parse_codex_review_verdict("Looks fine to me"); + + assert_eq!(approved.action, ReviewDecisionAction::Approve); + assert_eq!(changes.action, ReviewDecisionAction::RequestChanges); + assert_eq!(escalated.action, ReviewDecisionAction::Escalate); + assert_eq!(unknown.action, ReviewDecisionAction::Escalate); + assert!(unknown.summary.contains("explicit VERDICT")); + } + + #[test] + fn forbidden_done_path_scan_catches_deployment_sensitive_edits() { + let status = "\ + M dd-dashboards/temperpaw-overview.json + M os-apps/paw-channels/wasm/transport_reconcile/Cargo.lock +?? .proofs/patrol.md +"; + + let violation = forbidden_done_path_violations( + status, + "os-apps/paw-agent/,os-apps/paw-channels/,crates/paw-triggers/", + ); + + assert_eq!( + violation, + vec!["os-apps/paw-channels/wasm/transport_reconcile/Cargo.lock"] + ); + } + + #[test] + fn evaluation_commands_default_to_temperpaw_foundation_check() { + let commands = evaluation_commands(None); + assert_eq!( + commands, + vec!["cargo test -p temperpaw --test paw_patrol_foundation -- --nocapture"] + ); + + let configured = + evaluation_commands(Some("cargo check -p paw-codex-worker\n git diff --check ")); + assert_eq!( + configured, + vec!["cargo check -p paw-codex-worker", "git diff --check"] + ); + } + + #[test] + fn codex_success_summary_includes_git_evidence_for_proof_packets() { + let worker_run = WorkerRunState { + id: "wr-proof".to_string(), + status: "Running".to_string(), + task: "Fix a Discord trace leak".to_string(), + work_cycle_id: "wc-proof".to_string(), + worktree_path: "/tmp/paw-worktree".to_string(), + branch_name: "codex/trace-leak".to_string(), + runner_kind: "local_codex".to_string(), + allowed_worker_id: "mac-mini-codex-1".to_string(), + worker_id: "mac-mini-codex-1".to_string(), + provider_id: "local-codex".to_string(), + required_capabilities: "local_codex,repo_write".to_string(), + }; + let evidence = WorktreeEvidence { + status_short: " M crates/temperpaw/src/discord.rs\n?? docs/proofs/trace.md\n" + .to_string(), + diff_stat: "crates/temperpaw/src/discord.rs | 12 ++++++------".to_string(), + }; + + let summary = format_codex_success_summary( + &worker_run, + Path::new("/tmp/paw-worktree"), + "implemented the fix", + &evidence, + None, + ); + + assert!(summary.contains("codex exec completed for WorkerRun wr-proof")); + assert!(summary.contains("Worktree: /tmp/paw-worktree")); + assert!(summary.contains("```git-status")); + assert!(summary.contains(" M crates/temperpaw/src/discord.rs")); + assert!(summary.contains("?? docs/proofs/trace.md")); + assert!(summary.contains("```git-diff-stat")); + assert!(summary.contains("Codex stdout")); + } + + #[tokio::test] + async fn codex_exec_bypasses_sandbox_for_assigned_worktree() { + let _guard = ENV_LOCK.lock().await; + let _datadog_mcp = + EnvOverride::set("PAW_CODEX_ENABLE_DATADOG_MCP", OsString::from("0")); + let _model = EnvOverride::remove("PAW_CODEX_MODEL"); + let args = codex_exec_args(Path::new("/tmp/paw-worktree"), "Create the proof file"); + let args = args + .iter() + .map(|arg| arg.to_string_lossy().to_string()) + .collect::>(); + + assert_eq!( + args, + vec![ + "exec", + "--ignore-user-config", + "--ephemeral", + "--dangerously-bypass-approvals-and-sandbox", + "--cd", + "/tmp/paw-worktree", + "--skip-git-repo-check", + "Create the proof file" + ] + ); + } + + #[tokio::test] + async fn config_accepts_multiple_concurrent_runs_for_process_pool_slots() { + let _guard = ENV_LOCK.lock().await; + let _temper_url = EnvOverride::set("TEMPER_URL", OsString::from("http://127.0.0.1:3000")); + let _worker_id = EnvOverride::set("WORKER_ID", OsString::from("local-codex-slot-1")); + let _max_concurrent_runs = EnvOverride::set("MAX_CONCURRENT_RUNS", OsString::from("4")); + + let config = Config::from_env().expect("worker config should parse"); + + assert_eq!(config.max_concurrent_runs, 4); + } + + #[tokio::test] + async fn worker_agent_registration_body_uses_slot_identity() { + let _guard = ENV_LOCK.lock().await; + let _provider = EnvOverride::set("WORKER_PROVIDER_ID", OsString::from("local-codex")); + let _host = EnvOverride::set("HOSTNAME", OsString::from("worker-host")); + let config = Config { + temper_url: "https://temperpaw.example.test".to_string(), + tenant: "prod".to_string(), + worker_id: "mac-mini-codex-prod-slot-02".to_string(), + worker_token: Some("secret".to_string()), + workspace_root: PathBuf::from("/Users/me/Development/temperpaw-worktrees"), + repo_root: PathBuf::from("/Users/me/Development/temperpaw"), + codex_bin: "codex".to_string(), + max_concurrent_runs: 1, + enable_execution: true, + poll_on_start: true, + codex_exec_smoke: false, + codex_exec_timeout: Duration::from_secs(90), + }; + + let body = worker_agent_registration_body(&config); + + assert_eq!(body["id"], "mac-mini-codex-prod-slot-02"); + assert_eq!(body["worker_id"], "mac-mini-codex-prod-slot-02"); + assert_eq!(body["provider_id"], "local-codex"); + assert_eq!(body["host"], "worker-host"); + assert_eq!( + body["worktree_root"], + "/Users/me/Development/temperpaw-worktrees" + ); + } + + #[tokio::test] + async fn worker_agent_registration_is_idempotent_when_slot_exists() { + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + use tokio::net::TcpListener; + + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind test listener"); + let addr = listener.local_addr().expect("listener addr"); + let requests = std::sync::Arc::new(tokio::sync::Mutex::new(Vec::::new())); + let captured = requests.clone(); + let server = tokio::spawn(async move { + loop { + let Ok((mut socket, _)) = listener.accept().await else { + break; + }; + let requests = captured.clone(); + tokio::spawn(async move { + let mut buf = [0_u8; 4096]; + let n = socket.read(&mut buf).await.expect("read request"); + let request = String::from_utf8_lossy(&buf[..n]).to_string(); + requests.lock().await.push(request.clone()); + let body = if request.starts_with( + "GET /tdata/WorkerAgents('mac-mini-codex-prod-slot-02')", + ) { + r#"{"entity_id":"mac-mini-codex-prod-slot-02","fields":{"Id":"mac-mini-codex-prod-slot-02","Status":"Registered"}}"# + } else { + r#"{"error":"unexpected request"}"# + }; + let status = if request.starts_with( + "GET /tdata/WorkerAgents('mac-mini-codex-prod-slot-02')", + ) { + "200 OK" + } else { + "500 Internal Server Error" + }; + let response = format!( + "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ); + socket + .write_all(response.as_bytes()) + .await + .expect("write response"); + }); + } + }); + let config = Config { + temper_url: format!("http://{addr}"), + tenant: "prod".to_string(), + worker_id: "mac-mini-codex-prod-slot-02".to_string(), + worker_token: Some("secret".to_string()), + workspace_root: PathBuf::from("/Users/me/Development/temperpaw-worktrees"), + repo_root: PathBuf::from("/Users/me/Development/temperpaw"), + codex_bin: "codex".to_string(), + max_concurrent_runs: 1, + enable_execution: true, + poll_on_start: true, + codex_exec_smoke: false, + codex_exec_timeout: Duration::from_secs(90), + }; + let client = reqwest::Client::new(); + + ensure_worker_agent_registered(&client, &config) + .await + .expect("existing WorkerAgent should satisfy registration"); + + server.abort(); + let requests = requests.lock().await; + assert_eq!(requests.len(), 1, "registration should not POST when the slot exists"); + assert!(requests[0].starts_with( + "GET /tdata/WorkerAgents('mac-mini-codex-prod-slot-02')" + )); + } + + #[tokio::test] + async fn worker_agent_heartbeat_falls_back_to_installed_patrol_namespace() { + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + use tokio::net::TcpListener; + + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind test listener"); + let addr = listener.local_addr().expect("listener addr"); + let requests = std::sync::Arc::new(tokio::sync::Mutex::new(Vec::::new())); + let captured = requests.clone(); + let server = tokio::spawn(async move { + loop { + let Ok((mut socket, _)) = listener.accept().await else { + break; + }; + let requests = captured.clone(); + tokio::spawn(async move { + let mut buf = [0_u8; 4096]; + let n = socket.read(&mut buf).await.expect("read request"); + let request = String::from_utf8_lossy(&buf[..n]).to_string(); + requests.lock().await.push(request.clone()); + let new_namespace = request.starts_with( + "POST /tdata/WorkerAgents('mac-mini-codex-prod-slot-02')/Temper.PawOrchestration.ReportHeartbeat", + ); + let legacy_namespace = request.starts_with( + "POST /tdata/WorkerAgents('mac-mini-codex-prod-slot-02')/TemperPaw.Patrol.ReportHeartbeat", + ); + let (status, body) = if new_namespace { + ("404 Not Found", r#"{"error":"action not installed yet"}"#) + } else if legacy_namespace { + ("200 OK", r#"{"ok":true}"#) + } else { + ("500 Internal Server Error", r#"{"error":"unexpected request"}"#) + }; + let response = format!( + "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ); + socket + .write_all(response.as_bytes()) + .await + .expect("write response"); + }); + } + }); + let config = Config { + temper_url: format!("http://{addr}"), + tenant: "prod".to_string(), + worker_id: "mac-mini-codex-prod-slot-02".to_string(), + worker_token: Some("secret".to_string()), + workspace_root: PathBuf::from("/Users/me/Development/temperpaw-worktrees"), + repo_root: PathBuf::from("/Users/me/Development/temperpaw"), + codex_bin: "codex".to_string(), + max_concurrent_runs: 1, + enable_execution: true, + poll_on_start: true, + codex_exec_smoke: false, + codex_exec_timeout: Duration::from_secs(90), + }; + let client = reqwest::Client::new(); + + report_worker_heartbeat(&client, &config) + .await + .expect("legacy installed Patrol heartbeat namespace should still work"); + + server.abort(); + let requests = requests.lock().await; + assert_eq!(requests.len(), 2, "heartbeat should retry once via Patrol"); + assert!(requests[0].starts_with( + "POST /tdata/WorkerAgents('mac-mini-codex-prod-slot-02')/Temper.PawOrchestration.ReportHeartbeat" + )); + assert!(requests[1].starts_with( + "POST /tdata/WorkerAgents('mac-mini-codex-prod-slot-02')/TemperPaw.Patrol.ReportHeartbeat" + )); + } + + #[tokio::test] + async fn codex_exec_keeps_datadog_mcp_disabled_when_env_unset() { + let _guard = ENV_LOCK.lock().await; + let _datadog_mcp = EnvOverride::remove("PAW_CODEX_ENABLE_DATADOG_MCP"); + let _datadog_url = EnvOverride::remove("PAW_CODEX_DATADOG_MCP_URL"); + let _model = EnvOverride::remove("PAW_CODEX_MODEL"); + let args = codex_exec_args(Path::new("/tmp/paw-worktree"), "Create the proof file"); + let args = args + .iter() + .map(|arg| arg.to_string_lossy().to_string()) + .collect::>(); + + assert_eq!(args[0], "exec"); + assert_eq!(args[1], "--ignore-user-config"); + assert!(!args.contains(&"-c".to_string())); + assert!(!args.contains(&"--model".to_string())); + assert!( + !args + .iter() + .any(|arg| arg.contains("mcp_servers.datadog.url")) + ); + } + + #[tokio::test] + async fn codex_exec_can_enable_datadog_mcp_without_inheriting_user_config() { + let _guard = ENV_LOCK.lock().await; + let _datadog_mcp = + EnvOverride::set("PAW_CODEX_ENABLE_DATADOG_MCP", OsString::from("1")); + let _model = EnvOverride::remove("PAW_CODEX_MODEL"); + let _datadog_url = EnvOverride::set( + "PAW_CODEX_DATADOG_MCP_URL", + OsString::from("https://mcp.datadoghq.test/mcp?toolsets=logs"), + ); + let args = codex_exec_args( + Path::new("/tmp/paw-worktree"), + "Inspect Datadog evidence for the evolution direction", + ); + let args = args + .iter() + .map(|arg| arg.to_string_lossy().to_string()) + .collect::>(); + + assert_eq!( + &args[0..4], + &[ + "exec".to_string(), + "--ignore-user-config".to_string(), + "-c".to_string(), + "mcp_servers.datadog.url=\"https://mcp.datadoghq.test/mcp?toolsets=logs\"" + .to_string(), + ] + ); + assert!(args.contains(&"--ephemeral".to_string())); + assert!(args.contains(&"--dangerously-bypass-approvals-and-sandbox".to_string())); + assert!(args.contains(&"--skip-git-repo-check".to_string())); + assert_eq!( + args.last().map(String::as_str), + Some("Inspect Datadog evidence for the evolution direction") + ); + } + + #[tokio::test] + async fn codex_exec_can_force_datadog_mcp_for_telemetry_roles() { + let _guard = ENV_LOCK.lock().await; + let _datadog_mcp = + EnvOverride::set("PAW_CODEX_ENABLE_DATADOG_MCP", OsString::from("0")); + let _model = EnvOverride::remove("PAW_CODEX_MODEL"); + let _datadog_url = EnvOverride::set( + "PAW_CODEX_DATADOG_MCP_URL", + OsString::from("https://mcp.datadoghq.test/mcp?toolsets=logs"), + ); + let args = codex_exec_args_with_datadog_mcp( + Path::new("/tmp/paw-worktree"), + "Evaluate Datadog telemetry", + ); + let args = args + .iter() + .map(|arg| arg.to_string_lossy().to_string()) + .collect::>(); + + assert_eq!( + &args[0..4], + &[ + "exec".to_string(), + "--ignore-user-config".to_string(), + "-c".to_string(), + "mcp_servers.datadog.url=\"https://mcp.datadoghq.test/mcp?toolsets=logs\"" + .to_string(), + ] + ); + assert_eq!(args.last().map(String::as_str), Some("Evaluate Datadog telemetry")); + } + + #[tokio::test] + async fn codex_exec_uses_default_datadog_mcp_url_when_enabled() { + let _guard = ENV_LOCK.lock().await; + let _datadog_mcp = + EnvOverride::set("PAW_CODEX_ENABLE_DATADOG_MCP", OsString::from("1")); + let _datadog_url = EnvOverride::remove("PAW_CODEX_DATADOG_MCP_URL"); + let _model = EnvOverride::remove("PAW_CODEX_MODEL"); + let args = codex_exec_args( + Path::new("/tmp/paw-worktree"), + "Inspect Datadog evidence for the evolution direction", + ); + let args = args + .iter() + .map(|arg| arg.to_string_lossy().to_string()) + .collect::>(); + + assert!(args.contains( + &"mcp_servers.datadog.url=\"https://mcp.datadoghq.com/api/unstable/mcp-server/mcp?toolsets=all\"" + .to_string() + )); + } + + #[tokio::test] + async fn codex_exec_can_pin_worker_model_from_env() { + let _guard = ENV_LOCK.lock().await; + let _datadog_mcp = + EnvOverride::set("PAW_CODEX_ENABLE_DATADOG_MCP", OsString::from("0")); + let _model = EnvOverride::set("PAW_CODEX_MODEL", OsString::from("gpt-5-codex")); + let args = codex_exec_args(Path::new("/tmp/paw-worktree"), "Create the proof file"); + let args = args + .iter() + .map(|arg| arg.to_string_lossy().to_string()) + .collect::>(); + + assert_eq!( + &args[0..4], + &[ + "exec".to_string(), + "--ignore-user-config".to_string(), + "--model".to_string(), + "gpt-5-codex".to_string(), + ] + ); + } + + #[test] + fn toml_basic_string_escapes_codex_config_values() { + assert_eq!( + toml_basic_string("https://mcp.example.test/a\"b\\c\n"), + "\"https://mcp.example.test/a\\\"b\\\\c\\n\"" + ); + } + + #[test] + fn worker_proof_text_does_not_call_assigned_worktree_current_checkout() { + let worker_run = WorkerRunState { + id: "wr-existing-worktree".to_string(), + status: "Running".to_string(), + task: "Inspect an existing assigned worktree".to_string(), + work_cycle_id: "wc-existing-worktree".to_string(), + worktree_path: "/tmp/paw-existing-worktree".to_string(), + branch_name: String::new(), + runner_kind: "local_codex".to_string(), + allowed_worker_id: "mac-mini-codex-1".to_string(), + worker_id: "mac-mini-codex-1".to_string(), + provider_id: "local-codex".to_string(), + required_capabilities: "local_codex,repo_write".to_string(), + }; + let evidence = WorktreeEvidence { + status_short: String::new(), + diff_stat: String::new(), + }; + + let summary = format_codex_success_summary( + &worker_run, + Path::new("/tmp/paw-existing-worktree"), + "inspected", + &evidence, + None, + ); + let review = ReviewRunState { + status: "Requested".to_string(), + worker_run_id: worker_run.id.clone(), + proof_packet_id: "proof-1".to_string(), + }; + let prompt = codex_review_prompt(&worker_run, &review); + + assert!(summary.contains("Branch: (assigned worktree without branch)")); + assert!(prompt.contains("Branch: (assigned worktree without branch)")); + assert!(!summary.contains("Branch: (current checkout)")); + assert!(!prompt.contains("Branch: (current checkout)")); + } + + #[test] + fn review_prompt_allows_pr_ready_changes_with_deployment_pending() { + let worker_run = WorkerRunState { + id: "wr-dashboard".to_string(), + status: "Done".to_string(), + task: "Update Datadog dashboard JSON but do not deploy production.".to_string(), + work_cycle_id: "wc-dashboard".to_string(), + worktree_path: "/tmp/paw-dashboard".to_string(), + branch_name: "codex/dashboard".to_string(), + runner_kind: "local_codex".to_string(), + allowed_worker_id: "mac-mini-codex-1".to_string(), + worker_id: "mac-mini-codex-1".to_string(), + provider_id: "local-codex".to_string(), + required_capabilities: "local_codex,repo_write,review".to_string(), + }; + let review = ReviewRunState { + status: "Requested".to_string(), + worker_run_id: worker_run.id.clone(), + proof_packet_id: "proof-dashboard".to_string(), + }; + + let prompt = codex_review_prompt(&worker_run, &review); + + assert!(prompt.contains("deployment-pending residual risk")); + assert!(prompt.contains("Do not require production deployment")); + } + + #[test] + fn repo_health_review_prompt_reviews_scan_contract_not_patch_contract() { + let worker_run = WorkerRunState { + id: "wr-repo-scan".to_string(), + status: "Done".to_string(), + task: "RepoGraphSnapshot: snap-1\nWorkCycle: wc-1\nRun an agent-led repo health patrol.".to_string(), + work_cycle_id: "wc-1".to_string(), + worktree_path: "/tmp/paw-repo-scan".to_string(), + branch_name: "codex/paw-repo-sweep-snap-1".to_string(), + runner_kind: "local_codex".to_string(), + allowed_worker_id: "mac-mini-codex-1".to_string(), + worker_id: "mac-mini-codex-1".to_string(), + provider_id: "local-codex".to_string(), + required_capabilities: "local_codex,repo_write,evaluation".to_string(), + }; + let review = ReviewRunState { + status: "Requested".to_string(), + worker_run_id: worker_run.id.clone(), + proof_packet_id: "proof-1".to_string(), + }; + + let prompt = codex_review_prompt(&worker_run, &review); + + assert!(prompt.contains("repo-health Patrol scan reviewer")); + assert!(prompt.contains("RepoGraphSnapshot.ScanComplete")); + assert!(prompt.contains("do not require an implementation patch")); + assert!(!prompt.contains("Inspect the git diff, changed files, tests/proofs")); + } + + #[test] + fn review_evaluation_and_work_cycle_state_read_temper_odata_fields() { + let review = review_run_from_odata_value(json!({ + "entity_id": "rev-1", + "status": "Requested", + "fields": { + "work_cycle_id": "wc-1", + "worker_run_id": "wr-1", + "proof_packet_id": "proof-1" + } + })) + .expect("review run"); + let evaluation = evaluation_run_from_odata_value(json!({ + "entity_id": "eval-1", + "status": "Queued", + "fields": { + "work_cycle_id": "wc-1", + "required_checks": "[\"repo-sweep\"]" + } + })) + .expect("evaluation run"); + let work_cycle = work_cycle_from_odata_value(json!({ + "entity_id": "wc-1", + "status": "Reviewing", + "fields": { + "implementer_worker_run_id": "wr-1", + "reviewer_run_id": "rev-1", + "evaluation_run_id": "eval-1", + "review_passed": "true" + } + })) + .expect("work cycle"); + + assert_eq!(review.worker_run_id, "wr-1"); + assert_eq!(evaluation.work_cycle_id, "wc-1"); + assert_eq!(work_cycle.status, "Reviewing"); + assert_eq!(work_cycle.implementer_worker_run_id, "wr-1"); + assert!(work_cycle.review_passed); + } + + fn unique_temp_dir() -> PathBuf { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("time") + .as_nanos(); + env::temp_dir().join(format!( + "paw-codex-worker-test-{}-{nanos}", + std::process::id() + )) + } + + struct EnvOverride { + key: &'static str, + previous: Option, + } + + impl EnvOverride { + fn set(key: &'static str, value: OsString) -> Self { + let previous = env::var_os(key); + unsafe { + env::set_var(key, value); + } + Self { key, previous } + } + + fn remove(key: &'static str) -> Self { + let previous = env::var_os(key); + unsafe { + env::remove_var(key); + } + Self { key, previous } + } + } + + impl Drop for EnvOverride { + fn drop(&mut self) { + unsafe { + if let Some(value) = &self.previous { + env::set_var(self.key, value); + } else { + env::remove_var(self.key); + } + } + } + } + + fn run_raw_git_for_test(args: &[&str]) { + let output = std::process::Command::new("git") + .args(args) + .output() + .expect("run git"); + assert!( + output.status.success(), + "git {} failed: {}{}{}", + args.join(" "), + String::from_utf8_lossy(&output.stdout), + if output.stdout.is_empty() || output.stderr.is_empty() { + "" + } else { + "\n" + }, + String::from_utf8_lossy(&output.stderr) + ); + } + + fn run_git_for_test(workdir: &Path, args: &[&str]) { + let output = std::process::Command::new("git") + .arg("-C") + .arg(workdir) + .args(args) + .output() + .expect("run git"); + assert!( + output.status.success(), + "git -C {} {} failed: {}{}{}", + workdir.display(), + args.join(" "), + String::from_utf8_lossy(&output.stdout), + if output.stdout.is_empty() || output.stderr.is_empty() { + "" + } else { + "\n" + }, + String::from_utf8_lossy(&output.stderr) + ); + } + + fn run_git_capture_for_test(workdir: &Path, args: &[&str]) -> String { + let output = std::process::Command::new("git") + .arg("-C") + .arg(workdir) + .args(args) + .output() + .expect("run git"); + assert!( + output.status.success(), + "git -C {} {} failed: {}{}{}", + workdir.display(), + args.join(" "), + String::from_utf8_lossy(&output.stdout), + if output.stdout.is_empty() || output.stderr.is_empty() { + "" + } else { + "\n" + }, + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8_lossy(&output.stdout) + .trim_end() + .to_string() + } + + #[cfg(unix)] + fn make_executable(path: &Path) { + let mut permissions = fs::metadata(path).expect("metadata").permissions(); + permissions.set_mode(0o755); + fs::set_permissions(path, permissions).expect("chmod"); + } + + #[cfg(not(unix))] + fn make_executable(_path: &Path) {} +} diff --git a/crates/paw-codex-worker/src/worker_types.rs b/crates/paw-codex-worker/src/worker_types.rs index fd5570b98..af9819425 100644 --- a/crates/paw-codex-worker/src/worker_types.rs +++ b/crates/paw-codex-worker/src/worker_types.rs @@ -7,6 +7,11 @@ const EVALUATION_START_LABEL: &str = "EvaluationRun.Start"; const EVALUATION_PASS_LABEL: &str = "EvaluationRun.Pass"; const EVALUATION_FAIL_LABEL: &str = "EvaluationRun.Fail"; const DIRECTED_EVOLUTION_NAMESPACE: &str = "Temper.DirectedEvolution"; +const PAW_ORCHESTRATION_NAMESPACE: &str = "Temper.PawOrchestration"; + +fn action_conflict_message(message: &str) -> bool { + message.contains(" returned 409") || message.contains("409 Conflict") +} #[derive(Clone, Debug)] struct Config { @@ -29,6 +34,7 @@ enum WorkerCommand { Run, Doctor, LaunchdPlist, + RegisterWorkerAgent, DirectedEvolutionStartEpisode { contract_path: Option }, } @@ -67,6 +73,7 @@ impl Config { let max_concurrent_runs = env::var("MAX_CONCURRENT_RUNS") .ok() .and_then(|value| value.parse().ok()) + .filter(|value| *value > 0) .unwrap_or(1); let enable_execution = env::var("PAW_CODEX_ENABLE_EXECUTION") .map(|value| value == "1" || value.eq_ignore_ascii_case("true")) @@ -84,10 +91,6 @@ impl Config { .map(Duration::from_secs) .unwrap_or_else(|| Duration::from_secs(20 * 60)); - if max_concurrent_runs != 1 { - bail!("MAX_CONCURRENT_RUNS must be 1 in v1 so WorkerRun review remains simple"); - } - Ok(Self { temper_url, tenant, diff --git a/crates/temperpaw/Cargo.toml b/crates/temperpaw/Cargo.toml index 821bc872f..a577fbded 100644 --- a/crates/temperpaw/Cargo.toml +++ b/crates/temperpaw/Cargo.toml @@ -12,14 +12,14 @@ path = "src/main.rs" [dependencies] # Temper platform engine -temper-platform = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } -temper-observe = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } -temper-server = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c", features = ["observe"] } -temper-runtime = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } -temper-jit = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } -temper-authz = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } -temper-store-postgres = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } -temper-store-turso = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-platform = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } +temper-observe = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } +temper-server = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23", features = ["observe"] } +temper-runtime = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } +temper-jit = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } +temper-authz = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } +temper-store-postgres = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } +temper-store-turso = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } # Paw transport (local) paw-transport = { path = "../paw-transport" } diff --git a/crates/temperpaw/tests/datadog_observability_contract.rs b/crates/temperpaw/tests/datadog_observability_contract.rs index ae6d17f90..3ea0783ce 100644 --- a/crates/temperpaw/tests/datadog_observability_contract.rs +++ b/crates/temperpaw/tests/datadog_observability_contract.rs @@ -59,7 +59,7 @@ fn collect_cargo_manifests(root: &Path, relative_dir: &Path, files: &mut Vec PathBuf { diff --git a/crates/temperpaw/tests/paw_patrol_foundation.rs b/crates/temperpaw/tests/paw_patrol_foundation.rs index 9fc31a0af..798b4865f 100644 --- a/crates/temperpaw/tests/paw_patrol_foundation.rs +++ b/crates/temperpaw/tests/paw_patrol_foundation.rs @@ -1,6 +1,8 @@ use std::collections::HashMap; +use std::env; use std::fs; use std::path::{Path, PathBuf}; +use std::process::Command; use temper_authz::{AuthzEngine, SecurityContext}; @@ -1351,6 +1353,361 @@ fn datadog_patrol_smoke_proves_mcp_agent_fanout() { } } +#[test] +fn directed_evolution_agent_answers_live_proof_requires_datadog_evidence() { + let root = repo_root(); + let script = read( + root.join("crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh"), + ); + let readme = read(root.join("crates/paw-codex-worker/README.md")); + let ci = read(root.join(".github/workflows/ci.yml")); + + for needle in [ + "DIRECTED_EVOLUTION_DIRECTION_ID", + "DIRECTED_EVOLUTION_ORGANISM_ID", + "Agent Answers", + "directed-evolution-start-episode", + "PAW_CODEX_ENABLE_EXECUTION=1", + "START_LOCAL_WORKER", + "WORKER_SLOT_COUNT", + "WORKER_PIDS", + "slot_worker_id", + "worker_slot_count", + "worker_slot_ids", + "local_worker_logs", + "process_pool_model", + "MAX_CONCURRENT_RUNS=\"1\"", + "local worker process pool", + "START_LOCAL_WORKER=1 requires WORKER_SLOT_COUNT >= 2", + "EvidenceArtifacts", + "WorkerRuns", + "WorkItems", + "datadog-measured", + "datadog_url", + "present(f(\"ResultCount\"))", + "REQUIRE_DATADOG_EVIDENCE must remain 1", + "CONFIRM_PRODUCTION_DATADOG_QUERY_SECRETS", + "DD_API_KEY or DATADOG_API_KEY", + "DD_APP_KEY", + "REQUIRE_EPISODE_SUCCESS must remain 1", + "PRECHECK_ONLY", + "precheck ready", + "would_create_live_episode", + "no_mutation", + "datadog_count=\"$(wait_for_datadog_evidence", + "wait_for_datadog_role_evidence \"$episode_id\" observer", + "wait_for_datadog_role_evidence \"$episode_id\" telemetry_evaluator", + "datadog_observer_evidence_artifacts", + "datadog_telemetry_evaluator_evidence_artifacts", + "mandatory_datadog_observer_evidence", + "mandatory_datadog_telemetry_evaluator_evidence", + "Datadog observer/evaluator evidence polling", + "summary.json", + "proof.md", + "human-blockers.json", + "live Agent Answers Directed Evolution proof", + ] { + assert!( + script.contains(needle), + "Agent Answers live proof script should enforce production proof contract: {needle}" + ); + } + + assert!( + ci.contains( + "bash -n crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh" + ), + "CI should syntax-check the Agent Answers Directed Evolution live proof driver" + ); + assert!( + ci.contains("Agent Answers proof precheck") + && ci.contains("PRECHECK_ONLY=1") + && ci.contains("\"no_mutation\": true") + && ci.contains("\"mandatory_datadog_observer_evidence\": true") + && ci.contains("\"mandatory_datadog_telemetry_evaluator_evidence\": true"), + "CI should run the Agent Answers proof driver in no-mutation precheck mode" + ); + assert!( + readme.contains("directed-evolution-agent-answers-live-proof.sh"), + "worker README should document the Agent Answers Directed Evolution live proof driver" + ); + assert!( + readme.contains("CONFIRM_PRODUCTION_DATADOG_QUERY_SECRETS=1"), + "worker README should require production Datadog query-secret confirmation" + ); + assert!( + readme.contains("DD_API_KEY") && readme.contains("DD_APP_KEY"), + "worker README should document local Datadog credentials for local-worker proof mode" + ); + assert!( + readme.contains("WORKER_SLOT_COUNT") + && readme.contains("-slot-02") + && readme.contains("local-worker-logs"), + "worker README should document the local worker process pool proof mode" + ); + assert!( + readme.contains("observer") && readme.contains("telemetry_evaluator"), + "worker README should require distinct observer and telemetry_evaluator Datadog evidence" + ); +} + +#[test] +fn directed_evolution_agent_answers_live_proof_preflights_datadog_credentials() { + let root = repo_root(); + let proof_dir = env::temp_dir().join(format!( + "directed-evolution-agent-answers-live-proof-blocked-{}", + std::process::id() + )); + fs::remove_dir_all(&proof_dir).ok(); + + let output = Command::new( + root.join("crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh"), + ) + .current_dir(&root) + .env("PROOF_DIR", &proof_dir) + .env("ALLOW_PRODUCTION_WRITE", "1") + .env("CONFIRM_AGENT_ANSWERS_LIVE_PROOF", "1") + .env("TEMPER_URL", "https://temperpaw.example.invalid") + .env("WORKER_TOKEN", "worker-token") + .env("DIRECTED_EVOLUTION_DIRECTION_ID", "direction-agent-answers") + .env("DIRECTED_EVOLUTION_ORGANISM_ID", "organism-agent-answers") + .env("START_LOCAL_WORKER", "0") + .output() + .expect("run blocked Agent Answers Directed Evolution proof preflight"); + + assert!( + !output.status.success(), + "proof driver should block when production Datadog query secrets are not confirmed" + ); + let blockers = read(proof_dir.join("human-blockers.json")); + assert!( + blockers.contains("confirm:production_datadog_query_secrets"), + "blocked proof should name the missing production Datadog query-secret confirmation: {blockers}" + ); + + fs::remove_dir_all(&proof_dir).ok(); + + let local_proof_dir = env::temp_dir().join(format!( + "directed-evolution-agent-answers-live-proof-local-blocked-{}", + std::process::id() + )); + fs::remove_dir_all(&local_proof_dir).ok(); + + let local_output = Command::new( + root.join("crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh"), + ) + .current_dir(&root) + .env("PROOF_DIR", &local_proof_dir) + .env("ALLOW_PRODUCTION_WRITE", "1") + .env("CONFIRM_AGENT_ANSWERS_LIVE_PROOF", "1") + .env("TEMPER_URL", "https://temperpaw.example.invalid") + .env("WORKER_TOKEN", "worker-token") + .env("DIRECTED_EVOLUTION_DIRECTION_ID", "direction-agent-answers") + .env("DIRECTED_EVOLUTION_ORGANISM_ID", "organism-agent-answers") + .env("START_LOCAL_WORKER", "1") + .env("PAW_CODEX_ENABLE_EXECUTION", "1") + .output() + .expect("run blocked local-worker Agent Answers Directed Evolution proof preflight"); + + assert!( + !local_output.status.success(), + "local-worker proof driver should block without Datadog query credentials" + ); + let local_blockers = read(local_proof_dir.join("human-blockers.json")); + assert!( + local_blockers.contains("env:dd_api_key") && local_blockers.contains("env:dd_app_key"), + "blocked local proof should name missing Datadog query credentials: {local_blockers}" + ); + fs::remove_dir_all(&local_proof_dir).ok(); + + let single_slot_proof_dir = env::temp_dir().join(format!( + "directed-evolution-agent-answers-live-proof-single-slot-blocked-{}", + std::process::id() + )); + fs::remove_dir_all(&single_slot_proof_dir).ok(); + + let single_slot_output = Command::new( + root.join("crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh"), + ) + .current_dir(&root) + .env("PROOF_DIR", &single_slot_proof_dir) + .env("ALLOW_PRODUCTION_WRITE", "1") + .env("CONFIRM_AGENT_ANSWERS_LIVE_PROOF", "1") + .env("TEMPER_URL", "https://temperpaw.example.invalid") + .env("WORKER_TOKEN", "worker-token") + .env("DIRECTED_EVOLUTION_DIRECTION_ID", "direction-agent-answers") + .env("DIRECTED_EVOLUTION_ORGANISM_ID", "organism-agent-answers") + .env("START_LOCAL_WORKER", "1") + .env("PAW_CODEX_ENABLE_EXECUTION", "1") + .env("WORKER_SLOT_COUNT", "1") + .env("DD_API_KEY", "dd-api") + .env("DD_APP_KEY", "dd-app") + .output() + .expect("run blocked single-slot Agent Answers Directed Evolution proof preflight"); + + assert!( + !single_slot_output.status.success(), + "local-worker proof driver should block a single-worker local proof" + ); + let single_slot_blockers = read(single_slot_proof_dir.join("human-blockers.json")); + assert!( + single_slot_blockers.contains("env:worker_slot_count") + && single_slot_blockers.contains("WORKER_SLOT_COUNT >= 2"), + "blocked local proof should require a real worker process pool: {single_slot_blockers}" + ); + fs::remove_dir_all(&single_slot_proof_dir).ok(); +} + +#[test] +fn directed_evolution_agent_answers_live_proof_precheck_can_pass_without_mutation() { + let root = repo_root(); + let proof_dir = env::temp_dir().join(format!( + "directed-evolution-agent-answers-live-proof-precheck-{}", + std::process::id() + )); + fs::remove_dir_all(&proof_dir).ok(); + + let output = Command::new( + root.join("crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh"), + ) + .current_dir(&root) + .env("PROOF_DIR", &proof_dir) + .env("PRECHECK_ONLY", "1") + .env("ALLOW_PRODUCTION_WRITE", "1") + .env("CONFIRM_AGENT_ANSWERS_LIVE_PROOF", "1") + .env("CONFIRM_PRODUCTION_DATADOG_QUERY_SECRETS", "1") + .env("TEMPER_URL", "https://temperpaw.example.invalid") + .env("WORKER_TOKEN", "worker-token") + .env("DIRECTED_EVOLUTION_DIRECTION_ID", "direction-agent-answers") + .env("DIRECTED_EVOLUTION_ORGANISM_ID", "organism-agent-answers") + .env("START_LOCAL_WORKER", "0") + .output() + .expect("run Agent Answers Directed Evolution proof precheck"); + + assert!( + output.status.success(), + "proof precheck should pass without mutating production\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!( + String::from_utf8_lossy(&output.stdout).contains("precheck ready"), + "precheck should report readiness" + ); + + let summary = read(proof_dir.join("summary.json")); + assert!( + summary.contains("\"status\": \"ready\"") + && summary.contains("\"would_create_live_episode\": true") + && summary.contains("\"no_mutation\": true") + && summary.contains("\"mandatory_datadog_observer_evidence\": true") + && summary.contains("\"mandatory_datadog_telemetry_evaluator_evidence\": true") + && summary + .contains("\"process_pool_model\": \"already-running production worker pool\""), + "precheck summary should expose ready no-mutation state: {summary}" + ); + let proof = read(proof_dir.join("proof.md")); + assert!( + proof.contains("Datadog observer/evaluator evidence"), + "precheck proof markdown should name the role-specific Datadog evidence gate: {proof}" + ); + assert!( + !proof_dir.join("episode-start.json").exists(), + "precheck must not call directed-evolution-start-episode" + ); + assert!( + !proof_dir + .join("agent-answers-episode-contract.json") + .exists(), + "precheck must not write a live episode contract" + ); + fs::remove_dir_all(&proof_dir).ok(); + + let local_pool_proof_dir = env::temp_dir().join(format!( + "directed-evolution-agent-answers-live-proof-local-pool-precheck-{}", + std::process::id() + )); + fs::remove_dir_all(&local_pool_proof_dir).ok(); + + let local_pool_output = Command::new( + root.join("crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh"), + ) + .current_dir(&root) + .env("PROOF_DIR", &local_pool_proof_dir) + .env("PRECHECK_ONLY", "1") + .env("ALLOW_PRODUCTION_WRITE", "1") + .env("CONFIRM_AGENT_ANSWERS_LIVE_PROOF", "1") + .env("TEMPER_URL", "https://temperpaw.example.invalid") + .env("WORKER_TOKEN", "worker-token") + .env("DIRECTED_EVOLUTION_DIRECTION_ID", "direction-agent-answers") + .env("DIRECTED_EVOLUTION_ORGANISM_ID", "organism-agent-answers") + .env("START_LOCAL_WORKER", "1") + .env("PAW_CODEX_ENABLE_EXECUTION", "1") + .env("WORKER_SLOT_COUNT", "3") + .env("DD_API_KEY", "dd-api") + .env("DD_APP_KEY", "dd-app") + .output() + .expect("run local worker pool Agent Answers Directed Evolution proof precheck"); + + assert!( + local_pool_output.status.success(), + "local worker pool proof precheck should pass without mutating production\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&local_pool_output.stdout), + String::from_utf8_lossy(&local_pool_output.stderr) + ); + let local_pool_summary = read(local_pool_proof_dir.join("summary.json")); + assert!( + local_pool_summary.contains("\"worker_slot_count\": 3") + && local_pool_summary.contains("\"mac-mini-codex-prod-slot-02\"") + && local_pool_summary.contains("\"local_worker_logs\"") + && local_pool_summary.contains("\"process_pool_model\": \"local worker process pool\"") + && local_pool_summary.contains("\"local_worker_process_pool\": true"), + "local precheck summary should expose the local worker pool contract: {local_pool_summary}" + ); + assert!( + !local_pool_proof_dir.join("episode-start.json").exists(), + "local precheck must not call directed-evolution-start-episode" + ); + assert!( + !local_pool_proof_dir + .join("agent-answers-episode-contract.json") + .exists(), + "local precheck must not write a live episode contract" + ); + fs::remove_dir_all(&local_pool_proof_dir).ok(); +} + +#[test] +fn directed_evolution_agent_answers_live_proof_self_test_exercises_datadog_gate() { + let root = repo_root(); + let proof_dir = env::temp_dir().join(format!( + "directed-evolution-agent-answers-live-proof-self-test-{}", + std::process::id() + )); + fs::remove_dir_all(&proof_dir).ok(); + + let output = Command::new( + root.join("crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh"), + ) + .current_dir(&root) + .env("SCRIPT_SELF_TEST", "1") + .env("PROOF_DIR", &proof_dir) + .output() + .expect("run Agent Answers Directed Evolution proof self-test"); + + assert!( + output.status.success(), + "proof driver self-test should pass\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!( + String::from_utf8_lossy(&output.stdout).contains("self-test passed"), + "self-test should report success" + ); + fs::remove_dir_all(&proof_dir).ok(); +} + #[test] fn repo_sweep_brief_smoke_exports_visual_proof_bundle() { let root = repo_root(); @@ -1461,6 +1818,10 @@ fn production_readiness_script_keeps_mac_mini_activation_checkable() { "INSTALL_LAUNCHD=1", "PAW_CODEX_ENABLE_EXECUTION=0", "PAW_CODEX_DOCTOR_EXEC_SMOKE", + "WORKER_SLOT_COUNT", + "slot_worker_id", + "slot_launchd_label", + "registering worker slot identities", ] { assert!( script.contains(needle), @@ -1535,6 +1896,14 @@ fn production_readiness_script_keeps_mac_mini_activation_checkable() { "INSTALL_LAUNCHD=0", "PAW_CODEX_ENABLE_EXECUTION=0", "PAW_CODEX_DOCTOR_EXEC_SMOKE=1", + "WORKER_SLOT_COUNT=\"${WORKER_SLOT_COUNT:-3}\"", + "worker_slot_count", + "worker_slot_ids", + "process_pool_model", + "launchd worker slot per local Codex process", + "REQUIRE_MAIN_ANCESTRY=0", + "real production readiness script keeps ancestry checking enabled by default", + "slot-${slot_suffix}.plist", "token_not_printed_to_readiness_log", "launchd_not_loaded", "production readiness smoke passed", @@ -1551,6 +1920,9 @@ fn production_readiness_script_keeps_mac_mini_activation_checkable() { "scripts/production-readiness-smoke.sh", "WRITE_LAUNCHD_PLIST=1", "INSTALL_LAUNCHD=1", + "WORKER_SLOT_COUNT=3", + "process-pool rendering", + "isolated worker slot identities", "does not print `WORKER_TOKEN`", ] { assert!( diff --git a/docs/adrs/0055-directed-evolution-hotload-variant-runtimes.md b/docs/adrs/0055-directed-evolution-hotload-variant-runtimes.md index 476e2e99a..824b18112 100644 --- a/docs/adrs/0055-directed-evolution-hotload-variant-runtimes.md +++ b/docs/adrs/0055-directed-evolution-hotload-variant-runtimes.md @@ -6,7 +6,7 @@ Accepted. ## Context -Directed Evolution now creates real `WorkItem` brain runs and can generate +Directed Evolution now creates real `paw-orchestration.WorkItem`s and can generate candidate app changes in local Codex worktrees. That is not enough for the intended pipeline: simulated users and reviewers must evaluate running variants, not just inspect local diffs. Railway deployments are costly and @@ -39,14 +39,14 @@ a hot-loaded Genesis runtime: target `StageResult` or `Variant` has already been eliminated. Generation can queue multiple stages up front, so late queued work must not waste background Codex runs or add confusing evidence after a candidate is dead. -- Background Codex brain runs use isolated `codex exec` sessions +- Background Codex worker runs use isolated `codex exec` sessions (`--ignore-user-config --ephemeral`) so tiny JSON-producing work items do not inherit unrelated local MCP servers, hooks, or long-lived session state. This is an agent capability surface, not a hidden orchestration layer. The episode still advances through Directed Evolution entities and WASM transitions; the worker only supplies the external agent work product and self-reports it -through the existing `WorkItem` and `BrainRun` actions. +through shared `WorkItem` and `WorkerRun` actions. ## Consequences @@ -63,6 +63,6 @@ through the existing `WorkItem` and `BrainRun` actions. survive hot-load, simulated-user trials, review, selection, and promotion. - Stale work cancellation keeps evaluation throughput focused on live variants while preserving entity-level auditability through `CancelWorkItem`. -- Isolated Codex invocation means background brain runs rely on the Codex auth +- Isolated Codex invocation means background worker runs rely on the Codex auth environment but not the operator's interactive config; any needed tool surface must be provided explicitly by the worker prompt or execution environment. diff --git a/docs/adrs/0056-directed-evolution-promotion-materializer.md b/docs/adrs/0056-directed-evolution-promotion-materializer.md index c1b319950..dbd697cd3 100644 --- a/docs/adrs/0056-directed-evolution-promotion-materializer.md +++ b/docs/adrs/0056-directed-evolution-promotion-materializer.md @@ -12,7 +12,7 @@ the live proof showed a gap: advancing the canonical Genesis ref, publishing the app version, and hot-loading the winner into the production tenant still required manual operator commands. -Promotion materialization is not a reasoning task. The selector brain already +Promotion materialization is not a reasoning task. The selector worker already chooses the winner from evaluated evidence. The worker needs to execute bounded external side effects and report evidence back through Directed Evolution entities. @@ -35,7 +35,7 @@ deterministic materialization role: `runtime_ref`, evidence, and digest so the Directed Evolution app can record materialization on the `Promotion`. -This role runs through the same WorkItem/BrainRun reporting path as other +This role runs through the same WorkItem/WorkerRun reporting path as other Directed Evolution worker roles, but it does not launch a background Codex session. diff --git a/docs/adrs/0057-directed-evolution-datadog-evidence-contract.md b/docs/adrs/0057-directed-evolution-datadog-evidence-contract.md index 89d9aa0cb..719d57838 100644 --- a/docs/adrs/0057-directed-evolution-datadog-evidence-contract.md +++ b/docs/adrs/0057-directed-evolution-datadog-evidence-contract.md @@ -7,13 +7,13 @@ ## Context Directed Evolution worker roles already record `EvidenceArtifact` entities for -Codex brain runs, but the evidence can be too generic. The artifact URI falls +Codex worker runs, but the evidence can be too generic. The artifact URI falls back to a TemperPaw-local placeholder unless the agent happens to return a single `evidence_uri`, and the role prompts do not require a structured Datadog evidence scope. That is not enough for the Directed Evolution product contract. Observer, -reviewer, and simulated-user brains must be able to explain what they learned +reviewer, and simulated-user workers must be able to explain what they learned from production telemetry and live variant execution. Mission Control should show evidence that can be inspected, not just summaries. @@ -39,9 +39,9 @@ environment, and the Datadog query used to find matching worker telemetry. ## Consequences -- Directed Evolution evidence becomes Datadog-inspectable when a brain used +- Directed Evolution evidence becomes Datadog-inspectable when a worker used Datadog, rather than only locally summarized. -- Mission Control can surface a real observability URL for a variant or brain +- Mission Control can surface a real observability URL for a variant or worker result without inventing UI state. - The contract stays backward-compatible with existing worker outputs. diff --git a/docs/adrs/0060-directed-evolution-public-runtime-prompts.md b/docs/adrs/0060-directed-evolution-public-runtime-prompts.md index cd8dd343d..34a19d71b 100644 --- a/docs/adrs/0060-directed-evolution-public-runtime-prompts.md +++ b/docs/adrs/0060-directed-evolution-public-runtime-prompts.md @@ -15,7 +15,7 @@ Evaluation work items carried a `RuntimeRef` for a hot-loaded Genesis tenant, bu their prompt still said `TemperApiBase: http://127.0.0.1:8080` when the app tenant did not have a resolved `temper_public_api_url` secret. One simulated-user agent compensated by starting `temper serve` locally and leaving it in the -foreground, which stalled the brain run until the local server process was +foreground, which stalled the worker run until the local server process was terminated. The local worker already knows the public Genesis URL used for hot-loading. diff --git a/docs/adrs/0062-directed-evolution-worker-concurrency.md b/docs/adrs/0062-directed-evolution-worker-concurrency.md new file mode 100644 index 000000000..362d1d1f4 --- /dev/null +++ b/docs/adrs/0062-directed-evolution-worker-concurrency.md @@ -0,0 +1,42 @@ +# ADR-0062: Directed Evolution Worker Concurrency + +- Status: Accepted +- Date: 2026-06-08 +- Deciders: TemperPaw maintainers + +## Context + +The local `paw-codex-worker` can queue many Directed Evolution `WorkItems`, +but boot/backlog pickup processed queued work items with a sequential +`for ... await` loop. Setting `MAX_CONCURRENT_RUNS=4` therefore did not make a +single worker process run four simulated users at once; it only changed the +reported worker capacity. Operators had to start multiple worker processes to +see parallel seed journeys. + +## Decision + +Directed Evolution boot/backlog pickup will process queued `WorkItems` with a +bounded unordered async executor capped by `MAX_CONCURRENT_RUNS`. Each work item +still uses the existing claim action before execution, so concurrent workers +can race safely and losers observe the already-claimed state. + +The worker event stream remains token-only by default. Local Genesis/proxy +setups that require an observe principal for SSE may set +`PAW_CODEX_EVENT_STREAM_PRINCIPAL_KIND` and +`PAW_CODEX_EVENT_STREAM_PRINCIPAL_ID`; those headers are applied only to event +stream requests and do not change WorkItem action identity. + +## Consequences + +A single local worker process can run multiple simulated-user journeys in +parallel. Worker process pools remain useful for isolation or capacity, but are +no longer required just to satisfy the concurrency setting. + +Local workers can also connect to observe-protected streams without weakening +normal WorkItem mutation headers. + +## Non-Goals + +This does not change `WorkerRun`, `ReviewRun`, or `EvaluationRun` pickup +semantics. It also does not replace polling fallback; event-stream delivery is +handled separately by the runtime/proxy path. diff --git a/docs/adrs/0064-directed-evolution-proof-diffs-and-telemetry.md b/docs/adrs/0064-directed-evolution-proof-diffs-and-telemetry.md new file mode 100644 index 000000000..7ab6a08ed --- /dev/null +++ b/docs/adrs/0064-directed-evolution-proof-diffs-and-telemetry.md @@ -0,0 +1,22 @@ +# ADR 0064: Directed Evolution Proof Diffs And Telemetry + +## Status + +Accepted + +## Context + +Directed Evolution proof runs must be reviewable from Temper/Genesis state without relying on chat narration. Variant generators currently commit app changes, but mutation entities only receive changed filenames and a diff reference. Telemetry evaluators also need stable Datadog join fields that line up with the runtime headers sent by simulated users and evaluators. + +## Decision + +- Variant-generator finalization stages the repo, captures the staged unified patch, commits, and returns `diff_patch` with `changed_files` and `diff_ref`. +- Worker evidence summaries include Datadog context derived from the WorkItem correlation graph, not only the WorkItem id. +- Worker prompts keep simulated users observational: they use the app and report journeys, while evaluator roles decide pass/fail. +- Default human episode setup treats Datadog runtime errors as a hard elimination metric alongside state/spec regressions. + +## Consequences + +- Genesis can materialize GitHub-like code diffs from mutation entities. +- Datadog queries are reproducible from evidence records and can be joined by tenant, episode, generation, variant, stage, trial, and work item. +- Missing Datadog evidence can be failed by the app router instead of remaining an opaque link. diff --git a/docs/adrs/0065-canonical-genesis-worker-pool.md b/docs/adrs/0065-canonical-genesis-worker-pool.md new file mode 100644 index 000000000..f74f1976d --- /dev/null +++ b/docs/adrs/0065-canonical-genesis-worker-pool.md @@ -0,0 +1,31 @@ +# ADR 0065: Canonical Genesis Worker Pool + +## Status + +Accepted + +## Context + +TemperPaw currently contains both platform worker code and several Temper-native app bundles. Directed Evolution review made this confusing: the user wants Genesis to be the canonical home for app bundles, while TemperPaw remains the worker/agent platform that can install pinned app refs and run worker processes. + +The Codex worker also had a V1-only `MAX_CONCURRENT_RUNS == 1` restriction. That blocks real simulated-user and evaluator concurrency, and it makes local Codex look like a singleton instead of a pool of Temper-claimed worker slots. + +## Decision + +- Production app bundles move to Genesis. TemperPaw keeps platform code, worker crates, test fixtures, bootstrap/install references, and docs. +- The shared execution app is published as `temperpaw/paw-orchestration`. Its + bundle-local app name remains `paw-orchestration`, and it owns + `WorkerProvider`, `WorkerAgent`, `WorkItem`, and `WorkerRun`. +- Paw Patrol no longer owns generic worker concepts long-term. It links to shared `WorkItemId` and `WorkerRunId` for patrol-specific work. +- `paw-agent` keeps native TemperPaw session entities. `WorkerRun.SessionId` is optional and only set for TemperPaw-native agent execution. +- Local Codex concurrency is modeled as multiple worker processes. Each process registers as one `WorkerAgent` slot and claims one `WorkItem` at a time. +- Each local Codex process uses an explicit worker profile. `PAW_CODEX_MODEL` pins the model used by `codex exec`, and `PAW_CODEX_EXEC_TIMEOUT_SECS` bounds each run. This keeps process-pool behavior reproducible and makes WorkerRun provenance say which model profile was actually used. +- Selector and promoter work uses exclusive keys so it is serialized per episode/organism. Observer, simulated-user, and evaluator work can run concurrently. +- Directed Evolution observer and telemetry evaluator roles require structured Datadog evidence. Missing Datadog query/window/result/interpretation evidence is a failure, not an informational warning. + +## Consequences + +- Users can scale local Codex by starting more worker slots instead of relying on hidden concurrency inside one process. +- Execution provenance uses worker/provider/run language everywhere, avoiding `BrainRun` as an entity name. +- TemperPaw app directories become migration/bootstrap surfaces until their canonical Genesis refs are published and pinned. +- Existing local proof data may still contain old names; worker and UI code should remain tolerant while presenting new language. diff --git a/docs/deployment.md b/docs/deployment.md index a8e4bb834..563d55ffe 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -42,6 +42,11 @@ continues to readiness using the installed state already present in the database; operators can repair the pinned Genesis app version and redeploy without any database reset. -## Optional Datadog +## Datadog Deploy the collector config in `scripts/otel-collector-railway.yaml` as a second Railway service and point `OTEL_EXPORTER_OTLP_ENDPOINT` at the collector's internal hostname. + +Datadog is mandatory for Directed Evolution observer and telemetry-evaluator +roles. Those roles must record the query, time window, result count, +interpretation, zero-result meaning, and Datadog URL before they can propose a +direction or pass a telemetry stage. diff --git a/os-apps/README.md b/os-apps/README.md new file mode 100644 index 000000000..15c33c7ee --- /dev/null +++ b/os-apps/README.md @@ -0,0 +1,10 @@ +# Repo-Local Paw Apps + +Genesis is the canonical source for production TemperPaw app bundles. + +This `os-apps/` directory remains only for bootstrap, migration, local +development, and tests while canonical Genesis refs are published and pinned. It +is not a production app catalog and it is not an app-source mirror. + +Production TemperPaw instances should install configured pinned Genesis refs and +recover already-installed app state from the backing Temper store. diff --git a/os-apps/paw-agent/wasm/agent_reply/Cargo.toml b/os-apps/paw-agent/wasm/agent_reply/Cargo.toml index 43b53ad78..348025049 100644 --- a/os-apps/paw-agent/wasm/agent_reply/Cargo.toml +++ b/os-apps/paw-agent/wasm/agent_reply/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/coding_agent_runner/Cargo.toml b/os-apps/paw-agent/wasm/coding_agent_runner/Cargo.toml index 87f8f203f..fc4997299 100644 --- a/os-apps/paw-agent/wasm/coding_agent_runner/Cargo.toml +++ b/os-apps/paw-agent/wasm/coding_agent_runner/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/context_compactor/Cargo.lock b/os-apps/paw-agent/wasm/context_compactor/Cargo.lock index f92452918..3042a92fe 100644 --- a/os-apps/paw-agent/wasm/context_compactor/Cargo.lock +++ b/os-apps/paw-agent/wasm/context_compactor/Cargo.lock @@ -113,7 +113,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-agent/wasm/context_compactor/Cargo.toml b/os-apps/paw-agent/wasm/context_compactor/Cargo.toml index bb7f28e8f..0bd00bb92 100644 --- a/os-apps/paw-agent/wasm/context_compactor/Cargo.toml +++ b/os-apps/paw-agent/wasm/context_compactor/Cargo.toml @@ -9,7 +9,7 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } session-tree-lib = { path = "../session-tree-lib" } wasm-helpers = { path = "../wasm-helpers" } openai-codex-wire = { path = "../openai-codex-wire" } diff --git a/os-apps/paw-agent/wasm/context_preparer/Cargo.lock b/os-apps/paw-agent/wasm/context_preparer/Cargo.lock index 65b400f1f..bd365b5e3 100644 --- a/os-apps/paw-agent/wasm/context_preparer/Cargo.lock +++ b/os-apps/paw-agent/wasm/context_preparer/Cargo.lock @@ -117,7 +117,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-agent/wasm/context_preparer/Cargo.toml b/os-apps/paw-agent/wasm/context_preparer/Cargo.toml index 0709e07bf..d97f08dc9 100644 --- a/os-apps/paw-agent/wasm/context_preparer/Cargo.toml +++ b/os-apps/paw-agent/wasm/context_preparer/Cargo.toml @@ -9,7 +9,7 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } session-tree-lib = { path = "../session-tree-lib" } session-turn-artifacts = { path = "../session-turn-artifacts" } tool-catalog = { path = "../tool-catalog" } diff --git a/os-apps/paw-agent/wasm/cron_compute_next/Cargo.toml b/os-apps/paw-agent/wasm/cron_compute_next/Cargo.toml index 00071ceca..d3abc541f 100644 --- a/os-apps/paw-agent/wasm/cron_compute_next/Cargo.toml +++ b/os-apps/paw-agent/wasm/cron_compute_next/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/emit_ots_trajectory/Cargo.lock b/os-apps/paw-agent/wasm/emit_ots_trajectory/Cargo.lock index 5d9fe62ac..cc93977d1 100644 --- a/os-apps/paw-agent/wasm/emit_ots_trajectory/Cargo.lock +++ b/os-apps/paw-agent/wasm/emit_ots_trajectory/Cargo.lock @@ -98,7 +98,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-agent/wasm/emit_ots_trajectory/Cargo.toml b/os-apps/paw-agent/wasm/emit_ots_trajectory/Cargo.toml index a5ab2cbf5..fd1f4523d 100644 --- a/os-apps/paw-agent/wasm/emit_ots_trajectory/Cargo.toml +++ b/os-apps/paw-agent/wasm/emit_ots_trajectory/Cargo.toml @@ -9,6 +9,6 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } wasm-helpers = { path = "../wasm-helpers" } serde_json = "1" diff --git a/os-apps/paw-agent/wasm/guest_observability_probe/Cargo.toml b/os-apps/paw-agent/wasm/guest_observability_probe/Cargo.toml index bcaa66493..31b4b366b 100644 --- a/os-apps/paw-agent/wasm/guest_observability_probe/Cargo.toml +++ b/os-apps/paw-agent/wasm/guest_observability_probe/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } serde_json = "1" diff --git a/os-apps/paw-agent/wasm/monty_repl/Cargo.toml b/os-apps/paw-agent/wasm/monty_repl/Cargo.toml index 5e86a8b4f..d56f3f0a1 100644 --- a/os-apps/paw-agent/wasm/monty_repl/Cargo.toml +++ b/os-apps/paw-agent/wasm/monty_repl/Cargo.toml @@ -9,7 +9,7 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } session-tree-lib = { path = "../session-tree-lib" } wasm-helpers = { path = "../wasm-helpers" } tool-catalog = { path = "../tool-catalog" } diff --git a/os-apps/paw-agent/wasm/openai_codex_auth/Cargo.toml b/os-apps/paw-agent/wasm/openai_codex_auth/Cargo.toml index 3f7f31d4c..6d1865b07 100644 --- a/os-apps/paw-agent/wasm/openai_codex_auth/Cargo.toml +++ b/os-apps/paw-agent/wasm/openai_codex_auth/Cargo.toml @@ -9,7 +9,7 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } wasm-helpers = { path = "../wasm-helpers" } openai-codex-wire = { path = "../openai-codex-wire" } serde_json = "1" diff --git a/os-apps/paw-agent/wasm/plan_approval_handler/Cargo.toml b/os-apps/paw-agent/wasm/plan_approval_handler/Cargo.toml index e5c44092b..c5d800fee 100644 --- a/os-apps/paw-agent/wasm/plan_approval_handler/Cargo.toml +++ b/os-apps/paw-agent/wasm/plan_approval_handler/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/plan_review_feedback_handler/Cargo.toml b/os-apps/paw-agent/wasm/plan_review_feedback_handler/Cargo.toml index 8cc3d341b..5d93a442c 100644 --- a/os-apps/paw-agent/wasm/plan_review_feedback_handler/Cargo.toml +++ b/os-apps/paw-agent/wasm/plan_review_feedback_handler/Cargo.toml @@ -11,5 +11,5 @@ crate-type = ["cdylib"] [dependencies] serde_json = "1" session-tree-lib = { path = "../session-tree-lib" } -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/provider_auth_gate/Cargo.toml b/os-apps/paw-agent/wasm/provider_auth_gate/Cargo.toml index 748bebce6..75206f3fb 100644 --- a/os-apps/paw-agent/wasm/provider_auth_gate/Cargo.toml +++ b/os-apps/paw-agent/wasm/provider_auth_gate/Cargo.toml @@ -9,6 +9,6 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } wasm-helpers = { path = "../wasm-helpers" } serde_json = "1" diff --git a/os-apps/paw-agent/wasm/provider_caller/Cargo.lock b/os-apps/paw-agent/wasm/provider_caller/Cargo.lock index 9ba1bec2c..0a96cd7db 100644 --- a/os-apps/paw-agent/wasm/provider_caller/Cargo.lock +++ b/os-apps/paw-agent/wasm/provider_caller/Cargo.lock @@ -124,7 +124,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-agent/wasm/provider_caller/Cargo.toml b/os-apps/paw-agent/wasm/provider_caller/Cargo.toml index a2c2fae27..a89ff8bd7 100644 --- a/os-apps/paw-agent/wasm/provider_caller/Cargo.toml +++ b/os-apps/paw-agent/wasm/provider_caller/Cargo.toml @@ -9,7 +9,7 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } session-tree-lib = { path = "../session-tree-lib" } session-turn-artifacts = { path = "../session-turn-artifacts" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/provider_response_applier/Cargo.lock b/os-apps/paw-agent/wasm/provider_response_applier/Cargo.lock index 9c191b43f..4784d7fc2 100644 --- a/os-apps/paw-agent/wasm/provider_response_applier/Cargo.lock +++ b/os-apps/paw-agent/wasm/provider_response_applier/Cargo.lock @@ -116,7 +116,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-agent/wasm/provider_response_applier/Cargo.toml b/os-apps/paw-agent/wasm/provider_response_applier/Cargo.toml index 590cdcd1a..e456b12be 100644 --- a/os-apps/paw-agent/wasm/provider_response_applier/Cargo.toml +++ b/os-apps/paw-agent/wasm/provider_response_applier/Cargo.toml @@ -9,7 +9,7 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } session-tree-lib = { path = "../session-tree-lib" } session-turn-artifacts = { path = "../session-turn-artifacts" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/request_approval/Cargo.toml b/os-apps/paw-agent/wasm/request_approval/Cargo.toml index 49ebf0f15..154e2262b 100644 --- a/os-apps/paw-agent/wasm/request_approval/Cargo.toml +++ b/os-apps/paw-agent/wasm/request_approval/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/request_plan_review/Cargo.toml b/os-apps/paw-agent/wasm/request_plan_review/Cargo.toml index 026030664..e2f1e03db 100644 --- a/os-apps/paw-agent/wasm/request_plan_review/Cargo.toml +++ b/os-apps/paw-agent/wasm/request_plan_review/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/sandbox_provisioner/Cargo.toml b/os-apps/paw-agent/wasm/sandbox_provisioner/Cargo.toml index 59f1e2d69..d2cfd4665 100644 --- a/os-apps/paw-agent/wasm/sandbox_provisioner/Cargo.toml +++ b/os-apps/paw-agent/wasm/sandbox_provisioner/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/session_link_monitor/Cargo.toml b/os-apps/paw-agent/wasm/session_link_monitor/Cargo.toml index f4f43d948..c5afe42bc 100644 --- a/os-apps/paw-agent/wasm/session_link_monitor/Cargo.toml +++ b/os-apps/paw-agent/wasm/session_link_monitor/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/session_recoverer/Cargo.toml b/os-apps/paw-agent/wasm/session_recoverer/Cargo.toml index c0252a3ba..0daba1b67 100644 --- a/os-apps/paw-agent/wasm/session_recoverer/Cargo.toml +++ b/os-apps/paw-agent/wasm/session_recoverer/Cargo.toml @@ -9,6 +9,6 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } session-tree-lib = { path = "../session-tree-lib" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/steering_checker/Cargo.toml b/os-apps/paw-agent/wasm/steering_checker/Cargo.toml index fd2c78a1e..e7d72f042 100644 --- a/os-apps/paw-agent/wasm/steering_checker/Cargo.toml +++ b/os-apps/paw-agent/wasm/steering_checker/Cargo.toml @@ -9,6 +9,6 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } session-tree-lib = { path = "../session-tree-lib" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/wasm-helpers/Cargo.toml b/os-apps/paw-agent/wasm/wasm-helpers/Cargo.toml index 68c135f4e..02ecc93f6 100644 --- a/os-apps/paw-agent/wasm/wasm-helpers/Cargo.toml +++ b/os-apps/paw-agent/wasm/wasm-helpers/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["rlib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } serde_json = "1" diff --git a/os-apps/paw-agent/wasm/workspace_provisioner/Cargo.toml b/os-apps/paw-agent/wasm/workspace_provisioner/Cargo.toml index cf696944d..c9204a828 100644 --- a/os-apps/paw-agent/wasm/workspace_provisioner/Cargo.toml +++ b/os-apps/paw-agent/wasm/workspace_provisioner/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/workspace_restorer/Cargo.toml b/os-apps/paw-agent/wasm/workspace_restorer/Cargo.toml index d65b372ca..89244b809 100644 --- a/os-apps/paw-agent/wasm/workspace_restorer/Cargo.toml +++ b/os-apps/paw-agent/wasm/workspace_restorer/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-autoreason/wasm/initialize_tournament/Cargo.lock b/os-apps/paw-autoreason/wasm/initialize_tournament/Cargo.lock index 74e331b04..33d304b57 100644 --- a/os-apps/paw-autoreason/wasm/initialize_tournament/Cargo.lock +++ b/os-apps/paw-autoreason/wasm/initialize_tournament/Cargo.lock @@ -97,7 +97,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-autoreason/wasm/initialize_tournament/Cargo.toml b/os-apps/paw-autoreason/wasm/initialize_tournament/Cargo.toml index 4a84547d9..bc04d4c62 100644 --- a/os-apps/paw-autoreason/wasm/initialize_tournament/Cargo.toml +++ b/os-apps/paw-autoreason/wasm/initialize_tournament/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } serde_json = "1" diff --git a/os-apps/paw-autoreason/wasm/run_round/Cargo.lock b/os-apps/paw-autoreason/wasm/run_round/Cargo.lock index ada597109..5d85b2085 100644 --- a/os-apps/paw-autoreason/wasm/run_round/Cargo.lock +++ b/os-apps/paw-autoreason/wasm/run_round/Cargo.lock @@ -97,7 +97,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-autoreason/wasm/run_round/Cargo.toml b/os-apps/paw-autoreason/wasm/run_round/Cargo.toml index 0513de596..b4b4c3623 100644 --- a/os-apps/paw-autoreason/wasm/run_round/Cargo.toml +++ b/os-apps/paw-autoreason/wasm/run_round/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } serde_json = "1" diff --git a/os-apps/paw-autoreason/wasm/tally_votes/Cargo.lock b/os-apps/paw-autoreason/wasm/tally_votes/Cargo.lock index a18200edf..d0ef590f4 100644 --- a/os-apps/paw-autoreason/wasm/tally_votes/Cargo.lock +++ b/os-apps/paw-autoreason/wasm/tally_votes/Cargo.lock @@ -97,7 +97,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-autoreason/wasm/tally_votes/Cargo.toml b/os-apps/paw-autoreason/wasm/tally_votes/Cargo.toml index 170fdb097..822edc1b2 100644 --- a/os-apps/paw-autoreason/wasm/tally_votes/Cargo.toml +++ b/os-apps/paw-autoreason/wasm/tally_votes/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } serde_json = "1" diff --git a/os-apps/paw-channels/wasm/channel_connect/Cargo.toml b/os-apps/paw-channels/wasm/channel_connect/Cargo.toml index 538c5a2c4..e1f28017c 100644 --- a/os-apps/paw-channels/wasm/channel_connect/Cargo.toml +++ b/os-apps/paw-channels/wasm/channel_connect/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-channels/wasm/route_message/Cargo.toml b/os-apps/paw-channels/wasm/route_message/Cargo.toml index 87701eed2..0ffc0dff6 100644 --- a/os-apps/paw-channels/wasm/route_message/Cargo.toml +++ b/os-apps/paw-channels/wasm/route_message/Cargo.toml @@ -11,4 +11,4 @@ crate-type = ["cdylib", "rlib"] [dependencies] session-tree-lib = { path = "../../../paw-agent/wasm/session-tree-lib" } wasm-helpers = { path = "../../../paw-agent/wasm/wasm-helpers" } -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-channels/wasm/send_reply/Cargo.toml b/os-apps/paw-channels/wasm/send_reply/Cargo.toml index 811c9828b..2ea80774b 100644 --- a/os-apps/paw-channels/wasm/send_reply/Cargo.toml +++ b/os-apps/paw-channels/wasm/send_reply/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-channels/wasm/transport_reconcile/Cargo.lock b/os-apps/paw-channels/wasm/transport_reconcile/Cargo.lock index 082050d48..ada76bd21 100644 --- a/os-apps/paw-channels/wasm/transport_reconcile/Cargo.lock +++ b/os-apps/paw-channels/wasm/transport_reconcile/Cargo.lock @@ -89,7 +89,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-channels/wasm/transport_reconcile/Cargo.toml b/os-apps/paw-channels/wasm/transport_reconcile/Cargo.toml index 1bccf687a..2f95013bc 100644 --- a/os-apps/paw-channels/wasm/transport_reconcile/Cargo.toml +++ b/os-apps/paw-channels/wasm/transport_reconcile/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-consilium/wasm/check_and_synthesize/Cargo.lock b/os-apps/paw-consilium/wasm/check_and_synthesize/Cargo.lock index 17a710afd..f612abaa4 100644 --- a/os-apps/paw-consilium/wasm/check_and_synthesize/Cargo.lock +++ b/os-apps/paw-consilium/wasm/check_and_synthesize/Cargo.lock @@ -97,7 +97,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-consilium/wasm/check_and_synthesize/Cargo.toml b/os-apps/paw-consilium/wasm/check_and_synthesize/Cargo.toml index 634148f1c..cf7271874 100644 --- a/os-apps/paw-consilium/wasm/check_and_synthesize/Cargo.toml +++ b/os-apps/paw-consilium/wasm/check_and_synthesize/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } serde_json = "1" diff --git a/os-apps/paw-consilium/wasm/spawn_perspectives/Cargo.lock b/os-apps/paw-consilium/wasm/spawn_perspectives/Cargo.lock index 8f79d8514..19e588279 100644 --- a/os-apps/paw-consilium/wasm/spawn_perspectives/Cargo.lock +++ b/os-apps/paw-consilium/wasm/spawn_perspectives/Cargo.lock @@ -97,7 +97,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-consilium/wasm/spawn_perspectives/Cargo.toml b/os-apps/paw-consilium/wasm/spawn_perspectives/Cargo.toml index d6afd6ccd..2d6e38922 100644 --- a/os-apps/paw-consilium/wasm/spawn_perspectives/Cargo.toml +++ b/os-apps/paw-consilium/wasm/spawn_perspectives/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } serde_json = "1" diff --git a/os-apps/paw-foresight/wasm/advance_step/Cargo.toml b/os-apps/paw-foresight/wasm/advance_step/Cargo.toml index 517070f5f..32ff4401f 100644 --- a/os-apps/paw-foresight/wasm/advance_step/Cargo.toml +++ b/os-apps/paw-foresight/wasm/advance_step/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-foresight/wasm/handle_convergence/Cargo.lock b/os-apps/paw-foresight/wasm/handle_convergence/Cargo.lock index b9f3a86f9..d8aa7d15e 100644 --- a/os-apps/paw-foresight/wasm/handle_convergence/Cargo.lock +++ b/os-apps/paw-foresight/wasm/handle_convergence/Cargo.lock @@ -96,7 +96,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-foresight/wasm/handle_convergence/Cargo.toml b/os-apps/paw-foresight/wasm/handle_convergence/Cargo.toml index a74303d03..7b86645fc 100644 --- a/os-apps/paw-foresight/wasm/handle_convergence/Cargo.toml +++ b/os-apps/paw-foresight/wasm/handle_convergence/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-foresight/wasm/handle_probe_done/Cargo.lock b/os-apps/paw-foresight/wasm/handle_probe_done/Cargo.lock index ce277b4ee..d2464e5f4 100644 --- a/os-apps/paw-foresight/wasm/handle_probe_done/Cargo.lock +++ b/os-apps/paw-foresight/wasm/handle_probe_done/Cargo.lock @@ -96,7 +96,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-foresight/wasm/handle_probe_done/Cargo.toml b/os-apps/paw-foresight/wasm/handle_probe_done/Cargo.toml index d44fb19c1..32229d5fc 100644 --- a/os-apps/paw-foresight/wasm/handle_probe_done/Cargo.toml +++ b/os-apps/paw-foresight/wasm/handle_probe_done/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-foresight/wasm/handle_projection_updated/Cargo.lock b/os-apps/paw-foresight/wasm/handle_projection_updated/Cargo.lock index 30c8c3662..caf3d4e56 100644 --- a/os-apps/paw-foresight/wasm/handle_projection_updated/Cargo.lock +++ b/os-apps/paw-foresight/wasm/handle_projection_updated/Cargo.lock @@ -96,7 +96,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-foresight/wasm/handle_projection_updated/Cargo.toml b/os-apps/paw-foresight/wasm/handle_projection_updated/Cargo.toml index 0fcd33320..97099ddd9 100644 --- a/os-apps/paw-foresight/wasm/handle_projection_updated/Cargo.toml +++ b/os-apps/paw-foresight/wasm/handle_projection_updated/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-foresight/wasm/seed_model/Cargo.toml b/os-apps/paw-foresight/wasm/seed_model/Cargo.toml index c605fc377..ac704a315 100644 --- a/os-apps/paw-foresight/wasm/seed_model/Cargo.toml +++ b/os-apps/paw-foresight/wasm/seed_model/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-foresight/wasm/spawn_probes/Cargo.lock b/os-apps/paw-foresight/wasm/spawn_probes/Cargo.lock index c6a57114c..9f28bb9b4 100644 --- a/os-apps/paw-foresight/wasm/spawn_probes/Cargo.lock +++ b/os-apps/paw-foresight/wasm/spawn_probes/Cargo.lock @@ -96,7 +96,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-foresight/wasm/spawn_probes/Cargo.toml b/os-apps/paw-foresight/wasm/spawn_probes/Cargo.toml index b3467ea0f..4eae3ea1e 100644 --- a/os-apps/paw-foresight/wasm/spawn_probes/Cargo.toml +++ b/os-apps/paw-foresight/wasm/spawn_probes/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-foresight/wasm/spawn_seed_agent/Cargo.lock b/os-apps/paw-foresight/wasm/spawn_seed_agent/Cargo.lock index d1a21c538..d3902b225 100644 --- a/os-apps/paw-foresight/wasm/spawn_seed_agent/Cargo.lock +++ b/os-apps/paw-foresight/wasm/spawn_seed_agent/Cargo.lock @@ -97,7 +97,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-foresight/wasm/spawn_seed_agent/Cargo.toml b/os-apps/paw-foresight/wasm/spawn_seed_agent/Cargo.toml index 2181eb5bc..7bbe2177f 100644 --- a/os-apps/paw-foresight/wasm/spawn_seed_agent/Cargo.toml +++ b/os-apps/paw-foresight/wasm/spawn_seed_agent/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } serde_json = "1" diff --git a/os-apps/paw-fs/wasm/artifact_batch_apply/Cargo.toml b/os-apps/paw-fs/wasm/artifact_batch_apply/Cargo.toml index 6666e1ce7..47c16ba3c 100644 --- a/os-apps/paw-fs/wasm/artifact_batch_apply/Cargo.toml +++ b/os-apps/paw-fs/wasm/artifact_batch_apply/Cargo.toml @@ -9,6 +9,6 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/os-apps/paw-fs/wasm/workspace_fs/Cargo.toml b/os-apps/paw-fs/wasm/workspace_fs/Cargo.toml index d989c6bbf..e91f99f68 100644 --- a/os-apps/paw-fs/wasm/workspace_fs/Cargo.toml +++ b/os-apps/paw-fs/wasm/workspace_fs/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } serde_json = "1" diff --git a/os-apps/paw-heal/wasm/alert_opener/Cargo.toml b/os-apps/paw-heal/wasm/alert_opener/Cargo.toml index 6398debdd..ce7298c4a 100644 --- a/os-apps/paw-heal/wasm/alert_opener/Cargo.toml +++ b/os-apps/paw-heal/wasm/alert_opener/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-heal/wasm/alert_verifier/Cargo.toml b/os-apps/paw-heal/wasm/alert_verifier/Cargo.toml index cbe9ed4a8..90fbf02ec 100644 --- a/os-apps/paw-heal/wasm/alert_verifier/Cargo.toml +++ b/os-apps/paw-heal/wasm/alert_verifier/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-heal/wasm/cicd_initiator/Cargo.toml b/os-apps/paw-heal/wasm/cicd_initiator/Cargo.toml index 3141e149e..a54d19f8d 100644 --- a/os-apps/paw-heal/wasm/cicd_initiator/Cargo.toml +++ b/os-apps/paw-heal/wasm/cicd_initiator/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-heal/wasm/cicd_merger/Cargo.toml b/os-apps/paw-heal/wasm/cicd_merger/Cargo.toml index 0f13205fa..1c56ce48c 100644 --- a/os-apps/paw-heal/wasm/cicd_merger/Cargo.toml +++ b/os-apps/paw-heal/wasm/cicd_merger/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-heal/wasm/deployment_tracker/Cargo.toml b/os-apps/paw-heal/wasm/deployment_tracker/Cargo.toml index f5618b681..3abbcde20 100644 --- a/os-apps/paw-heal/wasm/deployment_tracker/Cargo.toml +++ b/os-apps/paw-heal/wasm/deployment_tracker/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-heal/wasm/heal_reporter/Cargo.toml b/os-apps/paw-heal/wasm/heal_reporter/Cargo.toml index 72dfcf488..9486b5177 100644 --- a/os-apps/paw-heal/wasm/heal_reporter/Cargo.toml +++ b/os-apps/paw-heal/wasm/heal_reporter/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-ingest/wasm/process_webhook/Cargo.toml b/os-apps/paw-ingest/wasm/process_webhook/Cargo.toml index 834adc50a..9fb6abf5e 100644 --- a/os-apps/paw-ingest/wasm/process_webhook/Cargo.toml +++ b/os-apps/paw-ingest/wasm/process_webhook/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-ingest/wasm/route_webhook/Cargo.toml b/os-apps/paw-ingest/wasm/route_webhook/Cargo.toml index d193c3e9c..fe6e8379f 100644 --- a/os-apps/paw-ingest/wasm/route_webhook/Cargo.toml +++ b/os-apps/paw-ingest/wasm/route_webhook/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-ingest/wasm/validate_webhook/Cargo.toml b/os-apps/paw-ingest/wasm/validate_webhook/Cargo.toml index 2d9d00bd6..ec23197bd 100644 --- a/os-apps/paw-ingest/wasm/validate_webhook/Cargo.toml +++ b/os-apps/paw-ingest/wasm/validate_webhook/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-managed-agents/adrs/004-directed-evolution-local-codex-worker.md b/os-apps/paw-managed-agents/adrs/004-directed-evolution-local-codex-worker.md index 489a9940d..51ec49d0f 100644 --- a/os-apps/paw-managed-agents/adrs/004-directed-evolution-local-codex-worker.md +++ b/os-apps/paw-managed-agents/adrs/004-directed-evolution-local-codex-worker.md @@ -5,9 +5,9 @@ ## Context -Directed Evolution needs many brain instances: observer, direction framer, +Directed Evolution needs many worker-backed agent instances: observer, direction framer, variant generator, simulated user, reviewer, selector, and narrator. The human -facing director brain remains the live Codex chat session, but the background +facing director remains the live Codex chat session, but the background roles must run as bounded jobs and write structured results back to Temper. Deployed Genesis or Railway services should not run Codex directly in v1. @@ -20,14 +20,14 @@ entities, while TemperPaw claims explicit work and self-reports completion. Add a Directed Evolution worker path to the managed-agent layer. The worker claims `WorkItem` entities from the Directed Evolution control plane, starts a -bounded Codex session for the requested brain role, captures evidence and +bounded Codex session for the requested worker role, captures evidence and observability metadata, and records results back through entity actions. The worker does not own the Directed Evolution state machine. It only moves -state by dispatching explicit actions on `WorkItem`, `BrainRun`, `Variant`, +state by dispatching explicit actions on shared `WorkItem`, `WorkerRun`, `Variant`, `StageResult`, `Trial`, `Direction`, `Episode`, or `Promotion` entities. -### Brain Roles +### Worker Roles The worker supports role-specific prompts and output schemas for: @@ -55,7 +55,7 @@ winning behavior. Every claimed work item records: - `work_item_id` -- `brain_run_id` +- `worker_run_id` - role - parent session id, if any - target entity ids @@ -79,7 +79,7 @@ entities accept the selection and dispatch the promotion action. becoming a second orchestration engine. - The same Codex-session machinery can run variant generation, simulated users, review, selection, and narration. -- Mission Control can link brain output to entity state and observability +- Mission Control can link worker output to entity state and observability evidence. ## Verification diff --git a/os-apps/paw-managed-agents/wasm/event_emitter/Cargo.lock b/os-apps/paw-managed-agents/wasm/event_emitter/Cargo.lock index 4dd8c01f6..19885fca5 100644 --- a/os-apps/paw-managed-agents/wasm/event_emitter/Cargo.lock +++ b/os-apps/paw-managed-agents/wasm/event_emitter/Cargo.lock @@ -105,7 +105,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-managed-agents/wasm/event_emitter/Cargo.toml b/os-apps/paw-managed-agents/wasm/event_emitter/Cargo.toml index bdc85e898..7418b9364 100644 --- a/os-apps/paw-managed-agents/wasm/event_emitter/Cargo.toml +++ b/os-apps/paw-managed-agents/wasm/event_emitter/Cargo.toml @@ -9,6 +9,6 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } session-tree-lib = { path = "../../../paw-agent/wasm/session-tree-lib" } wasm-helpers = { path = "../../../paw-agent/wasm/wasm-helpers" } diff --git a/os-apps/paw-managed-agents/wasm/managed_agent_updater/Cargo.lock b/os-apps/paw-managed-agents/wasm/managed_agent_updater/Cargo.lock index 4a1ae2d5d..6cf8a36d3 100644 --- a/os-apps/paw-managed-agents/wasm/managed_agent_updater/Cargo.lock +++ b/os-apps/paw-managed-agents/wasm/managed_agent_updater/Cargo.lock @@ -97,7 +97,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-managed-agents/wasm/managed_agent_updater/Cargo.toml b/os-apps/paw-managed-agents/wasm/managed_agent_updater/Cargo.toml index 5ac37e0fd..afc53186e 100644 --- a/os-apps/paw-managed-agents/wasm/managed_agent_updater/Cargo.toml +++ b/os-apps/paw-managed-agents/wasm/managed_agent_updater/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } wasm-helpers = { path = "../../../paw-agent/wasm/wasm-helpers" } diff --git a/os-apps/paw-managed-agents/wasm/session_orchestrator/Cargo.lock b/os-apps/paw-managed-agents/wasm/session_orchestrator/Cargo.lock index f9a6f5d3b..61235102a 100644 --- a/os-apps/paw-managed-agents/wasm/session_orchestrator/Cargo.lock +++ b/os-apps/paw-managed-agents/wasm/session_orchestrator/Cargo.lock @@ -97,7 +97,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-managed-agents/wasm/session_orchestrator/Cargo.toml b/os-apps/paw-managed-agents/wasm/session_orchestrator/Cargo.toml index 7edfe087e..cda22f8c4 100644 --- a/os-apps/paw-managed-agents/wasm/session_orchestrator/Cargo.toml +++ b/os-apps/paw-managed-agents/wasm/session_orchestrator/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } wasm-helpers = { path = "../../../paw-agent/wasm/wasm-helpers" } diff --git a/os-apps/paw-managed-agents/wasm/session_terminator/Cargo.lock b/os-apps/paw-managed-agents/wasm/session_terminator/Cargo.lock index 2881dc289..f197a8372 100644 --- a/os-apps/paw-managed-agents/wasm/session_terminator/Cargo.lock +++ b/os-apps/paw-managed-agents/wasm/session_terminator/Cargo.lock @@ -97,7 +97,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-managed-agents/wasm/session_terminator/Cargo.toml b/os-apps/paw-managed-agents/wasm/session_terminator/Cargo.toml index c22bc30ea..043f92500 100644 --- a/os-apps/paw-managed-agents/wasm/session_terminator/Cargo.toml +++ b/os-apps/paw-managed-agents/wasm/session_terminator/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } wasm-helpers = { path = "../../../paw-agent/wasm/wasm-helpers" } diff --git a/os-apps/paw-patrol/wasm/daily_brief_lifecycle/Cargo.toml b/os-apps/paw-patrol/wasm/daily_brief_lifecycle/Cargo.toml index 1aa7cb246..ad77e56da 100644 --- a/os-apps/paw-patrol/wasm/daily_brief_lifecycle/Cargo.toml +++ b/os-apps/paw-patrol/wasm/daily_brief_lifecycle/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-patrol/wasm/finding_lifecycle/Cargo.toml b/os-apps/paw-patrol/wasm/finding_lifecycle/Cargo.toml index 5d69b1727..732406c77 100644 --- a/os-apps/paw-patrol/wasm/finding_lifecycle/Cargo.toml +++ b/os-apps/paw-patrol/wasm/finding_lifecycle/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-patrol/wasm/patrol_request_router/Cargo.toml b/os-apps/paw-patrol/wasm/patrol_request_router/Cargo.toml index 54f2aaca3..d719f6f05 100644 --- a/os-apps/paw-patrol/wasm/patrol_request_router/Cargo.toml +++ b/os-apps/paw-patrol/wasm/patrol_request_router/Cargo.toml @@ -11,4 +11,4 @@ test = false [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-patrol/wasm/patrol_run_lifecycle/Cargo.toml b/os-apps/paw-patrol/wasm/patrol_run_lifecycle/Cargo.toml index 599773fbe..01d207631 100644 --- a/os-apps/paw-patrol/wasm/patrol_run_lifecycle/Cargo.toml +++ b/os-apps/paw-patrol/wasm/patrol_run_lifecycle/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-patrol/wasm/patrol_schedule_lifecycle/Cargo.toml b/os-apps/paw-patrol/wasm/patrol_schedule_lifecycle/Cargo.toml index 1060c5115..5e0ca2d2e 100644 --- a/os-apps/paw-patrol/wasm/patrol_schedule_lifecycle/Cargo.toml +++ b/os-apps/paw-patrol/wasm/patrol_schedule_lifecycle/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-patrol/wasm/repo_sweep_lifecycle/Cargo.toml b/os-apps/paw-patrol/wasm/repo_sweep_lifecycle/Cargo.toml index bf1811e82..b94fb942c 100644 --- a/os-apps/paw-patrol/wasm/repo_sweep_lifecycle/Cargo.toml +++ b/os-apps/paw-patrol/wasm/repo_sweep_lifecycle/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-patrol/wasm/review_gate_lifecycle/Cargo.toml b/os-apps/paw-patrol/wasm/review_gate_lifecycle/Cargo.toml index 6c4230109..62a4e8b5f 100644 --- a/os-apps/paw-patrol/wasm/review_gate_lifecycle/Cargo.toml +++ b/os-apps/paw-patrol/wasm/review_gate_lifecycle/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-patrol/wasm/signal_router/Cargo.toml b/os-apps/paw-patrol/wasm/signal_router/Cargo.toml index 8144ecd0b..8801b2573 100644 --- a/os-apps/paw-patrol/wasm/signal_router/Cargo.toml +++ b/os-apps/paw-patrol/wasm/signal_router/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-patrol/wasm/work_cycle_lifecycle/Cargo.toml b/os-apps/paw-patrol/wasm/work_cycle_lifecycle/Cargo.toml index e29bdf73a..393d69651 100644 --- a/os-apps/paw-patrol/wasm/work_cycle_lifecycle/Cargo.toml +++ b/os-apps/paw-patrol/wasm/work_cycle_lifecycle/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-patrol/wasm/worker_run_lifecycle/Cargo.toml b/os-apps/paw-patrol/wasm/worker_run_lifecycle/Cargo.toml index 873fd24d1..40d28ae0a 100644 --- a/os-apps/paw-patrol/wasm/worker_run_lifecycle/Cargo.toml +++ b/os-apps/paw-patrol/wasm/worker_run_lifecycle/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-research/wasm/web_fetch/Cargo.toml b/os-apps/paw-research/wasm/web_fetch/Cargo.toml index 4df5352a5..87a108e3e 100644 --- a/os-apps/paw-research/wasm/web_fetch/Cargo.toml +++ b/os-apps/paw-research/wasm/web_fetch/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } serde_json = "1" diff --git a/os-apps/paw-research/wasm/web_search/Cargo.toml b/os-apps/paw-research/wasm/web_search/Cargo.toml index 6a46f1f84..5e3138e32 100644 --- a/os-apps/paw-research/wasm/web_search/Cargo.toml +++ b/os-apps/paw-research/wasm/web_search/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } serde_json = "1" diff --git a/os-apps/paw-skills/wasm/skill_installer/Cargo.toml b/os-apps/paw-skills/wasm/skill_installer/Cargo.toml index d7dd2b015..60d692cc5 100644 --- a/os-apps/paw-skills/wasm/skill_installer/Cargo.toml +++ b/os-apps/paw-skills/wasm/skill_installer/Cargo.toml @@ -12,4 +12,4 @@ crate-type = ["cdylib"] hex = "0.4" serde_json = "1" sha2 = "0.10" -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-wiki/wasm/build_session_message/Cargo.lock b/os-apps/paw-wiki/wasm/build_session_message/Cargo.lock index 5f408ab64..2efbe1621 100644 --- a/os-apps/paw-wiki/wasm/build_session_message/Cargo.lock +++ b/os-apps/paw-wiki/wasm/build_session_message/Cargo.lock @@ -97,7 +97,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-wiki/wasm/build_session_message/Cargo.toml b/os-apps/paw-wiki/wasm/build_session_message/Cargo.toml index 0e8bc91a8..6d9f2c511 100644 --- a/os-apps/paw-wiki/wasm/build_session_message/Cargo.toml +++ b/os-apps/paw-wiki/wasm/build_session_message/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } serde_json = "1" diff --git a/os-apps/paw-wiki/wasm/finalize_spawned_session/Cargo.lock b/os-apps/paw-wiki/wasm/finalize_spawned_session/Cargo.lock index 020d95c58..448f9703a 100644 --- a/os-apps/paw-wiki/wasm/finalize_spawned_session/Cargo.lock +++ b/os-apps/paw-wiki/wasm/finalize_spawned_session/Cargo.lock @@ -97,7 +97,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-wiki/wasm/finalize_spawned_session/Cargo.toml b/os-apps/paw-wiki/wasm/finalize_spawned_session/Cargo.toml index a6da2711f..39f0e95f1 100644 --- a/os-apps/paw-wiki/wasm/finalize_spawned_session/Cargo.toml +++ b/os-apps/paw-wiki/wasm/finalize_spawned_session/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } serde_json = "1" diff --git a/railway.toml b/railway.toml index 2cff68d94..3be836899 100644 --- a/railway.toml +++ b/railway.toml @@ -1,6 +1,6 @@ [build] builder = "DOCKERFILE" -dockerfilePath = "Dockerfile.deploy" +dockerfilePath = "Dockerfile" [deploy] healthcheckPath = "/healthz"