From 3c09fd2f0f6c9f44236d711625b89355adf25726 Mon Sep 17 00:00:00 2001 From: rita-aga Date: Fri, 29 May 2026 00:25:42 -0400 Subject: [PATCH 01/16] Close directed evolution worker proof gaps --- crates/paw-codex-worker/src/boot_watch.rs | 3 + crates/paw-codex-worker/src/datadog_patrol.rs | 5 +- .../src/directed_evolution.rs | 76 +++- .../src/directed_evolution/evidence.rs | 50 ++- .../human_episode_defaults.rs | 14 + .../directed_evolution/human_episode_plan.rs | 104 +++-- .../mechanical_evaluator.rs | 41 ++ .../src/directed_evolution/prompt.rs | 31 +- .../src/directed_evolution/recovery.rs | 370 ++++++++++++++++++ .../src/directed_evolution/tests.rs | 214 +++++++++- .../src/directed_evolution/workdir.rs | 111 ++++-- crates/paw-codex-worker/src/execution.rs | 33 +- crates/paw-codex-worker/src/main.rs | 1 + crates/paw-codex-worker/src/tests.rs | 31 ++ ...ted-evolution-proof-diffs-and-telemetry.md | 22 ++ 15 files changed, 998 insertions(+), 108 deletions(-) create mode 100644 crates/paw-codex-worker/src/directed_evolution/recovery.rs create mode 100644 docs/adrs/0064-directed-evolution-proof-diffs-and-telemetry.md diff --git a/crates/paw-codex-worker/src/boot_watch.rs b/crates/paw-codex-worker/src/boot_watch.rs index a81a1b105..5331c9e2f 100644 --- a/crates/paw-codex-worker/src/boot_watch.rs +++ b/crates/paw-codex-worker/src/boot_watch.rs @@ -76,6 +76,9 @@ async fn claim_boot_queued_directed_evolution_work_items( client: &reqwest::Client, config: &Config, ) -> Result<()> { + if let Err(error) = recover_pending_directed_evolution_stage_results(client, config).await { + warn!(%error, "Directed Evolution stage-result recovery pass failed"); + } let ids = query_boot_entity_ids(client, config, "WorkItems", "Queued").await?; for work_item_id in ids { handle_queued_directed_evolution_work_item(client, config, &work_item_id).await?; diff --git a/crates/paw-codex-worker/src/datadog_patrol.rs b/crates/paw-codex-worker/src/datadog_patrol.rs index 82917e75d..0aff7b122 100644 --- a/crates/paw-codex-worker/src/datadog_patrol.rs +++ b/crates/paw-codex-worker/src/datadog_patrol.rs @@ -99,8 +99,9 @@ async fn investigate_datadog_with_codex( let workdir = ensure_worktree(config, worker_run).await?; let prompt = datadog_mcp_patrol_prompt(patrol_run_id, worker_run); - let output = run_codex_exec_command(config, &workdir, prompt, "run Codex Datadog MCP Patrol") - .await?; + let output = + run_codex_exec_command_with_datadog_mcp(config, &workdir, prompt, "run Codex Datadog MCP Patrol") + .await?; let stdout = String::from_utf8_lossy(&output.stdout); let stderr = String::from_utf8_lossy(&output.stderr); diff --git a/crates/paw-codex-worker/src/directed_evolution.rs b/crates/paw-codex-worker/src/directed_evolution.rs index 0b2698221..1a4aff2cb 100644 --- a/crates/paw-codex-worker/src/directed_evolution.rs +++ b/crates/paw-codex-worker/src/directed_evolution.rs @@ -20,9 +20,18 @@ async fn handle_queued_directed_evolution_work_item( if let Some(reason) = stale_directed_evolution_work_item_reason(client, config, &work_item).await? { - eliminate_stale_directed_evolution_stage_result(client, config, &work_item, &reason) - .await?; - post_directed_evolution_action( + if let Err(error) = + eliminate_stale_directed_evolution_stage_result(client, config, &work_item, &reason) + .await + { + warn!( + work_item_id = %work_item.id, + role = %work_item.role, + %error, + "stale Directed Evolution WorkItem cleanup could not mutate StageResult; cancelling WorkItem only" + ); + } + if let Err(error) = post_directed_evolution_action( client, config, "WorkItems", @@ -30,7 +39,12 @@ async fn handle_queued_directed_evolution_work_item( "CancelWorkItem", json!({ "Reason": reason }), ) - .await?; + .await + { + if !error.to_string().contains("not valid from state 'Cancelled'") { + return Err(error); + } + } info!( work_item_id = %work_item.id, role = %work_item.role, @@ -39,8 +53,7 @@ async fn handle_queued_directed_evolution_work_item( return Ok(()); } - let brain_run_id = create_entity(client, config, "BrainRuns", json!({})).await?; - post_directed_evolution_action( + if let Err(error) = post_directed_evolution_action( client, config, "WorkItems", @@ -51,7 +64,30 @@ async fn handle_queued_directed_evolution_work_item( "ClaimedBy": config.worker_id, }), ) - .await?; + .await + { + let message = error.to_string(); + if directed_evolution_claim_conflict_message(&message) { + let current = fetch_directed_evolution_work_item(client, config, &work_item.id).await?; + info!( + work_item_id = %work_item.id, + role = %work_item.role, + status = %current.status, + "Directed Evolution WorkItem claim lost to another worker" + ); + if current.status != "Queued" { + return Ok(()); + } + } + return Err(error).with_context(|| { + format!( + "claim Directed Evolution WorkItem {} as {}", + work_item.id, config.worker_id + ) + }); + } + + let brain_run_id = create_entity(client, config, "BrainRuns", json!({})).await?; post_directed_evolution_action( client, config, @@ -195,6 +231,11 @@ async fn handle_queued_directed_evolution_work_item( } } +fn directed_evolution_claim_conflict_message(message: &str) -> bool { + message.contains("WorkItems.ClaimWorkItem returned 409") + && message.contains("not valid from state") +} + async fn stale_directed_evolution_work_item_reason( client: &reqwest::Client, config: &Config, @@ -378,14 +419,18 @@ async fn run_directed_evolution_codex_role( } else { directed_evolution_git_status_snapshot(&workdir.path).await? }; - let output = match run_codex_exec_command( - config, - &workdir.path, - prompt, - "run Directed Evolution Codex role", - ) - .await - { + let output = match if work_item.role == "telemetry_evaluator" { + run_codex_exec_command_with_datadog_mcp( + config, + &workdir.path, + prompt, + "run Directed Evolution Datadog telemetry evaluator", + ) + .await + } else { + run_codex_exec_command(config, &workdir.path, prompt, "run Directed Evolution Codex role") + .await + } { Ok(output) => output, Err(error) => { return recover_directed_evolution_variant_output( @@ -509,6 +554,7 @@ async fn recover_directed_evolution_variant_output( include!("directed_evolution/workdir.rs"); +include!("directed_evolution/recovery.rs"); include!("directed_evolution/mechanical_evaluator.rs"); include!("directed_evolution/prompt.rs"); include!("directed_evolution/tests.rs"); diff --git a/crates/paw-codex-worker/src/directed_evolution/evidence.rs b/crates/paw-codex-worker/src/directed_evolution/evidence.rs index b2ee68d2a..8d21614ef 100644 --- a/crates/paw-codex-worker/src/directed_evolution/evidence.rs +++ b/crates/paw-codex-worker/src/directed_evolution/evidence.rs @@ -214,7 +214,20 @@ fn directed_evolution_datadog_context(work_item: &DirectedEvolutionWorkItemState let service = env::var("DD_SERVICE").unwrap_or_else(|_| "temperpaw".to_string()); let env_name = env::var("DD_ENV").unwrap_or_else(|_| "local".to_string()); let site = env::var("DD_SITE").unwrap_or_else(|_| "datadoghq.com".to_string()); - let query = format!("service:{service} env:{env_name} @work_item_id:{}", work_item.id); + let correlation = serde_json::from_str::(&work_item.correlation_json) + .unwrap_or_else(|_| json!({})); + let join_fields = directed_evolution_datadog_join_fields(work_item, &correlation); + let mut query_parts = vec![ + format!("service:{service}"), + format!("env:{env_name}"), + format!("@work_item_id:{}", work_item.id), + ]; + for (key, value) in &join_fields { + if !value.trim().is_empty() { + query_parts.push(format!("@{key}:{value}")); + } + } + let query = query_parts.join(" "); json!({ "service": service, "env": env_name, @@ -223,6 +236,7 @@ fn directed_evolution_datadog_context(work_item: &DirectedEvolutionWorkItemState "target_entity_type": work_item.target_entity_type, "target_entity_id": work_item.target_entity_id, "control_tenant": config_tenant_label(), + "join_fields": join_fields, "query": query, "logs_url": format!( "https://app.{site}/logs?query={}", @@ -231,6 +245,40 @@ fn directed_evolution_datadog_context(work_item: &DirectedEvolutionWorkItemState }) } +fn directed_evolution_datadog_join_fields( + work_item: &DirectedEvolutionWorkItemState, + correlation: &Value, +) -> BTreeMap { + let mut fields = BTreeMap::new(); + for key in [ + "episode_id", + "direction_id", + "generation_id", + "variant_id", + "evaluation_stage_id", + "stage_result_id", + "trial_id", + "simulated_user_plan_id", + "persona_index", + "run_index", + "runtime_ref", + "app_ref", + ] { + let value = correlation + .get(key) + .and_then(Value::as_str) + .map(str::to_string) + .or_else(|| correlation.get(key).map(Value::to_string)) + .unwrap_or_default(); + if !value.trim().is_empty() { + fields.insert(key.to_string(), value); + } + } + fields.insert("work_item_id".to_string(), work_item.id.clone()); + fields.insert("role".to_string(), work_item.role.clone()); + fields +} + fn config_tenant_label() -> String { env::var("TEMPER_TENANT").unwrap_or_else(|_| "default".to_string()) } diff --git a/crates/paw-codex-worker/src/directed_evolution/human_episode_defaults.rs b/crates/paw-codex-worker/src/directed_evolution/human_episode_defaults.rs index 5aa40341c..9eb071254 100644 --- a/crates/paw-codex-worker/src/directed_evolution/human_episode_defaults.rs +++ b/crates/paw-codex-worker/src/directed_evolution/human_episode_defaults.rs @@ -113,6 +113,18 @@ fn default_human_episode_metrics() -> Vec { interpretation: "Simulated users record observations; evaluator decides whether friction is acceptable.".to_string(), hard_constraint: false, }, + DirectedEvolutionMetricPlan { + name: "simulated_user_app_blocker_count".to_string(), + kind: "journey".to_string(), + unit: "trials".to_string(), + higher_is_better: false, + description: "Simulated-user blockers classified as app behavior rather than runtime access.".to_string(), + provenance_kind: "state-verified".to_string(), + evaluator_ref: String::new(), + evaluator_module: "trial-state-verifier".to_string(), + interpretation: "The router counts terminal Trial entities and only hard-eliminates app-behavior blockers; runtime-access blockers remain visible evidence.".to_string(), + hard_constraint: true, + }, DirectedEvolutionMetricPlan { name: "runtime_error_count".to_string(), kind: "telemetry".to_string(), @@ -162,6 +174,8 @@ fn default_human_episode_elimination_rules() -> Vec, - #[serde(default, alias = "Description")] + #[serde(default, alias = "Description", alias = "description")] description: String, - #[serde(default, alias = "ProvenanceKind", alias = "provenanceKind")] + #[serde( + default, + alias = "ProvenanceKind", + alias = "provenanceKind", + alias = "provenance_kind" + )] provenance_kind: String, - #[serde(default, alias = "EvaluatorRef", alias = "evaluatorRef")] + #[serde(default, alias = "EvaluatorRef", alias = "evaluatorRef", alias = "evaluator_ref")] evaluator_ref: String, - #[serde(default, alias = "EvaluatorModule", alias = "evaluatorModule")] + #[serde( + default, + alias = "EvaluatorModule", + alias = "evaluatorModule", + alias = "evaluator_module" + )] evaluator_module: String, - #[serde(default, alias = "Interpretation")] + #[serde(default, alias = "Interpretation", alias = "interpretation")] interpretation: String, - #[serde(default, alias = "HardConstraint", alias = "hardConstraint")] + #[serde( + default, + alias = "HardConstraint", + alias = "hardConstraint", + alias = "hard_constraint" + )] hard_constraint: Option, } #[derive(Clone, Debug, Deserialize)] struct DirectedEvolutionEliminationRuleInput { - #[serde(default, alias = "RuleStatement", alias = "ruleStatement")] + #[serde(default, alias = "RuleStatement", alias = "ruleStatement", alias = "rule_statement")] statement: String, - #[serde(default, alias = "MetricNames", alias = "metricNames")] + #[serde(default, alias = "MetricNames", alias = "metricNames", alias = "metric_names")] metric_names: Vec, - #[serde(default, alias = "MetricIds", alias = "metricIds")] + #[serde(default, alias = "MetricIds", alias = "metricIds", alias = "metric_ids")] metric_ids: Vec, #[serde( default = "empty_directed_evolution_json_object", @@ -100,33 +130,53 @@ struct DirectedEvolutionEliminationRuleInput { #[derive(Clone, Debug, Deserialize)] struct DirectedEvolutionScoringRuleInput { - #[serde(default, alias = "RuleStatement", alias = "ruleStatement")] + #[serde(default, alias = "RuleStatement", alias = "ruleStatement", alias = "rule_statement")] statement: String, - #[serde(default, alias = "MetricNames", alias = "metricNames")] + #[serde(default, alias = "MetricNames", alias = "metricNames", alias = "metric_names")] metric_names: Vec, - #[serde(default, alias = "MetricIds", alias = "metricIds")] + #[serde(default, alias = "MetricIds", alias = "metricIds", alias = "metric_ids")] metric_ids: Vec, - #[serde(default, alias = "Weight")] + #[serde(default, alias = "Weight", alias = "weight")] weight: String, } #[derive(Clone, Debug, Deserialize)] struct DirectedEvolutionEvaluationStageInput { - #[serde(default, alias = "StageName", alias = "stageName")] + #[serde(default, alias = "StageName", alias = "stageName", alias = "stage_name")] name: String, - #[serde(default, alias = "StageKind", alias = "stageKind")] + #[serde(default, alias = "StageKind", alias = "stageKind", alias = "stage_kind")] kind: String, - #[serde(default, alias = "ExecutorKind", alias = "executorKind")] + #[serde(default, alias = "ExecutorKind", alias = "executorKind", alias = "executor_kind")] executor: String, - #[serde(default, alias = "RequiredEvidence", alias = "requiredEvidence")] + #[serde( + default, + alias = "RequiredEvidence", + alias = "requiredEvidence", + alias = "required_evidence" + )] required_evidence: Vec, - #[serde(default, alias = "MeasurementProvenance", alias = "measurementProvenance")] + #[serde( + default, + alias = "MeasurementProvenance", + alias = "measurementProvenance", + alias = "measurement_provenance" + )] measurement_provenance: String, - #[serde(default, alias = "EvaluatorRef", alias = "evaluatorRef")] + #[serde(default, alias = "EvaluatorRef", alias = "evaluatorRef", alias = "evaluator_ref")] evaluator_ref: String, - #[serde(default, alias = "EvaluatorModule", alias = "evaluatorModule")] + #[serde( + default, + alias = "EvaluatorModule", + alias = "evaluatorModule", + alias = "evaluator_module" + )] evaluator_module: String, - #[serde(default, alias = "DecisionAuthority", alias = "decisionAuthority")] + #[serde( + default, + alias = "DecisionAuthority", + alias = "decisionAuthority", + alias = "decision_authority" + )] decision_authority: String, } diff --git a/crates/paw-codex-worker/src/directed_evolution/mechanical_evaluator.rs b/crates/paw-codex-worker/src/directed_evolution/mechanical_evaluator.rs index bb325086d..24975a027 100644 --- a/crates/paw-codex-worker/src/directed_evolution/mechanical_evaluator.rs +++ b/crates/paw-codex-worker/src/directed_evolution/mechanical_evaluator.rs @@ -137,6 +137,15 @@ fn directed_evolution_state_verifier_output( )); } + let missing_required_changed_files = + directed_evolution_missing_required_changed_files(evaluation_stage, &changed_files); + if !missing_required_changed_files.is_empty() { + findings.push(format!( + "Variant did not change required file(s): {}", + missing_required_changed_files.join(", ") + )); + } + let passed = findings.is_empty(); let viability_regression_count = if passed { 0 } else { findings.len() }; let summary = if passed { @@ -181,6 +190,12 @@ fn directed_evolution_state_verifier_output( "unit": "files", "provenance_kind": provenance, "interpretation": "Number of changed files under the pinned evaluator bundle." + }, + "required_changed_file_missing_count": { + "value": missing_required_changed_files.len(), + "unit": "files", + "provenance_kind": provenance, + "interpretation": "Number of stage RequiredEvidence changed_file: requirements not present on the Mutation." } }, "decision_basis": { @@ -247,6 +262,32 @@ fn directed_evolution_changed_files(fields: &Value) -> Option> { ) } +fn directed_evolution_missing_required_changed_files( + evaluation_stage: &Value, + changed_files: &[String], +) -> Vec { + parse_json_string_array(&value_field_string( + evaluation_stage, + &["RequiredEvidenceJson", "required_evidence_json"], + )) + .into_iter() + .filter_map(|requirement| { + requirement + .strip_prefix("changed_file:") + .or_else(|| requirement.strip_prefix("changed-path:")) + .map(str::trim) + .filter(|path| !path.is_empty()) + .map(str::to_string) + }) + .filter(|required_path| { + !changed_files.iter().any(|changed_path| { + let normalized = changed_path.trim_start_matches("./"); + normalized == required_path || normalized.ends_with(&format!("/{required_path}")) + }) + }) + .collect() +} + fn directed_evolution_evaluator_path(path: &str) -> bool { let normalized = path.trim_start_matches("./"); normalized.starts_with("apps/agent-answers-evaluation/") diff --git a/crates/paw-codex-worker/src/directed_evolution/prompt.rs b/crates/paw-codex-worker/src/directed_evolution/prompt.rs index 8259ea2da..0b7a76756 100644 --- a/crates/paw-codex-worker/src/directed_evolution/prompt.rs +++ b/crates/paw-codex-worker/src/directed_evolution/prompt.rs @@ -10,16 +10,16 @@ fn directed_evolution_prompt(work_item: &DirectedEvolutionWorkItemState) -> Stri "Generate one bounded candidate variant for the target organism. Make the concrete app-bundle changes, avoid long exploratory verification, and return the concise JSON object immediately after the mutation is complete." } "simulated_user" => { - "Act as an AI simulated user against the target organism. Exercise the live runtime when a RuntimeRef is provided. Do not judge viability, do not pass/fail the variant, and do not select a winner. Return journeys, observations, friction, unmet intents, user-visible measurements, and evidence_scope entries from the app interaction only." + "Act as an AI simulated user against the target organism. Exercise the live runtime when a RuntimeRef is provided. Start with /tdata and /tdata/$metadata for the parsed runtime tenant; a 404 from / or decorative app routes is not a blocker when OData works. Do not judge viability, do not pass/fail the variant, and do not select a winner. Return journeys, observations, friction, unmet intents, user-visible measurements, blocker_kind, and evidence_scope entries from the app interaction only." } "reviewer" | "viability_evaluator" => { - "Evaluate a variant against the Adaptation Goal, Viability Constraints, Selection Protocol, and recorded evidence. Use authenticated Datadog MCP tools only when the stage requires telemetry evidence. Return pass/fail reasoning, explicit provenance_kind, metrics, structured evidence_scope entries, and risk notes." + "Evaluate a variant against the Adaptation Goal, Viability Constraints, Selection Protocol, and recorded evidence. This is not a telemetry role by default: do not fail because Datadog runtime-request logs are absent unless the literal stage prompt explicitly says the stage is Datadog-measured. Return pass/fail reasoning, explicit non-telemetry provenance_kind, metrics, structured evidence_scope entries, and risk notes." } "state_verifier" => { "Run deterministic state/spec verification against the variant. Use specs, CSDL, Cedar, runtime state, and transition checks rather than subjective judgment. Return pass/fail, provenance_kind=state-verified, metrics, decision_basis, and concrete evidence." } "telemetry_evaluator" => { - "Evaluate Datadog or runtime telemetry evidence for the variant. Query logs, traces, metrics, or monitors when available. Return pass/fail, provenance_kind=datadog-measured or runtime-measured, metrics, query summaries, result counts, zero-result meaning, and Datadog URLs when available." + "Evaluate Datadog telemetry evidence for the variant. Datadog is the primary judging surface: query logs for service:temper-platform \"directed evolution runtime request\" scoped by directed_evolution.episode_id, directed_evolution.variant_id, and the runtime tenant parsed from RuntimeRef. Do not require directed_evolution.runtime_ref unless Datadog field discovery proves it is indexed for these logs. Return top-level provenance_kind=datadog-measured. The first evidence_scope item must include surface=logs, the exact Datadog query, time_window, result_count, interpretation, zero_result_meaning, and a Datadog URL. Zero matching runtime-request logs is failure for this stage; runtime OData probes may be supporting evidence but must not replace Datadog." } "wasm_evaluator" => { "Run or inspect the pinned deterministic evaluator bundle for this stage. Do not let the variant alter the evaluator judging it. Return pass/fail, provenance_kind=wasm-computed, metrics, evaluator inputs, and evidence." @@ -99,6 +99,7 @@ fn directed_evolution_output_contract(role: &str) -> &'static str { "runtime_ref": "runnable variant URL or local ref", "changed_files": ["..."], "diff_ref": "...", + "diff_patch": "unified diff patch if available", "verification_notes": "...", "reasoning_summary": "..." }"# @@ -115,17 +116,34 @@ fn directed_evolution_output_contract(role: &str) -> &'static str { "evidence_scope": [{"surface":"runtime|app","query":"...","result_summary":"..."}], "evidence_refs": ["..."], "blocker": "", + "blocker_kind": "none|runtime-access|app-behavior|ambiguous", "reasoning_summary": "..." }"# } - "reviewer" | "viability_evaluator" | "state_verifier" | "telemetry_evaluator" | "wasm_evaluator" => { + "telemetry_evaluator" => { + r#"{ + "passed": true, + "status": "passed|failed", + "summary": "...", + "failure_reason": "", + "evaluator_role": "telemetry_evaluator", + "provenance_kind": "datadog-measured", + "metrics": {"runtime_request_log_count":{"value":0,"unit":"logs","provenance_kind":"datadog-measured","interpretation":"..."}}, + "decision_basis": {"why":"Datadog query result is the pass/fail source; runtime OData probes are supporting only.","tradeoffs":["Do not over-scope by runtime_ref unless field discovery shows it is indexed."]}, + "inputs": {"telemetry":["Datadog logs query scoped by directed_evolution episode, variant, and tenant fields"],"observations":[],"state":[]}, + "evidence_scope": [{"surface":"logs","query":"service:temper-platform \"directed evolution runtime request\" ...","time_window":"...","result_count":1,"interpretation":"...","zero_result_meaning":"failure","datadog_url":"https://app.datadoghq.com/logs?query=..."}], + "evidence_refs": ["..."], + "reasoning_summary": "..." +}"# + } + "reviewer" | "viability_evaluator" | "state_verifier" | "wasm_evaluator" => { r#"{ "passed": true, "status": "passed|failed", "summary": "...", "failure_reason": "", "evaluator_role": "reviewer|viability_evaluator|state_verifier|telemetry_evaluator|wasm_evaluator", - "provenance_kind": "brain-judged|state-verified|wasm-computed|runtime-measured|datadog-measured", + "provenance_kind": "brain-judged|state-verified|wasm-computed|runtime-measured", "metrics": {"metric_name":{"value":0,"unit":"score","provenance_kind":"...","interpretation":"..."}}, "decision_basis": {"why":"...","tradeoffs":["..."]}, "inputs": {"observations":["..."],"telemetry":["..."],"state":["..."]}, @@ -202,7 +220,8 @@ fn directed_evolution_worker_prompt_body(role: &str, prompt_body: &str) -> Strin - Do not start a foreground long-lived server. If a local server is absolutely required, start it in the background, stop it before returning, and include the cleanup in evidence_scope.\n\ - If runtime execution is unavailable, fail the stage with clear evidence instead of hanging.\n\ - Simulated-user roles must only report observations and must not include passed, failed, viable, score, winner, or selection fields.\n\ -- Evaluator roles may pass/fail only from recorded observations, deterministic checks, runtime measurements, or telemetry evidence with explicit provenance_kind.\n", +- Evaluator roles may pass/fail only from recorded observations, deterministic checks, runtime measurements, or telemetry evidence with explicit provenance_kind.\n\ +- For reviewer, viability_evaluator, state_verifier, and wasm_evaluator roles, Datadog absence is not a failure unless the literal stage prompt names Datadog as required evidence.\n", ); body } diff --git a/crates/paw-codex-worker/src/directed_evolution/recovery.rs b/crates/paw-codex-worker/src/directed_evolution/recovery.rs new file mode 100644 index 000000000..8dc658d4f --- /dev/null +++ b/crates/paw-codex-worker/src/directed_evolution/recovery.rs @@ -0,0 +1,370 @@ +async fn recover_pending_directed_evolution_stage_results( + client: &reqwest::Client, + config: &Config, +) -> Result<()> { + let stage_results = + query_directed_evolution_rows(client, config, "StageResults", "Status eq 'Pending'", 300) + .await?; + let trials = query_directed_evolution_rows(client, config, "Trials", "", 1000).await?; + let director_headers = directed_evolution_director_headers(config)?; + + for stage_result in stage_results { + let stage_result_id = directed_evolution_row_id(&stage_result); + let fields = directed_evolution_row_fields(&stage_result); + let episode_id = value_field_string(&fields, &["EpisodeId", "episode_id"]); + let generation_id = value_field_string(&fields, &["GenerationId", "generation_id"]); + let variant_id = value_field_string(&fields, &["VariantId", "variant_id"]); + let stage_id = value_field_string(&fields, &["EvaluationStageId", "evaluation_stage_id"]); + if stage_result_id.is_empty() + || episode_id.is_empty() + || generation_id.is_empty() + || variant_id.is_empty() + || stage_id.is_empty() + { + continue; + } + + let stage_fields = + fetch_directed_evolution_entity_fields(client, config, "EvaluationStages", &stage_id) + .await?; + let stage_kind = value_field_string(&stage_fields, &["StageKind", "stage_kind"]) + .to_ascii_lowercase(); + let role = recovered_stage_role(&stage_fields); + let ready = if stage_kind.contains("simulated") { + directed_evolution_trials_terminal_for_stage(&trials, &stage_result_id) + } else if role == "telemetry_evaluator" { + directed_evolution_trials_terminal_for_variant(&trials, &variant_id) + } else { + false + }; + if !ready { + continue; + } + if recovered_stage_work_item_exists(client, config, &stage_result_id, &role).await? { + continue; + } + + let variant_fields = + fetch_directed_evolution_entity_fields(client, config, "Variants", &variant_id) + .await?; + if value_field_string(&variant_fields, &["Status", "status"]) != "Active" { + continue; + } + + let work_item_id = create_directed_evolution_entity_with_headers( + client, + config, + &director_headers, + "WorkItems", + ) + .await?; + let prompt = recovered_stage_prompt( + &role, + &episode_id, + &generation_id, + &variant_id, + &stage_id, + &stage_result_id, + &work_item_id, + &stage_fields, + &variant_fields, + &trials, + ); + if let Err(error) = post_directed_evolution_action_with_headers( + client, + config, + &director_headers, + "StageResults", + &stage_result_id, + "StartStageResult", + json!({ + "EpisodeId": episode_id, + "GenerationId": generation_id, + "VariantId": variant_id, + "EvaluationStageId": stage_id, + "WorkItemId": work_item_id, + }), + ) + .await + { + let cancel_reason = + format!("recovery skipped StageResult start because it changed state: {error}"); + if let Err(cancel_error) = post_directed_evolution_action_with_headers( + client, + config, + &director_headers, + "WorkItems", + &work_item_id, + "CancelWorkItem", + json!({ "Reason": cancel_reason }), + ) + .await + { + warn!( + stage_result_id, + work_item_id, + %cancel_error, + "could not cancel placeholder Directed Evolution recovery WorkItem" + ); + } + info!( + stage_result_id, + work_item_id, + role, + %error, + "skipped Directed Evolution stage-result recovery because another actor changed the stage" + ); + continue; + } + post_directed_evolution_action_with_headers( + client, + config, + &director_headers, + "WorkItems", + &work_item_id, + "QueueWorkItem", + json!({ + "Role": role, + "TargetEntityType": "StageResult", + "TargetEntityId": stage_result_id, + "PromptRef": format!("literal:{prompt}"), + "ContextRef": format!("stage_result:{stage_result_id}"), + "OutputSchemaRef": "directed-evolution.stage-evaluation.v1", + "CorrelationJson": json!({ + "episode_id": episode_id, + "generation_id": generation_id, + "variant_id": variant_id, + "evaluation_stage_id": stage_id, + "stage_result_id": stage_result_id, + "role": role, + "recovered_after_terminal_trials": true, + }) + .to_string(), + }), + ) + .await?; + info!( + stage_result_id, + work_item_id, + role, + "recovered pending Directed Evolution stage result after terminal trials" + ); + } + + Ok(()) +} + +fn recovered_stage_role(stage_fields: &Value) -> String { + let executor = value_field_string(stage_fields, &["ExecutorKind", "executor_kind"]); + if directed_evolution_stage_evaluator_role(&executor) { + return executor; + } + let kind = value_field_string(stage_fields, &["StageKind", "stage_kind"]).to_ascii_lowercase(); + let provenance = + value_field_string(stage_fields, &["MeasurementProvenance", "measurement_provenance"]) + .to_ascii_lowercase(); + if kind.contains("telemetry") || kind.contains("datadog") || provenance.contains("datadog") { + "telemetry_evaluator".to_string() + } else if kind.contains("simulated") { + "viability_evaluator".to_string() + } else { + "reviewer".to_string() + } +} + +fn directed_evolution_trials_terminal_for_stage(trials: &[Value], stage_result_id: &str) -> bool { + let matching = trials + .iter() + .filter(|trial| { + value_field_string( + &directed_evolution_row_fields(trial), + &["StageResultId", "stage_result_id"], + ) == stage_result_id + }) + .collect::>(); + !matching.is_empty() && matching.iter().all(|trial| directed_evolution_trial_terminal(trial)) +} + +fn directed_evolution_trials_terminal_for_variant(trials: &[Value], variant_id: &str) -> bool { + let matching = trials + .iter() + .filter(|trial| { + value_field_string( + &directed_evolution_row_fields(trial), + &["VariantId", "variant_id"], + ) == variant_id + }) + .collect::>(); + !matching.is_empty() && matching.iter().all(|trial| directed_evolution_trial_terminal(trial)) +} + +fn directed_evolution_trial_terminal(trial: &&Value) -> bool { + matches!( + value_field_string( + &directed_evolution_row_fields(trial), + &["Status", "status"] + ) + .as_str(), + "Succeeded" | "Failed" | "Archived" + ) +} + +async fn recovered_stage_work_item_exists( + client: &reqwest::Client, + config: &Config, + stage_result_id: &str, + role: &str, +) -> Result { + let work_items = query_directed_evolution_rows(client, config, "WorkItems", "", 1000).await?; + Ok(work_items.iter().any(|item| { + let fields = directed_evolution_row_fields(item); + value_field_string(&fields, &["TargetEntityType", "target_entity_type"]) == "StageResult" + && value_field_string(&fields, &["TargetEntityId", "target_entity_id"]) == stage_result_id + && value_field_string(&fields, &["Role", "role"]) == role + && matches!( + value_field_string(&fields, &["Status", "status"]).as_str(), + "Queued" | "Claimed" | "Running" | "Succeeded" + ) + })) +} + +fn recovered_stage_prompt( + role: &str, + episode_id: &str, + generation_id: &str, + variant_id: &str, + stage_id: &str, + stage_result_id: &str, + work_item_id: &str, + stage_fields: &Value, + variant_fields: &Value, + trials: &[Value], +) -> String { + let stage_name = value_field_string(stage_fields, &["StageName", "stage_name"]); + let stage_kind = value_field_string(stage_fields, &["StageKind", "stage_kind"]); + let app_ref = value_field_string(variant_fields, &["AppRef", "app_ref"]); + let runtime_ref = value_field_string(variant_fields, &["RuntimeRef", "runtime_ref"]); + let variant_summary = value_field_string(variant_fields, &["Summary", "summary"]); + let trial_context = recovered_trial_context(trials, variant_id); + let header_block = format!( + "x-de-episode-id: {episode_id}\n\ +x-de-generation-id: {generation_id}\n\ +x-de-variant-id: {variant_id}\n\ +x-de-stage-id: {stage_id}\n\ +x-de-stage-result-id: {stage_result_id}\n\ +x-de-work-item-id: {work_item_id}\n\ +x-de-runtime-ref: {runtime_ref}\n\ +x-de-app-ref: {app_ref}" + ); + + if role == "telemetry_evaluator" { + return format!( + "Evaluate Directed Evolution Datadog telemetry after simulated users completed.\n\ +EpisodeId: {episode_id}\n\ +GenerationId: {generation_id}\n\ +VariantId: {variant_id}\n\ +EvaluationStageId: {stage_id}\n\ +StageResultId: {stage_result_id}\n\ +StageName: {stage_name}\n\ +StageKind: {stage_kind}\n\ +TemperApiBase: {base}\n\ +AppRef: {app_ref}\n\ +RuntimeRef: {runtime_ref}\n\ +VariantSummary: {variant_summary}\n\n\ +RecordedTrialEvidence:\n{trial_context}\n\n\ +DirectedEvolutionHeaders:\n{header_block}\n\n\ +This is a Datadog-measured telemetry stage. Use Datadog MCP aggregate/SQL evidence over brittle log-explorer field syntax: filter for \"directed evolution runtime request\", select directed_evolution.episode_id, directed_evolution.variant_id, and tenant, and count rows for this exact episode, variant, and runtime tenant parsed from RuntimeRef. Return provenance_kind=datadog-measured. The first evidence_scope item must include query, time_window, result_count, interpretation, zero_result_meaning, and datadog_url. Zero matching runtime-request logs is failure; runtime probes may support but must not replace Datadog. Return exactly one concise JSON object with passed/status/summary/failure_reason/evaluator_role/provenance_kind/metrics/decision_basis/inputs/evidence_scope/evidence_refs/reasoning_summary.", + base = configless_public_temper_api_base(), + ); + } + + format!( + "Evaluate the completed simulated-user trial observations for a Directed Evolution variant.\n\ +EpisodeId: {episode_id}\n\ +GenerationId: {generation_id}\n\ +VariantId: {variant_id}\n\ +EvaluationStageId: {stage_id}\n\ +StageResultId: {stage_result_id}\n\ +StageName: {stage_name}\n\ +StageKind: {stage_kind}\n\ +TemperApiBase: {base}\n\ +AppRef: {app_ref}\n\ +RuntimeRef: {runtime_ref}\n\ +VariantSummary: {variant_summary}\n\n\ +RecordedTrialEvidence:\n{trial_context}\n\n\ +Use only recorded simulated-user observations, blockers, friction, and live-runtime evidence. Do not query Datadog for this non-telemetry stage, do not select a winner, and do not modify files or evaluators. Return exactly one concise JSON object with passed/status/summary/failure_reason/evaluator_role=viability_evaluator/provenance_kind=brain-judged/metrics/decision_basis/inputs/evidence_scope/evidence_refs/reasoning_summary.", + base = configless_public_temper_api_base(), + ) +} + +fn recovered_trial_context(trials: &[Value], variant_id: &str) -> String { + let mut rows = Vec::new(); + for trial in trials { + let fields = directed_evolution_row_fields(trial); + if value_field_string(&fields, &["VariantId", "variant_id"]) != variant_id { + continue; + } + rows.push(format!( + "- {} {} run={} summary={} blocker={} intent={}", + directed_evolution_row_id(trial), + value_field_string(&fields, &["Status", "status"]), + value_field_string(&fields, &["RunIndex", "run_index"]), + value_field_string(&fields, &["Summary", "summary", "FailureReason", "failure_reason"]), + value_field_string(&fields, &["Blocker", "blocker"]), + value_field_string(&fields, &["IntentSatisfied", "intent_satisfied"]), + )); + } + if rows.is_empty() { + "No trial evidence found for this variant.".to_string() + } else { + rows.join("\n") + } +} + +fn configless_public_temper_api_base() -> String { + env::var("DIRECTED_EVOLUTION_PUBLIC_API_URL") + .ok() + .or_else(|| env::var("DIRECTED_EVOLUTION_GENESIS_URL").ok()) + .or_else(|| env::var("TEMPER_URL").ok()) + .unwrap_or_else(|| "https://genesis-production-164d.up.railway.app".to_string()) +} + +async fn query_directed_evolution_rows( + client: &reqwest::Client, + config: &Config, + entity_set: &str, + filter: &str, + top: usize, +) -> Result> { + let mut url = format!("{}/tdata/{entity_set}?$top={top}", config.temper_url); + if !filter.trim().is_empty() { + url.push_str("&$filter="); + url.push_str(&filter.replace(' ', "%20").replace('\'', "%27")); + } + let response = client + .get(url) + .headers(headers(config)?) + .send() + .await + .with_context(|| format!("query Directed Evolution {entity_set}"))?; + if !response.status().is_success() { + let status = response.status(); + let text = response.text().await.unwrap_or_default(); + bail!("query Directed Evolution {entity_set} returned {status}: {text}"); + } + let body: Value = response.json().await.context("parse Directed Evolution query")?; + Ok(body + .get("value") + .and_then(Value::as_array) + .cloned() + .unwrap_or_default()) +} + +fn directed_evolution_row_fields(row: &Value) -> Value { + row.get("fields").cloned().unwrap_or_else(|| json!({})) +} + +fn directed_evolution_row_id(row: &Value) -> String { + let fields = directed_evolution_row_fields(row); + first_string(row, &fields, &["entity_id", "id", "Id"], &["Id", "id"]) +} diff --git a/crates/paw-codex-worker/src/directed_evolution/tests.rs b/crates/paw-codex-worker/src/directed_evolution/tests.rs index 94bf8c112..c14b0b2cd 100644 --- a/crates/paw-codex-worker/src/directed_evolution/tests.rs +++ b/crates/paw-codex-worker/src/directed_evolution/tests.rs @@ -92,7 +92,8 @@ mod directed_evolution_tests { let prompt = directed_evolution_prompt(&work_item); - assert!(prompt.contains("Query logs, traces, metrics, or monitors")); + assert!(prompt.contains("Datadog is the primary judging surface")); + assert!(prompt.contains("directed_evolution.variant_id")); assert!(prompt.contains("provenance_kind=datadog-measured")); assert!(prompt.contains("result_count")); assert!(prompt.contains("zero_result_meaning")); @@ -318,6 +319,45 @@ mod directed_evolution_tests { ); } + #[test] + fn mechanical_state_verifier_rejects_missing_required_changed_file() { + let work_item = DirectedEvolutionWorkItemState { + id: "wi-state".to_string(), + status: "Running".to_string(), + role: "state_verifier".to_string(), + target_entity_type: "StageResult".to_string(), + target_entity_id: "sr-required-file".to_string(), + prompt_ref: String::new(), + context_ref: String::new(), + output_schema_ref: String::new(), + correlation_json: "{}".to_string(), + }; + + let output = directed_evolution_state_verifier_output( + &work_item, + &json!({ "VariantId": "var-required-file", "EvaluationStageId": "stage-required-file" }), + &json!({ "MutationId": "mut-required-file" }), + &json!({ + "ChangedFilesJson": "[\"APP.md\",\"specs/question.ioa.toml\",\"specs/model.csdl.xml\"]" + }), + &json!({ + "RequiredEvidenceJson": "[\"changed_file:specs/answer.ioa.toml\",\"no_evaluator_mutation\"]" + }), + ); + + assert_eq!(output["passed"], false); + assert_eq!( + output["metrics"]["required_changed_file_missing_count"]["value"], + 1 + ); + assert!( + output["failure_reason"] + .as_str() + .unwrap() + .contains("specs/answer.ioa.toml") + ); + } + #[test] fn mechanical_evaluator_roles_are_not_codex_brains() { assert!(directed_evolution_mechanical_evaluator_role("state_verifier")); @@ -359,6 +399,30 @@ mod directed_evolution_tests { } } + #[test] + fn directed_evolution_repo_mapping_accepts_object_path() { + let previous = env::var_os("DIRECTED_EVOLUTION_ORGANISM_REPOS_JSON"); + unsafe { + env::set_var( + "DIRECTED_EVOLUTION_ORGANISM_REPOS_JSON", + r#"{"org-agent-answers":{"path":"/tmp/agent-answers","owner":"arni-labs","app":"agent-answers"}}"#, + ); + } + + let path = + directed_evolution_repo_from_mapping("org-agent-answers", "arni-labs/agent-answers@abc123") + .expect("repo should resolve from object path"); + + assert_eq!(path, PathBuf::from("/tmp/agent-answers")); + unsafe { + if let Some(value) = previous { + env::set_var("DIRECTED_EVOLUTION_ORGANISM_REPOS_JSON", value); + } else { + env::remove_var("DIRECTED_EVOLUTION_ORGANISM_REPOS_JSON"); + } + } + } + #[test] fn directed_evolution_app_ref_maps_to_genesis_app_id() { let app = directed_evolution_genesis_app_from_ref("nerdsane/agent-answers@abc123") @@ -466,26 +530,34 @@ mod directed_evolution_tests { } #[test] - fn directed_evolution_canonical_push_targets_main_with_registry_tenant() { - let app = directed_evolution_genesis_app_from_ref("nerdsane/agent-answers@abc123") - .expect("app ref should parse"); + fn directed_evolution_registry_tenant_defaults_to_default() { + let previous = env::var_os("DIRECTED_EVOLUTION_REGISTRY_TENANT"); + unsafe { + env::remove_var("DIRECTED_EVOLUTION_REGISTRY_TENANT"); + } - let args = directed_evolution_canonical_push_args( - "https://genesis-production-164d.up.railway.app/", - &app, - "default", - ); + let config = Config { + temper_url: "http://temper.test".to_string(), + tenant: "de-control".to_string(), + worker_id: "worker".to_string(), + worker_token: None, + workspace_root: PathBuf::from("/tmp/workspaces"), + repo_root: PathBuf::from("/tmp/repo"), + codex_bin: "codex".to_string(), + max_concurrent_runs: 1, + enable_execution: true, + poll_on_start: true, + codex_exec_smoke: false, + codex_exec_timeout: Duration::from_secs(60), + }; - assert_eq!( - args, - vec![ - "-c", - "http.extraHeader=x-tenant-id: default", - "push", - "https://genesis-production-164d.up.railway.app/nerdsane/agent-answers.git", - "HEAD:refs/heads/main", - ] - ); + assert_eq!(directed_evolution_registry_tenant(&config), "default"); + + unsafe { + if let Some(value) = previous { + env::set_var("DIRECTED_EVOLUTION_REGISTRY_TENANT", value); + } + } } #[test] @@ -576,6 +648,26 @@ mod directed_evolution_tests { assert!(!is_datadog_app_url("https://example.com/logs")); } + #[test] + fn directed_evolution_evidence_summary_preserves_numeric_result_count() { + let summary = directed_evolution_first_evidence_scope_summary(&json!({ + "evidence_scope": [ + { + "surface": "logs", + "query": "service:temper-platform", + "time_window": "2026-05-28T00:00:00Z to 2026-05-29T23:59:59Z", + "result_count": 78, + "interpretation": "Datadog returned matching runtime-request logs.", + "zero_result_meaning": "failure" + } + ] + })); + + assert_eq!(summary.query, "service:temper-platform"); + assert_eq!(summary.result_count, "78"); + assert_eq!(summary.zero_result_meaning, "failure"); + } + #[test] fn directed_evolution_summary_uses_agent_summary_when_available() { let work_item = DirectedEvolutionWorkItemState { @@ -632,6 +724,17 @@ mod directed_evolution_tests { assert!(context["query"].as_str().unwrap().contains("@work_item_id:wi-dd")); } + #[test] + fn directed_evolution_claim_conflict_is_benign_race_signature() { + let message = + "WorkItems.ClaimWorkItem returned 409 Conflict: Action 'ClaimWorkItem' not valid from state 'Running'"; + + assert!(directed_evolution_claim_conflict_message(message)); + assert!(!directed_evolution_claim_conflict_message( + "WorkItems.StartWorkItem returned 409 Conflict: Action 'StartWorkItem' not valid from state 'Queued'" + )); + } + #[test] fn human_episode_contract_uses_direction_and_organism_defaults() { let input: DirectedEvolutionHumanEpisodeInput = serde_json::from_value(json!({ @@ -666,7 +769,7 @@ mod directed_evolution_tests { "Humans compare candidate answers before acceptance." ); assert_eq!(plan.viability_constraints[0].statement, "Answer creation still works."); - assert_eq!(plan.metrics.len(), 4); + assert_eq!(plan.metrics.len(), 5); assert_eq!(plan.simulated_user_plan.users_per_variant, 3); assert_eq!(plan.simulated_user_plan.runs_per_persona, 2); assert!( @@ -738,6 +841,77 @@ mod directed_evolution_tests { assert!(!plan.evaluator_ref.trim().is_empty()); } + #[test] + fn human_episode_contract_accepts_snake_case_semantics() { + let input: DirectedEvolutionHumanEpisodeInput = serde_json::from_value(json!({ + "direction_id": "direction-growth", + "adaptation_goal": "Improve answer comparison.", + "metrics": [ + { + "metric_name": "required_changed_file_missing_count", + "metric_kind": "state", + "unit": "files", + "higher_is_better": false, + "provenance_kind": "state-verified", + "evaluator_ref": "genesis://nerdsane/agent-answers-evaluation@frozen", + "evaluator_module": "state-verifier", + "hard_constraint": true + } + ], + "viability_constraints": [ + { + "constraint_statement": "Answer spec must change.", + "constraint_kind": "state-verified" + } + ], + "elimination_rules": [ + { + "rule_statement": "Eliminate missing answer spec mutation.", + "metric_names": ["required_changed_file_missing_count"], + "threshold": { "max": 0 } + } + ], + "scoring_rules": [ + { + "rule_statement": "Prefer exact answer-spec fit.", + "metric_names": ["required_changed_file_missing_count"], + "weight": "0.20" + } + ], + "evaluation_stages": [ + { + "stage_name": "Deterministic state verification", + "stage_kind": "state_verification", + "executor_kind": "state_verifier", + "required_evidence": ["changed_file:specs/answer.ioa.toml"], + "measurement_provenance": "state-verified", + "evaluator_ref": "genesis://nerdsane/agent-answers-evaluation@frozen", + "evaluator_module": "state-verifier", + "decision_authority": "mechanical" + } + ] + })) + .expect("contract input should parse"); + + let plan = directed_evolution_episode_plan_from_input( + input, + &json!({ + "OrganismId": "org-agent-answers", + "ProposedAdaptationGoal": "Direction goal", + }), + &json!({ "ParentVersionId": "ov-parent" }), + ) + .expect("episode plan should resolve"); + + assert_eq!(plan.metrics[0].name, "required_changed_file_missing_count"); + assert_eq!(plan.metrics[0].higher_is_better, false); + assert_eq!(plan.metrics[0].hard_constraint, true); + assert_eq!(plan.viability_constraints[0].statement, "Answer spec must change."); + assert_eq!(plan.elimination_rules[0].metric_names[0], "required_changed_file_missing_count"); + assert_eq!(plan.scoring_rules[0].weight, "0.20"); + assert_eq!(plan.evaluation_stages[0].required_evidence[0], "changed_file:specs/answer.ioa.toml"); + } + #[test] fn directed_evolution_start_episode_command_accepts_contract_path() { let command = parse_worker_command([ diff --git a/crates/paw-codex-worker/src/directed_evolution/workdir.rs b/crates/paw-codex-worker/src/directed_evolution/workdir.rs index f16197963..f64ae23af 100644 --- a/crates/paw-codex-worker/src/directed_evolution/workdir.rs +++ b/crates/paw-codex-worker/src/directed_evolution/workdir.rs @@ -260,6 +260,13 @@ fn directed_evolution_repo_from_mapping(organism_id: &str, app_ref: &str) -> Opt if let Some(path) = mapping.get(key).and_then(Value::as_str) { return Some(PathBuf::from(path)); } + if let Some(path) = mapping + .get(key) + .and_then(|value| value.get("path")) + .and_then(Value::as_str) + { + return Some(PathBuf::from(path)); + } } None } @@ -286,6 +293,9 @@ async fn ensure_directed_evolution_worktree( if worktree.join(".git").exists() { return Ok(worktree); } + if let Some(existing) = existing_directed_evolution_worktree_for_branch(base_repo, &branch).await? { + return Ok(existing); + } fs::create_dir_all(worktree.parent().unwrap_or(&config.workspace_root)) .with_context(|| format!("create {}", worktree.display()))?; let inside = git_capture(base_repo, &["rev-parse", "--is-inside-work-tree"]).await?; @@ -307,6 +317,32 @@ async fn ensure_directed_evolution_worktree( Ok(worktree) } +async fn existing_directed_evolution_worktree_for_branch( + base_repo: &Path, + branch: &str, +) -> Result> { + let list = git_capture(base_repo, &["worktree", "list", "--porcelain"]).await?; + let expected_ref = format!("refs/heads/{branch}"); + let mut current_path: Option = None; + + for line in list.lines().chain(std::iter::once("")) { + if let Some(path) = line.strip_prefix("worktree ") { + current_path = Some(PathBuf::from(path)); + continue; + } + if let Some(active_branch) = line.strip_prefix("branch ") { + if active_branch == expected_ref { + return Ok(current_path); + } + } + if line.trim().is_empty() { + current_path = None; + } + } + + Ok(None) +} + async fn finalize_directed_evolution_output( client: &reqwest::Client, config: &Config, @@ -326,6 +362,7 @@ async fn finalize_directed_evolution_output( ); } git_capture(&workdir.path, &["add", "-A"]).await?; + let diff_patch = git_capture(&workdir.path, &["diff", "--cached", "--binary"]).await?; git_capture_owned( &workdir.path, vec![ @@ -355,6 +392,7 @@ async fn finalize_directed_evolution_output( "diff_ref".to_string(), json!(format!("git:{}:{commit}", workdir.branch_ref)), ); + object.insert("diff_patch".to_string(), json!(diff_patch)); } if let Some(runtime) = hotload_directed_evolution_variant(client, config, work_item, workdir, app_name, &commit) @@ -452,6 +490,8 @@ async fn hotload_directed_evolution_variant( let registry_tenant = directed_evolution_registry_tenant(config); push_directed_evolution_variant_commit(&workdir.path, &genesis_url, &app, &branch, ®istry_tenant) .await?; + publish_directed_evolution_app_version(client, config, &genesis_url, &app, ®istry_tenant, &branch) + .await?; let tenant_prefix = env::var("DIRECTED_EVOLUTION_VARIANT_TENANT_PREFIX") .unwrap_or_else(|_| "de-variant".to_string()); let tenant = directed_evolution_variant_tenant(&tenant_prefix, work_item); @@ -503,10 +543,20 @@ async fn materialize_directed_evolution_promotion( } let registry_tenant = directed_evolution_registry_tenant(config); - push_directed_evolution_canonical_commit(&workdir.path, &genesis_url, &app, ®istry_tenant) - .await?; - publish_directed_evolution_app_version(client, config, &genesis_url, &app, ®istry_tenant) - .await?; + let ref_name = if workdir.branch_ref.trim().is_empty() { + directed_evolution_variant_branch(work_item) + } else { + workdir.branch_ref.clone() + }; + publish_directed_evolution_app_version( + client, + config, + &genesis_url, + &app, + ®istry_tenant, + &ref_name, + ) + .await?; let production_tenant = directed_evolution_production_tenant(); install_directed_evolution_variant( client, @@ -522,8 +572,8 @@ async fn materialize_directed_evolution_promotion( Ok(DirectedEvolutionPromotionMaterialization { runtime_ref: format!("temper://tenant/{production_tenant}/app/{canonical_app_ref}"), summary: format!( - "Published {} to Genesis main and hot-loaded it into tenant {}.", - canonical_app_ref, production_tenant + "Published {} from winner ref {} and hot-loaded it into tenant {}.", + canonical_app_ref, ref_name, production_tenant ), digest: app.hash.clone(), canonical_app_ref, @@ -570,28 +620,6 @@ async fn push_directed_evolution_variant_commit( Ok(()) } -async fn push_directed_evolution_canonical_commit( - workdir: &Path, - genesis_url: &str, - app: &DirectedEvolutionGenesisApp, - registry_tenant: &str, -) -> Result<()> { - git_capture_owned( - workdir, - directed_evolution_canonical_push_args(genesis_url, app, registry_tenant), - ) - .await?; - Ok(()) -} - -fn directed_evolution_canonical_push_args( - genesis_url: &str, - app: &DirectedEvolutionGenesisApp, - registry_tenant: &str, -) -> Vec { - directed_evolution_variant_push_args(genesis_url, app, "main", registry_tenant) -} - fn directed_evolution_variant_push_args( genesis_url: &str, app: &DirectedEvolutionGenesisApp, @@ -643,6 +671,14 @@ async fn install_directed_evolution_variant( let status = response.status(); let body = response.text().await.unwrap_or_default(); if !status.is_success() { + if status == reqwest::StatusCode::CONFLICT && body.contains("state 'Installed'") { + tracing::info!( + app_ref = %app.pinned_ref(), + target_tenant, + "Directed Evolution install already materialized; treating Installed conflict as success" + ); + return Ok(()); + } bail!( "Directed Evolution variant install failed for {} into tenant {}: HTTP {} body={}", app.pinned_ref(), @@ -660,6 +696,7 @@ async fn publish_directed_evolution_app_version( genesis_url: &str, app: &DirectedEvolutionGenesisApp, registry_tenant: &str, + ref_name: &str, ) -> Result<()> { let url = format!( "{}/tdata/Apps('{}')/App.PublishNewVersion?await_integration=true", @@ -671,7 +708,7 @@ async fn publish_directed_evolution_app_version( .headers(headers_for_tenant(config, registry_tenant)?) .json(&json!({ "NewHash": app.hash, - "RefName": "main", + "RefName": ref_name, })) .send() .await @@ -689,8 +726,8 @@ async fn publish_directed_evolution_app_version( Ok(()) } -fn directed_evolution_registry_tenant(config: &Config) -> String { - env::var("DIRECTED_EVOLUTION_REGISTRY_TENANT").unwrap_or_else(|_| config.tenant.clone()) +fn directed_evolution_registry_tenant(_config: &Config) -> String { + env::var("DIRECTED_EVOLUTION_REGISTRY_TENANT").unwrap_or_else(|_| "default".to_string()) } fn directed_evolution_production_tenant() -> String { @@ -817,9 +854,15 @@ fn sanitize_id_component(value: &str) -> String { fn value_field_string(fields: &Value, keys: &[&str]) -> String { keys.iter() - .find_map(|key| fields.get(*key).and_then(Value::as_str)) - .unwrap_or("") - .to_string() + .find_map(|key| { + fields.get(*key).and_then(|value| match value { + Value::String(value) => Some(value.clone()), + Value::Number(value) => Some(value.to_string()), + Value::Bool(value) => Some(value.to_string()), + _ => None, + }) + }) + .unwrap_or_default() } fn env_flag(key: &str) -> Option { diff --git a/crates/paw-codex-worker/src/execution.rs b/crates/paw-codex-worker/src/execution.rs index 00289cc88..3f20a0caa 100644 --- a/crates/paw-codex-worker/src/execution.rs +++ b/crates/paw-codex-worker/src/execution.rs @@ -378,6 +378,21 @@ async fn run_codex_exec_command( .await } +async fn run_codex_exec_command_with_datadog_mcp( + config: &Config, + workdir: &Path, + prompt: String, + context_label: &str, +) -> Result { + run_codex_exec_command_with_args( + config, + workdir, + codex_exec_args_with_datadog_mcp(workdir, &prompt), + context_label, + ) + .await +} + async fn run_codex_exec_command_with_args( config: &Config, workdir: &Path, @@ -488,8 +503,20 @@ fn signal_process_group(pid: u32, signal: libc::c_int, label: &str, context: &st async fn terminate_process_group(_pid: u32, _context: &str) {} fn codex_exec_args(workdir: &Path, prompt: &str) -> Vec { + codex_exec_args_with_datadog_option(workdir, prompt, false) +} + +fn codex_exec_args_with_datadog_mcp(workdir: &Path, prompt: &str) -> Vec { + codex_exec_args_with_datadog_option(workdir, prompt, true) +} + +fn codex_exec_args_with_datadog_option( + workdir: &Path, + prompt: &str, + force_datadog_mcp: bool, +) -> Vec { let mut args = vec!["exec".into(), "--ignore-user-config".into()]; - if let Some(datadog_mcp_url) = codex_datadog_mcp_url() { + if let Some(datadog_mcp_url) = codex_datadog_mcp_url(force_datadog_mcp) { args.extend([ "-c".into(), format!( @@ -510,11 +537,11 @@ fn codex_exec_args(workdir: &Path, prompt: &str) -> Vec { args } -fn codex_datadog_mcp_url() -> Option { +fn codex_datadog_mcp_url(force_enabled: bool) -> Option { let enabled = env::var("PAW_CODEX_ENABLE_DATADOG_MCP") .map(|value| value == "1" || value.eq_ignore_ascii_case("true")) .unwrap_or(false); - if !enabled { + if !force_enabled && !enabled { return None; } Some( diff --git a/crates/paw-codex-worker/src/main.rs b/crates/paw-codex-worker/src/main.rs index 8ce231f5c..71bafc49e 100644 --- a/crates/paw-codex-worker/src/main.rs +++ b/crates/paw-codex-worker/src/main.rs @@ -7,6 +7,7 @@ use repo_health::{ use reqwest::header::{ACCEPT, AUTHORIZATION, CONTENT_TYPE, HeaderMap, HeaderValue}; use serde::Deserialize; use serde_json::{Value, json}; +use std::collections::BTreeMap; use std::env; use std::fs; use std::io::{self, Read}; diff --git a/crates/paw-codex-worker/src/tests.rs b/crates/paw-codex-worker/src/tests.rs index b2a912b39..10982a2f3 100644 --- a/crates/paw-codex-worker/src/tests.rs +++ b/crates/paw-codex-worker/src/tests.rs @@ -604,6 +604,37 @@ mod tests { ); } + #[tokio::test] + async fn codex_exec_can_force_datadog_mcp_for_telemetry_roles() { + let _guard = ENV_LOCK.lock().await; + let _datadog_mcp = + EnvOverride::set("PAW_CODEX_ENABLE_DATADOG_MCP", OsString::from("0")); + let _datadog_url = EnvOverride::set( + "PAW_CODEX_DATADOG_MCP_URL", + OsString::from("https://mcp.datadoghq.test/mcp?toolsets=logs"), + ); + let args = codex_exec_args_with_datadog_mcp( + Path::new("/tmp/paw-worktree"), + "Evaluate Datadog telemetry", + ); + let args = args + .iter() + .map(|arg| arg.to_string_lossy().to_string()) + .collect::>(); + + assert_eq!( + &args[0..4], + &[ + "exec".to_string(), + "--ignore-user-config".to_string(), + "-c".to_string(), + "mcp_servers.datadog.url=\"https://mcp.datadoghq.test/mcp?toolsets=logs\"" + .to_string(), + ] + ); + assert_eq!(args.last().map(String::as_str), Some("Evaluate Datadog telemetry")); + } + #[tokio::test] async fn codex_exec_uses_default_datadog_mcp_url_when_enabled() { let _guard = ENV_LOCK.lock().await; diff --git a/docs/adrs/0064-directed-evolution-proof-diffs-and-telemetry.md b/docs/adrs/0064-directed-evolution-proof-diffs-and-telemetry.md new file mode 100644 index 000000000..7ab6a08ed --- /dev/null +++ b/docs/adrs/0064-directed-evolution-proof-diffs-and-telemetry.md @@ -0,0 +1,22 @@ +# ADR 0064: Directed Evolution Proof Diffs And Telemetry + +## Status + +Accepted + +## Context + +Directed Evolution proof runs must be reviewable from Temper/Genesis state without relying on chat narration. Variant generators currently commit app changes, but mutation entities only receive changed filenames and a diff reference. Telemetry evaluators also need stable Datadog join fields that line up with the runtime headers sent by simulated users and evaluators. + +## Decision + +- Variant-generator finalization stages the repo, captures the staged unified patch, commits, and returns `diff_patch` with `changed_files` and `diff_ref`. +- Worker evidence summaries include Datadog context derived from the WorkItem correlation graph, not only the WorkItem id. +- Worker prompts keep simulated users observational: they use the app and report journeys, while evaluator roles decide pass/fail. +- Default human episode setup treats Datadog runtime errors as a hard elimination metric alongside state/spec regressions. + +## Consequences + +- Genesis can materialize GitHub-like code diffs from mutation entities. +- Datadog queries are reproducible from evidence records and can be joined by tenant, episode, generation, variant, stage, trial, and work item. +- Missing Datadog evidence can be failed by the app router instead of remaining an opaque link. From b9e16d6a47643d8b167b6c584d46f5d02e130d09 Mon Sep 17 00:00:00 2001 From: rita-aga Date: Tue, 2 Jun 2026 18:07:28 -0400 Subject: [PATCH 02/16] Close directed evolution worker proof gaps --- Cargo.lock | 1 + crates/paw-codex-worker/Cargo.toml | 1 + .../paw-codex-worker/fixtures/fake-codex.sh | 8 + .../scripts/production-readiness.sh | 120 ++++- crates/paw-codex-worker/src/boot_watch.rs | 11 +- crates/paw-codex-worker/src/cli.rs | 3 + .../src/directed_evolution.rs | 432 ++++++++++++++++-- .../src/directed_evolution/evidence.rs | 75 ++- .../src/directed_evolution/human_episode.rs | 10 +- .../directed_evolution/human_episode_plan.rs | 18 +- .../src/directed_evolution/prompt.rs | 36 +- .../src/directed_evolution/recovery.rs | 4 +- .../src/directed_evolution/tests.rs | 319 ++++++++++++- .../src/directed_evolution/workdir.rs | 264 ++++++++++- crates/paw-codex-worker/src/event_loop.rs | 46 +- crates/paw-codex-worker/src/execution.rs | 10 + crates/paw-codex-worker/src/http_headers.rs | 14 +- crates/paw-codex-worker/src/launchd.rs | 23 +- crates/paw-codex-worker/src/main.rs | 9 + crates/paw-codex-worker/src/temper_api.rs | 122 ++++- crates/paw-codex-worker/src/tests.rs | 237 +++++++++- crates/paw-codex-worker/src/worker_types.rs | 11 +- ...cted-evolution-hotload-variant-runtimes.md | 8 +- ...rected-evolution-promotion-materializer.md | 4 +- ...ted-evolution-datadog-evidence-contract.md | 8 +- ...rected-evolution-public-runtime-prompts.md | 2 +- .../0065-canonical-genesis-worker-pool.md | 31 ++ docs/deployment.md | 7 +- os-apps/README.md | 10 + ...4-directed-evolution-local-codex-worker.md | 14 +- 30 files changed, 1717 insertions(+), 141 deletions(-) create mode 100644 docs/adrs/0065-canonical-genesis-worker-pool.md create mode 100644 os-apps/README.md diff --git a/Cargo.lock b/Cargo.lock index 36918f934..4be0fb141 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3951,6 +3951,7 @@ name = "paw-codex-worker" version = "0.1.0" dependencies = [ "anyhow", + "base64 0.22.1", "futures-util", "libc", "reqwest", diff --git a/crates/paw-codex-worker/Cargo.toml b/crates/paw-codex-worker/Cargo.toml index 5d06702fe..1897ca797 100644 --- a/crates/paw-codex-worker/Cargo.toml +++ b/crates/paw-codex-worker/Cargo.toml @@ -12,6 +12,7 @@ path = "src/main.rs" [dependencies] anyhow = { workspace = true } +base64 = "0.22" futures-util = "0.3" libc = "0.2" reqwest = { version = "0.12", features = ["json", "stream"] } diff --git a/crates/paw-codex-worker/fixtures/fake-codex.sh b/crates/paw-codex-worker/fixtures/fake-codex.sh index ff44ef894..961a57160 100755 --- a/crates/paw-codex-worker/fixtures/fake-codex.sh +++ b/crates/paw-codex-worker/fixtures/fake-codex.sh @@ -25,6 +25,14 @@ while [ "$#" -gt 0 ]; do fi shift ;; + --model) + shift + if [ "$#" -eq 0 ]; then + echo "--model requires a value" >&2 + exit 2 + fi + shift + ;; --cd) shift if [ "$#" -eq 0 ]; then diff --git a/crates/paw-codex-worker/scripts/production-readiness.sh b/crates/paw-codex-worker/scripts/production-readiness.sh index 4fa617e09..0e987f0e7 100755 --- a/crates/paw-codex-worker/scripts/production-readiness.sh +++ b/crates/paw-codex-worker/scripts/production-readiness.sh @@ -15,6 +15,8 @@ PAW_CODEX_POLL_ON_START="${PAW_CODEX_POLL_ON_START:-1}" PAW_CODEX_DOCTOR_EXEC_SMOKE="${PAW_CODEX_DOCTOR_EXEC_SMOKE:-0}" PAW_CODEX_EXEC_TIMEOUT_SECS="${PAW_CODEX_EXEC_TIMEOUT_SECS:-1200}" MAX_CONCURRENT_RUNS="${MAX_CONCURRENT_RUNS:-1}" +WORKER_SLOT_COUNT="${WORKER_SLOT_COUNT:-$MAX_CONCURRENT_RUNS}" +LAUNCHD_STALE_SLOT_SCAN_MAX="${LAUNCHD_STALE_SLOT_SCAN_MAX:-64}" WRITE_LAUNCHD_PLIST="${WRITE_LAUNCHD_PLIST:-0}" INSTALL_LAUNCHD="${INSTALL_LAUNCHD:-0}" LAUNCHD_PLIST="${LAUNCHD_PLIST:-$HOME/Library/LaunchAgents/com.temperpaw.paw-codex-worker.plist}" @@ -41,10 +43,13 @@ require_env() { } worker_env() { + local worker_id="$1" + local launchd_label="$2" + shift 2 env \ TEMPER_URL="$TEMPER_URL" \ TEMPER_TENANT="$TEMPER_TENANT" \ - WORKER_ID="$WORKER_ID" \ + WORKER_ID="$worker_id" \ WORKER_TOKEN="$WORKER_TOKEN" \ REPO_ROOT="$REPO_ROOT" \ WORKSPACE_ROOT="$WORKSPACE_ROOT" \ @@ -53,15 +58,91 @@ worker_env() { PAW_CODEX_POLL_ON_START="$PAW_CODEX_POLL_ON_START" \ PAW_CODEX_DOCTOR_EXEC_SMOKE="$PAW_CODEX_DOCTOR_EXEC_SMOKE" \ PAW_CODEX_EXEC_TIMEOUT_SECS="$PAW_CODEX_EXEC_TIMEOUT_SECS" \ - MAX_CONCURRENT_RUNS="$MAX_CONCURRENT_RUNS" \ + MAX_CONCURRENT_RUNS="1" \ + PAW_CODEX_LAUNCHD_LABEL="$launchd_label" \ "$@" } +slot_worker_id() { + local slot="$1" + if [[ "$slot" == "1" ]]; then + printf '%s\n' "$WORKER_ID" + else + printf '%s-slot-%02d\n' "$WORKER_ID" "$slot" + fi +} + +slot_launchd_label() { + local slot="$1" + if [[ "$WORKER_SLOT_COUNT" == "1" ]]; then + printf '%s\n' "com.temperpaw.paw-codex-worker" + else + slot_launchd_label_numbered "$slot" + fi +} + +slot_launchd_label_numbered() { + local slot="$1" + printf 'com.temperpaw.paw-codex-worker.slot-%02d\n' "$slot" +} + +slot_launchd_plist() { + local slot="$1" + if [[ "$WORKER_SLOT_COUNT" == "1" ]]; then + printf '%s\n' "$LAUNCHD_PLIST" + else + slot_launchd_plist_numbered "$slot" + fi +} + +slot_launchd_plist_numbered() { + local slot="$1" + local dir + dir="$(dirname "$LAUNCHD_PLIST")" + printf '%s/com.temperpaw.paw-codex-worker.slot-%02d.plist\n' "$dir" "$slot" +} + +launchd_bootout_agent() { + local label="$1" + local plist="$2" + launchctl bootout "gui/$(id -u)/${label}" >/dev/null 2>&1 || true + if [[ -f "$plist" ]]; then + launchctl bootout "gui/$(id -u)" "$plist" >/dev/null 2>&1 || true + fi +} + +unload_stale_launchd_agents() { + local max_slot="$LAUNCHD_STALE_SLOT_SCAN_MAX" + if [[ "$WORKER_SLOT_COUNT" -gt "$max_slot" ]]; then + max_slot="$WORKER_SLOT_COUNT" + fi + + log "unloading stale paw-codex-worker launchd agents" + launchd_bootout_agent "com.temperpaw.paw-codex-worker" "$LAUNCHD_PLIST" + for slot in $(seq 1 "$max_slot"); do + launchd_bootout_agent "$(slot_launchd_label_numbered "$slot")" "$(slot_launchd_plist_numbered "$slot")" + done + + local launchd_dir + launchd_dir="$(dirname "$LAUNCHD_PLIST")" + for plist in "${launchd_dir}"/com.temperpaw.paw-codex-worker.slot-*.plist; do + [[ -e "$plist" ]] || continue + launchctl bootout "gui/$(id -u)" "$plist" >/dev/null 2>&1 || true + done +} + require_cmd cargo require_cmd git require_env TEMPER_URL require_env WORKER_TOKEN +if ! [[ "$WORKER_SLOT_COUNT" =~ ^[0-9]+$ ]] || [[ "$WORKER_SLOT_COUNT" -lt 1 ]]; then + fail "WORKER_SLOT_COUNT must be a positive integer" +fi +if ! [[ "$LAUNCHD_STALE_SLOT_SCAN_MAX" =~ ^[0-9]+$ ]] || [[ "$LAUNCHD_STALE_SLOT_SCAN_MAX" -lt 1 ]]; then + fail "LAUNCHD_STALE_SLOT_SCAN_MAX must be a positive integer" +fi + if [[ "$REQUIRE_MAIN_ANCESTRY" == "1" ]]; then bash "${ROOT}/crates/paw-codex-worker/scripts/production-git-ancestry-guard.sh" fi @@ -70,6 +151,7 @@ log "repo root: ${ROOT}" log "temper url: ${TEMPER_URL}" log "tenant: ${TEMPER_TENANT}" log "worker id: ${WORKER_ID}" +log "worker slot count: ${WORKER_SLOT_COUNT}" log "repo checkout: ${REPO_ROOT}" log "worktree root: ${WORKSPACE_ROOT}" log "codex binary: ${CODEX_BIN}" @@ -80,23 +162,39 @@ log "building paw-codex-worker release binary" cargo build -p paw-codex-worker --release log "running paw-codex-worker doctor" -worker_env "$WORKER_BIN" doctor +worker_env "$(slot_worker_id 1)" "$(slot_launchd_label 1)" "$WORKER_BIN" doctor + +log "registering worker slot identities" +for slot in $(seq 1 "$WORKER_SLOT_COUNT"); do + slot_worker="$(slot_worker_id "$slot")" + slot_label="$(slot_launchd_label "$slot")" + worker_env "$slot_worker" "$slot_label" "$WORKER_BIN" register-worker-agent +done if [[ "$WRITE_LAUNCHD_PLIST" == "1" ]]; then - log "rendering launchd-plist to ${LAUNCHD_PLIST}" - mkdir -p "$(dirname "$LAUNCHD_PLIST")" - worker_env "$WORKER_BIN" launchd-plist >"$LAUNCHD_PLIST" - chmod 600 "$LAUNCHD_PLIST" + for slot in $(seq 1 "$WORKER_SLOT_COUNT"); do + slot_worker="$(slot_worker_id "$slot")" + slot_label="$(slot_launchd_label "$slot")" + slot_plist="$(slot_launchd_plist "$slot")" + log "rendering launchd plist ${slot} for ${slot_worker} to ${slot_plist}" + mkdir -p "$(dirname "$slot_plist")" + worker_env "$slot_worker" "$slot_label" "$WORKER_BIN" launchd-plist >"$slot_plist" + chmod 600 "$slot_plist" + done elif [[ "$INSTALL_LAUNCHD" == "1" ]]; then fail "INSTALL_LAUNCHD=1 requires WRITE_LAUNCHD_PLIST=1 so the exact plist is rendered first" fi if [[ "$INSTALL_LAUNCHD" == "1" ]]; then require_cmd launchctl - log "loading launchd agent ${LAUNCHD_PLIST}" - launchctl bootout "gui/$(id -u)" "$LAUNCHD_PLIST" >/dev/null 2>&1 || true - launchctl bootstrap "gui/$(id -u)" "$LAUNCHD_PLIST" - launchctl kickstart -k "gui/$(id -u)/com.temperpaw.paw-codex-worker" + unload_stale_launchd_agents + for slot in $(seq 1 "$WORKER_SLOT_COUNT"); do + slot_label="$(slot_launchd_label "$slot")" + slot_plist="$(slot_launchd_plist "$slot")" + log "loading launchd agent ${slot_label} from ${slot_plist}" + launchctl bootstrap "gui/$(id -u)" "$slot_plist" + launchctl kickstart -k "gui/$(id -u)/${slot_label}" + done fi log "production readiness check passed" diff --git a/crates/paw-codex-worker/src/boot_watch.rs b/crates/paw-codex-worker/src/boot_watch.rs index 5331c9e2f..237fc326e 100644 --- a/crates/paw-codex-worker/src/boot_watch.rs +++ b/crates/paw-codex-worker/src/boot_watch.rs @@ -103,7 +103,16 @@ async fn query_boot_entity_ids( .await .with_context(|| format!("query {entity_set} with status {status} on boot"))?; if !response.status().is_success() { - warn!(entity_set, status, http_status = %response.status(), "boot query failed"); + if response.status() == reqwest::StatusCode::NOT_FOUND { + debug!( + entity_set, + status, + http_status = %response.status(), + "optional boot query entity set is not installed" + ); + } else { + warn!(entity_set, status, http_status = %response.status(), "boot query failed"); + } return Ok(Vec::new()); } diff --git a/crates/paw-codex-worker/src/cli.rs b/crates/paw-codex-worker/src/cli.rs index 01b0ffe0d..1c738aaa5 100644 --- a/crates/paw-codex-worker/src/cli.rs +++ b/crates/paw-codex-worker/src/cli.rs @@ -6,6 +6,9 @@ fn parse_worker_command(args: impl IntoIterator) -> WorkerCommand "launchd-plist" | "plist" | "--launchd-plist" => { return WorkerCommand::LaunchdPlist; } + "register-worker-agent" | "--register-worker-agent" => { + return WorkerCommand::RegisterWorkerAgent; + } "directed-evolution-start-episode" | "de-start-episode" => { return WorkerCommand::DirectedEvolutionStartEpisode { contract_path: args.next(), diff --git a/crates/paw-codex-worker/src/directed_evolution.rs b/crates/paw-codex-worker/src/directed_evolution.rs index 1a4aff2cb..4129e7133 100644 --- a/crates/paw-codex-worker/src/directed_evolution.rs +++ b/crates/paw-codex-worker/src/directed_evolution.rs @@ -14,7 +14,31 @@ async fn handle_queued_directed_evolution_work_item( return Ok(()); } if work_item.role.trim().is_empty() { - debug!(work_item_id, "Directed Evolution WorkItem has no brain role yet"); + debug!(work_item_id, "Directed Evolution WorkItem has no worker role yet"); + return Ok(()); + } + if let Some(conflict) = + directed_evolution_exclusive_key_conflict(client, config, &work_item).await? + { + let reason = format!( + "Exclusive key {} is already active on WorkItem {}; cancelling duplicate {} work", + conflict.exclusive_key, conflict.active_work_item_id, work_item.role + ); + post_paw_orchestration_action( + client, + config, + "WorkItems", + &work_item.id, + "CancelWorkItem", + json!({ "Reason": reason }), + ) + .await?; + info!( + work_item_id = %work_item.id, + active_work_item_id = %conflict.active_work_item_id, + exclusive_key = %conflict.exclusive_key, + "cancelled duplicate exclusive Directed Evolution WorkItem" + ); return Ok(()); } if let Some(reason) = @@ -31,7 +55,7 @@ async fn handle_queued_directed_evolution_work_item( "stale Directed Evolution WorkItem cleanup could not mutate StageResult; cancelling WorkItem only" ); } - if let Err(error) = post_directed_evolution_action( + if let Err(error) = post_paw_orchestration_action( client, config, "WorkItems", @@ -53,7 +77,7 @@ async fn handle_queued_directed_evolution_work_item( return Ok(()); } - if let Err(error) = post_directed_evolution_action( + if let Err(error) = post_paw_orchestration_action( client, config, "WorkItems", @@ -86,36 +110,63 @@ async fn handle_queued_directed_evolution_work_item( ) }); } + if let Some(conflict) = + directed_evolution_exclusive_key_post_claim_conflict(client, config, &work_item).await? + { + let reason = format!( + "Exclusive key {} is already claimed by lower-priority WorkItem {}; cancelling duplicate {} work before WorkerRun start", + conflict.exclusive_key, conflict.active_work_item_id, work_item.role + ); + post_paw_orchestration_action( + client, + config, + "WorkItems", + &work_item.id, + "CancelWorkItem", + json!({ "Reason": reason }), + ) + .await?; + info!( + work_item_id = %work_item.id, + active_work_item_id = %conflict.active_work_item_id, + exclusive_key = %conflict.exclusive_key, + "cancelled duplicate exclusive Directed Evolution WorkItem after claim" + ); + return Ok(()); + } - let brain_run_id = create_entity(client, config, "BrainRuns", json!({})).await?; - post_directed_evolution_action( + let worker_run_id = create_entity(client, config, "WorkerRuns", json!({})).await?; + post_paw_orchestration_action( client, config, - "BrainRuns", - &brain_run_id, - "StartBrainRun", + "WorkerRuns", + &worker_run_id, + "StartWorkerRun", json!({ "Role": work_item.role, "WorkItemId": work_item.id, + "WorkerId": config.worker_id, + "ProviderId": directed_evolution_worker_provider_id(), "AgentKind": directed_evolution_agent_kind_for_role(&work_item.role), "Model": directed_evolution_model_for_role(&work_item.role), + "SessionId": "", "ParentSessionId": env::var("CODEX_SESSION_ID").unwrap_or_default(), "CorrelationJson": work_item.correlation_json, }), ) .await?; - post_directed_evolution_action( + post_paw_orchestration_action( client, config, "WorkItems", &work_item.id, "StartWorkItem", - json!({ "BrainRunId": brain_run_id }), + json!({ "WorkerRunId": worker_run_id }), ) .await?; info!( work_item_id = %work_item.id, - brain_run_id = %brain_run_id, + worker_run_id = %worker_run_id, role = %work_item.role, target_entity_type = %work_item.target_entity_type, target_entity_id = %work_item.target_entity_id, @@ -125,22 +176,22 @@ async fn handle_queued_directed_evolution_work_item( match run_directed_evolution_codex_role(client, config, &work_item).await { Ok(output_json) => { let summary = directed_evolution_summary(&work_item, &output_json); - let evidence_artifact_id = record_directed_evolution_brain_evidence( + let evidence_artifact_id = record_directed_evolution_worker_evidence( client, config, &work_item, - &brain_run_id, - "codex_brain_run", + &worker_run_id, + "codex_worker_run", &output_json, &summary, ) .await?; - post_directed_evolution_action( + post_paw_orchestration_action( client, config, - "BrainRuns", - &brain_run_id, - "SucceedBrainRun", + "WorkerRuns", + &worker_run_id, + "SucceedWorkerRun", json!({ "OutputJson": output_json, "EvidenceArtifactId": evidence_artifact_id, @@ -148,7 +199,43 @@ async fn handle_queued_directed_evolution_work_item( }), ) .await?; - post_directed_evolution_action( + if let Err(route_error) = route_directed_evolution_work_item_success( + client, + config, + &work_item, + &worker_run_id, + &output_json, + &evidence_artifact_id, + &summary, + ) + .await + { + let failure_reason = format!( + "Directed Evolution receipt routing failed after WorkerRun success: {route_error}" + ); + warn!( + work_item_id = %work_item.id, + worker_run_id = %worker_run_id, + role = %work_item.role, + evidence_artifact_id = %evidence_artifact_id, + %route_error, + "Directed Evolution WorkItem receipt routing failed after WorkerRun success" + ); + post_paw_orchestration_action( + client, + config, + "WorkItems", + &work_item.id, + "FailWorkItem", + json!({ + "FailureReason": failure_reason, + "EvidenceArtifactId": evidence_artifact_id, + }), + ) + .await?; + return Ok(()); + } + post_paw_orchestration_action( client, config, "WorkItems", @@ -163,21 +250,21 @@ async fn handle_queued_directed_evolution_work_item( .await?; info!( work_item_id = %work_item.id, - brain_run_id = %brain_run_id, + worker_run_id = %worker_run_id, role = %work_item.role, evidence_artifact_id = %evidence_artifact_id, - "completed Directed Evolution Codex brain run" + "completed Directed Evolution Codex worker run" ); Ok(()) } Err(error) => { let failure_reason = format!("Directed Evolution Codex role failed: {error}"); - let evidence_artifact_id = match record_directed_evolution_brain_evidence( + let evidence_artifact_id = match record_directed_evolution_worker_evidence( client, config, &work_item, - &brain_run_id, - "codex_brain_run_failure", + &worker_run_id, + "codex_worker_run_failure", &serde_json::to_string(&json!({ "status": "failed", "failure_reason": failure_reason, @@ -188,16 +275,16 @@ async fn handle_queued_directed_evolution_work_item( { Ok(id) => id, Err(report_error) => { - warn!(%report_error, work_item_id, brain_run_id, "failed to record Directed Evolution failure evidence"); + warn!(%report_error, work_item_id, worker_run_id, "failed to record Directed Evolution failure evidence"); String::new() } }; - if let Err(report_error) = post_directed_evolution_action( + if let Err(report_error) = post_paw_orchestration_action( client, config, - "BrainRuns", - &brain_run_id, - "FailBrainRun", + "WorkerRuns", + &worker_run_id, + "FailWorkerRun", json!({ "FailureReason": failure_reason, "EvidenceArtifactId": evidence_artifact_id, @@ -205,9 +292,28 @@ async fn handle_queued_directed_evolution_work_item( ) .await { - warn!(%report_error, work_item_id, brain_run_id, "failed to report BrainRun failure"); + warn!(%report_error, work_item_id, worker_run_id, "failed to report WorkerRun failure"); } - post_directed_evolution_action( + if let Err(route_error) = route_directed_evolution_work_item_failure( + client, + config, + &work_item, + &worker_run_id, + &failure_reason, + &evidence_artifact_id, + ) + .await + { + warn!( + work_item_id = %work_item.id, + worker_run_id = %worker_run_id, + role = %work_item.role, + evidence_artifact_id = %evidence_artifact_id, + %route_error, + "Directed Evolution failure receipt routing failed; failing WorkItem anyway" + ); + } + post_paw_orchestration_action( client, config, "WorkItems", @@ -221,10 +327,10 @@ async fn handle_queued_directed_evolution_work_item( .await?; warn!( work_item_id = %work_item.id, - brain_run_id = %brain_run_id, + worker_run_id = %worker_run_id, role = %work_item.role, evidence_artifact_id = %evidence_artifact_id, - "failed Directed Evolution Codex brain run" + "failed Directed Evolution Codex worker run" ); Ok(()) } @@ -236,6 +342,105 @@ fn directed_evolution_claim_conflict_message(message: &str) -> bool { && message.contains("not valid from state") } +struct DirectedEvolutionExclusiveKeyConflict { + exclusive_key: String, + active_work_item_id: String, +} + +async fn directed_evolution_exclusive_key_conflict( + client: &reqwest::Client, + config: &Config, + work_item: &DirectedEvolutionWorkItemState, +) -> Result> { + directed_evolution_exclusive_key_conflict_with_policy( + client, + config, + work_item, + ExclusiveKeyConflictPolicy::AnyActivePeer, + ) + .await +} + +async fn directed_evolution_exclusive_key_post_claim_conflict( + client: &reqwest::Client, + config: &Config, + work_item: &DirectedEvolutionWorkItemState, +) -> Result> { + directed_evolution_exclusive_key_conflict_with_policy( + client, + config, + work_item, + ExclusiveKeyConflictPolicy::LowerActivePeer, + ) + .await +} + +#[derive(Clone, Copy)] +enum ExclusiveKeyConflictPolicy { + AnyActivePeer, + LowerActivePeer, +} + +async fn directed_evolution_exclusive_key_conflict_with_policy( + client: &reqwest::Client, + config: &Config, + work_item: &DirectedEvolutionWorkItemState, + policy: ExclusiveKeyConflictPolicy, +) -> Result> { + let fields = + fetch_directed_evolution_entity_fields(client, config, "WorkItems", &work_item.id).await?; + let exclusive_key = value_field_string(&fields, &["ExclusiveKey", "exclusive_key"]); + if exclusive_key.trim().is_empty() { + return Ok(None); + } + + let escaped_key = exclusive_key.replace('\'', "''"); + let rows = query_directed_evolution_rows( + client, + config, + "WorkItems", + &format!("ExclusiveKey eq '{escaped_key}'"), + 100, + ) + .await?; + if let Some(active_work_item_id) = + directed_evolution_exclusive_key_conflict_from_rows(&work_item.id, &rows, policy) + { + return Ok(Some(DirectedEvolutionExclusiveKeyConflict { + exclusive_key, + active_work_item_id, + })); + } + + Ok(None) +} + +fn directed_evolution_exclusive_key_conflict_from_rows( + work_item_id: &str, + rows: &[Value], + policy: ExclusiveKeyConflictPolicy, +) -> Option { + let mut active_ids = Vec::new(); + for row in rows { + let row_fields = directed_evolution_row_fields(row); + let row_id = directed_evolution_row_id(row); + if row_id == work_item_id { + continue; + } + let status = value_field_string(&row_fields, &["Status", "status"]); + if matches!(status.as_str(), "Claimed" | "Running") { + active_ids.push(row_id); + } + } + active_ids.sort(); + match policy { + ExclusiveKeyConflictPolicy::AnyActivePeer => active_ids.into_iter().next(), + ExclusiveKeyConflictPolicy::LowerActivePeer => active_ids + .into_iter() + .find(|active_id| active_id.as_str() < work_item_id), + } +} + async fn stale_directed_evolution_work_item_reason( client: &reqwest::Client, config: &Config, @@ -374,6 +579,116 @@ async fn post_directed_evolution_action( .await } +async fn post_paw_orchestration_action( + client: &reqwest::Client, + config: &Config, + entity_set: &str, + entity_id: &str, + action: &str, + body: Value, +) -> Result<()> { + post_entity_action_with_namespace( + client, + config, + entity_set, + entity_id, + PAW_ORCHESTRATION_NAMESPACE, + action, + body, + ) + .await +} + +async fn post_paw_orchestration_action_with_headers( + client: &reqwest::Client, + config: &Config, + request_headers: &HeaderMap, + entity_set: &str, + entity_id: &str, + action: &str, + body: Value, +) -> Result<()> { + let url = config.entity_action_url_with_namespace( + entity_set, + entity_id, + PAW_ORCHESTRATION_NAMESPACE, + action, + ); + let response = client + .post(url) + .headers(request_headers.clone()) + .json(&body) + .send() + .await + .with_context(|| format!("dispatch PawOrchestration {entity_set}.{action}"))?; + if !response.status().is_success() { + let status = response.status(); + let text = response.text().await.unwrap_or_default(); + bail!("PawOrchestration {entity_set}.{action} returned {status}: {text}"); + } + Ok(()) +} + +async fn route_directed_evolution_work_item_success( + client: &reqwest::Client, + config: &Config, + work_item: &DirectedEvolutionWorkItemState, + worker_run_id: &str, + output_json: &str, + evidence_artifact_id: &str, + summary: &str, +) -> Result<()> { + let receipt_id = create_entity(client, config, "WorkItemReceipts", json!({})).await?; + post_directed_evolution_action( + client, + config, + "WorkItemReceipts", + &receipt_id, + "RouteSucceededWorkItem", + json!({ + "WorkItemId": work_item.id, + "Role": work_item.role, + "TargetEntityType": work_item.target_entity_type, + "TargetEntityId": work_item.target_entity_id, + "WorkerRunId": worker_run_id, + "ResultJson": output_json, + "EvidenceArtifactId": evidence_artifact_id, + "Summary": summary, + "CorrelationJson": work_item.correlation_json, + }), + ) + .await +} + +async fn route_directed_evolution_work_item_failure( + client: &reqwest::Client, + config: &Config, + work_item: &DirectedEvolutionWorkItemState, + worker_run_id: &str, + failure_reason: &str, + evidence_artifact_id: &str, +) -> Result<()> { + let receipt_id = create_entity(client, config, "WorkItemReceipts", json!({})).await?; + post_directed_evolution_action( + client, + config, + "WorkItemReceipts", + &receipt_id, + "RouteFailedWorkItem", + json!({ + "WorkItemId": work_item.id, + "Role": work_item.role, + "TargetEntityType": work_item.target_entity_type, + "TargetEntityId": work_item.target_entity_id, + "WorkerRunId": worker_run_id, + "FailureReason": failure_reason, + "EvidenceArtifactId": evidence_artifact_id, + "CorrelationJson": work_item.correlation_json, + }), + ) + .await +} + async fn run_directed_evolution_codex_role( client: &reqwest::Client, config: &Config, @@ -389,6 +704,12 @@ async fn run_directed_evolution_codex_role( "executing Directed Evolution Codex role" ); if !config.enable_execution { + if directed_evolution_work_item_requires_datadog_evidence(work_item) { + bail!( + "Directed Evolution role {} requires live Codex execution and Datadog evidence", + work_item.role + ); + } return serde_json::to_string(&json!({ "status": "dry_run", "role": work_item.role, @@ -419,12 +740,12 @@ async fn run_directed_evolution_codex_role( } else { directed_evolution_git_status_snapshot(&workdir.path).await? }; - let output = match if work_item.role == "telemetry_evaluator" { + let output = match if directed_evolution_work_item_requires_datadog_evidence(work_item) { run_codex_exec_command_with_datadog_mcp( config, &workdir.path, prompt, - "run Directed Evolution Datadog telemetry evaluator", + "run Directed Evolution Datadog-backed role", ) .await } else { @@ -487,6 +808,7 @@ async fn run_directed_evolution_codex_role( }) }); } + ensure_directed_evolution_required_datadog_evidence(work_item, &payload)?; serde_json::to_string(&payload) .context("serialize Directed Evolution Codex output") } @@ -499,14 +821,46 @@ fn directed_evolution_agent_kind_for_role(role: &str) -> &'static str { } } -fn directed_evolution_model_for_role(role: &str) -> &'static str { +fn directed_evolution_role_requires_datadog_mcp(role: &str) -> bool { + matches!(role, "observer" | "telemetry_evaluator") +} + +fn directed_evolution_work_item_requires_datadog_evidence( + work_item: &DirectedEvolutionWorkItemState, +) -> bool { + directed_evolution_role_requires_datadog_mcp(&work_item.role) + || directed_evolution_text_requires_datadog_evidence(&work_item.prompt_ref) + || directed_evolution_text_requires_datadog_evidence(&work_item.context_ref) + || directed_evolution_text_requires_datadog_evidence(&work_item.output_schema_ref) + || directed_evolution_text_requires_datadog_evidence(&work_item.correlation_json) +} + +fn directed_evolution_text_requires_datadog_evidence(value: &str) -> bool { + value + .to_ascii_lowercase() + .contains("datadog_evidence_scope") +} + +fn directed_evolution_worker_provider_id() -> String { + worker_provider_id() +} + +fn directed_evolution_model_for_role(role: &str) -> String { if matches!(role, "promoter" | "state_verifier" | "wasm_evaluator") { - "deterministic-worker" + "deterministic-worker".to_string() } else { - "codex-cli" + directed_evolution_codex_model_label() } } +fn directed_evolution_codex_model_label() -> String { + env::var("PAW_CODEX_MODEL") + .ok() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| "codex-cli".to_string()) +} + async fn recover_directed_evolution_variant_output( client: &reqwest::Client, config: &Config, @@ -540,7 +894,7 @@ async fn recover_directed_evolution_variant_output( "verification_notes": format!( "Codex produced repository changes but did not finish the structured response: {error_summary}. Later evaluation stages must validate the hot-loaded runtime before selection." ), - "reasoning_summary": "The worker recovered concrete file mutations left by the variant-generator brain and will publish them as a candidate for downstream evaluation.", + "reasoning_summary": "The worker recovered concrete file mutations left by the variant-generator worker run and will publish them as a candidate for downstream evaluation.", "recovery": { "reason": "codex_exec_error_after_file_changes", "error": error_summary, diff --git a/crates/paw-codex-worker/src/directed_evolution/evidence.rs b/crates/paw-codex-worker/src/directed_evolution/evidence.rs index 8d21614ef..c7a26427a 100644 --- a/crates/paw-codex-worker/src/directed_evolution/evidence.rs +++ b/crates/paw-codex-worker/src/directed_evolution/evidence.rs @@ -1,8 +1,8 @@ -async fn record_directed_evolution_brain_evidence( +async fn record_directed_evolution_worker_evidence( client: &reqwest::Client, config: &Config, work_item: &DirectedEvolutionWorkItemState, - brain_run_id: &str, + worker_run_id: &str, artifact_kind: &str, output_json: &str, summary: &str, @@ -16,7 +16,7 @@ async fn record_directed_evolution_brain_evidence( let uri = directed_evolution_evidence_uri(work_item, &output_value); let correlation = json!({ "work_item_id": work_item.id, - "brain_run_id": brain_run_id, + "worker_run_id": worker_run_id, "role": work_item.role, "target_entity_type": work_item.target_entity_type, "target_entity_id": work_item.target_entity_id, @@ -54,8 +54,8 @@ async fn record_directed_evolution_brain_evidence( &evidence_id, "LinkEvidenceArtifact", json!({ - "TargetEntityType": "BrainRun", - "TargetEntityId": brain_run_id, + "TargetEntityType": "WorkerRun", + "TargetEntityId": worker_run_id, }), ) .await?; @@ -81,7 +81,11 @@ fn directed_evolution_first_evidence_scope_summary( else { return DirectedEvolutionEvidenceScopeSummary::default(); }; - let Some(first) = items.first() else { + let Some(first) = items + .iter() + .find(|item| directed_evolution_scope_satisfies_datadog_contract(item)) + .or_else(|| items.first()) + else { return DirectedEvolutionEvidenceScopeSummary::default(); }; DirectedEvolutionEvidenceScopeSummary { @@ -108,6 +112,65 @@ fn directed_evolution_first_evidence_scope_summary( } } +fn ensure_directed_evolution_required_datadog_evidence( + work_item: &DirectedEvolutionWorkItemState, + output: &Value, +) -> Result<()> { + if !directed_evolution_work_item_requires_datadog_evidence(work_item) { + return Ok(()); + } + let role = work_item.role.as_str(); + let Some(items) = output + .get("evidence_scope") + .or_else(|| output.get("evidenceScope")) + .and_then(Value::as_array) + else { + bail!("{role} output missing mandatory Datadog evidence_scope"); + }; + if items + .iter() + .any(directed_evolution_scope_satisfies_datadog_contract) + { + return Ok(()); + } + bail!("{role} output missing complete Datadog evidence_scope entry") +} + +fn directed_evolution_scope_satisfies_datadog_contract(scope: &Value) -> bool { + let required = [ + ("query", &["query", "Query"][..]), + ("time_window", &["time_window", "timeWindow", "TimeWindow"][..]), + ( + "result_count", + &["result_count", "resultCount", "ResultCount", "count"][..], + ), + ("interpretation", &["interpretation", "Interpretation"][..]), + ( + "zero_result_meaning", + &[ + "zero_result_meaning", + "zeroResultMeaning", + "ZeroResultMeaning", + ][..], + ), + ]; + for (_, aliases) in required { + if value_field_string(scope, aliases).trim().is_empty() { + return false; + } + } + let Some(url) = scope + .get("datadog_url") + .or_else(|| scope.get("datadogUrl")) + .and_then(Value::as_str) + .map(str::trim) + .filter(|value| !value.is_empty()) + else { + return false; + }; + is_datadog_app_url(url) +} + fn directed_evolution_evidence_provenance(role: &str, output: &Value) -> String { let fallback = match role { "simulated_user" => "agent-observed", diff --git a/crates/paw-codex-worker/src/directed_evolution/human_episode.rs b/crates/paw-codex-worker/src/directed_evolution/human_episode.rs index 3dcba9b72..165fce7cb 100644 --- a/crates/paw-codex-worker/src/directed_evolution/human_episode.rs +++ b/crates/paw-codex-worker/src/directed_evolution/human_episode.rs @@ -78,7 +78,7 @@ async fn start_directed_evolution_human_episode( json!({ "EpisodeId": episode_id, "GoalStatement": plan.adaptation_goal, - "CreatedByBrainRunId": plan.created_by_brain_run_id, + "CreatedByWorkerRunId": plan.created_by_worker_run_id, "HumanNotes": plan.human_notes, }), ) @@ -140,7 +140,7 @@ async fn start_directed_evolution_human_episode( "EpisodeId": episode_id, "ConstraintStatement": constraint.statement, "ConstraintKind": constraint.kind, - "CreatedByBrainRunId": plan.created_by_brain_run_id, + "CreatedByWorkerRunId": plan.created_by_worker_run_id, }), ) .await?; @@ -202,7 +202,7 @@ async fn start_directed_evolution_human_episode( "RuleStatement": rule.statement, "MetricIdsJson": json!(metric_ids_for_rule).to_string(), "ThresholdJson": rule.threshold.to_string(), - "CreatedByBrainRunId": plan.created_by_brain_run_id, + "CreatedByWorkerRunId": plan.created_by_worker_run_id, }), ) .await?; @@ -231,7 +231,7 @@ async fn start_directed_evolution_human_episode( "RuleStatement": rule.statement, "MetricIdsJson": json!(metric_ids_for_rule).to_string(), "Weight": rule.weight, - "CreatedByBrainRunId": plan.created_by_brain_run_id, + "CreatedByWorkerRunId": plan.created_by_worker_run_id, }), ) .await?; @@ -299,7 +299,7 @@ async fn start_directed_evolution_human_episode( "ScoringRuleIdsJson": json!(scoring_rule_ids).to_string(), "EvaluatorRef": plan.evaluator_ref, "DecisionPolicy": plan.selection_notes, - "CreatedByBrainRunId": plan.created_by_brain_run_id, + "CreatedByWorkerRunId": plan.created_by_worker_run_id, }), ) .await?; diff --git a/crates/paw-codex-worker/src/directed_evolution/human_episode_plan.rs b/crates/paw-codex-worker/src/directed_evolution/human_episode_plan.rs index 3314e777f..74f82ac0d 100644 --- a/crates/paw-codex-worker/src/directed_evolution/human_episode_plan.rs +++ b/crates/paw-codex-worker/src/directed_evolution/human_episode_plan.rs @@ -12,8 +12,14 @@ struct DirectedEvolutionHumanEpisodeInput { adaptation_goal: String, #[serde(default, alias = "HumanNotes", alias = "humanNotes")] human_notes: String, - #[serde(default, alias = "CreatedByBrainRunId", alias = "createdByBrainRunId")] - created_by_brain_run_id: String, + #[serde( + default, + alias = "CreatedByWorkerRunId", + alias = "createdByWorkerRunId", + alias = "CreatedByBrainRunId", + alias = "createdByBrainRunId" + )] + created_by_worker_run_id: String, #[serde(default, alias = "EvaluatorRef", alias = "evaluatorRef")] evaluator_ref: String, #[serde(default, alias = "Metrics", alias = "metricDefinitions")] @@ -188,7 +194,7 @@ struct DirectedEvolutionEpisodePlan { autonomy_lane: String, adaptation_goal: String, human_notes: String, - created_by_brain_run_id: String, + created_by_worker_run_id: String, evaluator_ref: String, metrics: Vec, viability_constraints: Vec, @@ -377,8 +383,8 @@ fn directed_evolution_episode_plan_from_input( input.human_notes, "Human and Codex agreed to this episode contract in chat.".to_string(), ), - created_by_brain_run_id: nonempty( - input.created_by_brain_run_id.clone(), + created_by_worker_run_id: nonempty( + input.created_by_worker_run_id.clone(), env::var("CODEX_SESSION_ID").unwrap_or_else(|_| "chat-codex".to_string()), ), evaluator_ref: nonempty( @@ -412,7 +418,7 @@ fn directed_evolution_episode_plan_from_input( selected_by: nonempty(input.selected_by, "human+codex-chat".to_string()), selection_notes: nonempty( input.selection_notes, - "Selected after human-brain negotiation in Codex chat.".to_string(), + "Selected after human-agent negotiation in Codex chat.".to_string(), ), started_by: nonempty(input.started_by, "codex".to_string()), start_reason: nonempty( diff --git a/crates/paw-codex-worker/src/directed_evolution/prompt.rs b/crates/paw-codex-worker/src/directed_evolution/prompt.rs index 0b7a76756..ce79bd711 100644 --- a/crates/paw-codex-worker/src/directed_evolution/prompt.rs +++ b/crates/paw-codex-worker/src/directed_evolution/prompt.rs @@ -1,7 +1,7 @@ fn directed_evolution_prompt(work_item: &DirectedEvolutionWorkItemState) -> String { let role_contract = match work_item.role.as_str() { "observer" => { - "Observe real signals and infer pressures. If the signal source or correlation mentions Datadog, use authenticated Datadog MCP tools to inspect the relevant logs, traces, metrics, monitors, or dashboards. Do not treat the signal summary as proof. Return candidate pressures, rejected interpretations, structured evidence_scope entries with datadog_url values when available, and confidence." + "Observe real signals and infer pressures. Datadog inspection is mandatory: use authenticated Datadog MCP tools to inspect relevant logs, traces, metrics, monitors, or dashboards alongside Temper runtime evidence and Genesis/app history. Do not treat the signal summary as proof. Return multiple candidate pressures or directions when evidence supports them, rejected interpretations, structured evidence_scope entries with query, time_window, result_count, interpretation, zero_result_meaning, datadog_url, and confidence. If usable Datadog evidence is unavailable, return actionable=false and explain the missing evidence." } "direction_framer" => { "Frame human-legible directions from pressures. Return direction title, rationale, source signals, required human gate, and likely adaptation goals." @@ -34,14 +34,14 @@ fn directed_evolution_prompt(work_item: &DirectedEvolutionWorkItemState) -> Stri "Explain the episode outcome for Mission Control. Return concise human-facing narrative, lineage impact, and evidence links." } _ => { - "Execute the Directed Evolution brain role described by this WorkItem. Return structured evidence and next-state recommendations." + "Execute the Directed Evolution worker role described by this WorkItem. Return structured evidence and next-state recommendations." } }; let output_contract = directed_evolution_output_contract(&work_item.role); let prompt_body = directed_evolution_worker_prompt_body(&work_item.role, &literal_prompt_ref(&work_item.prompt_ref)); format!( - r#"You are a Codex brain run executing a Directed Evolution WorkItem. + r#"You are a Codex WorkerRun executing a Directed Evolution WorkItem. Role: {role} WorkItemId: {work_item_id} @@ -79,14 +79,28 @@ fn directed_evolution_output_contract(role: &str) -> &'static str { "observer" => { r#"{ "actionable": true, - "pressure_class": "growth|repair|performance|policy|ux", - "pressure_summary": "...", - "title": "...", - "direction_summary": "...", - "autonomy_lane": "human-approval|repair-auto", - "proposed_adaptation_goal": "...", - "proposed_viability_constraints": ["..."], - "evidence_scope": [{"surface":"logs|traces|metrics|monitors|runtime","query":"...","result_summary":"...","datadog_url":"https://app.datadoghq.com/..."}], + "directions": [ + { + "pressure_class": "growth|repair|performance|policy|ux", + "pressure_summary": "...", + "title": "...", + "direction_summary": "...", + "autonomy_lane": "human-approval|repair-auto", + "proposed_adaptation_goal": "...", + "proposed_viability_constraints": ["..."] + }, + { + "pressure_class": "growth|repair|performance|policy|ux", + "pressure_summary": "...", + "title": "...", + "direction_summary": "...", + "autonomy_lane": "human-approval|repair-auto", + "proposed_adaptation_goal": "...", + "proposed_viability_constraints": ["..."] + } + ], + "rejected_interpretations": [{"interpretation":"...","why_rejected":"..."}], + "evidence_scope": [{"surface":"logs|traces|metrics|monitors|runtime|genesis","query":"...","time_window":"...","result_count":0,"interpretation":"...","zero_result_meaning":"success|failure|neutral","datadog_url":"https://app.datadoghq.com/..."}], "evidence_refs": ["..."], "reasoning_summary": "..." }"# diff --git a/crates/paw-codex-worker/src/directed_evolution/recovery.rs b/crates/paw-codex-worker/src/directed_evolution/recovery.rs index 8dc658d4f..cd898ea28 100644 --- a/crates/paw-codex-worker/src/directed_evolution/recovery.rs +++ b/crates/paw-codex-worker/src/directed_evolution/recovery.rs @@ -89,7 +89,7 @@ async fn recover_pending_directed_evolution_stage_results( { let cancel_reason = format!("recovery skipped StageResult start because it changed state: {error}"); - if let Err(cancel_error) = post_directed_evolution_action_with_headers( + if let Err(cancel_error) = post_paw_orchestration_action_with_headers( client, config, &director_headers, @@ -116,7 +116,7 @@ async fn recover_pending_directed_evolution_stage_results( ); continue; } - post_directed_evolution_action_with_headers( + post_paw_orchestration_action_with_headers( client, config, &director_headers, diff --git a/crates/paw-codex-worker/src/directed_evolution/tests.rs b/crates/paw-codex-worker/src/directed_evolution/tests.rs index c14b0b2cd..b4d110d49 100644 --- a/crates/paw-codex-worker/src/directed_evolution/tests.rs +++ b/crates/paw-codex-worker/src/directed_evolution/tests.rs @@ -359,7 +359,7 @@ mod directed_evolution_tests { } #[test] - fn mechanical_evaluator_roles_are_not_codex_brains() { + fn mechanical_evaluator_roles_do_not_spawn_codex_agents() { assert!(directed_evolution_mechanical_evaluator_role("state_verifier")); assert!(directed_evolution_mechanical_evaluator_role("wasm_evaluator")); assert_eq!( @@ -435,6 +435,108 @@ mod directed_evolution_tests { assert_eq!(app.pinned_ref(), "nerdsane/agent-answers@abc123"); } + #[test] + fn directed_evolution_genesis_bundle_url_targets_pinned_version() { + let app = directed_evolution_genesis_app_from_ref("nerdsane/agent-answers@abc123") + .expect("app ref should parse"); + + let url = directed_evolution_genesis_bundle_url( + "https://genesis-production-164d.up.railway.app/", + &app, + ); + + assert_eq!( + url, + "https://genesis-production-164d.up.railway.app/api/genesis/apps/nerdsane/agent-answers/versions/abc123/bundle" + ); + } + + #[test] + fn directed_evolution_bundle_path_rejects_traversal() { + assert!(safe_directed_evolution_bundle_path("specs/answer.ioa.toml").is_ok()); + assert!(safe_directed_evolution_bundle_path("../answer.ioa.toml").is_err()); + assert!(safe_directed_evolution_bundle_path("/tmp/answer.ioa.toml").is_err()); + assert!(safe_directed_evolution_bundle_path(".git/config").is_err()); + assert!(safe_directed_evolution_bundle_path(".Git/config").is_err()); + assert!(safe_directed_evolution_bundle_path("specs/.git/config").is_err()); + assert!(safe_directed_evolution_bundle_path("specs/.GIT/config").is_err()); + } + + #[tokio::test] + async fn directed_evolution_bundle_materializes_git_repo() { + let repo = env::temp_dir().join(format!( + "paw-codex-worker-bundle-test-{}-{}", + std::process::id(), + SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("time") + .as_nanos() + )); + let app = directed_evolution_genesis_app_from_ref("nerdsane/agent-answers@abc123") + .expect("app ref should parse"); + let bundle = json!({ + "apps": [{ + "files": [ + { + "path": "APP.md", + "content_base64": "IyBBZ2VudCBBbnN3ZXJzCg==" + }, + { + "path": "specs/answer.ioa.toml", + "content_base64": "W2F1dG9tYXRvbl0KbmFtZSA9ICJBbnN3ZXIiCg==" + }, + { + "path": ".gitignore", + "content_base64": "ZGlzdC8K" + }, + { + "path": "dist/app.wasm", + "content_base64": "AAECAw==" + } + ] + }] + }); + + materialize_directed_evolution_bundle_repo(&repo, &app, &bundle) + .await + .expect("bundle materializes"); + + assert!(repo.join(".git").exists()); + assert_eq!( + fs::read_to_string(repo.join("APP.md")).expect("read APP.md"), + "# Agent Answers\n" + ); + let head = std::process::Command::new("git") + .arg("-C") + .arg(&repo) + .args(["rev-parse", "HEAD"]) + .output() + .expect("run git"); + assert!( + head.status.success(), + "git rev-parse failed: {}", + String::from_utf8_lossy(&head.stderr) + ); + assert!(!String::from_utf8_lossy(&head.stdout).trim().is_empty()); + let listed = std::process::Command::new("git") + .arg("-C") + .arg(&repo) + .args(["ls-files", "--", "dist/app.wasm"]) + .output() + .expect("run git ls-files"); + assert!( + listed.status.success(), + "git ls-files failed: {}", + String::from_utf8_lossy(&listed.stderr) + ); + assert_eq!( + String::from_utf8_lossy(&listed.stdout).trim(), + "dist/app.wasm", + "forced bundle materialization should commit files ignored by bundled .gitignore" + ); + fs::remove_dir_all(repo).ok(); + } + #[test] fn directed_evolution_variant_tenant_is_stable_and_safe() { let work_item = DirectedEvolutionWorkItemState { @@ -668,6 +770,149 @@ mod directed_evolution_tests { assert_eq!(summary.zero_result_meaning, "failure"); } + #[test] + fn directed_evolution_evidence_summary_prefers_complete_datadog_scope() { + let summary = directed_evolution_first_evidence_scope_summary(&json!({ + "evidence_scope": [ + { + "surface": "genesis", + "query": "app history", + "result_summary": "Found prior changes." + }, + { + "surface": "logs", + "query": "service:temper-platform @directed_evolution.variant_id:v3", + "time_window": "2026-06-02T14:00:00Z/2026-06-02T15:00:00Z", + "result_count": 31, + "interpretation": "Runtime logs were present for v3.", + "zero_result_meaning": "failure", + "datadog_url": "https://app.datadoghq.com/logs?query=v3" + } + ] + })); + + assert_eq!( + summary.query, + "service:temper-platform @directed_evolution.variant_id:v3" + ); + assert_eq!(summary.result_count, "31"); + assert_eq!(summary.interpretation, "Runtime logs were present for v3."); + } + + #[test] + fn directed_evolution_datadog_roles_require_structured_evidence() { + fn work_item(role: &str, prompt_ref: &str) -> DirectedEvolutionWorkItemState { + DirectedEvolutionWorkItemState { + id: format!("wi-{role}"), + status: "Queued".to_string(), + role: role.to_string(), + target_entity_type: "StageResult".to_string(), + target_entity_id: "sr-1".to_string(), + prompt_ref: prompt_ref.to_string(), + context_ref: "stage-result:sr-1".to_string(), + output_schema_ref: "schema-1".to_string(), + correlation_json: "{}".to_string(), + } + } + let observer = work_item("observer", "literal:observe"); + let telemetry = work_item("telemetry_evaluator", "literal:evaluate telemetry"); + let plain_reviewer = work_item("reviewer", "literal:review without Datadog gate"); + let datadog_reviewer = work_item( + "reviewer", + "literal:RequiredEvidence: [\"datadog_evidence_scope\"]", + ); + let datadog_viability = work_item( + "viability_evaluator", + "literal:RequiredEvidence: [\"datadog_evidence_scope\"]", + ); + + assert!(directed_evolution_work_item_requires_datadog_evidence( + &observer + )); + assert!(directed_evolution_work_item_requires_datadog_evidence( + &telemetry + )); + assert!(!directed_evolution_work_item_requires_datadog_evidence( + &plain_reviewer + )); + assert!(directed_evolution_work_item_requires_datadog_evidence( + &datadog_reviewer + )); + assert!(directed_evolution_work_item_requires_datadog_evidence( + &datadog_viability + )); + + assert!( + ensure_directed_evolution_required_datadog_evidence( + &observer, + &json!({ + "evidence_scope": [ + { + "surface": "genesis", + "query": "app history", + "result_summary": "Found prior answer changes." + }, + { + "surface": "logs", + "query": "service:temper-platform", + "time_window": "2026-06-02T14:00:00Z/2026-06-02T15:00:00Z", + "result_count": 12, + "interpretation": "Runtime request logs were present.", + "zero_result_meaning": "failure", + "datadog_url": "https://app.datadoghq.com/logs?query=service%3Atemper-platform" + } + ] + }) + ) + .is_ok() + ); + assert!( + ensure_directed_evolution_required_datadog_evidence( + &telemetry, + &json!({ + "evidence_scope": [{ + "query": "service:temper-platform", + "time_window": "2026-06-02T14:00:00Z/2026-06-02T15:00:00Z", + "result_count": 0, + "interpretation": "No runtime logs were present.", + "zero_result_meaning": "failure" + }] + }) + ) + .is_err() + ); + assert!( + ensure_directed_evolution_required_datadog_evidence( + &plain_reviewer, + &json!({"summary": "Reviewer evidence can be non-Datadog."}) + ) + .is_ok() + ); + assert!( + ensure_directed_evolution_required_datadog_evidence( + &datadog_reviewer, + &json!({"summary": "Reviewer prompt required Datadog but omitted it."}) + ) + .is_err() + ); + assert!( + ensure_directed_evolution_required_datadog_evidence( + &datadog_viability, + &json!({ + "evidence_scope": [{ + "query": "service:temper-platform @directed_evolution.variant_id:v2", + "time_window": "2026-06-02T14:00:00Z/2026-06-02T15:00:00Z", + "result_count": 4, + "interpretation": "Viability Datadog check returned runtime logs.", + "zero_result_meaning": "failure", + "datadog_url": "https://app.datadoghq.com/logs?query=v2" + }] + }) + ) + .is_ok() + ); + } + #[test] fn directed_evolution_summary_uses_agent_summary_when_available() { let work_item = DirectedEvolutionWorkItemState { @@ -735,6 +980,78 @@ mod directed_evolution_tests { )); } + #[test] + fn directed_evolution_exclusive_key_conflict_detects_active_peer() { + let rows = vec![ + json!({ + "entity_id": "wi-selector-current", + "fields": {"Status": "Queued"} + }), + json!({ + "entity_id": "wi-selector-active", + "fields": {"Status": "Running"} + }), + json!({ + "entity_id": "wi-selector-done", + "fields": {"Status": "Succeeded"} + }), + ]; + + assert_eq!( + directed_evolution_exclusive_key_conflict_from_rows( + "wi-selector-current", + &rows, + ExclusiveKeyConflictPolicy::AnyActivePeer, + ) + .as_deref(), + Some("wi-selector-active") + ); + assert_eq!( + directed_evolution_exclusive_key_conflict_from_rows( + "wi-selector-active", + &rows, + ExclusiveKeyConflictPolicy::AnyActivePeer, + ), + None + ); + } + + #[test] + fn directed_evolution_post_claim_conflict_lets_lowest_active_work_item_win() { + let rows = vec![ + json!({ + "entity_id": "wi-selector-003", + "fields": {"Status": "Claimed"} + }), + json!({ + "entity_id": "wi-selector-001", + "fields": {"Status": "Claimed"} + }), + json!({ + "entity_id": "wi-selector-004", + "fields": {"Status": "Running"} + }), + ]; + + assert_eq!( + directed_evolution_exclusive_key_conflict_from_rows( + "wi-selector-003", + &rows, + ExclusiveKeyConflictPolicy::LowerActivePeer, + ) + .as_deref(), + Some("wi-selector-001") + ); + assert_eq!( + directed_evolution_exclusive_key_conflict_from_rows( + "wi-selector-001", + &rows, + ExclusiveKeyConflictPolicy::LowerActivePeer, + ), + None + ); + } + #[test] fn human_episode_contract_uses_direction_and_organism_defaults() { let input: DirectedEvolutionHumanEpisodeInput = serde_json::from_value(json!({ diff --git a/crates/paw-codex-worker/src/directed_evolution/workdir.rs b/crates/paw-codex-worker/src/directed_evolution/workdir.rs index f64ae23af..4c3c7257e 100644 --- a/crates/paw-codex-worker/src/directed_evolution/workdir.rs +++ b/crates/paw-codex-worker/src/directed_evolution/workdir.rs @@ -10,6 +10,7 @@ async fn resolve_directed_evolution_workdir( path: config.repo_root.clone(), app_ref: String::new(), branch_ref: String::new(), + canonical_genesis_bundle: false, }); }; if let Some(path) = @@ -23,6 +24,7 @@ async fn resolve_directed_evolution_workdir( path: worktree, app_ref: context.app_ref, branch_ref, + canonical_genesis_bundle: false, }); } if !context.branch_ref.trim().is_empty() { @@ -33,19 +35,55 @@ async fn resolve_directed_evolution_workdir( path: worktree, app_ref: context.app_ref, branch_ref: context.branch_ref, + canonical_genesis_bundle: false, }); } return Ok(DirectedEvolutionWorkdir { path, app_ref: context.app_ref, branch_ref: context.branch_ref, + canonical_genesis_bundle: false, + }); + } + if !context.branch_ref.trim().is_empty() { + if let Some(worktree) = + directed_evolution_existing_workspace_worktree(config, &context.branch_ref).await? + { + return Ok(DirectedEvolutionWorkdir { + path: worktree, + app_ref: context.app_ref, + branch_ref: context.branch_ref, + canonical_genesis_bundle: false, + }); + } + } + if let Some(path) = + directed_evolution_repo_from_genesis_bundle(client, config, &context.app_ref, &work_item.id) + .await? + { + if directed_evolution_role_may_write_repo(&work_item.role) { + let branch_ref = directed_evolution_variant_branch(work_item); + let worktree = + ensure_directed_evolution_worktree(config, &path, &branch_ref, true).await?; + return Ok(DirectedEvolutionWorkdir { + path: worktree, + app_ref: context.app_ref, + branch_ref, + canonical_genesis_bundle: false, + }); + } + return Ok(DirectedEvolutionWorkdir { + path, + app_ref: context.app_ref, + branch_ref: context.branch_ref, + canonical_genesis_bundle: true, }); } if directed_evolution_role_may_write_repo(&work_item.role) && env_flag("PAW_DE_ALLOW_GLOBAL_REPO_FOR_VARIANTS") != Some(true) { bail!( - "Directed Evolution {} WorkItem {} resolved app_ref '{}' for organism '{}' but no repository mapping was configured. Set DIRECTED_EVOLUTION_ORGANISM_REPOS_JSON or PAW_DE_ALLOW_GLOBAL_REPO_FOR_VARIANTS=true.", + "Directed Evolution {} WorkItem {} resolved app_ref '{}' for organism '{}' but no repository mapping or canonical Genesis bundle worktree was available. Set DIRECTED_EVOLUTION_ORGANISM_REPOS_JSON, DIRECTED_EVOLUTION_GENESIS_URL, or PAW_DE_ALLOW_GLOBAL_REPO_FOR_VARIANTS=true.", work_item.role, work_item.id, context.app_ref, @@ -56,6 +94,7 @@ async fn resolve_directed_evolution_workdir( path: config.repo_root.clone(), app_ref: context.app_ref, branch_ref: context.branch_ref, + canonical_genesis_bundle: false, }) } @@ -64,6 +103,7 @@ struct DirectedEvolutionWorkdir { path: PathBuf, app_ref: String, branch_ref: String, + canonical_genesis_bundle: bool, } #[derive(Clone, Debug)] @@ -279,6 +319,169 @@ fn directed_evolution_variant_branch(work_item: &DirectedEvolutionWorkItemState) format!("directed-evolution/{}", sanitize_git_ref_component(&work_item.id)) } +async fn directed_evolution_repo_from_genesis_bundle( + client: &reqwest::Client, + config: &Config, + app_ref: &str, + work_item_id: &str, +) -> Result> { + if app_ref.trim().is_empty() { + return Ok(None); + } + let Some(genesis_url) = directed_evolution_genesis_url() else { + return Ok(None); + }; + let app = directed_evolution_genesis_app_from_ref(app_ref)?; + let repo = directed_evolution_materialized_bundle_repo_path(config, &app, work_item_id); + if repo.join(".git").exists() { + return Ok(Some(repo)); + } + + let url = directed_evolution_genesis_bundle_url(&genesis_url, &app); + let registry_tenant = directed_evolution_registry_tenant(config); + let response = client + .get(&url) + .headers(headers_for_tenant(config, ®istry_tenant)?) + .send() + .await + .with_context(|| format!("fetch canonical Genesis bundle {}", app.pinned_ref()))?; + let status = response.status(); + let body = response + .text() + .await + .context("read canonical Genesis bundle response")?; + if !status.is_success() { + bail!( + "fetch canonical Genesis bundle {} failed: HTTP {} body={}", + app.pinned_ref(), + status, + body + ); + } + let bundle: Value = serde_json::from_str(&body) + .with_context(|| format!("parse canonical Genesis bundle {}", app.pinned_ref()))?; + materialize_directed_evolution_bundle_repo(&repo, &app, &bundle).await?; + Ok(Some(repo)) +} + +fn directed_evolution_materialized_bundle_repo_path( + config: &Config, + app: &DirectedEvolutionGenesisApp, + work_item_id: &str, +) -> PathBuf { + config + .workspace_root + .join("directed-evolution") + .join("canonical-bundles") + .join(app.app_id()) + .join(sanitize_path_component(&app.hash)) + .join(sanitize_path_component(work_item_id)) +} + +fn directed_evolution_genesis_bundle_url(genesis_url: &str, app: &DirectedEvolutionGenesisApp) -> String { + format!( + "{}/api/genesis/apps/{}/{}/versions/{}/bundle", + genesis_url.trim_end_matches('/'), + app.owner, + app.name, + app.hash + ) +} + +async fn materialize_directed_evolution_bundle_repo( + repo: &Path, + app: &DirectedEvolutionGenesisApp, + bundle: &Value, +) -> Result<()> { + if repo.exists() { + fs::remove_dir_all(repo) + .with_context(|| format!("remove stale canonical bundle repo {}", repo.display()))?; + } + fs::create_dir_all(repo) + .with_context(|| format!("create canonical bundle repo {}", repo.display()))?; + + for (path, content) in directed_evolution_bundle_files(bundle)? { + let file_path = repo.join(&path); + if let Some(parent) = file_path.parent() { + fs::create_dir_all(parent) + .with_context(|| format!("create bundle directory {}", parent.display()))?; + } + fs::write(&file_path, content) + .with_context(|| format!("write bundle file {}", file_path.display()))?; + } + + git_capture(repo, &["init"]).await?; + ensure_git_identity(repo).await?; + git_capture(repo, &["add", "-f", "-A"]).await?; + git_capture_owned( + repo, + vec![ + "commit".to_string(), + "-m".to_string(), + format!("Materialize canonical Genesis bundle {}", app.pinned_ref()), + ], + ) + .await?; + Ok(()) +} + +fn directed_evolution_bundle_files(bundle: &Value) -> Result)>> { + let apps = bundle + .get("apps") + .and_then(Value::as_array) + .context("canonical Genesis bundle missing apps array")?; + let app = apps + .first() + .context("canonical Genesis bundle has no apps")?; + let files = app + .get("files") + .and_then(Value::as_array) + .context("canonical Genesis bundle app missing files array")?; + let mut out = Vec::new(); + for file in files { + let path = file + .get("path") + .and_then(Value::as_str) + .context("canonical Genesis bundle file missing path")?; + let path = safe_directed_evolution_bundle_path(path)?; + let content_base64 = file + .get("content_base64") + .and_then(Value::as_str) + .context("canonical Genesis bundle file missing content_base64")?; + let content = base64::engine::general_purpose::STANDARD + .decode(content_base64) + .context("decode canonical Genesis bundle file")?; + out.push((path, content)); + } + Ok(out) +} + +fn safe_directed_evolution_bundle_path(path: &str) -> Result { + let path = Path::new(path); + if path.as_os_str().is_empty() { + bail!("canonical Genesis bundle file path is empty"); + } + let mut safe = PathBuf::new(); + for component in path.components() { + match component { + std::path::Component::Normal(part) => { + if part.to_string_lossy().eq_ignore_ascii_case(".git") { + bail!( + "canonical Genesis bundle file path may not contain .git: {}", + path.display() + ); + } + safe.push(part) + } + _ => bail!("canonical Genesis bundle file path is unsafe: {}", path.display()), + } + } + if safe.as_os_str().is_empty() { + bail!("canonical Genesis bundle file path is empty"); + } + Ok(safe) +} + async fn ensure_directed_evolution_worktree( config: &Config, base_repo: &Path, @@ -343,6 +546,25 @@ async fn existing_directed_evolution_worktree_for_branch( Ok(None) } +async fn directed_evolution_existing_workspace_worktree( + config: &Config, + branch_ref: &str, +) -> Result> { + let branch = sanitize_git_branch(branch_ref); + let worktree = config + .workspace_root + .join("directed-evolution") + .join(sanitize_path_component(&branch)); + if !worktree.join(".git").exists() { + return Ok(None); + } + let current_branch = git_capture(&worktree, &["rev-parse", "--abbrev-ref", "HEAD"]).await?; + if current_branch.trim() == branch { + return Ok(Some(worktree)); + } + Ok(None) +} + async fn finalize_directed_evolution_output( client: &reqwest::Client, config: &Config, @@ -533,7 +755,7 @@ async fn materialize_directed_evolution_promotion( let workdir = resolve_directed_evolution_workdir(client, config, work_item).await?; let head = git_capture(&workdir.path, &["rev-parse", "HEAD"]).await?; let head = head.trim(); - if head != app.hash { + if head != app.hash && !workdir.canonical_genesis_bundle { bail!( "Directed Evolution promoter expected worktree {} at {}, found {}", workdir.path.display(), @@ -548,15 +770,17 @@ async fn materialize_directed_evolution_promotion( } else { workdir.branch_ref.clone() }; - publish_directed_evolution_app_version( - client, - config, - &genesis_url, - &app, - ®istry_tenant, - &ref_name, - ) - .await?; + if !workdir.canonical_genesis_bundle { + publish_directed_evolution_app_version( + client, + config, + &genesis_url, + &app, + ®istry_tenant, + &ref_name, + ) + .await?; + } let production_tenant = directed_evolution_production_tenant(); install_directed_evolution_variant( client, @@ -569,12 +793,20 @@ async fn materialize_directed_evolution_promotion( .await?; let canonical_app_ref = app.pinned_ref(); - Ok(DirectedEvolutionPromotionMaterialization { - runtime_ref: format!("temper://tenant/{production_tenant}/app/{canonical_app_ref}"), - summary: format!( + let summary = if workdir.canonical_genesis_bundle { + format!( + "Hot-loaded existing canonical Genesis app ref {} into tenant {}.", + canonical_app_ref, production_tenant + ) + } else { + format!( "Published {} from winner ref {} and hot-loaded it into tenant {}.", canonical_app_ref, ref_name, production_tenant - ), + ) + }; + Ok(DirectedEvolutionPromotionMaterialization { + runtime_ref: format!("temper://tenant/{production_tenant}/app/{canonical_app_ref}"), + summary, digest: app.hash.clone(), canonical_app_ref, production_tenant, @@ -777,7 +1009,7 @@ async fn ensure_directed_evolution_readonly_workdir_unchanged( let status_after = git_capture(workdir, &["status", "--porcelain"]).await?; if status_after != status_before { bail!( - "Directed Evolution read-only brain role modified {}. Before status: {:?}. After status: {:?}", + "Directed Evolution read-only worker role modified {}. Before status: {:?}. After status: {:?}", workdir.display(), status_before, status_after diff --git a/crates/paw-codex-worker/src/event_loop.rs b/crates/paw-codex-worker/src/event_loop.rs index 7e442d7e8..01397b388 100644 --- a/crates/paw-codex-worker/src/event_loop.rs +++ b/crates/paw-codex-worker/src/event_loop.rs @@ -197,7 +197,7 @@ async fn handle_requested_review_run( worker_run_id = %worker_run.id, "claiming ReviewRun for local Codex reviewer" ); - post_entity_action( + if let Err(error) = post_entity_action( client, config, "ReviewRuns", @@ -205,7 +205,20 @@ async fn handle_requested_review_run( "Claim", json!({ "reviewer_id": config.worker_id }), ) - .await?; + .await + { + let message = error.to_string(); + if action_conflict_message(&message) { + info!( + review_run_id, + worker_run_id = %worker_run.id, + worker_id = %config.worker_id, + "ReviewRun claim lost to another worker" + ); + return Ok(()); + } + return Err(error); + } post_entity_action( client, config, @@ -348,7 +361,18 @@ async fn handle_queued_evaluation_run( return Ok(()); } - claim_evaluation_run(client, config, evaluation_run_id).await?; + if let Err(error) = claim_evaluation_run(client, config, evaluation_run_id).await { + let message = error.to_string(); + if action_conflict_message(&message) { + info!( + evaluation_run_id, + worker_id = %config.worker_id, + "EvaluationRun claim lost to another worker" + ); + return Ok(()); + } + return Err(error); + } if !is_repo_sweep { return run_code_change_evaluation(client, config, evaluation_run_id, &worker_run).await; @@ -416,7 +440,21 @@ async fn fail_terminal_queued_evaluation_if_needed( }; if evaluation_run_is_claimable_by_worker(evaluation_run, &config.worker_id) { - fail_queued_evaluation_run(client, config, evaluation_run_id, &blocker).await?; + if let Err(error) = + fail_queued_evaluation_run(client, config, evaluation_run_id, &blocker).await + { + let message = error.to_string(); + if action_conflict_message(&message) { + info!( + evaluation_run_id, + worker_id = %config.worker_id, + failure_classification = %blocker.failure_classification, + "EvaluationRun terminal claim lost to another worker" + ); + return Ok(true); + } + return Err(error); + } } else { warn!( evaluation_run_id, diff --git a/crates/paw-codex-worker/src/execution.rs b/crates/paw-codex-worker/src/execution.rs index 3f20a0caa..ee25cba3c 100644 --- a/crates/paw-codex-worker/src/execution.rs +++ b/crates/paw-codex-worker/src/execution.rs @@ -516,6 +516,9 @@ fn codex_exec_args_with_datadog_option( force_datadog_mcp: bool, ) -> Vec { let mut args = vec!["exec".into(), "--ignore-user-config".into()]; + if let Some(model) = codex_exec_model() { + args.extend(["--model".into(), model.into()]); + } if let Some(datadog_mcp_url) = codex_datadog_mcp_url(force_datadog_mcp) { args.extend([ "-c".into(), @@ -537,6 +540,13 @@ fn codex_exec_args_with_datadog_option( args } +fn codex_exec_model() -> Option { + env::var("PAW_CODEX_MODEL") + .ok() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) +} + fn codex_datadog_mcp_url(force_enabled: bool) -> Option { let enabled = env::var("PAW_CODEX_ENABLE_DATADOG_MCP") .map(|value| value == "1" || value.eq_ignore_ascii_case("true")) diff --git a/crates/paw-codex-worker/src/http_headers.rs b/crates/paw-codex-worker/src/http_headers.rs index d38d4416b..f0cc6113e 100644 --- a/crates/paw-codex-worker/src/http_headers.rs +++ b/crates/paw-codex-worker/src/http_headers.rs @@ -12,6 +12,9 @@ fn extract_sse_data(frame: &str) -> Option { fn headers(config: &Config) -> Result { let mut headers = HeaderMap::new(); + let principal_kind = + env::var("PAW_CODEX_PRINCIPAL_KIND").unwrap_or_else(|_| "agent".to_string()); + let agent_type = env::var("PAW_CODEX_AGENT_TYPE").unwrap_or_else(|_| "worker".to_string()); headers.insert( "x-tenant-id", HeaderValue::from_str(&config.tenant).context("invalid TEMPER_TENANT")?, @@ -20,8 +23,14 @@ fn headers(config: &Config) -> Result { "x-temper-principal-id", HeaderValue::from_str(&config.worker_id).context("invalid WORKER_ID")?, ); - headers.insert("x-temper-principal-kind", HeaderValue::from_static("agent")); - headers.insert("x-temper-agent-type", HeaderValue::from_static("worker")); + headers.insert( + "x-temper-principal-kind", + HeaderValue::from_str(&principal_kind).context("invalid PAW_CODEX_PRINCIPAL_KIND")?, + ); + headers.insert( + "x-temper-agent-type", + HeaderValue::from_str(&agent_type).context("invalid PAW_CODEX_AGENT_TYPE")?, + ); headers.insert( "x-agent-id", HeaderValue::from_str(&config.worker_id).context("invalid WORKER_ID")?, @@ -59,4 +68,3 @@ fn event_stream_headers(config: &Config) -> Result { fn required_env(key: &str) -> Result { env::var(key).with_context(|| format!("{key} is required")) } - diff --git a/crates/paw-codex-worker/src/launchd.rs b/crates/paw-codex-worker/src/launchd.rs index 899d79f79..8eacbe1e8 100644 --- a/crates/paw-codex-worker/src/launchd.rs +++ b/crates/paw-codex-worker/src/launchd.rs @@ -10,6 +10,11 @@ fn launchd_worker_binary_path() -> PathBuf { } fn render_launchd_plist(config: &Config, worker_bin: &Path, eval_commands: Option<&str>) -> String { + let launchd_label = env::var("PAW_CODEX_LAUNCHD_LABEL") + .ok() + .filter(|value| !value.trim().is_empty()) + .unwrap_or_else(|| "com.temperpaw.paw-codex-worker".to_string()); + let max_concurrent_runs = config.max_concurrent_runs.to_string(); let mut env_vars = vec![ ("TEMPER_URL", config.temper_url.as_str()), ("TEMPER_TENANT", config.tenant.as_str()), @@ -18,7 +23,7 @@ fn render_launchd_plist(config: &Config, worker_bin: &Path, eval_commands: Optio ("REPO_ROOT", path_str(&config.repo_root)), ("CODEX_BIN", config.codex_bin.as_str()), ("PATH", launchd_path().leak()), - ("MAX_CONCURRENT_RUNS", "1"), + ("MAX_CONCURRENT_RUNS", max_concurrent_runs.as_str()), ( "PAW_CODEX_ENABLE_EXECUTION", if config.enable_execution { "1" } else { "0" }, @@ -86,6 +91,11 @@ fn render_launchd_plist(config: &Config, worker_bin: &Path, eval_commands: Optio if let Some(commands) = eval_commands.filter(|value| !value.trim().is_empty()) { env_vars.push(("PAW_CODEX_EVAL_COMMANDS", commands)); } + if let Ok(model) = env::var("PAW_CODEX_MODEL") + && !model.trim().is_empty() + { + env_vars.push(("PAW_CODEX_MODEL", model.leak())); + } let environment = env_vars .into_iter() @@ -105,7 +115,7 @@ fn render_launchd_plist(config: &Config, worker_bin: &Path, eval_commands: Optio Label - com.temperpaw.paw-codex-worker + {} ProgramArguments {} @@ -120,13 +130,16 @@ fn render_launchd_plist(config: &Config, worker_bin: &Path, eval_commands: Optio {} StandardOutPath - /tmp/paw-codex-worker.out.log + /tmp/{}.out.log StandardErrorPath - /tmp/paw-codex-worker.err.log + /tmp/{}.err.log "#, + escape_plist(&launchd_label), escape_plist(&worker_bin.display().to_string()), - environment + environment, + escape_plist(&launchd_label), + escape_plist(&launchd_label) ) } diff --git a/crates/paw-codex-worker/src/main.rs b/crates/paw-codex-worker/src/main.rs index 71bafc49e..ba9b60dd8 100644 --- a/crates/paw-codex-worker/src/main.rs +++ b/crates/paw-codex-worker/src/main.rs @@ -1,4 +1,5 @@ use anyhow::{Context, Result, anyhow, bail}; +use base64::Engine; use futures_util::StreamExt; use repo_health::{ extract_repo_sweep_snapshot_id, parse_repo_health_agent_output, repo_health_agent_prompt, @@ -44,6 +45,11 @@ async fn main() -> Result<()> { ); return Ok(()); } + if command == WorkerCommand::RegisterWorkerAgent { + ensure_worker_agent_registered(&client, &config).await?; + report_worker_heartbeat(&client, &config).await?; + return Ok(()); + } if let WorkerCommand::DirectedEvolutionStartEpisode { contract_path } = &command { let result = run_directed_evolution_human_episode_command( &client, @@ -64,6 +70,9 @@ async fn main() -> Result<()> { worker_capabilities = %worker_capabilities().join(","), "paw-codex-worker starting" ); + if let Err(error) = ensure_worker_agent_registered(&client, &config).await { + warn!(%error, "failed to ensure WorkerAgent registration"); + } if let Err(error) = report_worker_heartbeat(&client, &config).await { warn!(%error, "failed to report WorkerAgent heartbeat"); } diff --git a/crates/paw-codex-worker/src/temper_api.rs b/crates/paw-codex-worker/src/temper_api.rs index 81d12e40a..9bcba3c3c 100644 --- a/crates/paw-codex-worker/src/temper_api.rs +++ b/crates/paw-codex-worker/src/temper_api.rs @@ -157,19 +157,129 @@ async fn claim_worker_run( } async fn report_worker_heartbeat(client: &reqwest::Client, config: &Config) -> Result<()> { - post_entity_action( + let body = json!({ + "last_seen_at": generated_at_label(), + "status_summary": "paw-codex-worker running under openclaw launchd", + "capabilities": worker_capabilities().join(","), + }); + match post_entity_action_with_namespace( client, config, "WorkerAgents", &config.worker_id, + PAW_ORCHESTRATION_NAMESPACE, "ReportHeartbeat", - json!({ - "last_seen_at": generated_at_label(), - "status_summary": "paw-codex-worker running under openclaw launchd", - "capabilities": worker_capabilities().join(","), - }), + body.clone(), ) .await + { + Ok(()) => Ok(()), + Err(error) => { + warn!( + %error, + worker_id = %config.worker_id, + "WorkerAgent heartbeat via PawOrchestration failed; falling back to installed Patrol namespace" + ); + post_entity_action( + client, + config, + "WorkerAgents", + &config.worker_id, + "ReportHeartbeat", + body, + ) + .await + } + } +} + +async fn ensure_worker_agent_registered(client: &reqwest::Client, config: &Config) -> Result<()> { + if worker_agent_is_registered(client, config).await? { + return Ok(()); + } + let id = match create_entity( + client, + config, + "WorkerAgents", + worker_agent_registration_body(config), + ) + .await + { + Ok(id) => id, + Err(error) => { + if action_conflict_message(&error.to_string()) + && worker_agent_is_registered(client, config).await? + { + return Ok(()); + } + return Err(error); + } + }; + if id != config.worker_id { + bail!( + "WorkerAgent registration returned unexpected id {id}; expected {}", + config.worker_id + ); + } + Ok(()) +} + +async fn worker_agent_is_registered(client: &reqwest::Client, config: &Config) -> Result { + let response = client + .get(config.entity_url("WorkerAgents", &config.worker_id)) + .headers(headers(config)?) + .send() + .await + .with_context(|| format!("fetch WorkerAgent {}", config.worker_id))?; + if response.status().is_success() { + return Ok(true); + } + if response.status() == reqwest::StatusCode::NOT_FOUND { + return Ok(false); + } + let status = response.status(); + let text = response.text().await.unwrap_or_default(); + bail!( + "fetch WorkerAgent {} returned {}: {}", + config.worker_id, + status, + text + ) +} + +fn worker_agent_registration_body(config: &Config) -> Value { + json!({ + "id": config.worker_id, + "Id": config.worker_id, + "worker_id": config.worker_id, + "WorkerId": config.worker_id, + "provider_id": worker_provider_id(), + "ProviderId": worker_provider_id(), + "display_name": config.worker_id, + "DisplayName": config.worker_id, + "capabilities": worker_capabilities().join(","), + "Capabilities": worker_capabilities().join(","), + "host": worker_host_label(), + "Host": worker_host_label(), + "worktree_root": path_str(&config.workspace_root), + "WorktreeRoot": path_str(&config.workspace_root), + }) +} + +fn worker_provider_id() -> String { + env::var("WORKER_PROVIDER_ID") + .ok() + .filter(|value| !value.trim().is_empty()) + .unwrap_or_else(|| "local-codex".to_string()) +} + +fn worker_host_label() -> String { + env::var("HOSTNAME") + .or_else(|_| env::var("COMPUTERNAME")) + .ok() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| "local".to_string()) } async fn start_local_worker_run( diff --git a/crates/paw-codex-worker/src/tests.rs b/crates/paw-codex-worker/src/tests.rs index 10982a2f3..687cbe0b1 100644 --- a/crates/paw-codex-worker/src/tests.rs +++ b/crates/paw-codex-worker/src/tests.rs @@ -223,14 +223,20 @@ mod tests { parse_worker_command(vec!["--launchd-plist".to_string()]), WorkerCommand::LaunchdPlist ); + assert_eq!( + parse_worker_command(vec!["register-worker-agent".to_string()]), + WorkerCommand::RegisterWorkerAgent + ); assert_eq!( parse_worker_command(vec!["run".to_string()]), WorkerCommand::Run ); } - #[test] - fn launchd_plist_renders_concrete_worker_environment() { + #[tokio::test] + async fn launchd_plist_renders_concrete_worker_environment() { + let _guard = ENV_LOCK.lock().await; + let _model = EnvOverride::set("PAW_CODEX_MODEL", OsString::from("gpt-5-codex")); let config = Config { temper_url: "https://temperpaw.example.test".to_string(), tenant: "prod".to_string(), @@ -277,6 +283,8 @@ mod tests { "PAW_CODEX_FORBIDDEN_DONE_PATHS", "PAW_CODEX_EVAL_COMMANDS", "cargo test -p temperpaw --test paw_patrol_foundation", + "PAW_CODEX_MODEL", + "gpt-5-codex", ] { assert!(plist.contains(needle), "plist should contain {needle}"); } @@ -525,6 +533,7 @@ mod tests { let _guard = ENV_LOCK.lock().await; let _datadog_mcp = EnvOverride::set("PAW_CODEX_ENABLE_DATADOG_MCP", OsString::from("0")); + let _model = EnvOverride::remove("PAW_CODEX_MODEL"); let args = codex_exec_args(Path::new("/tmp/paw-worktree"), "Create the proof file"); let args = args .iter() @@ -546,11 +555,208 @@ mod tests { ); } + #[tokio::test] + async fn config_accepts_multiple_concurrent_runs_for_process_pool_slots() { + let _guard = ENV_LOCK.lock().await; + let _temper_url = EnvOverride::set("TEMPER_URL", OsString::from("http://127.0.0.1:3000")); + let _worker_id = EnvOverride::set("WORKER_ID", OsString::from("local-codex-slot-1")); + let _max_concurrent_runs = EnvOverride::set("MAX_CONCURRENT_RUNS", OsString::from("4")); + + let config = Config::from_env().expect("worker config should parse"); + + assert_eq!(config.max_concurrent_runs, 4); + } + + #[tokio::test] + async fn worker_agent_registration_body_uses_slot_identity() { + let _guard = ENV_LOCK.lock().await; + let _provider = EnvOverride::set("WORKER_PROVIDER_ID", OsString::from("local-codex")); + let _host = EnvOverride::set("HOSTNAME", OsString::from("worker-host")); + let config = Config { + temper_url: "https://temperpaw.example.test".to_string(), + tenant: "prod".to_string(), + worker_id: "mac-mini-codex-prod-slot-02".to_string(), + worker_token: Some("secret".to_string()), + workspace_root: PathBuf::from("/Users/me/Development/temperpaw-worktrees"), + repo_root: PathBuf::from("/Users/me/Development/temperpaw"), + codex_bin: "codex".to_string(), + max_concurrent_runs: 1, + enable_execution: true, + poll_on_start: true, + codex_exec_smoke: false, + codex_exec_timeout: Duration::from_secs(90), + }; + + let body = worker_agent_registration_body(&config); + + assert_eq!(body["id"], "mac-mini-codex-prod-slot-02"); + assert_eq!(body["worker_id"], "mac-mini-codex-prod-slot-02"); + assert_eq!(body["provider_id"], "local-codex"); + assert_eq!(body["host"], "worker-host"); + assert_eq!( + body["worktree_root"], + "/Users/me/Development/temperpaw-worktrees" + ); + } + + #[tokio::test] + async fn worker_agent_registration_is_idempotent_when_slot_exists() { + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + use tokio::net::TcpListener; + + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind test listener"); + let addr = listener.local_addr().expect("listener addr"); + let requests = std::sync::Arc::new(tokio::sync::Mutex::new(Vec::::new())); + let captured = requests.clone(); + let server = tokio::spawn(async move { + loop { + let Ok((mut socket, _)) = listener.accept().await else { + break; + }; + let requests = captured.clone(); + tokio::spawn(async move { + let mut buf = [0_u8; 4096]; + let n = socket.read(&mut buf).await.expect("read request"); + let request = String::from_utf8_lossy(&buf[..n]).to_string(); + requests.lock().await.push(request.clone()); + let body = if request.starts_with( + "GET /tdata/WorkerAgents('mac-mini-codex-prod-slot-02')", + ) { + r#"{"entity_id":"mac-mini-codex-prod-slot-02","fields":{"Id":"mac-mini-codex-prod-slot-02","Status":"Registered"}}"# + } else { + r#"{"error":"unexpected request"}"# + }; + let status = if request.starts_with( + "GET /tdata/WorkerAgents('mac-mini-codex-prod-slot-02')", + ) { + "200 OK" + } else { + "500 Internal Server Error" + }; + let response = format!( + "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ); + socket + .write_all(response.as_bytes()) + .await + .expect("write response"); + }); + } + }); + let config = Config { + temper_url: format!("http://{addr}"), + tenant: "prod".to_string(), + worker_id: "mac-mini-codex-prod-slot-02".to_string(), + worker_token: Some("secret".to_string()), + workspace_root: PathBuf::from("/Users/me/Development/temperpaw-worktrees"), + repo_root: PathBuf::from("/Users/me/Development/temperpaw"), + codex_bin: "codex".to_string(), + max_concurrent_runs: 1, + enable_execution: true, + poll_on_start: true, + codex_exec_smoke: false, + codex_exec_timeout: Duration::from_secs(90), + }; + let client = reqwest::Client::new(); + + ensure_worker_agent_registered(&client, &config) + .await + .expect("existing WorkerAgent should satisfy registration"); + + server.abort(); + let requests = requests.lock().await; + assert_eq!(requests.len(), 1, "registration should not POST when the slot exists"); + assert!(requests[0].starts_with( + "GET /tdata/WorkerAgents('mac-mini-codex-prod-slot-02')" + )); + } + + #[tokio::test] + async fn worker_agent_heartbeat_falls_back_to_installed_patrol_namespace() { + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + use tokio::net::TcpListener; + + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind test listener"); + let addr = listener.local_addr().expect("listener addr"); + let requests = std::sync::Arc::new(tokio::sync::Mutex::new(Vec::::new())); + let captured = requests.clone(); + let server = tokio::spawn(async move { + loop { + let Ok((mut socket, _)) = listener.accept().await else { + break; + }; + let requests = captured.clone(); + tokio::spawn(async move { + let mut buf = [0_u8; 4096]; + let n = socket.read(&mut buf).await.expect("read request"); + let request = String::from_utf8_lossy(&buf[..n]).to_string(); + requests.lock().await.push(request.clone()); + let new_namespace = request.starts_with( + "POST /tdata/WorkerAgents('mac-mini-codex-prod-slot-02')/Temper.PawOrchestration.ReportHeartbeat", + ); + let legacy_namespace = request.starts_with( + "POST /tdata/WorkerAgents('mac-mini-codex-prod-slot-02')/TemperPaw.Patrol.ReportHeartbeat", + ); + let (status, body) = if new_namespace { + ("404 Not Found", r#"{"error":"action not installed yet"}"#) + } else if legacy_namespace { + ("200 OK", r#"{"ok":true}"#) + } else { + ("500 Internal Server Error", r#"{"error":"unexpected request"}"#) + }; + let response = format!( + "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ); + socket + .write_all(response.as_bytes()) + .await + .expect("write response"); + }); + } + }); + let config = Config { + temper_url: format!("http://{addr}"), + tenant: "prod".to_string(), + worker_id: "mac-mini-codex-prod-slot-02".to_string(), + worker_token: Some("secret".to_string()), + workspace_root: PathBuf::from("/Users/me/Development/temperpaw-worktrees"), + repo_root: PathBuf::from("/Users/me/Development/temperpaw"), + codex_bin: "codex".to_string(), + max_concurrent_runs: 1, + enable_execution: true, + poll_on_start: true, + codex_exec_smoke: false, + codex_exec_timeout: Duration::from_secs(90), + }; + let client = reqwest::Client::new(); + + report_worker_heartbeat(&client, &config) + .await + .expect("legacy installed Patrol heartbeat namespace should still work"); + + server.abort(); + let requests = requests.lock().await; + assert_eq!(requests.len(), 2, "heartbeat should retry once via Patrol"); + assert!(requests[0].starts_with( + "POST /tdata/WorkerAgents('mac-mini-codex-prod-slot-02')/Temper.PawOrchestration.ReportHeartbeat" + )); + assert!(requests[1].starts_with( + "POST /tdata/WorkerAgents('mac-mini-codex-prod-slot-02')/TemperPaw.Patrol.ReportHeartbeat" + )); + } + #[tokio::test] async fn codex_exec_keeps_datadog_mcp_disabled_when_env_unset() { let _guard = ENV_LOCK.lock().await; let _datadog_mcp = EnvOverride::remove("PAW_CODEX_ENABLE_DATADOG_MCP"); let _datadog_url = EnvOverride::remove("PAW_CODEX_DATADOG_MCP_URL"); + let _model = EnvOverride::remove("PAW_CODEX_MODEL"); let args = codex_exec_args(Path::new("/tmp/paw-worktree"), "Create the proof file"); let args = args .iter() @@ -560,6 +766,7 @@ mod tests { assert_eq!(args[0], "exec"); assert_eq!(args[1], "--ignore-user-config"); assert!(!args.contains(&"-c".to_string())); + assert!(!args.contains(&"--model".to_string())); assert!( !args .iter() @@ -572,6 +779,7 @@ mod tests { let _guard = ENV_LOCK.lock().await; let _datadog_mcp = EnvOverride::set("PAW_CODEX_ENABLE_DATADOG_MCP", OsString::from("1")); + let _model = EnvOverride::remove("PAW_CODEX_MODEL"); let _datadog_url = EnvOverride::set( "PAW_CODEX_DATADOG_MCP_URL", OsString::from("https://mcp.datadoghq.test/mcp?toolsets=logs"), @@ -609,6 +817,7 @@ mod tests { let _guard = ENV_LOCK.lock().await; let _datadog_mcp = EnvOverride::set("PAW_CODEX_ENABLE_DATADOG_MCP", OsString::from("0")); + let _model = EnvOverride::remove("PAW_CODEX_MODEL"); let _datadog_url = EnvOverride::set( "PAW_CODEX_DATADOG_MCP_URL", OsString::from("https://mcp.datadoghq.test/mcp?toolsets=logs"), @@ -641,6 +850,7 @@ mod tests { let _datadog_mcp = EnvOverride::set("PAW_CODEX_ENABLE_DATADOG_MCP", OsString::from("1")); let _datadog_url = EnvOverride::remove("PAW_CODEX_DATADOG_MCP_URL"); + let _model = EnvOverride::remove("PAW_CODEX_MODEL"); let args = codex_exec_args( Path::new("/tmp/paw-worktree"), "Inspect Datadog evidence for the evolution direction", @@ -656,6 +866,29 @@ mod tests { )); } + #[tokio::test] + async fn codex_exec_can_pin_worker_model_from_env() { + let _guard = ENV_LOCK.lock().await; + let _datadog_mcp = + EnvOverride::set("PAW_CODEX_ENABLE_DATADOG_MCP", OsString::from("0")); + let _model = EnvOverride::set("PAW_CODEX_MODEL", OsString::from("gpt-5-codex")); + let args = codex_exec_args(Path::new("/tmp/paw-worktree"), "Create the proof file"); + let args = args + .iter() + .map(|arg| arg.to_string_lossy().to_string()) + .collect::>(); + + assert_eq!( + &args[0..4], + &[ + "exec".to_string(), + "--ignore-user-config".to_string(), + "--model".to_string(), + "gpt-5-codex".to_string(), + ] + ); + } + #[test] fn toml_basic_string_escapes_codex_config_values() { assert_eq!( diff --git a/crates/paw-codex-worker/src/worker_types.rs b/crates/paw-codex-worker/src/worker_types.rs index fd5570b98..af9819425 100644 --- a/crates/paw-codex-worker/src/worker_types.rs +++ b/crates/paw-codex-worker/src/worker_types.rs @@ -7,6 +7,11 @@ const EVALUATION_START_LABEL: &str = "EvaluationRun.Start"; const EVALUATION_PASS_LABEL: &str = "EvaluationRun.Pass"; const EVALUATION_FAIL_LABEL: &str = "EvaluationRun.Fail"; const DIRECTED_EVOLUTION_NAMESPACE: &str = "Temper.DirectedEvolution"; +const PAW_ORCHESTRATION_NAMESPACE: &str = "Temper.PawOrchestration"; + +fn action_conflict_message(message: &str) -> bool { + message.contains(" returned 409") || message.contains("409 Conflict") +} #[derive(Clone, Debug)] struct Config { @@ -29,6 +34,7 @@ enum WorkerCommand { Run, Doctor, LaunchdPlist, + RegisterWorkerAgent, DirectedEvolutionStartEpisode { contract_path: Option }, } @@ -67,6 +73,7 @@ impl Config { let max_concurrent_runs = env::var("MAX_CONCURRENT_RUNS") .ok() .and_then(|value| value.parse().ok()) + .filter(|value| *value > 0) .unwrap_or(1); let enable_execution = env::var("PAW_CODEX_ENABLE_EXECUTION") .map(|value| value == "1" || value.eq_ignore_ascii_case("true")) @@ -84,10 +91,6 @@ impl Config { .map(Duration::from_secs) .unwrap_or_else(|| Duration::from_secs(20 * 60)); - if max_concurrent_runs != 1 { - bail!("MAX_CONCURRENT_RUNS must be 1 in v1 so WorkerRun review remains simple"); - } - Ok(Self { temper_url, tenant, diff --git a/docs/adrs/0055-directed-evolution-hotload-variant-runtimes.md b/docs/adrs/0055-directed-evolution-hotload-variant-runtimes.md index 476e2e99a..824b18112 100644 --- a/docs/adrs/0055-directed-evolution-hotload-variant-runtimes.md +++ b/docs/adrs/0055-directed-evolution-hotload-variant-runtimes.md @@ -6,7 +6,7 @@ Accepted. ## Context -Directed Evolution now creates real `WorkItem` brain runs and can generate +Directed Evolution now creates real `paw-orchestration.WorkItem`s and can generate candidate app changes in local Codex worktrees. That is not enough for the intended pipeline: simulated users and reviewers must evaluate running variants, not just inspect local diffs. Railway deployments are costly and @@ -39,14 +39,14 @@ a hot-loaded Genesis runtime: target `StageResult` or `Variant` has already been eliminated. Generation can queue multiple stages up front, so late queued work must not waste background Codex runs or add confusing evidence after a candidate is dead. -- Background Codex brain runs use isolated `codex exec` sessions +- Background Codex worker runs use isolated `codex exec` sessions (`--ignore-user-config --ephemeral`) so tiny JSON-producing work items do not inherit unrelated local MCP servers, hooks, or long-lived session state. This is an agent capability surface, not a hidden orchestration layer. The episode still advances through Directed Evolution entities and WASM transitions; the worker only supplies the external agent work product and self-reports it -through the existing `WorkItem` and `BrainRun` actions. +through shared `WorkItem` and `WorkerRun` actions. ## Consequences @@ -63,6 +63,6 @@ through the existing `WorkItem` and `BrainRun` actions. survive hot-load, simulated-user trials, review, selection, and promotion. - Stale work cancellation keeps evaluation throughput focused on live variants while preserving entity-level auditability through `CancelWorkItem`. -- Isolated Codex invocation means background brain runs rely on the Codex auth +- Isolated Codex invocation means background worker runs rely on the Codex auth environment but not the operator's interactive config; any needed tool surface must be provided explicitly by the worker prompt or execution environment. diff --git a/docs/adrs/0056-directed-evolution-promotion-materializer.md b/docs/adrs/0056-directed-evolution-promotion-materializer.md index c1b319950..dbd697cd3 100644 --- a/docs/adrs/0056-directed-evolution-promotion-materializer.md +++ b/docs/adrs/0056-directed-evolution-promotion-materializer.md @@ -12,7 +12,7 @@ the live proof showed a gap: advancing the canonical Genesis ref, publishing the app version, and hot-loading the winner into the production tenant still required manual operator commands. -Promotion materialization is not a reasoning task. The selector brain already +Promotion materialization is not a reasoning task. The selector worker already chooses the winner from evaluated evidence. The worker needs to execute bounded external side effects and report evidence back through Directed Evolution entities. @@ -35,7 +35,7 @@ deterministic materialization role: `runtime_ref`, evidence, and digest so the Directed Evolution app can record materialization on the `Promotion`. -This role runs through the same WorkItem/BrainRun reporting path as other +This role runs through the same WorkItem/WorkerRun reporting path as other Directed Evolution worker roles, but it does not launch a background Codex session. diff --git a/docs/adrs/0057-directed-evolution-datadog-evidence-contract.md b/docs/adrs/0057-directed-evolution-datadog-evidence-contract.md index 89d9aa0cb..719d57838 100644 --- a/docs/adrs/0057-directed-evolution-datadog-evidence-contract.md +++ b/docs/adrs/0057-directed-evolution-datadog-evidence-contract.md @@ -7,13 +7,13 @@ ## Context Directed Evolution worker roles already record `EvidenceArtifact` entities for -Codex brain runs, but the evidence can be too generic. The artifact URI falls +Codex worker runs, but the evidence can be too generic. The artifact URI falls back to a TemperPaw-local placeholder unless the agent happens to return a single `evidence_uri`, and the role prompts do not require a structured Datadog evidence scope. That is not enough for the Directed Evolution product contract. Observer, -reviewer, and simulated-user brains must be able to explain what they learned +reviewer, and simulated-user workers must be able to explain what they learned from production telemetry and live variant execution. Mission Control should show evidence that can be inspected, not just summaries. @@ -39,9 +39,9 @@ environment, and the Datadog query used to find matching worker telemetry. ## Consequences -- Directed Evolution evidence becomes Datadog-inspectable when a brain used +- Directed Evolution evidence becomes Datadog-inspectable when a worker used Datadog, rather than only locally summarized. -- Mission Control can surface a real observability URL for a variant or brain +- Mission Control can surface a real observability URL for a variant or worker result without inventing UI state. - The contract stays backward-compatible with existing worker outputs. diff --git a/docs/adrs/0060-directed-evolution-public-runtime-prompts.md b/docs/adrs/0060-directed-evolution-public-runtime-prompts.md index cd8dd343d..34a19d71b 100644 --- a/docs/adrs/0060-directed-evolution-public-runtime-prompts.md +++ b/docs/adrs/0060-directed-evolution-public-runtime-prompts.md @@ -15,7 +15,7 @@ Evaluation work items carried a `RuntimeRef` for a hot-loaded Genesis tenant, bu their prompt still said `TemperApiBase: http://127.0.0.1:8080` when the app tenant did not have a resolved `temper_public_api_url` secret. One simulated-user agent compensated by starting `temper serve` locally and leaving it in the -foreground, which stalled the brain run until the local server process was +foreground, which stalled the worker run until the local server process was terminated. The local worker already knows the public Genesis URL used for hot-loading. diff --git a/docs/adrs/0065-canonical-genesis-worker-pool.md b/docs/adrs/0065-canonical-genesis-worker-pool.md new file mode 100644 index 000000000..f74f1976d --- /dev/null +++ b/docs/adrs/0065-canonical-genesis-worker-pool.md @@ -0,0 +1,31 @@ +# ADR 0065: Canonical Genesis Worker Pool + +## Status + +Accepted + +## Context + +TemperPaw currently contains both platform worker code and several Temper-native app bundles. Directed Evolution review made this confusing: the user wants Genesis to be the canonical home for app bundles, while TemperPaw remains the worker/agent platform that can install pinned app refs and run worker processes. + +The Codex worker also had a V1-only `MAX_CONCURRENT_RUNS == 1` restriction. That blocks real simulated-user and evaluator concurrency, and it makes local Codex look like a singleton instead of a pool of Temper-claimed worker slots. + +## Decision + +- Production app bundles move to Genesis. TemperPaw keeps platform code, worker crates, test fixtures, bootstrap/install references, and docs. +- The shared execution app is published as `temperpaw/paw-orchestration`. Its + bundle-local app name remains `paw-orchestration`, and it owns + `WorkerProvider`, `WorkerAgent`, `WorkItem`, and `WorkerRun`. +- Paw Patrol no longer owns generic worker concepts long-term. It links to shared `WorkItemId` and `WorkerRunId` for patrol-specific work. +- `paw-agent` keeps native TemperPaw session entities. `WorkerRun.SessionId` is optional and only set for TemperPaw-native agent execution. +- Local Codex concurrency is modeled as multiple worker processes. Each process registers as one `WorkerAgent` slot and claims one `WorkItem` at a time. +- Each local Codex process uses an explicit worker profile. `PAW_CODEX_MODEL` pins the model used by `codex exec`, and `PAW_CODEX_EXEC_TIMEOUT_SECS` bounds each run. This keeps process-pool behavior reproducible and makes WorkerRun provenance say which model profile was actually used. +- Selector and promoter work uses exclusive keys so it is serialized per episode/organism. Observer, simulated-user, and evaluator work can run concurrently. +- Directed Evolution observer and telemetry evaluator roles require structured Datadog evidence. Missing Datadog query/window/result/interpretation evidence is a failure, not an informational warning. + +## Consequences + +- Users can scale local Codex by starting more worker slots instead of relying on hidden concurrency inside one process. +- Execution provenance uses worker/provider/run language everywhere, avoiding `BrainRun` as an entity name. +- TemperPaw app directories become migration/bootstrap surfaces until their canonical Genesis refs are published and pinned. +- Existing local proof data may still contain old names; worker and UI code should remain tolerant while presenting new language. diff --git a/docs/deployment.md b/docs/deployment.md index a8e4bb834..563d55ffe 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -42,6 +42,11 @@ continues to readiness using the installed state already present in the database; operators can repair the pinned Genesis app version and redeploy without any database reset. -## Optional Datadog +## Datadog Deploy the collector config in `scripts/otel-collector-railway.yaml` as a second Railway service and point `OTEL_EXPORTER_OTLP_ENDPOINT` at the collector's internal hostname. + +Datadog is mandatory for Directed Evolution observer and telemetry-evaluator +roles. Those roles must record the query, time window, result count, +interpretation, zero-result meaning, and Datadog URL before they can propose a +direction or pass a telemetry stage. diff --git a/os-apps/README.md b/os-apps/README.md new file mode 100644 index 000000000..15c33c7ee --- /dev/null +++ b/os-apps/README.md @@ -0,0 +1,10 @@ +# Repo-Local Paw Apps + +Genesis is the canonical source for production TemperPaw app bundles. + +This `os-apps/` directory remains only for bootstrap, migration, local +development, and tests while canonical Genesis refs are published and pinned. It +is not a production app catalog and it is not an app-source mirror. + +Production TemperPaw instances should install configured pinned Genesis refs and +recover already-installed app state from the backing Temper store. diff --git a/os-apps/paw-managed-agents/adrs/004-directed-evolution-local-codex-worker.md b/os-apps/paw-managed-agents/adrs/004-directed-evolution-local-codex-worker.md index 489a9940d..51ec49d0f 100644 --- a/os-apps/paw-managed-agents/adrs/004-directed-evolution-local-codex-worker.md +++ b/os-apps/paw-managed-agents/adrs/004-directed-evolution-local-codex-worker.md @@ -5,9 +5,9 @@ ## Context -Directed Evolution needs many brain instances: observer, direction framer, +Directed Evolution needs many worker-backed agent instances: observer, direction framer, variant generator, simulated user, reviewer, selector, and narrator. The human -facing director brain remains the live Codex chat session, but the background +facing director remains the live Codex chat session, but the background roles must run as bounded jobs and write structured results back to Temper. Deployed Genesis or Railway services should not run Codex directly in v1. @@ -20,14 +20,14 @@ entities, while TemperPaw claims explicit work and self-reports completion. Add a Directed Evolution worker path to the managed-agent layer. The worker claims `WorkItem` entities from the Directed Evolution control plane, starts a -bounded Codex session for the requested brain role, captures evidence and +bounded Codex session for the requested worker role, captures evidence and observability metadata, and records results back through entity actions. The worker does not own the Directed Evolution state machine. It only moves -state by dispatching explicit actions on `WorkItem`, `BrainRun`, `Variant`, +state by dispatching explicit actions on shared `WorkItem`, `WorkerRun`, `Variant`, `StageResult`, `Trial`, `Direction`, `Episode`, or `Promotion` entities. -### Brain Roles +### Worker Roles The worker supports role-specific prompts and output schemas for: @@ -55,7 +55,7 @@ winning behavior. Every claimed work item records: - `work_item_id` -- `brain_run_id` +- `worker_run_id` - role - parent session id, if any - target entity ids @@ -79,7 +79,7 @@ entities accept the selection and dispatch the promotion action. becoming a second orchestration engine. - The same Codex-session machinery can run variant generation, simulated users, review, selection, and narration. -- Mission Control can link brain output to entity state and observability +- Mission Control can link worker output to entity state and observability evidence. ## Verification From 282a8ea44bb7fcbd6c07711d3bd10e7567fef799 Mon Sep 17 00:00:00 2001 From: rita-aga Date: Tue, 2 Jun 2026 18:28:54 -0400 Subject: [PATCH 03/16] Fix directed evolution worker CI lints --- .../src/directed_evolution.rs | 5 +- .../directed_evolution/human_episode_plan.rs | 4 +- .../src/directed_evolution/recovery.rs | 179 +++++++++++------- .../src/directed_evolution/tests.rs | 4 +- .../src/directed_evolution/workdir.rs | 27 ++- 5 files changed, 128 insertions(+), 91 deletions(-) diff --git a/crates/paw-codex-worker/src/directed_evolution.rs b/crates/paw-codex-worker/src/directed_evolution.rs index 4129e7133..2a169e024 100644 --- a/crates/paw-codex-worker/src/directed_evolution.rs +++ b/crates/paw-codex-worker/src/directed_evolution.rs @@ -64,10 +64,9 @@ async fn handle_queued_directed_evolution_work_item( json!({ "Reason": reason }), ) .await + && !error.to_string().contains("not valid from state 'Cancelled'") { - if !error.to_string().contains("not valid from state 'Cancelled'") { - return Err(error); - } + return Err(error); } info!( work_item_id = %work_item.id, diff --git a/crates/paw-codex-worker/src/directed_evolution/human_episode_plan.rs b/crates/paw-codex-worker/src/directed_evolution/human_episode_plan.rs index 74f82ac0d..78b75f90b 100644 --- a/crates/paw-codex-worker/src/directed_evolution/human_episode_plan.rs +++ b/crates/paw-codex-worker/src/directed_evolution/human_episode_plan.rs @@ -401,8 +401,8 @@ fn directed_evolution_episode_plan_from_input( scoring_rules, evaluation_stages, simulated_user_plan: DirectedEvolutionSimulatedUserPlan { - users_per_variant: input.users_per_variant.unwrap_or(3).max(1).min(12), - runs_per_persona: input.runs_per_persona.unwrap_or(2).max(1).min(12), + users_per_variant: input.users_per_variant.unwrap_or(3).clamp(1, 12), + runs_per_persona: input.runs_per_persona.unwrap_or(2).clamp(1, 12), personas: if input.personas.is_empty() { default_simulated_user_personas() } else { diff --git a/crates/paw-codex-worker/src/directed_evolution/recovery.rs b/crates/paw-codex-worker/src/directed_evolution/recovery.rs index cd898ea28..8f368ee5b 100644 --- a/crates/paw-codex-worker/src/directed_evolution/recovery.rs +++ b/crates/paw-codex-worker/src/directed_evolution/recovery.rs @@ -58,18 +58,18 @@ async fn recover_pending_directed_evolution_stage_results( "WorkItems", ) .await?; - let prompt = recovered_stage_prompt( - &role, - &episode_id, - &generation_id, - &variant_id, - &stage_id, - &stage_result_id, - &work_item_id, - &stage_fields, - &variant_fields, - &trials, - ); + let prompt = recovered_stage_prompt(RecoveredStagePrompt { + role: &role, + episode_id: &episode_id, + generation_id: &generation_id, + variant_id: &variant_id, + stage_id: &stage_id, + stage_result_id: &stage_result_id, + work_item_id: &work_item_id, + stage_fields: &stage_fields, + variant_fields: &variant_fields, + trials: &trials, + }); if let Err(error) = post_directed_evolution_action_with_headers( client, config, @@ -182,7 +182,7 @@ fn directed_evolution_trials_terminal_for_stage(trials: &[Value], stage_result_i ) == stage_result_id }) .collect::>(); - !matching.is_empty() && matching.iter().all(|trial| directed_evolution_trial_terminal(trial)) + !matching.is_empty() && matching.iter().all(directed_evolution_trial_terminal) } fn directed_evolution_trials_terminal_for_variant(trials: &[Value], variant_id: &str) -> bool { @@ -195,7 +195,7 @@ fn directed_evolution_trials_terminal_for_variant(trials: &[Value], variant_id: ) == variant_id }) .collect::>(); - !matching.is_empty() && matching.iter().all(|trial| directed_evolution_trial_terminal(trial)) + !matching.is_empty() && matching.iter().all(directed_evolution_trial_terminal) } fn directed_evolution_trial_terminal(trial: &&Value) -> bool { @@ -228,72 +228,111 @@ async fn recovered_stage_work_item_exists( })) } -fn recovered_stage_prompt( - role: &str, - episode_id: &str, - generation_id: &str, - variant_id: &str, - stage_id: &str, - stage_result_id: &str, - work_item_id: &str, - stage_fields: &Value, - variant_fields: &Value, - trials: &[Value], -) -> String { - let stage_name = value_field_string(stage_fields, &["StageName", "stage_name"]); - let stage_kind = value_field_string(stage_fields, &["StageKind", "stage_kind"]); - let app_ref = value_field_string(variant_fields, &["AppRef", "app_ref"]); - let runtime_ref = value_field_string(variant_fields, &["RuntimeRef", "runtime_ref"]); - let variant_summary = value_field_string(variant_fields, &["Summary", "summary"]); - let trial_context = recovered_trial_context(trials, variant_id); +struct RecoveredStagePrompt<'a> { + role: &'a str, + episode_id: &'a str, + generation_id: &'a str, + variant_id: &'a str, + stage_id: &'a str, + stage_result_id: &'a str, + work_item_id: &'a str, + stage_fields: &'a Value, + variant_fields: &'a Value, + trials: &'a [Value], +} + +fn recovered_stage_prompt(args: RecoveredStagePrompt<'_>) -> String { + let stage_name = value_field_string(args.stage_fields, &["StageName", "stage_name"]); + let stage_kind = value_field_string(args.stage_fields, &["StageKind", "stage_kind"]); + let app_ref = value_field_string(args.variant_fields, &["AppRef", "app_ref"]); + let runtime_ref = value_field_string(args.variant_fields, &["RuntimeRef", "runtime_ref"]); + let variant_summary = value_field_string(args.variant_fields, &["Summary", "summary"]); + let trial_context = recovered_trial_context(args.trials, args.variant_id); let header_block = format!( - "x-de-episode-id: {episode_id}\n\ -x-de-generation-id: {generation_id}\n\ -x-de-variant-id: {variant_id}\n\ -x-de-stage-id: {stage_id}\n\ -x-de-stage-result-id: {stage_result_id}\n\ -x-de-work-item-id: {work_item_id}\n\ -x-de-runtime-ref: {runtime_ref}\n\ -x-de-app-ref: {app_ref}" + concat!( + "x-de-episode-id: {episode_id}\n", + "x-de-generation-id: {generation_id}\n", + "x-de-variant-id: {variant_id}\n", + "x-de-stage-id: {stage_id}\n", + "x-de-stage-result-id: {stage_result_id}\n", + "x-de-work-item-id: {work_item_id}\n", + "x-de-runtime-ref: {runtime_ref}\n", + "x-de-app-ref: {app_ref}", + ), + episode_id = args.episode_id, + generation_id = args.generation_id, + variant_id = args.variant_id, + stage_id = args.stage_id, + stage_result_id = args.stage_result_id, + work_item_id = args.work_item_id, + runtime_ref = runtime_ref, + app_ref = app_ref ); - if role == "telemetry_evaluator" { + if args.role == "telemetry_evaluator" { return format!( - "Evaluate Directed Evolution Datadog telemetry after simulated users completed.\n\ -EpisodeId: {episode_id}\n\ -GenerationId: {generation_id}\n\ -VariantId: {variant_id}\n\ -EvaluationStageId: {stage_id}\n\ -StageResultId: {stage_result_id}\n\ -StageName: {stage_name}\n\ -StageKind: {stage_kind}\n\ -TemperApiBase: {base}\n\ -AppRef: {app_ref}\n\ -RuntimeRef: {runtime_ref}\n\ -VariantSummary: {variant_summary}\n\n\ -RecordedTrialEvidence:\n{trial_context}\n\n\ -DirectedEvolutionHeaders:\n{header_block}\n\n\ -This is a Datadog-measured telemetry stage. Use Datadog MCP aggregate/SQL evidence over brittle log-explorer field syntax: filter for \"directed evolution runtime request\", select directed_evolution.episode_id, directed_evolution.variant_id, and tenant, and count rows for this exact episode, variant, and runtime tenant parsed from RuntimeRef. Return provenance_kind=datadog-measured. The first evidence_scope item must include query, time_window, result_count, interpretation, zero_result_meaning, and datadog_url. Zero matching runtime-request logs is failure; runtime probes may support but must not replace Datadog. Return exactly one concise JSON object with passed/status/summary/failure_reason/evaluator_role/provenance_kind/metrics/decision_basis/inputs/evidence_scope/evidence_refs/reasoning_summary.", + concat!( + "Evaluate Directed Evolution Datadog telemetry after simulated users completed.\n", + "EpisodeId: {episode_id}\n", + "GenerationId: {generation_id}\n", + "VariantId: {variant_id}\n", + "EvaluationStageId: {stage_id}\n", + "StageResultId: {stage_result_id}\n", + "StageName: {stage_name}\n", + "StageKind: {stage_kind}\n", + "TemperApiBase: {base}\n", + "AppRef: {app_ref}\n", + "RuntimeRef: {runtime_ref}\n", + "VariantSummary: {variant_summary}\n\n", + "RecordedTrialEvidence:\n{trial_context}\n\n", + "DirectedEvolutionHeaders:\n{header_block}\n\n", + "This is a Datadog-measured telemetry stage. Use Datadog MCP aggregate/SQL evidence over brittle log-explorer field syntax: filter for \"directed evolution runtime request\", select directed_evolution.episode_id, directed_evolution.variant_id, and tenant, and count rows for this exact episode, variant, and runtime tenant parsed from RuntimeRef. Return provenance_kind=datadog-measured. The first evidence_scope item must include query, time_window, result_count, interpretation, zero_result_meaning, and datadog_url. Zero matching runtime-request logs is failure; runtime probes may support but must not replace Datadog. Return exactly one concise JSON object with passed/status/summary/failure_reason/evaluator_role/provenance_kind/metrics/decision_basis/inputs/evidence_scope/evidence_refs/reasoning_summary.", + ), base = configless_public_temper_api_base(), + episode_id = args.episode_id, + generation_id = args.generation_id, + variant_id = args.variant_id, + stage_id = args.stage_id, + stage_result_id = args.stage_result_id, + stage_name = stage_name, + stage_kind = stage_kind, + app_ref = app_ref, + runtime_ref = runtime_ref, + variant_summary = variant_summary, + trial_context = trial_context, + header_block = header_block ); } format!( - "Evaluate the completed simulated-user trial observations for a Directed Evolution variant.\n\ -EpisodeId: {episode_id}\n\ -GenerationId: {generation_id}\n\ -VariantId: {variant_id}\n\ -EvaluationStageId: {stage_id}\n\ -StageResultId: {stage_result_id}\n\ -StageName: {stage_name}\n\ -StageKind: {stage_kind}\n\ -TemperApiBase: {base}\n\ -AppRef: {app_ref}\n\ -RuntimeRef: {runtime_ref}\n\ -VariantSummary: {variant_summary}\n\n\ -RecordedTrialEvidence:\n{trial_context}\n\n\ -Use only recorded simulated-user observations, blockers, friction, and live-runtime evidence. Do not query Datadog for this non-telemetry stage, do not select a winner, and do not modify files or evaluators. Return exactly one concise JSON object with passed/status/summary/failure_reason/evaluator_role=viability_evaluator/provenance_kind=brain-judged/metrics/decision_basis/inputs/evidence_scope/evidence_refs/reasoning_summary.", + concat!( + "Evaluate the completed simulated-user trial observations for a Directed Evolution variant.\n", + "EpisodeId: {episode_id}\n", + "GenerationId: {generation_id}\n", + "VariantId: {variant_id}\n", + "EvaluationStageId: {stage_id}\n", + "StageResultId: {stage_result_id}\n", + "StageName: {stage_name}\n", + "StageKind: {stage_kind}\n", + "TemperApiBase: {base}\n", + "AppRef: {app_ref}\n", + "RuntimeRef: {runtime_ref}\n", + "VariantSummary: {variant_summary}\n\n", + "RecordedTrialEvidence:\n{trial_context}\n\n", + "Use only recorded simulated-user observations, blockers, friction, and live-runtime evidence. Do not query Datadog for this non-telemetry stage, do not select a winner, and do not modify files or evaluators. Return exactly one concise JSON object with passed/status/summary/failure_reason/evaluator_role=viability_evaluator/provenance_kind=brain-judged/metrics/decision_basis/inputs/evidence_scope/evidence_refs/reasoning_summary.", + ), base = configless_public_temper_api_base(), + episode_id = args.episode_id, + generation_id = args.generation_id, + variant_id = args.variant_id, + stage_id = args.stage_id, + stage_result_id = args.stage_result_id, + stage_name = stage_name, + stage_kind = stage_kind, + app_ref = app_ref, + runtime_ref = runtime_ref, + variant_summary = variant_summary, + trial_context = trial_context ) } diff --git a/crates/paw-codex-worker/src/directed_evolution/tests.rs b/crates/paw-codex-worker/src/directed_evolution/tests.rs index b4d110d49..2912e7b4b 100644 --- a/crates/paw-codex-worker/src/directed_evolution/tests.rs +++ b/crates/paw-codex-worker/src/directed_evolution/tests.rs @@ -1221,8 +1221,8 @@ mod directed_evolution_tests { .expect("episode plan should resolve"); assert_eq!(plan.metrics[0].name, "required_changed_file_missing_count"); - assert_eq!(plan.metrics[0].higher_is_better, false); - assert_eq!(plan.metrics[0].hard_constraint, true); + assert!(!plan.metrics[0].higher_is_better); + assert!(plan.metrics[0].hard_constraint); assert_eq!(plan.viability_constraints[0].statement, "Answer spec must change."); assert_eq!(plan.elimination_rules[0].metric_names[0], "required_changed_file_missing_count"); assert_eq!(plan.scoring_rules[0].weight, "0.20"); diff --git a/crates/paw-codex-worker/src/directed_evolution/workdir.rs b/crates/paw-codex-worker/src/directed_evolution/workdir.rs index 4c3c7257e..037d1a731 100644 --- a/crates/paw-codex-worker/src/directed_evolution/workdir.rs +++ b/crates/paw-codex-worker/src/directed_evolution/workdir.rs @@ -45,17 +45,16 @@ async fn resolve_directed_evolution_workdir( canonical_genesis_bundle: false, }); } - if !context.branch_ref.trim().is_empty() { - if let Some(worktree) = + if !context.branch_ref.trim().is_empty() + && let Some(worktree) = directed_evolution_existing_workspace_worktree(config, &context.branch_ref).await? - { - return Ok(DirectedEvolutionWorkdir { - path: worktree, - app_ref: context.app_ref, - branch_ref: context.branch_ref, - canonical_genesis_bundle: false, - }); - } + { + return Ok(DirectedEvolutionWorkdir { + path: worktree, + app_ref: context.app_ref, + branch_ref: context.branch_ref, + canonical_genesis_bundle: false, + }); } if let Some(path) = directed_evolution_repo_from_genesis_bundle(client, config, &context.app_ref, &work_item.id) @@ -533,10 +532,10 @@ async fn existing_directed_evolution_worktree_for_branch( current_path = Some(PathBuf::from(path)); continue; } - if let Some(active_branch) = line.strip_prefix("branch ") { - if active_branch == expected_ref { - return Ok(current_path); - } + if let Some(active_branch) = line.strip_prefix("branch ") + && active_branch == expected_ref + { + return Ok(current_path); } if line.trim().is_empty() { current_path = None; From 2771f3b39d5e8cb826ef7a1e31f9bedc1e67103c Mon Sep 17 00:00:00 2001 From: rita-aga Date: Tue, 2 Jun 2026 19:59:45 -0400 Subject: [PATCH 04/16] Fix directed evolution worker source budgets --- .../src/directed_evolution.rs | 908 +------------ .../src/directed_evolution/actions.rs | 129 ++ .../src/directed_evolution/codex_role.rs | 215 +++ .../directed_evolution/human_episode_plan.rs | 4 +- .../directed_evolution/work_item_guards.rs | 223 ++++ .../directed_evolution/work_item_handler.rs | 337 +++++ crates/paw-codex-worker/src/tests.rs | 1158 +---------------- crates/paw-codex-worker/src/tests/core.rs | 1157 ++++++++++++++++ 8 files changed, 2067 insertions(+), 2064 deletions(-) create mode 100644 crates/paw-codex-worker/src/directed_evolution/actions.rs create mode 100644 crates/paw-codex-worker/src/directed_evolution/codex_role.rs create mode 100644 crates/paw-codex-worker/src/directed_evolution/work_item_guards.rs create mode 100644 crates/paw-codex-worker/src/directed_evolution/work_item_handler.rs create mode 100644 crates/paw-codex-worker/src/tests/core.rs diff --git a/crates/paw-codex-worker/src/directed_evolution.rs b/crates/paw-codex-worker/src/directed_evolution.rs index 2a169e024..595046d14 100644 --- a/crates/paw-codex-worker/src/directed_evolution.rs +++ b/crates/paw-codex-worker/src/directed_evolution.rs @@ -1,911 +1,11 @@ -async fn handle_queued_directed_evolution_work_item( - client: &reqwest::Client, - config: &Config, - work_item_id: &str, -) -> Result<()> { - info!(work_item_id, "saw queued Directed Evolution WorkItem"); - let work_item = fetch_directed_evolution_work_item(client, config, work_item_id).await?; - if work_item.status != "Queued" { - debug!( - work_item_id, - status = %work_item.status, - "Directed Evolution WorkItem no longer queued" - ); - return Ok(()); - } - if work_item.role.trim().is_empty() { - debug!(work_item_id, "Directed Evolution WorkItem has no worker role yet"); - return Ok(()); - } - if let Some(conflict) = - directed_evolution_exclusive_key_conflict(client, config, &work_item).await? - { - let reason = format!( - "Exclusive key {} is already active on WorkItem {}; cancelling duplicate {} work", - conflict.exclusive_key, conflict.active_work_item_id, work_item.role - ); - post_paw_orchestration_action( - client, - config, - "WorkItems", - &work_item.id, - "CancelWorkItem", - json!({ "Reason": reason }), - ) - .await?; - info!( - work_item_id = %work_item.id, - active_work_item_id = %conflict.active_work_item_id, - exclusive_key = %conflict.exclusive_key, - "cancelled duplicate exclusive Directed Evolution WorkItem" - ); - return Ok(()); - } - if let Some(reason) = - stale_directed_evolution_work_item_reason(client, config, &work_item).await? - { - if let Err(error) = - eliminate_stale_directed_evolution_stage_result(client, config, &work_item, &reason) - .await - { - warn!( - work_item_id = %work_item.id, - role = %work_item.role, - %error, - "stale Directed Evolution WorkItem cleanup could not mutate StageResult; cancelling WorkItem only" - ); - } - if let Err(error) = post_paw_orchestration_action( - client, - config, - "WorkItems", - &work_item.id, - "CancelWorkItem", - json!({ "Reason": reason }), - ) - .await - && !error.to_string().contains("not valid from state 'Cancelled'") - { - return Err(error); - } - info!( - work_item_id = %work_item.id, - role = %work_item.role, - "cancelled stale Directed Evolution WorkItem" - ); - return Ok(()); - } - - if let Err(error) = post_paw_orchestration_action( - client, - config, - "WorkItems", - &work_item.id, - "ClaimWorkItem", - json!({ - "WorkerId": config.worker_id, - "ClaimedBy": config.worker_id, - }), - ) - .await - { - let message = error.to_string(); - if directed_evolution_claim_conflict_message(&message) { - let current = fetch_directed_evolution_work_item(client, config, &work_item.id).await?; - info!( - work_item_id = %work_item.id, - role = %work_item.role, - status = %current.status, - "Directed Evolution WorkItem claim lost to another worker" - ); - if current.status != "Queued" { - return Ok(()); - } - } - return Err(error).with_context(|| { - format!( - "claim Directed Evolution WorkItem {} as {}", - work_item.id, config.worker_id - ) - }); - } - if let Some(conflict) = - directed_evolution_exclusive_key_post_claim_conflict(client, config, &work_item).await? - { - let reason = format!( - "Exclusive key {} is already claimed by lower-priority WorkItem {}; cancelling duplicate {} work before WorkerRun start", - conflict.exclusive_key, conflict.active_work_item_id, work_item.role - ); - post_paw_orchestration_action( - client, - config, - "WorkItems", - &work_item.id, - "CancelWorkItem", - json!({ "Reason": reason }), - ) - .await?; - info!( - work_item_id = %work_item.id, - active_work_item_id = %conflict.active_work_item_id, - exclusive_key = %conflict.exclusive_key, - "cancelled duplicate exclusive Directed Evolution WorkItem after claim" - ); - return Ok(()); - } - - let worker_run_id = create_entity(client, config, "WorkerRuns", json!({})).await?; - post_paw_orchestration_action( - client, - config, - "WorkerRuns", - &worker_run_id, - "StartWorkerRun", - json!({ - "Role": work_item.role, - "WorkItemId": work_item.id, - "WorkerId": config.worker_id, - "ProviderId": directed_evolution_worker_provider_id(), - "AgentKind": directed_evolution_agent_kind_for_role(&work_item.role), - "Model": directed_evolution_model_for_role(&work_item.role), - "SessionId": "", - "ParentSessionId": env::var("CODEX_SESSION_ID").unwrap_or_default(), - "CorrelationJson": work_item.correlation_json, - }), - ) - .await?; - post_paw_orchestration_action( - client, - config, - "WorkItems", - &work_item.id, - "StartWorkItem", - json!({ "WorkerRunId": worker_run_id }), - ) - .await?; - info!( - work_item_id = %work_item.id, - worker_run_id = %worker_run_id, - role = %work_item.role, - target_entity_type = %work_item.target_entity_type, - target_entity_id = %work_item.target_entity_id, - "started Directed Evolution worker run" - ); - - match run_directed_evolution_codex_role(client, config, &work_item).await { - Ok(output_json) => { - let summary = directed_evolution_summary(&work_item, &output_json); - let evidence_artifact_id = record_directed_evolution_worker_evidence( - client, - config, - &work_item, - &worker_run_id, - "codex_worker_run", - &output_json, - &summary, - ) - .await?; - post_paw_orchestration_action( - client, - config, - "WorkerRuns", - &worker_run_id, - "SucceedWorkerRun", - json!({ - "OutputJson": output_json, - "EvidenceArtifactId": evidence_artifact_id, - "Summary": summary, - }), - ) - .await?; - if let Err(route_error) = route_directed_evolution_work_item_success( - client, - config, - &work_item, - &worker_run_id, - &output_json, - &evidence_artifact_id, - &summary, - ) - .await - { - let failure_reason = format!( - "Directed Evolution receipt routing failed after WorkerRun success: {route_error}" - ); - warn!( - work_item_id = %work_item.id, - worker_run_id = %worker_run_id, - role = %work_item.role, - evidence_artifact_id = %evidence_artifact_id, - %route_error, - "Directed Evolution WorkItem receipt routing failed after WorkerRun success" - ); - post_paw_orchestration_action( - client, - config, - "WorkItems", - &work_item.id, - "FailWorkItem", - json!({ - "FailureReason": failure_reason, - "EvidenceArtifactId": evidence_artifact_id, - }), - ) - .await?; - return Ok(()); - } - post_paw_orchestration_action( - client, - config, - "WorkItems", - &work_item.id, - "SucceedWorkItem", - json!({ - "ResultJson": output_json, - "EvidenceArtifactId": evidence_artifact_id, - "Summary": summary, - }), - ) - .await?; - info!( - work_item_id = %work_item.id, - worker_run_id = %worker_run_id, - role = %work_item.role, - evidence_artifact_id = %evidence_artifact_id, - "completed Directed Evolution Codex worker run" - ); - Ok(()) - } - Err(error) => { - let failure_reason = format!("Directed Evolution Codex role failed: {error}"); - let evidence_artifact_id = match record_directed_evolution_worker_evidence( - client, - config, - &work_item, - &worker_run_id, - "codex_worker_run_failure", - &serde_json::to_string(&json!({ - "status": "failed", - "failure_reason": failure_reason, - }))?, - &failure_reason, - ) - .await - { - Ok(id) => id, - Err(report_error) => { - warn!(%report_error, work_item_id, worker_run_id, "failed to record Directed Evolution failure evidence"); - String::new() - } - }; - if let Err(report_error) = post_paw_orchestration_action( - client, - config, - "WorkerRuns", - &worker_run_id, - "FailWorkerRun", - json!({ - "FailureReason": failure_reason, - "EvidenceArtifactId": evidence_artifact_id, - }), - ) - .await - { - warn!(%report_error, work_item_id, worker_run_id, "failed to report WorkerRun failure"); - } - if let Err(route_error) = route_directed_evolution_work_item_failure( - client, - config, - &work_item, - &worker_run_id, - &failure_reason, - &evidence_artifact_id, - ) - .await - { - warn!( - work_item_id = %work_item.id, - worker_run_id = %worker_run_id, - role = %work_item.role, - evidence_artifact_id = %evidence_artifact_id, - %route_error, - "Directed Evolution failure receipt routing failed; failing WorkItem anyway" - ); - } - post_paw_orchestration_action( - client, - config, - "WorkItems", - &work_item.id, - "FailWorkItem", - json!({ - "FailureReason": failure_reason, - "EvidenceArtifactId": evidence_artifact_id, - }), - ) - .await?; - warn!( - work_item_id = %work_item.id, - worker_run_id = %worker_run_id, - role = %work_item.role, - evidence_artifact_id = %evidence_artifact_id, - "failed Directed Evolution Codex worker run" - ); - Ok(()) - } - } -} - -fn directed_evolution_claim_conflict_message(message: &str) -> bool { - message.contains("WorkItems.ClaimWorkItem returned 409") - && message.contains("not valid from state") -} - -struct DirectedEvolutionExclusiveKeyConflict { - exclusive_key: String, - active_work_item_id: String, -} - -async fn directed_evolution_exclusive_key_conflict( - client: &reqwest::Client, - config: &Config, - work_item: &DirectedEvolutionWorkItemState, -) -> Result> { - directed_evolution_exclusive_key_conflict_with_policy( - client, - config, - work_item, - ExclusiveKeyConflictPolicy::AnyActivePeer, - ) - .await -} - -async fn directed_evolution_exclusive_key_post_claim_conflict( - client: &reqwest::Client, - config: &Config, - work_item: &DirectedEvolutionWorkItemState, -) -> Result> { - directed_evolution_exclusive_key_conflict_with_policy( - client, - config, - work_item, - ExclusiveKeyConflictPolicy::LowerActivePeer, - ) - .await -} - -#[derive(Clone, Copy)] -enum ExclusiveKeyConflictPolicy { - AnyActivePeer, - LowerActivePeer, -} - -async fn directed_evolution_exclusive_key_conflict_with_policy( - client: &reqwest::Client, - config: &Config, - work_item: &DirectedEvolutionWorkItemState, - policy: ExclusiveKeyConflictPolicy, -) -> Result> { - let fields = - fetch_directed_evolution_entity_fields(client, config, "WorkItems", &work_item.id).await?; - let exclusive_key = value_field_string(&fields, &["ExclusiveKey", "exclusive_key"]); - if exclusive_key.trim().is_empty() { - return Ok(None); - } - - let escaped_key = exclusive_key.replace('\'', "''"); - let rows = query_directed_evolution_rows( - client, - config, - "WorkItems", - &format!("ExclusiveKey eq '{escaped_key}'"), - 100, - ) - .await?; - if let Some(active_work_item_id) = - directed_evolution_exclusive_key_conflict_from_rows(&work_item.id, &rows, policy) - { - return Ok(Some(DirectedEvolutionExclusiveKeyConflict { - exclusive_key, - active_work_item_id, - })); - } - - Ok(None) -} - -fn directed_evolution_exclusive_key_conflict_from_rows( - work_item_id: &str, - rows: &[Value], - policy: ExclusiveKeyConflictPolicy, -) -> Option { - let mut active_ids = Vec::new(); - for row in rows { - let row_fields = directed_evolution_row_fields(row); - let row_id = directed_evolution_row_id(row); - if row_id == work_item_id { - continue; - } - let status = value_field_string(&row_fields, &["Status", "status"]); - if matches!(status.as_str(), "Claimed" | "Running") { - active_ids.push(row_id); - } - } - active_ids.sort(); - match policy { - ExclusiveKeyConflictPolicy::AnyActivePeer => active_ids.into_iter().next(), - ExclusiveKeyConflictPolicy::LowerActivePeer => active_ids - .into_iter() - .find(|active_id| active_id.as_str() < work_item_id), - } -} - -async fn stale_directed_evolution_work_item_reason( - client: &reqwest::Client, - config: &Config, - work_item: &DirectedEvolutionWorkItemState, -) -> Result> { - if !directed_evolution_stage_evaluator_role(&work_item.role) - || work_item.target_entity_type != "StageResult" - { - return Ok(None); - } - - let stage_fields = - fetch_directed_evolution_entity_fields(client, config, "StageResults", &work_item.target_entity_id) - .await?; - let variant_fields = { - let variant_id = value_field_string(&stage_fields, &["VariantId", "variant_id"]); - if variant_id.trim().is_empty() { - json!({}) - } else { - fetch_directed_evolution_entity_fields(client, config, "Variants", &variant_id).await? - } - }; - Ok(stale_directed_evolution_stage_work_reason( - work_item, - &stage_fields, - &variant_fields, - )) -} - -async fn eliminate_stale_directed_evolution_stage_result( - client: &reqwest::Client, - config: &Config, - work_item: &DirectedEvolutionWorkItemState, - reason: &str, -) -> Result<()> { - if !stale_stage_work_targets_stage_result(work_item) { - return Ok(()); - } - - let stage_fields = - fetch_directed_evolution_entity_fields(client, config, "StageResults", &work_item.target_entity_id) - .await?; - if !stale_stage_result_should_eliminate(&stage_fields) { - return Ok(()); - } - - post_directed_evolution_action( - client, - config, - "StageResults", - &work_item.target_entity_id, - "EliminateStageResult", - json!({ - "EliminationRuleId": "stale-after-variant-terminal", - "EvidenceArtifactId": value_field_string(&stage_fields, &["EvidenceArtifactId", "evidence_artifact_id"]), - "Reason": reason, - }), - ) - .await -} - -fn stale_directed_evolution_stage_work_reason( - work_item: &DirectedEvolutionWorkItemState, - stage_fields: &Value, - variant_fields: &Value, -) -> Option { - if !stale_stage_work_targets_stage_result(work_item) { - return None; - } - let stage_status = value_field_string(stage_fields, &["Status", "status"]); - if !stage_status.trim().is_empty() && stage_status != "Running" { - return Some(format!( - "Target StageResult {} is already {}; skipping stale {} work", - work_item.target_entity_id, stage_status, work_item.role - )); - } - let variant_status = value_field_string(variant_fields, &["Status", "status"]); - if matches!( - variant_status.as_str(), - "Eliminated" | "Promoted" | "Superseded" | "Failed" - ) { - return Some(format!( - "Target variant is already {}; skipping stale {} work for StageResult {}", - variant_status, work_item.role, work_item.target_entity_id - )); - } - None -} - -fn stale_stage_work_targets_stage_result(work_item: &DirectedEvolutionWorkItemState) -> bool { - directed_evolution_stage_evaluator_role(&work_item.role) - && work_item.target_entity_type == "StageResult" -} - -fn directed_evolution_stage_evaluator_role(role: &str) -> bool { - matches!( - role, - "reviewer" - | "viability_evaluator" - | "state_verifier" - | "telemetry_evaluator" - | "wasm_evaluator" - ) -} - -fn stale_stage_result_should_eliminate(stage_fields: &Value) -> bool { - matches!( - value_field_string(stage_fields, &["Status", "status"]).as_str(), - "Running" | "Failed" - ) -} - - +include!("directed_evolution/work_item_handler.rs"); +include!("directed_evolution/work_item_guards.rs"); include!("directed_evolution/evidence.rs"); include!("directed_evolution/human_episode_defaults.rs"); include!("directed_evolution/human_episode_plan.rs"); include!("directed_evolution/human_episode.rs"); - -async fn post_directed_evolution_action( - client: &reqwest::Client, - config: &Config, - entity_set: &str, - entity_id: &str, - action: &str, - body: Value, -) -> Result<()> { - post_entity_action_with_namespace( - client, - config, - entity_set, - entity_id, - DIRECTED_EVOLUTION_NAMESPACE, - action, - body, - ) - .await -} - -async fn post_paw_orchestration_action( - client: &reqwest::Client, - config: &Config, - entity_set: &str, - entity_id: &str, - action: &str, - body: Value, -) -> Result<()> { - post_entity_action_with_namespace( - client, - config, - entity_set, - entity_id, - PAW_ORCHESTRATION_NAMESPACE, - action, - body, - ) - .await -} - -async fn post_paw_orchestration_action_with_headers( - client: &reqwest::Client, - config: &Config, - request_headers: &HeaderMap, - entity_set: &str, - entity_id: &str, - action: &str, - body: Value, -) -> Result<()> { - let url = config.entity_action_url_with_namespace( - entity_set, - entity_id, - PAW_ORCHESTRATION_NAMESPACE, - action, - ); - let response = client - .post(url) - .headers(request_headers.clone()) - .json(&body) - .send() - .await - .with_context(|| format!("dispatch PawOrchestration {entity_set}.{action}"))?; - if !response.status().is_success() { - let status = response.status(); - let text = response.text().await.unwrap_or_default(); - bail!("PawOrchestration {entity_set}.{action} returned {status}: {text}"); - } - Ok(()) -} - -async fn route_directed_evolution_work_item_success( - client: &reqwest::Client, - config: &Config, - work_item: &DirectedEvolutionWorkItemState, - worker_run_id: &str, - output_json: &str, - evidence_artifact_id: &str, - summary: &str, -) -> Result<()> { - let receipt_id = create_entity(client, config, "WorkItemReceipts", json!({})).await?; - post_directed_evolution_action( - client, - config, - "WorkItemReceipts", - &receipt_id, - "RouteSucceededWorkItem", - json!({ - "WorkItemId": work_item.id, - "Role": work_item.role, - "TargetEntityType": work_item.target_entity_type, - "TargetEntityId": work_item.target_entity_id, - "WorkerRunId": worker_run_id, - "ResultJson": output_json, - "EvidenceArtifactId": evidence_artifact_id, - "Summary": summary, - "CorrelationJson": work_item.correlation_json, - }), - ) - .await -} - -async fn route_directed_evolution_work_item_failure( - client: &reqwest::Client, - config: &Config, - work_item: &DirectedEvolutionWorkItemState, - worker_run_id: &str, - failure_reason: &str, - evidence_artifact_id: &str, -) -> Result<()> { - let receipt_id = create_entity(client, config, "WorkItemReceipts", json!({})).await?; - post_directed_evolution_action( - client, - config, - "WorkItemReceipts", - &receipt_id, - "RouteFailedWorkItem", - json!({ - "WorkItemId": work_item.id, - "Role": work_item.role, - "TargetEntityType": work_item.target_entity_type, - "TargetEntityId": work_item.target_entity_id, - "WorkerRunId": worker_run_id, - "FailureReason": failure_reason, - "EvidenceArtifactId": evidence_artifact_id, - "CorrelationJson": work_item.correlation_json, - }), - ) - .await -} - -async fn run_directed_evolution_codex_role( - client: &reqwest::Client, - config: &Config, - work_item: &DirectedEvolutionWorkItemState, -) -> Result { - let prompt = directed_evolution_prompt(work_item); - info!( - work_item_id = %work_item.id, - role = %work_item.role, - target_entity_type = %work_item.target_entity_type, - target_entity_id = %work_item.target_entity_id, - execution_enabled = config.enable_execution, - "executing Directed Evolution Codex role" - ); - if !config.enable_execution { - if directed_evolution_work_item_requires_datadog_evidence(work_item) { - bail!( - "Directed Evolution role {} requires live Codex execution and Datadog evidence", - work_item.role - ); - } - return serde_json::to_string(&json!({ - "status": "dry_run", - "role": work_item.role, - "work_item_id": work_item.id, - "target": { - "entity_type": work_item.target_entity_type, - "entity_id": work_item.target_entity_id, - }, - "prompt_preview": truncate_middle(&prompt, 1200), - })) - .context("serialize Directed Evolution dry-run output"); - } - if directed_evolution_mechanical_evaluator_role(&work_item.role) { - let payload = run_directed_evolution_mechanical_evaluator(client, config, work_item).await?; - return serde_json::to_string(&payload) - .context("serialize Directed Evolution mechanical evaluator output"); - } - if work_item.role == "promoter" { - let materialization = - materialize_directed_evolution_promotion(client, config, work_item).await?; - return serde_json::to_string(&directed_evolution_promotion_output(&materialization)) - .context("serialize Directed Evolution promoter output"); - } - - let workdir = resolve_directed_evolution_workdir(client, config, work_item).await?; - let readonly_status_before = if directed_evolution_role_may_write_repo(&work_item.role) { - None - } else { - directed_evolution_git_status_snapshot(&workdir.path).await? - }; - let output = match if directed_evolution_work_item_requires_datadog_evidence(work_item) { - run_codex_exec_command_with_datadog_mcp( - config, - &workdir.path, - prompt, - "run Directed Evolution Datadog-backed role", - ) - .await - } else { - run_codex_exec_command(config, &workdir.path, prompt, "run Directed Evolution Codex role") - .await - } { - Ok(output) => output, - Err(error) => { - return recover_directed_evolution_variant_output( - client, config, work_item, &workdir, error, - ) - .await; - } - }; - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - if !output.status.success() { - return recover_directed_evolution_variant_output( - client, - config, - work_item, - &workdir, - anyhow!( - "codex role {} failed with status {:?}: {}", - work_item.role, - output.status.code(), - truncate_middle(&format!("{stdout}\n{stderr}"), 4_000) - ), - ) - .await; - } - let mut payload = parse_codex_jsonish(&stdout).unwrap_or_else(|| { - json!({ - "status": "succeeded", - "summary": truncate_middle(&stdout, 4_000), - }) - }); - if let Some(status_before) = readonly_status_before { - ensure_directed_evolution_readonly_workdir_unchanged(&workdir.path, &status_before).await?; - } - payload = finalize_directed_evolution_output(client, config, work_item, &workdir, payload).await?; - if let Some(object) = payload.as_object_mut() { - object - .entry("role".to_string()) - .or_insert_with(|| json!(work_item.role)); - object - .entry("work_item_id".to_string()) - .or_insert_with(|| json!(work_item.id)); - object.entry("target".to_string()).or_insert_with(|| { - json!({ - "entity_type": work_item.target_entity_type, - "entity_id": work_item.target_entity_id, - }) - }); - object.entry("execution".to_string()).or_insert_with(|| { - json!({ - "workdir": workdir.path.display().to_string(), - "stdout_bytes": stdout.len(), - "stderr_bytes": stderr.len(), - }) - }); - } - ensure_directed_evolution_required_datadog_evidence(work_item, &payload)?; - serde_json::to_string(&payload) - .context("serialize Directed Evolution Codex output") -} - -fn directed_evolution_agent_kind_for_role(role: &str) -> &'static str { - if matches!(role, "promoter" | "state_verifier" | "wasm_evaluator") { - "temperpaw-worker" - } else { - "codex" - } -} - -fn directed_evolution_role_requires_datadog_mcp(role: &str) -> bool { - matches!(role, "observer" | "telemetry_evaluator") -} - -fn directed_evolution_work_item_requires_datadog_evidence( - work_item: &DirectedEvolutionWorkItemState, -) -> bool { - directed_evolution_role_requires_datadog_mcp(&work_item.role) - || directed_evolution_text_requires_datadog_evidence(&work_item.prompt_ref) - || directed_evolution_text_requires_datadog_evidence(&work_item.context_ref) - || directed_evolution_text_requires_datadog_evidence(&work_item.output_schema_ref) - || directed_evolution_text_requires_datadog_evidence(&work_item.correlation_json) -} - -fn directed_evolution_text_requires_datadog_evidence(value: &str) -> bool { - value - .to_ascii_lowercase() - .contains("datadog_evidence_scope") -} - -fn directed_evolution_worker_provider_id() -> String { - worker_provider_id() -} - -fn directed_evolution_model_for_role(role: &str) -> String { - if matches!(role, "promoter" | "state_verifier" | "wasm_evaluator") { - "deterministic-worker".to_string() - } else { - directed_evolution_codex_model_label() - } -} - -fn directed_evolution_codex_model_label() -> String { - env::var("PAW_CODEX_MODEL") - .ok() - .map(|value| value.trim().to_string()) - .filter(|value| !value.is_empty()) - .unwrap_or_else(|| "codex-cli".to_string()) -} - -async fn recover_directed_evolution_variant_output( - client: &reqwest::Client, - config: &Config, - work_item: &DirectedEvolutionWorkItemState, - workdir: &DirectedEvolutionWorkdir, - error: anyhow::Error, -) -> Result { - if work_item.role != "variant_generator" - || env_flag("PAW_DE_RECOVER_VARIANT_CHANGES_ON_CODEX_ERROR") == Some(false) - { - return Err(error); - } - let changed_files = match git_changed_files(&workdir.path).await { - Ok(files) if !files.is_empty() => files, - _ => return Err(error), - }; - let error_summary = truncate_middle(&error.to_string(), 1_200); - warn!( - work_item_id = %work_item.id, - changed_file_count = changed_files.len(), - error = %error_summary, - "recovering Directed Evolution variant from git changes after Codex execution error" - ); - let payload = json!({ - "status": "recovered_after_codex_exec_error", - "summary": format!( - "Recovered a Directed Evolution variant from {} changed file(s) after Codex exited before returning JSON.", - changed_files.len() - ), - "changed_files": changed_files, - "verification_notes": format!( - "Codex produced repository changes but did not finish the structured response: {error_summary}. Later evaluation stages must validate the hot-loaded runtime before selection." - ), - "reasoning_summary": "The worker recovered concrete file mutations left by the variant-generator worker run and will publish them as a candidate for downstream evaluation.", - "recovery": { - "reason": "codex_exec_error_after_file_changes", - "error": error_summary, - "worker_id": config.worker_id, - } - }); - let payload = finalize_directed_evolution_output(client, config, work_item, workdir, payload) - .await?; - serde_json::to_string(&payload).context("serialize recovered Directed Evolution variant output") -} - - +include!("directed_evolution/actions.rs"); +include!("directed_evolution/codex_role.rs"); include!("directed_evolution/workdir.rs"); include!("directed_evolution/recovery.rs"); include!("directed_evolution/mechanical_evaluator.rs"); diff --git a/crates/paw-codex-worker/src/directed_evolution/actions.rs b/crates/paw-codex-worker/src/directed_evolution/actions.rs new file mode 100644 index 000000000..3ade9fb6a --- /dev/null +++ b/crates/paw-codex-worker/src/directed_evolution/actions.rs @@ -0,0 +1,129 @@ +async fn post_directed_evolution_action( + client: &reqwest::Client, + config: &Config, + entity_set: &str, + entity_id: &str, + action: &str, + body: Value, +) -> Result<()> { + post_entity_action_with_namespace( + client, + config, + entity_set, + entity_id, + DIRECTED_EVOLUTION_NAMESPACE, + action, + body, + ) + .await +} + +async fn post_paw_orchestration_action( + client: &reqwest::Client, + config: &Config, + entity_set: &str, + entity_id: &str, + action: &str, + body: Value, +) -> Result<()> { + post_entity_action_with_namespace( + client, + config, + entity_set, + entity_id, + PAW_ORCHESTRATION_NAMESPACE, + action, + body, + ) + .await +} + +async fn post_paw_orchestration_action_with_headers( + client: &reqwest::Client, + config: &Config, + request_headers: &HeaderMap, + entity_set: &str, + entity_id: &str, + action: &str, + body: Value, +) -> Result<()> { + let url = config.entity_action_url_with_namespace( + entity_set, + entity_id, + PAW_ORCHESTRATION_NAMESPACE, + action, + ); + let response = client + .post(url) + .headers(request_headers.clone()) + .json(&body) + .send() + .await + .with_context(|| format!("dispatch PawOrchestration {entity_set}.{action}"))?; + if !response.status().is_success() { + let status = response.status(); + let text = response.text().await.unwrap_or_default(); + bail!("PawOrchestration {entity_set}.{action} returned {status}: {text}"); + } + Ok(()) +} + +async fn route_directed_evolution_work_item_success( + client: &reqwest::Client, + config: &Config, + work_item: &DirectedEvolutionWorkItemState, + worker_run_id: &str, + output_json: &str, + evidence_artifact_id: &str, + summary: &str, +) -> Result<()> { + let receipt_id = create_entity(client, config, "WorkItemReceipts", json!({})).await?; + post_directed_evolution_action( + client, + config, + "WorkItemReceipts", + &receipt_id, + "RouteSucceededWorkItem", + json!({ + "WorkItemId": work_item.id, + "Role": work_item.role, + "TargetEntityType": work_item.target_entity_type, + "TargetEntityId": work_item.target_entity_id, + "WorkerRunId": worker_run_id, + "ResultJson": output_json, + "EvidenceArtifactId": evidence_artifact_id, + "Summary": summary, + "CorrelationJson": work_item.correlation_json, + }), + ) + .await +} + +async fn route_directed_evolution_work_item_failure( + client: &reqwest::Client, + config: &Config, + work_item: &DirectedEvolutionWorkItemState, + worker_run_id: &str, + failure_reason: &str, + evidence_artifact_id: &str, +) -> Result<()> { + let receipt_id = create_entity(client, config, "WorkItemReceipts", json!({})).await?; + post_directed_evolution_action( + client, + config, + "WorkItemReceipts", + &receipt_id, + "RouteFailedWorkItem", + json!({ + "WorkItemId": work_item.id, + "Role": work_item.role, + "TargetEntityType": work_item.target_entity_type, + "TargetEntityId": work_item.target_entity_id, + "WorkerRunId": worker_run_id, + "FailureReason": failure_reason, + "EvidenceArtifactId": evidence_artifact_id, + "CorrelationJson": work_item.correlation_json, + }), + ) + .await +} diff --git a/crates/paw-codex-worker/src/directed_evolution/codex_role.rs b/crates/paw-codex-worker/src/directed_evolution/codex_role.rs new file mode 100644 index 000000000..907fb8940 --- /dev/null +++ b/crates/paw-codex-worker/src/directed_evolution/codex_role.rs @@ -0,0 +1,215 @@ +async fn run_directed_evolution_codex_role( + client: &reqwest::Client, + config: &Config, + work_item: &DirectedEvolutionWorkItemState, +) -> Result { + let prompt = directed_evolution_prompt(work_item); + info!( + work_item_id = %work_item.id, + role = %work_item.role, + target_entity_type = %work_item.target_entity_type, + target_entity_id = %work_item.target_entity_id, + execution_enabled = config.enable_execution, + "executing Directed Evolution Codex role" + ); + if !config.enable_execution { + if directed_evolution_work_item_requires_datadog_evidence(work_item) { + bail!( + "Directed Evolution role {} requires live Codex execution and Datadog evidence", + work_item.role + ); + } + return serde_json::to_string(&json!({ + "status": "dry_run", + "role": work_item.role, + "work_item_id": work_item.id, + "target": { + "entity_type": work_item.target_entity_type, + "entity_id": work_item.target_entity_id, + }, + "prompt_preview": truncate_middle(&prompt, 1200), + })) + .context("serialize Directed Evolution dry-run output"); + } + if directed_evolution_mechanical_evaluator_role(&work_item.role) { + let payload = run_directed_evolution_mechanical_evaluator(client, config, work_item).await?; + return serde_json::to_string(&payload) + .context("serialize Directed Evolution mechanical evaluator output"); + } + if work_item.role == "promoter" { + let materialization = + materialize_directed_evolution_promotion(client, config, work_item).await?; + return serde_json::to_string(&directed_evolution_promotion_output(&materialization)) + .context("serialize Directed Evolution promoter output"); + } + + let workdir = resolve_directed_evolution_workdir(client, config, work_item).await?; + let readonly_status_before = if directed_evolution_role_may_write_repo(&work_item.role) { + None + } else { + directed_evolution_git_status_snapshot(&workdir.path).await? + }; + let output = match if directed_evolution_work_item_requires_datadog_evidence(work_item) { + run_codex_exec_command_with_datadog_mcp( + config, + &workdir.path, + prompt, + "run Directed Evolution Datadog-backed role", + ) + .await + } else { + run_codex_exec_command(config, &workdir.path, prompt, "run Directed Evolution Codex role") + .await + } { + Ok(output) => output, + Err(error) => { + return recover_directed_evolution_variant_output( + client, config, work_item, &workdir, error, + ) + .await; + } + }; + let stdout = String::from_utf8_lossy(&output.stdout).to_string(); + let stderr = String::from_utf8_lossy(&output.stderr).to_string(); + if !output.status.success() { + return recover_directed_evolution_variant_output( + client, + config, + work_item, + &workdir, + anyhow!( + "codex role {} failed with status {:?}: {}", + work_item.role, + output.status.code(), + truncate_middle(&format!("{stdout}\n{stderr}"), 4_000) + ), + ) + .await; + } + let mut payload = parse_codex_jsonish(&stdout).unwrap_or_else(|| { + json!({ + "status": "succeeded", + "summary": truncate_middle(&stdout, 4_000), + }) + }); + if let Some(status_before) = readonly_status_before { + ensure_directed_evolution_readonly_workdir_unchanged(&workdir.path, &status_before).await?; + } + payload = finalize_directed_evolution_output(client, config, work_item, &workdir, payload).await?; + if let Some(object) = payload.as_object_mut() { + object + .entry("role".to_string()) + .or_insert_with(|| json!(work_item.role)); + object + .entry("work_item_id".to_string()) + .or_insert_with(|| json!(work_item.id)); + object.entry("target".to_string()).or_insert_with(|| { + json!({ + "entity_type": work_item.target_entity_type, + "entity_id": work_item.target_entity_id, + }) + }); + object.entry("execution".to_string()).or_insert_with(|| { + json!({ + "workdir": workdir.path.display().to_string(), + "stdout_bytes": stdout.len(), + "stderr_bytes": stderr.len(), + }) + }); + } + ensure_directed_evolution_required_datadog_evidence(work_item, &payload)?; + serde_json::to_string(&payload).context("serialize Directed Evolution Codex output") +} + +fn directed_evolution_agent_kind_for_role(role: &str) -> &'static str { + if matches!(role, "promoter" | "state_verifier" | "wasm_evaluator") { + "temperpaw-worker" + } else { + "codex" + } +} + +fn directed_evolution_role_requires_datadog_mcp(role: &str) -> bool { + matches!(role, "observer" | "telemetry_evaluator") +} + +fn directed_evolution_work_item_requires_datadog_evidence( + work_item: &DirectedEvolutionWorkItemState, +) -> bool { + directed_evolution_role_requires_datadog_mcp(&work_item.role) + || directed_evolution_text_requires_datadog_evidence(&work_item.prompt_ref) + || directed_evolution_text_requires_datadog_evidence(&work_item.context_ref) + || directed_evolution_text_requires_datadog_evidence(&work_item.output_schema_ref) + || directed_evolution_text_requires_datadog_evidence(&work_item.correlation_json) +} + +fn directed_evolution_text_requires_datadog_evidence(value: &str) -> bool { + value + .to_ascii_lowercase() + .contains("datadog_evidence_scope") +} + +fn directed_evolution_worker_provider_id() -> String { + worker_provider_id() +} + +fn directed_evolution_model_for_role(role: &str) -> String { + if matches!(role, "promoter" | "state_verifier" | "wasm_evaluator") { + "deterministic-worker".to_string() + } else { + directed_evolution_codex_model_label() + } +} + +fn directed_evolution_codex_model_label() -> String { + env::var("PAW_CODEX_MODEL") + .ok() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| "codex-cli".to_string()) +} + +async fn recover_directed_evolution_variant_output( + client: &reqwest::Client, + config: &Config, + work_item: &DirectedEvolutionWorkItemState, + workdir: &DirectedEvolutionWorkdir, + error: anyhow::Error, +) -> Result { + if work_item.role != "variant_generator" + || env_flag("PAW_DE_RECOVER_VARIANT_CHANGES_ON_CODEX_ERROR") == Some(false) + { + return Err(error); + } + let changed_files = match git_changed_files(&workdir.path).await { + Ok(files) if !files.is_empty() => files, + _ => return Err(error), + }; + let error_summary = truncate_middle(&error.to_string(), 1_200); + warn!( + work_item_id = %work_item.id, + changed_file_count = changed_files.len(), + error = %error_summary, + "recovering Directed Evolution variant from git changes after Codex execution error" + ); + let payload = json!({ + "status": "recovered_after_codex_exec_error", + "summary": format!( + "Recovered a Directed Evolution variant from {} changed file(s) after Codex exited before returning JSON.", + changed_files.len() + ), + "changed_files": changed_files, + "verification_notes": format!( + "Codex produced repository changes but did not finish the structured response: {error_summary}. Later evaluation stages must validate the hot-loaded runtime before selection." + ), + "reasoning_summary": "The worker recovered concrete file mutations left by the variant-generator worker run and will publish them as a candidate for downstream evaluation.", + "recovery": { + "reason": "codex_exec_error_after_file_changes", + "error": error_summary, + "worker_id": config.worker_id, + } + }); + let payload = + finalize_directed_evolution_output(client, config, work_item, workdir, payload).await?; + serde_json::to_string(&payload).context("serialize recovered Directed Evolution variant output") +} diff --git a/crates/paw-codex-worker/src/directed_evolution/human_episode_plan.rs b/crates/paw-codex-worker/src/directed_evolution/human_episode_plan.rs index 78b75f90b..d1e6df701 100644 --- a/crates/paw-codex-worker/src/directed_evolution/human_episode_plan.rs +++ b/crates/paw-codex-worker/src/directed_evolution/human_episode_plan.rs @@ -15,9 +15,7 @@ struct DirectedEvolutionHumanEpisodeInput { #[serde( default, alias = "CreatedByWorkerRunId", - alias = "createdByWorkerRunId", - alias = "CreatedByBrainRunId", - alias = "createdByBrainRunId" + alias = "createdByWorkerRunId" )] created_by_worker_run_id: String, #[serde(default, alias = "EvaluatorRef", alias = "evaluatorRef")] diff --git a/crates/paw-codex-worker/src/directed_evolution/work_item_guards.rs b/crates/paw-codex-worker/src/directed_evolution/work_item_guards.rs new file mode 100644 index 000000000..ac6952ca3 --- /dev/null +++ b/crates/paw-codex-worker/src/directed_evolution/work_item_guards.rs @@ -0,0 +1,223 @@ +fn directed_evolution_claim_conflict_message(message: &str) -> bool { + message.contains("WorkItems.ClaimWorkItem returned 409") + && message.contains("not valid from state") +} + +struct DirectedEvolutionExclusiveKeyConflict { + exclusive_key: String, + active_work_item_id: String, +} + +async fn directed_evolution_exclusive_key_conflict( + client: &reqwest::Client, + config: &Config, + work_item: &DirectedEvolutionWorkItemState, +) -> Result> { + directed_evolution_exclusive_key_conflict_with_policy( + client, + config, + work_item, + ExclusiveKeyConflictPolicy::AnyActivePeer, + ) + .await +} + +async fn directed_evolution_exclusive_key_post_claim_conflict( + client: &reqwest::Client, + config: &Config, + work_item: &DirectedEvolutionWorkItemState, +) -> Result> { + directed_evolution_exclusive_key_conflict_with_policy( + client, + config, + work_item, + ExclusiveKeyConflictPolicy::LowerActivePeer, + ) + .await +} + +#[derive(Clone, Copy)] +enum ExclusiveKeyConflictPolicy { + AnyActivePeer, + LowerActivePeer, +} + +async fn directed_evolution_exclusive_key_conflict_with_policy( + client: &reqwest::Client, + config: &Config, + work_item: &DirectedEvolutionWorkItemState, + policy: ExclusiveKeyConflictPolicy, +) -> Result> { + let fields = + fetch_directed_evolution_entity_fields(client, config, "WorkItems", &work_item.id).await?; + let exclusive_key = value_field_string(&fields, &["ExclusiveKey", "exclusive_key"]); + if exclusive_key.trim().is_empty() { + return Ok(None); + } + + let escaped_key = exclusive_key.replace('\'', "''"); + let rows = query_directed_evolution_rows( + client, + config, + "WorkItems", + &format!("ExclusiveKey eq '{escaped_key}'"), + 100, + ) + .await?; + if let Some(active_work_item_id) = + directed_evolution_exclusive_key_conflict_from_rows(&work_item.id, &rows, policy) + { + return Ok(Some(DirectedEvolutionExclusiveKeyConflict { + exclusive_key, + active_work_item_id, + })); + } + + Ok(None) +} + +fn directed_evolution_exclusive_key_conflict_from_rows( + work_item_id: &str, + rows: &[Value], + policy: ExclusiveKeyConflictPolicy, +) -> Option { + let mut active_ids = Vec::new(); + for row in rows { + let row_fields = directed_evolution_row_fields(row); + let row_id = directed_evolution_row_id(row); + if row_id == work_item_id { + continue; + } + let status = value_field_string(&row_fields, &["Status", "status"]); + if matches!(status.as_str(), "Claimed" | "Running") { + active_ids.push(row_id); + } + } + active_ids.sort(); + match policy { + ExclusiveKeyConflictPolicy::AnyActivePeer => active_ids.into_iter().next(), + ExclusiveKeyConflictPolicy::LowerActivePeer => active_ids + .into_iter() + .find(|active_id| active_id.as_str() < work_item_id), + } +} + +async fn stale_directed_evolution_work_item_reason( + client: &reqwest::Client, + config: &Config, + work_item: &DirectedEvolutionWorkItemState, +) -> Result> { + if !directed_evolution_stage_evaluator_role(&work_item.role) + || work_item.target_entity_type != "StageResult" + { + return Ok(None); + } + + let stage_fields = fetch_directed_evolution_entity_fields( + client, + config, + "StageResults", + &work_item.target_entity_id, + ) + .await?; + let variant_fields = { + let variant_id = value_field_string(&stage_fields, &["VariantId", "variant_id"]); + if variant_id.trim().is_empty() { + json!({}) + } else { + fetch_directed_evolution_entity_fields(client, config, "Variants", &variant_id).await? + } + }; + Ok(stale_directed_evolution_stage_work_reason( + work_item, + &stage_fields, + &variant_fields, + )) +} + +async fn eliminate_stale_directed_evolution_stage_result( + client: &reqwest::Client, + config: &Config, + work_item: &DirectedEvolutionWorkItemState, + reason: &str, +) -> Result<()> { + if !stale_stage_work_targets_stage_result(work_item) { + return Ok(()); + } + + let stage_fields = fetch_directed_evolution_entity_fields( + client, + config, + "StageResults", + &work_item.target_entity_id, + ) + .await?; + if !stale_stage_result_should_eliminate(&stage_fields) { + return Ok(()); + } + + post_directed_evolution_action( + client, + config, + "StageResults", + &work_item.target_entity_id, + "EliminateStageResult", + json!({ + "EliminationRuleId": "stale-after-variant-terminal", + "EvidenceArtifactId": value_field_string(&stage_fields, &["EvidenceArtifactId", "evidence_artifact_id"]), + "Reason": reason, + }), + ) + .await +} + +fn stale_directed_evolution_stage_work_reason( + work_item: &DirectedEvolutionWorkItemState, + stage_fields: &Value, + variant_fields: &Value, +) -> Option { + if !stale_stage_work_targets_stage_result(work_item) { + return None; + } + let stage_status = value_field_string(stage_fields, &["Status", "status"]); + if !stage_status.trim().is_empty() && stage_status != "Running" { + return Some(format!( + "Target StageResult {} is already {}; skipping stale {} work", + work_item.target_entity_id, stage_status, work_item.role + )); + } + let variant_status = value_field_string(variant_fields, &["Status", "status"]); + if matches!( + variant_status.as_str(), + "Eliminated" | "Promoted" | "Superseded" | "Failed" + ) { + return Some(format!( + "Target variant is already {}; skipping stale {} work for StageResult {}", + variant_status, work_item.role, work_item.target_entity_id + )); + } + None +} + +fn stale_stage_work_targets_stage_result(work_item: &DirectedEvolutionWorkItemState) -> bool { + directed_evolution_stage_evaluator_role(&work_item.role) + && work_item.target_entity_type == "StageResult" +} + +fn directed_evolution_stage_evaluator_role(role: &str) -> bool { + matches!( + role, + "reviewer" + | "viability_evaluator" + | "state_verifier" + | "telemetry_evaluator" + | "wasm_evaluator" + ) +} + +fn stale_stage_result_should_eliminate(stage_fields: &Value) -> bool { + matches!( + value_field_string(stage_fields, &["Status", "status"]).as_str(), + "Running" | "Failed" + ) +} diff --git a/crates/paw-codex-worker/src/directed_evolution/work_item_handler.rs b/crates/paw-codex-worker/src/directed_evolution/work_item_handler.rs new file mode 100644 index 000000000..d314e5672 --- /dev/null +++ b/crates/paw-codex-worker/src/directed_evolution/work_item_handler.rs @@ -0,0 +1,337 @@ +async fn handle_queued_directed_evolution_work_item( + client: &reqwest::Client, + config: &Config, + work_item_id: &str, +) -> Result<()> { + info!(work_item_id, "saw queued Directed Evolution WorkItem"); + let work_item = fetch_directed_evolution_work_item(client, config, work_item_id).await?; + if work_item.status != "Queued" { + debug!( + work_item_id, + status = %work_item.status, + "Directed Evolution WorkItem no longer queued" + ); + return Ok(()); + } + if work_item.role.trim().is_empty() { + debug!(work_item_id, "Directed Evolution WorkItem has no worker role yet"); + return Ok(()); + } + if let Some(conflict) = + directed_evolution_exclusive_key_conflict(client, config, &work_item).await? + { + let reason = format!( + "Exclusive key {} is already active on WorkItem {}; cancelling duplicate {} work", + conflict.exclusive_key, conflict.active_work_item_id, work_item.role + ); + post_paw_orchestration_action( + client, + config, + "WorkItems", + &work_item.id, + "CancelWorkItem", + json!({ "Reason": reason }), + ) + .await?; + info!( + work_item_id = %work_item.id, + active_work_item_id = %conflict.active_work_item_id, + exclusive_key = %conflict.exclusive_key, + "cancelled duplicate exclusive Directed Evolution WorkItem" + ); + return Ok(()); + } + if let Some(reason) = + stale_directed_evolution_work_item_reason(client, config, &work_item).await? + { + if let Err(error) = + eliminate_stale_directed_evolution_stage_result(client, config, &work_item, &reason) + .await + { + warn!( + work_item_id = %work_item.id, + role = %work_item.role, + %error, + "stale Directed Evolution WorkItem cleanup could not mutate StageResult; cancelling WorkItem only" + ); + } + if let Err(error) = post_paw_orchestration_action( + client, + config, + "WorkItems", + &work_item.id, + "CancelWorkItem", + json!({ "Reason": reason }), + ) + .await + && !error.to_string().contains("not valid from state 'Cancelled'") + { + return Err(error); + } + info!( + work_item_id = %work_item.id, + role = %work_item.role, + "cancelled stale Directed Evolution WorkItem" + ); + return Ok(()); + } + + if let Err(error) = post_paw_orchestration_action( + client, + config, + "WorkItems", + &work_item.id, + "ClaimWorkItem", + json!({ + "WorkerId": config.worker_id, + "ClaimedBy": config.worker_id, + }), + ) + .await + { + let message = error.to_string(); + if directed_evolution_claim_conflict_message(&message) { + let current = fetch_directed_evolution_work_item(client, config, &work_item.id).await?; + info!( + work_item_id = %work_item.id, + role = %work_item.role, + status = %current.status, + "Directed Evolution WorkItem claim lost to another worker" + ); + if current.status != "Queued" { + return Ok(()); + } + } + return Err(error).with_context(|| { + format!( + "claim Directed Evolution WorkItem {} as {}", + work_item.id, config.worker_id + ) + }); + } + if let Some(conflict) = + directed_evolution_exclusive_key_post_claim_conflict(client, config, &work_item).await? + { + let reason = format!( + "Exclusive key {} is already claimed by lower-priority WorkItem {}; cancelling duplicate {} work before WorkerRun start", + conflict.exclusive_key, conflict.active_work_item_id, work_item.role + ); + post_paw_orchestration_action( + client, + config, + "WorkItems", + &work_item.id, + "CancelWorkItem", + json!({ "Reason": reason }), + ) + .await?; + info!( + work_item_id = %work_item.id, + active_work_item_id = %conflict.active_work_item_id, + exclusive_key = %conflict.exclusive_key, + "cancelled duplicate exclusive Directed Evolution WorkItem after claim" + ); + return Ok(()); + } + + let worker_run_id = create_entity(client, config, "WorkerRuns", json!({})).await?; + post_paw_orchestration_action( + client, + config, + "WorkerRuns", + &worker_run_id, + "StartWorkerRun", + json!({ + "Role": work_item.role, + "WorkItemId": work_item.id, + "WorkerId": config.worker_id, + "ProviderId": directed_evolution_worker_provider_id(), + "AgentKind": directed_evolution_agent_kind_for_role(&work_item.role), + "Model": directed_evolution_model_for_role(&work_item.role), + "SessionId": "", + "ParentSessionId": env::var("CODEX_SESSION_ID").unwrap_or_default(), + "CorrelationJson": work_item.correlation_json, + }), + ) + .await?; + post_paw_orchestration_action( + client, + config, + "WorkItems", + &work_item.id, + "StartWorkItem", + json!({ "WorkerRunId": worker_run_id }), + ) + .await?; + info!( + work_item_id = %work_item.id, + worker_run_id = %worker_run_id, + role = %work_item.role, + target_entity_type = %work_item.target_entity_type, + target_entity_id = %work_item.target_entity_id, + "started Directed Evolution worker run" + ); + + match run_directed_evolution_codex_role(client, config, &work_item).await { + Ok(output_json) => { + let summary = directed_evolution_summary(&work_item, &output_json); + let evidence_artifact_id = record_directed_evolution_worker_evidence( + client, + config, + &work_item, + &worker_run_id, + "codex_worker_run", + &output_json, + &summary, + ) + .await?; + post_paw_orchestration_action( + client, + config, + "WorkerRuns", + &worker_run_id, + "SucceedWorkerRun", + json!({ + "OutputJson": output_json, + "EvidenceArtifactId": evidence_artifact_id, + "Summary": summary, + }), + ) + .await?; + if let Err(route_error) = route_directed_evolution_work_item_success( + client, + config, + &work_item, + &worker_run_id, + &output_json, + &evidence_artifact_id, + &summary, + ) + .await + { + let failure_reason = format!( + "Directed Evolution receipt routing failed after WorkerRun success: {route_error}" + ); + warn!( + work_item_id = %work_item.id, + worker_run_id = %worker_run_id, + role = %work_item.role, + evidence_artifact_id = %evidence_artifact_id, + %route_error, + "Directed Evolution WorkItem receipt routing failed after WorkerRun success" + ); + post_paw_orchestration_action( + client, + config, + "WorkItems", + &work_item.id, + "FailWorkItem", + json!({ + "FailureReason": failure_reason, + "EvidenceArtifactId": evidence_artifact_id, + }), + ) + .await?; + return Ok(()); + } + post_paw_orchestration_action( + client, + config, + "WorkItems", + &work_item.id, + "SucceedWorkItem", + json!({ + "ResultJson": output_json, + "EvidenceArtifactId": evidence_artifact_id, + "Summary": summary, + }), + ) + .await?; + info!( + work_item_id = %work_item.id, + worker_run_id = %worker_run_id, + role = %work_item.role, + evidence_artifact_id = %evidence_artifact_id, + "completed Directed Evolution Codex worker run" + ); + Ok(()) + } + Err(error) => { + let failure_reason = format!("Directed Evolution Codex role failed: {error}"); + let evidence_artifact_id = match record_directed_evolution_worker_evidence( + client, + config, + &work_item, + &worker_run_id, + "codex_worker_run_failure", + &serde_json::to_string(&json!({ + "status": "failed", + "failure_reason": failure_reason, + }))?, + &failure_reason, + ) + .await + { + Ok(id) => id, + Err(report_error) => { + warn!(%report_error, work_item_id, worker_run_id, "failed to record Directed Evolution failure evidence"); + String::new() + } + }; + if let Err(report_error) = post_paw_orchestration_action( + client, + config, + "WorkerRuns", + &worker_run_id, + "FailWorkerRun", + json!({ + "FailureReason": failure_reason, + "EvidenceArtifactId": evidence_artifact_id, + }), + ) + .await + { + warn!(%report_error, work_item_id, worker_run_id, "failed to report WorkerRun failure"); + } + if let Err(route_error) = route_directed_evolution_work_item_failure( + client, + config, + &work_item, + &worker_run_id, + &failure_reason, + &evidence_artifact_id, + ) + .await + { + warn!( + work_item_id = %work_item.id, + worker_run_id = %worker_run_id, + role = %work_item.role, + evidence_artifact_id = %evidence_artifact_id, + %route_error, + "Directed Evolution failure receipt routing failed; failing WorkItem anyway" + ); + } + post_paw_orchestration_action( + client, + config, + "WorkItems", + &work_item.id, + "FailWorkItem", + json!({ + "FailureReason": failure_reason, + "EvidenceArtifactId": evidence_artifact_id, + }), + ) + .await?; + warn!( + work_item_id = %work_item.id, + worker_run_id = %worker_run_id, + role = %work_item.role, + evidence_artifact_id = %evidence_artifact_id, + "failed Directed Evolution Codex worker run" + ); + Ok(()) + } + } +} diff --git a/crates/paw-codex-worker/src/tests.rs b/crates/paw-codex-worker/src/tests.rs index 687cbe0b1..0168c5400 100644 --- a/crates/paw-codex-worker/src/tests.rs +++ b/crates/paw-codex-worker/src/tests.rs @@ -1,1157 +1 @@ -#[cfg(test)] -mod tests { - use super::*; - use std::ffi::OsString; - use std::fs; - #[cfg(unix)] - use std::os::unix::fs::PermissionsExt; - use std::time::{SystemTime, UNIX_EPOCH}; - use tokio::sync::Mutex; - - include!("datadog_patrol_tests.rs"); - include!("github_patrol_tests.rs"); - include!("evaluation_tests.rs"); - include!("event_stream_tests.rs"); - include!("fake_codex_tests.rs"); - include!("codex_plan_tests.rs"); - include!("pull_request_tests.rs"); - include!("repo_health_parser_tests.rs"); - include!("worker_http_tests.rs"); - - static ENV_LOCK: Mutex<()> = Mutex::const_new(()); - - #[test] - fn worker_run_state_reads_temper_odata_fields() { - let value = json!({ - "entity_id": "wr-1", - "status": "Queued", - "fields": { - "task": "Do useful work", - "work_cycle_id": "wc-1", - "worktree_path": "/tmp/worktree", - "branch_name": "codex/test", - "runner_kind": "local_codex", - "allowed_worker_id": "mac-mini-codex-1", - "worker_id": "mac-mini-codex-1", - "provider_id": "local-codex", - "required_capabilities": "local_codex,repo_write" - } - }); - - let worker_run = worker_run_from_odata_value(value).expect("worker run should parse"); - - assert_eq!(worker_run.id, "wr-1"); - assert_eq!(worker_run.status, "Queued"); - assert_eq!(worker_run.task, "Do useful work"); - assert_eq!(worker_run.work_cycle_id, "wc-1"); - assert_eq!(worker_run.worktree_path, "/tmp/worktree"); - assert_eq!(worker_run.branch_name, "codex/test"); - assert_eq!(worker_run.runner_kind, "local_codex"); - assert_eq!(worker_run.allowed_worker_id, "mac-mini-codex-1"); - assert_eq!(worker_run.worker_id, "mac-mini-codex-1"); - assert_eq!(worker_run.provider_id, "local-codex"); - assert_eq!(worker_run.required_capabilities, "local_codex,repo_write"); - } - - #[test] - fn local_worker_claims_only_configured_local_codex_runs() { - let configured = WorkerRunState { - id: "wr-1".to_string(), - status: "Queued".to_string(), - task: "Do useful work".to_string(), - work_cycle_id: "wc-1".to_string(), - worktree_path: "/tmp/worktree".to_string(), - branch_name: "codex/test".to_string(), - runner_kind: "local_codex".to_string(), - allowed_worker_id: "mac-mini-codex-1".to_string(), - worker_id: String::new(), - provider_id: "local-codex".to_string(), - required_capabilities: "local_codex,repo_write".to_string(), - }; - let unconfigured = WorkerRunState { - id: "wr-2".to_string(), - status: "Queued".to_string(), - task: String::new(), - work_cycle_id: String::new(), - worktree_path: String::new(), - branch_name: String::new(), - runner_kind: String::new(), - allowed_worker_id: String::new(), - worker_id: String::new(), - provider_id: String::new(), - required_capabilities: String::new(), - }; - let cloud = WorkerRunState { - id: "wr-3".to_string(), - status: "Queued".to_string(), - task: "Cloud overflow".to_string(), - work_cycle_id: "wc-cloud".to_string(), - worktree_path: String::new(), - branch_name: String::new(), - runner_kind: "codex_cloud".to_string(), - allowed_worker_id: "mac-mini-codex-1".to_string(), - worker_id: String::new(), - provider_id: "local-codex".to_string(), - required_capabilities: "local_codex,repo_write".to_string(), - }; - let no_worktree_assignment = WorkerRunState { - id: "wr-4".to_string(), - status: "Queued".to_string(), - task: "Fix a Discord trace leak".to_string(), - work_cycle_id: "wc-no-worktree".to_string(), - worktree_path: String::new(), - branch_name: String::new(), - runner_kind: "local_codex".to_string(), - allowed_worker_id: "mac-mini-codex-1".to_string(), - worker_id: String::new(), - provider_id: "local-codex".to_string(), - required_capabilities: "local_codex,repo_write".to_string(), - }; - - assert!(worker_run_is_claimable_by_local_codex( - &configured, - "mac-mini-codex-1" - )); - assert!(!worker_run_is_claimable_by_local_codex( - &configured, - "other-worker" - )); - assert!(!worker_run_is_claimable_by_local_codex( - &unconfigured, - "mac-mini-codex-1" - )); - assert!(!worker_run_is_claimable_by_local_codex( - &cloud, - "mac-mini-codex-1" - )); - assert!(!worker_run_is_claimable_by_local_codex( - &no_worktree_assignment, - "mac-mini-codex-1" - )); - } - - #[test] - fn local_worker_recovers_only_own_running_codex_runs() { - let recoverable = WorkerRunState { - id: "wr-1".to_string(), - status: "Running".to_string(), - task: "Resume this work".to_string(), - work_cycle_id: "wc-1".to_string(), - worktree_path: "/tmp/worktree".to_string(), - branch_name: "codex/test".to_string(), - runner_kind: "local_codex".to_string(), - allowed_worker_id: "mac-mini-codex-1".to_string(), - worker_id: "mac-mini-codex-1".to_string(), - provider_id: "local-codex".to_string(), - required_capabilities: "local_codex,repo_write".to_string(), - }; - let other_worker = WorkerRunState { - worker_id: "other-worker".to_string(), - ..recoverable.clone() - }; - let queued = WorkerRunState { - status: "Queued".to_string(), - ..recoverable.clone() - }; - - assert!(worker_run_is_recoverable_by_local_codex( - &recoverable, - "mac-mini-codex-1" - )); - assert!(!worker_run_is_recoverable_by_local_codex( - &other_worker, - "mac-mini-codex-1" - )); - assert!(!worker_run_is_recoverable_by_local_codex( - &queued, - "mac-mini-codex-1" - )); - } - - #[test] - fn repo_sweep_worker_runs_are_detected_for_review_and_evaluation() { - let worker_run = WorkerRunState { - id: "wr-1".to_string(), - status: "Done".to_string(), - task: "RepoGraphSnapshot: snap-1\nWorkCycle: wc-1".to_string(), - work_cycle_id: "wc-1".to_string(), - worktree_path: "/tmp/worktree".to_string(), - branch_name: "codex/repo-sweep".to_string(), - runner_kind: "local_codex".to_string(), - allowed_worker_id: "mac-mini-codex-1".to_string(), - worker_id: String::new(), - provider_id: "local-codex".to_string(), - required_capabilities: "local_codex,repo_write".to_string(), - }; - let normal_worker_run = WorkerRunState { - id: "wr-2".to_string(), - status: "Done".to_string(), - task: "Fix a Discord reply bug".to_string(), - work_cycle_id: "wc-2".to_string(), - worktree_path: "/tmp/worktree".to_string(), - branch_name: "codex/bugfix".to_string(), - runner_kind: "local_codex".to_string(), - allowed_worker_id: "mac-mini-codex-1".to_string(), - worker_id: String::new(), - provider_id: "local-codex".to_string(), - required_capabilities: "local_codex,repo_write".to_string(), - }; - - assert!(worker_run_is_repo_sweep(&worker_run)); - assert!(!worker_run_is_repo_sweep(&normal_worker_run)); - } - - #[test] - fn worker_command_defaults_to_run_and_accepts_doctor() { - assert_eq!( - parse_worker_command(Vec::::new()), - WorkerCommand::Run - ); - assert_eq!( - parse_worker_command(vec!["doctor".to_string()]), - WorkerCommand::Doctor - ); - assert_eq!( - parse_worker_command(vec!["--doctor".to_string()]), - WorkerCommand::Doctor - ); - assert_eq!( - parse_worker_command(vec!["launchd-plist".to_string()]), - WorkerCommand::LaunchdPlist - ); - assert_eq!( - parse_worker_command(vec!["--launchd-plist".to_string()]), - WorkerCommand::LaunchdPlist - ); - assert_eq!( - parse_worker_command(vec!["register-worker-agent".to_string()]), - WorkerCommand::RegisterWorkerAgent - ); - assert_eq!( - parse_worker_command(vec!["run".to_string()]), - WorkerCommand::Run - ); - } - - #[tokio::test] - async fn launchd_plist_renders_concrete_worker_environment() { - let _guard = ENV_LOCK.lock().await; - let _model = EnvOverride::set("PAW_CODEX_MODEL", OsString::from("gpt-5-codex")); - let config = Config { - temper_url: "https://temperpaw.example.test".to_string(), - tenant: "prod".to_string(), - worker_id: "mac-mini-codex-1".to_string(), - worker_token: Some("secret&token".to_string()), - workspace_root: PathBuf::from("/Users/me/Development/temperpaw-worktrees"), - repo_root: PathBuf::from("/Users/me/Development/temperpaw"), - codex_bin: "/Users/me/.local/bin/codex".to_string(), - max_concurrent_runs: 1, - enable_execution: true, - poll_on_start: false, - codex_exec_smoke: true, - codex_exec_timeout: Duration::from_secs(90), - }; - - let plist = render_launchd_plist( - &config, - Path::new("/Users/me/.local/bin/paw-codex-worker"), - Some("cargo test -p temperpaw --test paw_patrol_foundation"), - ); - - for needle in [ - "Label", - "com.temperpaw.paw-codex-worker", - "/Users/me/.local/bin/paw-codex-worker", - "TEMPER_URL", - "https://temperpaw.example.test", - "WORKER_ID", - "mac-mini-codex-1", - "PATH", - "/opt/homebrew/bin:/opt/homebrew/sbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Users/openclaw/.cargo/bin", - "PAW_CODEX_ENABLE_EXECUTION", - "1", - "PAW_CODEX_POLL_ON_START", - "0", - "PAW_CODEX_DOCTOR_EXEC_SMOKE", - "1", - "PAW_CODEX_EXEC_TIMEOUT_SECS", - "90", - "PAW_CODEX_WORKER_CAPABILITIES", - "local_codex,repo_write,review,evaluation,datadog_query,github_query", - "PAW_CODEX_WORKER_ENV_FILE", - "/Users/openclaw/.config/temperpaw/paw-codex-worker.env", - "PAW_CODEX_FORBIDDEN_DONE_PATHS", - "PAW_CODEX_EVAL_COMMANDS", - "cargo test -p temperpaw --test paw_patrol_foundation", - "PAW_CODEX_MODEL", - "gpt-5-codex", - ] { - assert!(plist.contains(needle), "plist should contain {needle}"); - } - assert!( - !plist.contains("WORKER_TOKEN"), - "launchd plist should keep WORKER_TOKEN in the 0600 env file, not in launchctl-visible environment" - ); - assert!( - !plist.contains("PAW_CODEX_INHERIT_USER_CONFIG"), - "tool profile opt-in should stay in the worker env file so launchd does not freeze a default" - ); - } - - #[test] - fn doctor_status_fails_only_on_failures() { - let pass_and_warn = vec![ - DoctorCheck::pass("repo", "ok".to_string()), - DoctorCheck::warn("token", "missing token".to_string()), - ]; - let fail = vec![DoctorCheck::fail("odata", "not reachable".to_string())]; - - assert!(!doctor_has_failures(&pass_and_warn)); - assert!(doctor_has_failures(&fail)); - assert_eq!(doctor_status_label(DoctorStatus::Pass), "pass"); - assert_eq!(doctor_status_label(DoctorStatus::Warn), "warn"); - assert_eq!(doctor_status_label(DoctorStatus::Fail), "fail"); - } - - #[tokio::test] - async fn doctor_codex_exec_smoke_runs_fixture_when_enabled() { - let fixture = Path::new(env!("CARGO_MANIFEST_DIR")).join("fixtures/fake-codex.sh"); - let root = unique_temp_dir(); - fs::create_dir_all(&root).expect("temp dir"); - let config = Config { - temper_url: "http://127.0.0.1:3497".to_string(), - tenant: "default".to_string(), - worker_id: "mac-mini-codex-1".to_string(), - worker_token: Some("secret".to_string()), - workspace_root: root.clone(), - repo_root: root.clone(), - codex_bin: fixture.display().to_string(), - max_concurrent_runs: 1, - enable_execution: false, - poll_on_start: true, - codex_exec_smoke: true, - codex_exec_timeout: Duration::from_secs(30), - }; - - let check = check_codex_exec_smoke(&config).await; - - assert_eq!(check.name, "codex_exec_smoke"); - assert_eq!(check.status, DoctorStatus::Pass); - assert!( - check.detail.contains("PAW_CODEX_DOCTOR_EXEC_OK"), - "unexpected detail: {}", - check.detail - ); - fs::remove_dir_all(root).ok(); - } - - #[tokio::test] - async fn run_codex_times_out_stuck_local_exec() { - let fixture = Path::new(env!("CARGO_MANIFEST_DIR")).join("fixtures/fake-codex.sh"); - let root = unique_temp_dir(); - fs::create_dir_all(&root).expect("temp dir"); - let config = Config { - temper_url: "http://127.0.0.1:3497".to_string(), - tenant: "default".to_string(), - worker_id: "mac-mini-codex-1".to_string(), - worker_token: Some("secret".to_string()), - workspace_root: root.clone(), - repo_root: root.clone(), - codex_bin: fixture.display().to_string(), - max_concurrent_runs: 1, - enable_execution: true, - poll_on_start: true, - codex_exec_smoke: false, - codex_exec_timeout: Duration::from_millis(100), - }; - let worker_run = WorkerRunState { - id: "wr-timeout".to_string(), - status: "Running".to_string(), - task: "PAW_FAKE_CODEX_HANG: simulate a stuck Codex child".to_string(), - work_cycle_id: "wc-timeout".to_string(), - worktree_path: root.display().to_string(), - branch_name: "codex/timeout".to_string(), - runner_kind: "local_codex".to_string(), - allowed_worker_id: "mac-mini-codex-1".to_string(), - worker_id: "mac-mini-codex-1".to_string(), - provider_id: "local-codex".to_string(), - required_capabilities: "local_codex,repo_write".to_string(), - }; - - let result = run_codex(&config, &worker_run).await; - - let error = result.expect_err("stuck codex exec should time out"); - let message = format!("{error:#}"); - assert!( - message.contains("codex exec timed out after"), - "unexpected timeout error: {message}" - ); - fs::remove_dir_all(root).ok(); - } - - #[tokio::test] - async fn run_codex_timeout_cleans_child_process_group() { - let fixture = Path::new(env!("CARGO_MANIFEST_DIR")).join("fixtures/fake-codex.sh"); - let root = unique_temp_dir(); - fs::create_dir_all(&root).expect("temp dir"); - let marker = root.join("orphan-survived"); - let config = Config { - temper_url: "http://127.0.0.1:3497".to_string(), - tenant: "default".to_string(), - worker_id: "mac-mini-codex-1".to_string(), - worker_token: Some("secret".to_string()), - workspace_root: root.clone(), - repo_root: root.clone(), - codex_bin: fixture.display().to_string(), - max_concurrent_runs: 1, - enable_execution: true, - poll_on_start: true, - codex_exec_smoke: false, - codex_exec_timeout: Duration::from_millis(100), - }; - let worker_run = WorkerRunState { - id: "wr-timeout-tree".to_string(), - status: "Running".to_string(), - task: format!("PAW_FAKE_CODEX_ORPHAN:{}", marker.display()), - work_cycle_id: "wc-timeout-tree".to_string(), - worktree_path: root.display().to_string(), - branch_name: "codex/timeout-tree".to_string(), - runner_kind: "local_codex".to_string(), - allowed_worker_id: "mac-mini-codex-1".to_string(), - worker_id: "mac-mini-codex-1".to_string(), - provider_id: "local-codex".to_string(), - required_capabilities: "local_codex,repo_write".to_string(), - }; - - let result = run_codex(&config, &worker_run).await; - - let error = result.expect_err("stuck codex exec should time out"); - assert!( - format!("{error:#}").contains("codex exec timed out after"), - "unexpected timeout error: {error:#}" - ); - sleep(Duration::from_millis(1_500)).await; - assert!( - !marker.exists(), - "timed-out codex exec must not leave descendant processes running" - ); - fs::remove_dir_all(root).ok(); - } - - #[test] - fn codex_review_verdict_requires_explicit_marker() { - let approved = - parse_codex_review_verdict("SUMMARY: Looks good\nVERDICT: approve\nLIVE_E2E: passed"); - let changes = - parse_codex_review_verdict("VERDICT: request_changes\nSUMMARY: Missing E2E proof"); - let escalated = - parse_codex_review_verdict("VERDICT: escalate\nSUMMARY: Cedar risk needs human"); - let unknown = parse_codex_review_verdict("Looks fine to me"); - - assert_eq!(approved.action, ReviewDecisionAction::Approve); - assert_eq!(changes.action, ReviewDecisionAction::RequestChanges); - assert_eq!(escalated.action, ReviewDecisionAction::Escalate); - assert_eq!(unknown.action, ReviewDecisionAction::Escalate); - assert!(unknown.summary.contains("explicit VERDICT")); - } - - #[test] - fn forbidden_done_path_scan_catches_deployment_sensitive_edits() { - let status = "\ - M dd-dashboards/temperpaw-overview.json - M os-apps/paw-channels/wasm/transport_reconcile/Cargo.lock -?? .proofs/patrol.md -"; - - let violation = forbidden_done_path_violations( - status, - "os-apps/paw-agent/,os-apps/paw-channels/,crates/paw-triggers/", - ); - - assert_eq!( - violation, - vec!["os-apps/paw-channels/wasm/transport_reconcile/Cargo.lock"] - ); - } - - #[test] - fn evaluation_commands_default_to_temperpaw_foundation_check() { - let commands = evaluation_commands(None); - assert_eq!( - commands, - vec!["cargo test -p temperpaw --test paw_patrol_foundation -- --nocapture"] - ); - - let configured = - evaluation_commands(Some("cargo check -p paw-codex-worker\n git diff --check ")); - assert_eq!( - configured, - vec!["cargo check -p paw-codex-worker", "git diff --check"] - ); - } - - #[test] - fn codex_success_summary_includes_git_evidence_for_proof_packets() { - let worker_run = WorkerRunState { - id: "wr-proof".to_string(), - status: "Running".to_string(), - task: "Fix a Discord trace leak".to_string(), - work_cycle_id: "wc-proof".to_string(), - worktree_path: "/tmp/paw-worktree".to_string(), - branch_name: "codex/trace-leak".to_string(), - runner_kind: "local_codex".to_string(), - allowed_worker_id: "mac-mini-codex-1".to_string(), - worker_id: "mac-mini-codex-1".to_string(), - provider_id: "local-codex".to_string(), - required_capabilities: "local_codex,repo_write".to_string(), - }; - let evidence = WorktreeEvidence { - status_short: " M crates/temperpaw/src/discord.rs\n?? docs/proofs/trace.md\n" - .to_string(), - diff_stat: "crates/temperpaw/src/discord.rs | 12 ++++++------".to_string(), - }; - - let summary = format_codex_success_summary( - &worker_run, - Path::new("/tmp/paw-worktree"), - "implemented the fix", - &evidence, - None, - ); - - assert!(summary.contains("codex exec completed for WorkerRun wr-proof")); - assert!(summary.contains("Worktree: /tmp/paw-worktree")); - assert!(summary.contains("```git-status")); - assert!(summary.contains(" M crates/temperpaw/src/discord.rs")); - assert!(summary.contains("?? docs/proofs/trace.md")); - assert!(summary.contains("```git-diff-stat")); - assert!(summary.contains("Codex stdout")); - } - - #[tokio::test] - async fn codex_exec_bypasses_sandbox_for_assigned_worktree() { - let _guard = ENV_LOCK.lock().await; - let _datadog_mcp = - EnvOverride::set("PAW_CODEX_ENABLE_DATADOG_MCP", OsString::from("0")); - let _model = EnvOverride::remove("PAW_CODEX_MODEL"); - let args = codex_exec_args(Path::new("/tmp/paw-worktree"), "Create the proof file"); - let args = args - .iter() - .map(|arg| arg.to_string_lossy().to_string()) - .collect::>(); - - assert_eq!( - args, - vec![ - "exec", - "--ignore-user-config", - "--ephemeral", - "--dangerously-bypass-approvals-and-sandbox", - "--cd", - "/tmp/paw-worktree", - "--skip-git-repo-check", - "Create the proof file" - ] - ); - } - - #[tokio::test] - async fn config_accepts_multiple_concurrent_runs_for_process_pool_slots() { - let _guard = ENV_LOCK.lock().await; - let _temper_url = EnvOverride::set("TEMPER_URL", OsString::from("http://127.0.0.1:3000")); - let _worker_id = EnvOverride::set("WORKER_ID", OsString::from("local-codex-slot-1")); - let _max_concurrent_runs = EnvOverride::set("MAX_CONCURRENT_RUNS", OsString::from("4")); - - let config = Config::from_env().expect("worker config should parse"); - - assert_eq!(config.max_concurrent_runs, 4); - } - - #[tokio::test] - async fn worker_agent_registration_body_uses_slot_identity() { - let _guard = ENV_LOCK.lock().await; - let _provider = EnvOverride::set("WORKER_PROVIDER_ID", OsString::from("local-codex")); - let _host = EnvOverride::set("HOSTNAME", OsString::from("worker-host")); - let config = Config { - temper_url: "https://temperpaw.example.test".to_string(), - tenant: "prod".to_string(), - worker_id: "mac-mini-codex-prod-slot-02".to_string(), - worker_token: Some("secret".to_string()), - workspace_root: PathBuf::from("/Users/me/Development/temperpaw-worktrees"), - repo_root: PathBuf::from("/Users/me/Development/temperpaw"), - codex_bin: "codex".to_string(), - max_concurrent_runs: 1, - enable_execution: true, - poll_on_start: true, - codex_exec_smoke: false, - codex_exec_timeout: Duration::from_secs(90), - }; - - let body = worker_agent_registration_body(&config); - - assert_eq!(body["id"], "mac-mini-codex-prod-slot-02"); - assert_eq!(body["worker_id"], "mac-mini-codex-prod-slot-02"); - assert_eq!(body["provider_id"], "local-codex"); - assert_eq!(body["host"], "worker-host"); - assert_eq!( - body["worktree_root"], - "/Users/me/Development/temperpaw-worktrees" - ); - } - - #[tokio::test] - async fn worker_agent_registration_is_idempotent_when_slot_exists() { - use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; - use tokio::net::TcpListener; - - let listener = TcpListener::bind("127.0.0.1:0") - .await - .expect("bind test listener"); - let addr = listener.local_addr().expect("listener addr"); - let requests = std::sync::Arc::new(tokio::sync::Mutex::new(Vec::::new())); - let captured = requests.clone(); - let server = tokio::spawn(async move { - loop { - let Ok((mut socket, _)) = listener.accept().await else { - break; - }; - let requests = captured.clone(); - tokio::spawn(async move { - let mut buf = [0_u8; 4096]; - let n = socket.read(&mut buf).await.expect("read request"); - let request = String::from_utf8_lossy(&buf[..n]).to_string(); - requests.lock().await.push(request.clone()); - let body = if request.starts_with( - "GET /tdata/WorkerAgents('mac-mini-codex-prod-slot-02')", - ) { - r#"{"entity_id":"mac-mini-codex-prod-slot-02","fields":{"Id":"mac-mini-codex-prod-slot-02","Status":"Registered"}}"# - } else { - r#"{"error":"unexpected request"}"# - }; - let status = if request.starts_with( - "GET /tdata/WorkerAgents('mac-mini-codex-prod-slot-02')", - ) { - "200 OK" - } else { - "500 Internal Server Error" - }; - let response = format!( - "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}", - body.len() - ); - socket - .write_all(response.as_bytes()) - .await - .expect("write response"); - }); - } - }); - let config = Config { - temper_url: format!("http://{addr}"), - tenant: "prod".to_string(), - worker_id: "mac-mini-codex-prod-slot-02".to_string(), - worker_token: Some("secret".to_string()), - workspace_root: PathBuf::from("/Users/me/Development/temperpaw-worktrees"), - repo_root: PathBuf::from("/Users/me/Development/temperpaw"), - codex_bin: "codex".to_string(), - max_concurrent_runs: 1, - enable_execution: true, - poll_on_start: true, - codex_exec_smoke: false, - codex_exec_timeout: Duration::from_secs(90), - }; - let client = reqwest::Client::new(); - - ensure_worker_agent_registered(&client, &config) - .await - .expect("existing WorkerAgent should satisfy registration"); - - server.abort(); - let requests = requests.lock().await; - assert_eq!(requests.len(), 1, "registration should not POST when the slot exists"); - assert!(requests[0].starts_with( - "GET /tdata/WorkerAgents('mac-mini-codex-prod-slot-02')" - )); - } - - #[tokio::test] - async fn worker_agent_heartbeat_falls_back_to_installed_patrol_namespace() { - use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; - use tokio::net::TcpListener; - - let listener = TcpListener::bind("127.0.0.1:0") - .await - .expect("bind test listener"); - let addr = listener.local_addr().expect("listener addr"); - let requests = std::sync::Arc::new(tokio::sync::Mutex::new(Vec::::new())); - let captured = requests.clone(); - let server = tokio::spawn(async move { - loop { - let Ok((mut socket, _)) = listener.accept().await else { - break; - }; - let requests = captured.clone(); - tokio::spawn(async move { - let mut buf = [0_u8; 4096]; - let n = socket.read(&mut buf).await.expect("read request"); - let request = String::from_utf8_lossy(&buf[..n]).to_string(); - requests.lock().await.push(request.clone()); - let new_namespace = request.starts_with( - "POST /tdata/WorkerAgents('mac-mini-codex-prod-slot-02')/Temper.PawOrchestration.ReportHeartbeat", - ); - let legacy_namespace = request.starts_with( - "POST /tdata/WorkerAgents('mac-mini-codex-prod-slot-02')/TemperPaw.Patrol.ReportHeartbeat", - ); - let (status, body) = if new_namespace { - ("404 Not Found", r#"{"error":"action not installed yet"}"#) - } else if legacy_namespace { - ("200 OK", r#"{"ok":true}"#) - } else { - ("500 Internal Server Error", r#"{"error":"unexpected request"}"#) - }; - let response = format!( - "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}", - body.len() - ); - socket - .write_all(response.as_bytes()) - .await - .expect("write response"); - }); - } - }); - let config = Config { - temper_url: format!("http://{addr}"), - tenant: "prod".to_string(), - worker_id: "mac-mini-codex-prod-slot-02".to_string(), - worker_token: Some("secret".to_string()), - workspace_root: PathBuf::from("/Users/me/Development/temperpaw-worktrees"), - repo_root: PathBuf::from("/Users/me/Development/temperpaw"), - codex_bin: "codex".to_string(), - max_concurrent_runs: 1, - enable_execution: true, - poll_on_start: true, - codex_exec_smoke: false, - codex_exec_timeout: Duration::from_secs(90), - }; - let client = reqwest::Client::new(); - - report_worker_heartbeat(&client, &config) - .await - .expect("legacy installed Patrol heartbeat namespace should still work"); - - server.abort(); - let requests = requests.lock().await; - assert_eq!(requests.len(), 2, "heartbeat should retry once via Patrol"); - assert!(requests[0].starts_with( - "POST /tdata/WorkerAgents('mac-mini-codex-prod-slot-02')/Temper.PawOrchestration.ReportHeartbeat" - )); - assert!(requests[1].starts_with( - "POST /tdata/WorkerAgents('mac-mini-codex-prod-slot-02')/TemperPaw.Patrol.ReportHeartbeat" - )); - } - - #[tokio::test] - async fn codex_exec_keeps_datadog_mcp_disabled_when_env_unset() { - let _guard = ENV_LOCK.lock().await; - let _datadog_mcp = EnvOverride::remove("PAW_CODEX_ENABLE_DATADOG_MCP"); - let _datadog_url = EnvOverride::remove("PAW_CODEX_DATADOG_MCP_URL"); - let _model = EnvOverride::remove("PAW_CODEX_MODEL"); - let args = codex_exec_args(Path::new("/tmp/paw-worktree"), "Create the proof file"); - let args = args - .iter() - .map(|arg| arg.to_string_lossy().to_string()) - .collect::>(); - - assert_eq!(args[0], "exec"); - assert_eq!(args[1], "--ignore-user-config"); - assert!(!args.contains(&"-c".to_string())); - assert!(!args.contains(&"--model".to_string())); - assert!( - !args - .iter() - .any(|arg| arg.contains("mcp_servers.datadog.url")) - ); - } - - #[tokio::test] - async fn codex_exec_can_enable_datadog_mcp_without_inheriting_user_config() { - let _guard = ENV_LOCK.lock().await; - let _datadog_mcp = - EnvOverride::set("PAW_CODEX_ENABLE_DATADOG_MCP", OsString::from("1")); - let _model = EnvOverride::remove("PAW_CODEX_MODEL"); - let _datadog_url = EnvOverride::set( - "PAW_CODEX_DATADOG_MCP_URL", - OsString::from("https://mcp.datadoghq.test/mcp?toolsets=logs"), - ); - let args = codex_exec_args( - Path::new("/tmp/paw-worktree"), - "Inspect Datadog evidence for the evolution direction", - ); - let args = args - .iter() - .map(|arg| arg.to_string_lossy().to_string()) - .collect::>(); - - assert_eq!( - &args[0..4], - &[ - "exec".to_string(), - "--ignore-user-config".to_string(), - "-c".to_string(), - "mcp_servers.datadog.url=\"https://mcp.datadoghq.test/mcp?toolsets=logs\"" - .to_string(), - ] - ); - assert!(args.contains(&"--ephemeral".to_string())); - assert!(args.contains(&"--dangerously-bypass-approvals-and-sandbox".to_string())); - assert!(args.contains(&"--skip-git-repo-check".to_string())); - assert_eq!( - args.last().map(String::as_str), - Some("Inspect Datadog evidence for the evolution direction") - ); - } - - #[tokio::test] - async fn codex_exec_can_force_datadog_mcp_for_telemetry_roles() { - let _guard = ENV_LOCK.lock().await; - let _datadog_mcp = - EnvOverride::set("PAW_CODEX_ENABLE_DATADOG_MCP", OsString::from("0")); - let _model = EnvOverride::remove("PAW_CODEX_MODEL"); - let _datadog_url = EnvOverride::set( - "PAW_CODEX_DATADOG_MCP_URL", - OsString::from("https://mcp.datadoghq.test/mcp?toolsets=logs"), - ); - let args = codex_exec_args_with_datadog_mcp( - Path::new("/tmp/paw-worktree"), - "Evaluate Datadog telemetry", - ); - let args = args - .iter() - .map(|arg| arg.to_string_lossy().to_string()) - .collect::>(); - - assert_eq!( - &args[0..4], - &[ - "exec".to_string(), - "--ignore-user-config".to_string(), - "-c".to_string(), - "mcp_servers.datadog.url=\"https://mcp.datadoghq.test/mcp?toolsets=logs\"" - .to_string(), - ] - ); - assert_eq!(args.last().map(String::as_str), Some("Evaluate Datadog telemetry")); - } - - #[tokio::test] - async fn codex_exec_uses_default_datadog_mcp_url_when_enabled() { - let _guard = ENV_LOCK.lock().await; - let _datadog_mcp = - EnvOverride::set("PAW_CODEX_ENABLE_DATADOG_MCP", OsString::from("1")); - let _datadog_url = EnvOverride::remove("PAW_CODEX_DATADOG_MCP_URL"); - let _model = EnvOverride::remove("PAW_CODEX_MODEL"); - let args = codex_exec_args( - Path::new("/tmp/paw-worktree"), - "Inspect Datadog evidence for the evolution direction", - ); - let args = args - .iter() - .map(|arg| arg.to_string_lossy().to_string()) - .collect::>(); - - assert!(args.contains( - &"mcp_servers.datadog.url=\"https://mcp.datadoghq.com/api/unstable/mcp-server/mcp?toolsets=all\"" - .to_string() - )); - } - - #[tokio::test] - async fn codex_exec_can_pin_worker_model_from_env() { - let _guard = ENV_LOCK.lock().await; - let _datadog_mcp = - EnvOverride::set("PAW_CODEX_ENABLE_DATADOG_MCP", OsString::from("0")); - let _model = EnvOverride::set("PAW_CODEX_MODEL", OsString::from("gpt-5-codex")); - let args = codex_exec_args(Path::new("/tmp/paw-worktree"), "Create the proof file"); - let args = args - .iter() - .map(|arg| arg.to_string_lossy().to_string()) - .collect::>(); - - assert_eq!( - &args[0..4], - &[ - "exec".to_string(), - "--ignore-user-config".to_string(), - "--model".to_string(), - "gpt-5-codex".to_string(), - ] - ); - } - - #[test] - fn toml_basic_string_escapes_codex_config_values() { - assert_eq!( - toml_basic_string("https://mcp.example.test/a\"b\\c\n"), - "\"https://mcp.example.test/a\\\"b\\\\c\\n\"" - ); - } - - #[test] - fn worker_proof_text_does_not_call_assigned_worktree_current_checkout() { - let worker_run = WorkerRunState { - id: "wr-existing-worktree".to_string(), - status: "Running".to_string(), - task: "Inspect an existing assigned worktree".to_string(), - work_cycle_id: "wc-existing-worktree".to_string(), - worktree_path: "/tmp/paw-existing-worktree".to_string(), - branch_name: String::new(), - runner_kind: "local_codex".to_string(), - allowed_worker_id: "mac-mini-codex-1".to_string(), - worker_id: "mac-mini-codex-1".to_string(), - provider_id: "local-codex".to_string(), - required_capabilities: "local_codex,repo_write".to_string(), - }; - let evidence = WorktreeEvidence { - status_short: String::new(), - diff_stat: String::new(), - }; - - let summary = format_codex_success_summary( - &worker_run, - Path::new("/tmp/paw-existing-worktree"), - "inspected", - &evidence, - None, - ); - let review = ReviewRunState { - status: "Requested".to_string(), - worker_run_id: worker_run.id.clone(), - proof_packet_id: "proof-1".to_string(), - }; - let prompt = codex_review_prompt(&worker_run, &review); - - assert!(summary.contains("Branch: (assigned worktree without branch)")); - assert!(prompt.contains("Branch: (assigned worktree without branch)")); - assert!(!summary.contains("Branch: (current checkout)")); - assert!(!prompt.contains("Branch: (current checkout)")); - } - - #[test] - fn review_prompt_allows_pr_ready_changes_with_deployment_pending() { - let worker_run = WorkerRunState { - id: "wr-dashboard".to_string(), - status: "Done".to_string(), - task: "Update Datadog dashboard JSON but do not deploy production.".to_string(), - work_cycle_id: "wc-dashboard".to_string(), - worktree_path: "/tmp/paw-dashboard".to_string(), - branch_name: "codex/dashboard".to_string(), - runner_kind: "local_codex".to_string(), - allowed_worker_id: "mac-mini-codex-1".to_string(), - worker_id: "mac-mini-codex-1".to_string(), - provider_id: "local-codex".to_string(), - required_capabilities: "local_codex,repo_write,review".to_string(), - }; - let review = ReviewRunState { - status: "Requested".to_string(), - worker_run_id: worker_run.id.clone(), - proof_packet_id: "proof-dashboard".to_string(), - }; - - let prompt = codex_review_prompt(&worker_run, &review); - - assert!(prompt.contains("deployment-pending residual risk")); - assert!(prompt.contains("Do not require production deployment")); - } - - #[test] - fn repo_health_review_prompt_reviews_scan_contract_not_patch_contract() { - let worker_run = WorkerRunState { - id: "wr-repo-scan".to_string(), - status: "Done".to_string(), - task: "RepoGraphSnapshot: snap-1\nWorkCycle: wc-1\nRun an agent-led repo health patrol.".to_string(), - work_cycle_id: "wc-1".to_string(), - worktree_path: "/tmp/paw-repo-scan".to_string(), - branch_name: "codex/paw-repo-sweep-snap-1".to_string(), - runner_kind: "local_codex".to_string(), - allowed_worker_id: "mac-mini-codex-1".to_string(), - worker_id: "mac-mini-codex-1".to_string(), - provider_id: "local-codex".to_string(), - required_capabilities: "local_codex,repo_write,evaluation".to_string(), - }; - let review = ReviewRunState { - status: "Requested".to_string(), - worker_run_id: worker_run.id.clone(), - proof_packet_id: "proof-1".to_string(), - }; - - let prompt = codex_review_prompt(&worker_run, &review); - - assert!(prompt.contains("repo-health Patrol scan reviewer")); - assert!(prompt.contains("RepoGraphSnapshot.ScanComplete")); - assert!(prompt.contains("do not require an implementation patch")); - assert!(!prompt.contains("Inspect the git diff, changed files, tests/proofs")); - } - - #[test] - fn review_evaluation_and_work_cycle_state_read_temper_odata_fields() { - let review = review_run_from_odata_value(json!({ - "entity_id": "rev-1", - "status": "Requested", - "fields": { - "work_cycle_id": "wc-1", - "worker_run_id": "wr-1", - "proof_packet_id": "proof-1" - } - })) - .expect("review run"); - let evaluation = evaluation_run_from_odata_value(json!({ - "entity_id": "eval-1", - "status": "Queued", - "fields": { - "work_cycle_id": "wc-1", - "required_checks": "[\"repo-sweep\"]" - } - })) - .expect("evaluation run"); - let work_cycle = work_cycle_from_odata_value(json!({ - "entity_id": "wc-1", - "status": "Reviewing", - "fields": { - "implementer_worker_run_id": "wr-1", - "reviewer_run_id": "rev-1", - "evaluation_run_id": "eval-1", - "review_passed": "true" - } - })) - .expect("work cycle"); - - assert_eq!(review.worker_run_id, "wr-1"); - assert_eq!(evaluation.work_cycle_id, "wc-1"); - assert_eq!(work_cycle.status, "Reviewing"); - assert_eq!(work_cycle.implementer_worker_run_id, "wr-1"); - assert!(work_cycle.review_passed); - } - - fn unique_temp_dir() -> PathBuf { - let nanos = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("time") - .as_nanos(); - env::temp_dir().join(format!( - "paw-codex-worker-test-{}-{nanos}", - std::process::id() - )) - } - - struct EnvOverride { - key: &'static str, - previous: Option, - } - - impl EnvOverride { - fn set(key: &'static str, value: OsString) -> Self { - let previous = env::var_os(key); - unsafe { - env::set_var(key, value); - } - Self { key, previous } - } - - fn remove(key: &'static str) -> Self { - let previous = env::var_os(key); - unsafe { - env::remove_var(key); - } - Self { key, previous } - } - } - - impl Drop for EnvOverride { - fn drop(&mut self) { - unsafe { - if let Some(value) = &self.previous { - env::set_var(self.key, value); - } else { - env::remove_var(self.key); - } - } - } - } - - fn run_raw_git_for_test(args: &[&str]) { - let output = std::process::Command::new("git") - .args(args) - .output() - .expect("run git"); - assert!( - output.status.success(), - "git {} failed: {}{}{}", - args.join(" "), - String::from_utf8_lossy(&output.stdout), - if output.stdout.is_empty() || output.stderr.is_empty() { - "" - } else { - "\n" - }, - String::from_utf8_lossy(&output.stderr) - ); - } - - fn run_git_for_test(workdir: &Path, args: &[&str]) { - let output = std::process::Command::new("git") - .arg("-C") - .arg(workdir) - .args(args) - .output() - .expect("run git"); - assert!( - output.status.success(), - "git -C {} {} failed: {}{}{}", - workdir.display(), - args.join(" "), - String::from_utf8_lossy(&output.stdout), - if output.stdout.is_empty() || output.stderr.is_empty() { - "" - } else { - "\n" - }, - String::from_utf8_lossy(&output.stderr) - ); - } - - fn run_git_capture_for_test(workdir: &Path, args: &[&str]) -> String { - let output = std::process::Command::new("git") - .arg("-C") - .arg(workdir) - .args(args) - .output() - .expect("run git"); - assert!( - output.status.success(), - "git -C {} {} failed: {}{}{}", - workdir.display(), - args.join(" "), - String::from_utf8_lossy(&output.stdout), - if output.stdout.is_empty() || output.stderr.is_empty() { - "" - } else { - "\n" - }, - String::from_utf8_lossy(&output.stderr) - ); - String::from_utf8_lossy(&output.stdout) - .trim_end() - .to_string() - } - - #[cfg(unix)] - fn make_executable(path: &Path) { - let mut permissions = fs::metadata(path).expect("metadata").permissions(); - permissions.set_mode(0o755); - fs::set_permissions(path, permissions).expect("chmod"); - } - - #[cfg(not(unix))] - fn make_executable(_path: &Path) {} -} +include!("tests/core.rs"); diff --git a/crates/paw-codex-worker/src/tests/core.rs b/crates/paw-codex-worker/src/tests/core.rs new file mode 100644 index 000000000..4faf2211f --- /dev/null +++ b/crates/paw-codex-worker/src/tests/core.rs @@ -0,0 +1,1157 @@ +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::OsString; + use std::fs; + #[cfg(unix)] + use std::os::unix::fs::PermissionsExt; + use std::time::{SystemTime, UNIX_EPOCH}; + use tokio::sync::Mutex; + + include!("../datadog_patrol_tests.rs"); + include!("../github_patrol_tests.rs"); + include!("../evaluation_tests.rs"); + include!("../event_stream_tests.rs"); + include!("../fake_codex_tests.rs"); + include!("../codex_plan_tests.rs"); + include!("../pull_request_tests.rs"); + include!("../repo_health_parser_tests.rs"); + include!("../worker_http_tests.rs"); + + static ENV_LOCK: Mutex<()> = Mutex::const_new(()); + + #[test] + fn worker_run_state_reads_temper_odata_fields() { + let value = json!({ + "entity_id": "wr-1", + "status": "Queued", + "fields": { + "task": "Do useful work", + "work_cycle_id": "wc-1", + "worktree_path": "/tmp/worktree", + "branch_name": "codex/test", + "runner_kind": "local_codex", + "allowed_worker_id": "mac-mini-codex-1", + "worker_id": "mac-mini-codex-1", + "provider_id": "local-codex", + "required_capabilities": "local_codex,repo_write" + } + }); + + let worker_run = worker_run_from_odata_value(value).expect("worker run should parse"); + + assert_eq!(worker_run.id, "wr-1"); + assert_eq!(worker_run.status, "Queued"); + assert_eq!(worker_run.task, "Do useful work"); + assert_eq!(worker_run.work_cycle_id, "wc-1"); + assert_eq!(worker_run.worktree_path, "/tmp/worktree"); + assert_eq!(worker_run.branch_name, "codex/test"); + assert_eq!(worker_run.runner_kind, "local_codex"); + assert_eq!(worker_run.allowed_worker_id, "mac-mini-codex-1"); + assert_eq!(worker_run.worker_id, "mac-mini-codex-1"); + assert_eq!(worker_run.provider_id, "local-codex"); + assert_eq!(worker_run.required_capabilities, "local_codex,repo_write"); + } + + #[test] + fn local_worker_claims_only_configured_local_codex_runs() { + let configured = WorkerRunState { + id: "wr-1".to_string(), + status: "Queued".to_string(), + task: "Do useful work".to_string(), + work_cycle_id: "wc-1".to_string(), + worktree_path: "/tmp/worktree".to_string(), + branch_name: "codex/test".to_string(), + runner_kind: "local_codex".to_string(), + allowed_worker_id: "mac-mini-codex-1".to_string(), + worker_id: String::new(), + provider_id: "local-codex".to_string(), + required_capabilities: "local_codex,repo_write".to_string(), + }; + let unconfigured = WorkerRunState { + id: "wr-2".to_string(), + status: "Queued".to_string(), + task: String::new(), + work_cycle_id: String::new(), + worktree_path: String::new(), + branch_name: String::new(), + runner_kind: String::new(), + allowed_worker_id: String::new(), + worker_id: String::new(), + provider_id: String::new(), + required_capabilities: String::new(), + }; + let cloud = WorkerRunState { + id: "wr-3".to_string(), + status: "Queued".to_string(), + task: "Cloud overflow".to_string(), + work_cycle_id: "wc-cloud".to_string(), + worktree_path: String::new(), + branch_name: String::new(), + runner_kind: "codex_cloud".to_string(), + allowed_worker_id: "mac-mini-codex-1".to_string(), + worker_id: String::new(), + provider_id: "local-codex".to_string(), + required_capabilities: "local_codex,repo_write".to_string(), + }; + let no_worktree_assignment = WorkerRunState { + id: "wr-4".to_string(), + status: "Queued".to_string(), + task: "Fix a Discord trace leak".to_string(), + work_cycle_id: "wc-no-worktree".to_string(), + worktree_path: String::new(), + branch_name: String::new(), + runner_kind: "local_codex".to_string(), + allowed_worker_id: "mac-mini-codex-1".to_string(), + worker_id: String::new(), + provider_id: "local-codex".to_string(), + required_capabilities: "local_codex,repo_write".to_string(), + }; + + assert!(worker_run_is_claimable_by_local_codex( + &configured, + "mac-mini-codex-1" + )); + assert!(!worker_run_is_claimable_by_local_codex( + &configured, + "other-worker" + )); + assert!(!worker_run_is_claimable_by_local_codex( + &unconfigured, + "mac-mini-codex-1" + )); + assert!(!worker_run_is_claimable_by_local_codex( + &cloud, + "mac-mini-codex-1" + )); + assert!(!worker_run_is_claimable_by_local_codex( + &no_worktree_assignment, + "mac-mini-codex-1" + )); + } + + #[test] + fn local_worker_recovers_only_own_running_codex_runs() { + let recoverable = WorkerRunState { + id: "wr-1".to_string(), + status: "Running".to_string(), + task: "Resume this work".to_string(), + work_cycle_id: "wc-1".to_string(), + worktree_path: "/tmp/worktree".to_string(), + branch_name: "codex/test".to_string(), + runner_kind: "local_codex".to_string(), + allowed_worker_id: "mac-mini-codex-1".to_string(), + worker_id: "mac-mini-codex-1".to_string(), + provider_id: "local-codex".to_string(), + required_capabilities: "local_codex,repo_write".to_string(), + }; + let other_worker = WorkerRunState { + worker_id: "other-worker".to_string(), + ..recoverable.clone() + }; + let queued = WorkerRunState { + status: "Queued".to_string(), + ..recoverable.clone() + }; + + assert!(worker_run_is_recoverable_by_local_codex( + &recoverable, + "mac-mini-codex-1" + )); + assert!(!worker_run_is_recoverable_by_local_codex( + &other_worker, + "mac-mini-codex-1" + )); + assert!(!worker_run_is_recoverable_by_local_codex( + &queued, + "mac-mini-codex-1" + )); + } + + #[test] + fn repo_sweep_worker_runs_are_detected_for_review_and_evaluation() { + let worker_run = WorkerRunState { + id: "wr-1".to_string(), + status: "Done".to_string(), + task: "RepoGraphSnapshot: snap-1\nWorkCycle: wc-1".to_string(), + work_cycle_id: "wc-1".to_string(), + worktree_path: "/tmp/worktree".to_string(), + branch_name: "codex/repo-sweep".to_string(), + runner_kind: "local_codex".to_string(), + allowed_worker_id: "mac-mini-codex-1".to_string(), + worker_id: String::new(), + provider_id: "local-codex".to_string(), + required_capabilities: "local_codex,repo_write".to_string(), + }; + let normal_worker_run = WorkerRunState { + id: "wr-2".to_string(), + status: "Done".to_string(), + task: "Fix a Discord reply bug".to_string(), + work_cycle_id: "wc-2".to_string(), + worktree_path: "/tmp/worktree".to_string(), + branch_name: "codex/bugfix".to_string(), + runner_kind: "local_codex".to_string(), + allowed_worker_id: "mac-mini-codex-1".to_string(), + worker_id: String::new(), + provider_id: "local-codex".to_string(), + required_capabilities: "local_codex,repo_write".to_string(), + }; + + assert!(worker_run_is_repo_sweep(&worker_run)); + assert!(!worker_run_is_repo_sweep(&normal_worker_run)); + } + + #[test] + fn worker_command_defaults_to_run_and_accepts_doctor() { + assert_eq!( + parse_worker_command(Vec::::new()), + WorkerCommand::Run + ); + assert_eq!( + parse_worker_command(vec!["doctor".to_string()]), + WorkerCommand::Doctor + ); + assert_eq!( + parse_worker_command(vec!["--doctor".to_string()]), + WorkerCommand::Doctor + ); + assert_eq!( + parse_worker_command(vec!["launchd-plist".to_string()]), + WorkerCommand::LaunchdPlist + ); + assert_eq!( + parse_worker_command(vec!["--launchd-plist".to_string()]), + WorkerCommand::LaunchdPlist + ); + assert_eq!( + parse_worker_command(vec!["register-worker-agent".to_string()]), + WorkerCommand::RegisterWorkerAgent + ); + assert_eq!( + parse_worker_command(vec!["run".to_string()]), + WorkerCommand::Run + ); + } + + #[tokio::test] + async fn launchd_plist_renders_concrete_worker_environment() { + let _guard = ENV_LOCK.lock().await; + let _model = EnvOverride::set("PAW_CODEX_MODEL", OsString::from("gpt-5-codex")); + let config = Config { + temper_url: "https://temperpaw.example.test".to_string(), + tenant: "prod".to_string(), + worker_id: "mac-mini-codex-1".to_string(), + worker_token: Some("secret&token".to_string()), + workspace_root: PathBuf::from("/Users/me/Development/temperpaw-worktrees"), + repo_root: PathBuf::from("/Users/me/Development/temperpaw"), + codex_bin: "/Users/me/.local/bin/codex".to_string(), + max_concurrent_runs: 1, + enable_execution: true, + poll_on_start: false, + codex_exec_smoke: true, + codex_exec_timeout: Duration::from_secs(90), + }; + + let plist = render_launchd_plist( + &config, + Path::new("/Users/me/.local/bin/paw-codex-worker"), + Some("cargo test -p temperpaw --test paw_patrol_foundation"), + ); + + for needle in [ + "Label", + "com.temperpaw.paw-codex-worker", + "/Users/me/.local/bin/paw-codex-worker", + "TEMPER_URL", + "https://temperpaw.example.test", + "WORKER_ID", + "mac-mini-codex-1", + "PATH", + "/opt/homebrew/bin:/opt/homebrew/sbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Users/openclaw/.cargo/bin", + "PAW_CODEX_ENABLE_EXECUTION", + "1", + "PAW_CODEX_POLL_ON_START", + "0", + "PAW_CODEX_DOCTOR_EXEC_SMOKE", + "1", + "PAW_CODEX_EXEC_TIMEOUT_SECS", + "90", + "PAW_CODEX_WORKER_CAPABILITIES", + "local_codex,repo_write,review,evaluation,datadog_query,github_query", + "PAW_CODEX_WORKER_ENV_FILE", + "/Users/openclaw/.config/temperpaw/paw-codex-worker.env", + "PAW_CODEX_FORBIDDEN_DONE_PATHS", + "PAW_CODEX_EVAL_COMMANDS", + "cargo test -p temperpaw --test paw_patrol_foundation", + "PAW_CODEX_MODEL", + "gpt-5-codex", + ] { + assert!(plist.contains(needle), "plist should contain {needle}"); + } + assert!( + !plist.contains("WORKER_TOKEN"), + "launchd plist should keep WORKER_TOKEN in the 0600 env file, not in launchctl-visible environment" + ); + assert!( + !plist.contains("PAW_CODEX_INHERIT_USER_CONFIG"), + "tool profile opt-in should stay in the worker env file so launchd does not freeze a default" + ); + } + + #[test] + fn doctor_status_fails_only_on_failures() { + let pass_and_warn = vec![ + DoctorCheck::pass("repo", "ok".to_string()), + DoctorCheck::warn("token", "missing token".to_string()), + ]; + let fail = vec![DoctorCheck::fail("odata", "not reachable".to_string())]; + + assert!(!doctor_has_failures(&pass_and_warn)); + assert!(doctor_has_failures(&fail)); + assert_eq!(doctor_status_label(DoctorStatus::Pass), "pass"); + assert_eq!(doctor_status_label(DoctorStatus::Warn), "warn"); + assert_eq!(doctor_status_label(DoctorStatus::Fail), "fail"); + } + + #[tokio::test] + async fn doctor_codex_exec_smoke_runs_fixture_when_enabled() { + let fixture = Path::new(env!("CARGO_MANIFEST_DIR")).join("fixtures/fake-codex.sh"); + let root = unique_temp_dir(); + fs::create_dir_all(&root).expect("temp dir"); + let config = Config { + temper_url: "http://127.0.0.1:3497".to_string(), + tenant: "default".to_string(), + worker_id: "mac-mini-codex-1".to_string(), + worker_token: Some("secret".to_string()), + workspace_root: root.clone(), + repo_root: root.clone(), + codex_bin: fixture.display().to_string(), + max_concurrent_runs: 1, + enable_execution: false, + poll_on_start: true, + codex_exec_smoke: true, + codex_exec_timeout: Duration::from_secs(30), + }; + + let check = check_codex_exec_smoke(&config).await; + + assert_eq!(check.name, "codex_exec_smoke"); + assert_eq!(check.status, DoctorStatus::Pass); + assert!( + check.detail.contains("PAW_CODEX_DOCTOR_EXEC_OK"), + "unexpected detail: {}", + check.detail + ); + fs::remove_dir_all(root).ok(); + } + + #[tokio::test] + async fn run_codex_times_out_stuck_local_exec() { + let fixture = Path::new(env!("CARGO_MANIFEST_DIR")).join("fixtures/fake-codex.sh"); + let root = unique_temp_dir(); + fs::create_dir_all(&root).expect("temp dir"); + let config = Config { + temper_url: "http://127.0.0.1:3497".to_string(), + tenant: "default".to_string(), + worker_id: "mac-mini-codex-1".to_string(), + worker_token: Some("secret".to_string()), + workspace_root: root.clone(), + repo_root: root.clone(), + codex_bin: fixture.display().to_string(), + max_concurrent_runs: 1, + enable_execution: true, + poll_on_start: true, + codex_exec_smoke: false, + codex_exec_timeout: Duration::from_millis(100), + }; + let worker_run = WorkerRunState { + id: "wr-timeout".to_string(), + status: "Running".to_string(), + task: "PAW_FAKE_CODEX_HANG: simulate a stuck Codex child".to_string(), + work_cycle_id: "wc-timeout".to_string(), + worktree_path: root.display().to_string(), + branch_name: "codex/timeout".to_string(), + runner_kind: "local_codex".to_string(), + allowed_worker_id: "mac-mini-codex-1".to_string(), + worker_id: "mac-mini-codex-1".to_string(), + provider_id: "local-codex".to_string(), + required_capabilities: "local_codex,repo_write".to_string(), + }; + + let result = run_codex(&config, &worker_run).await; + + let error = result.expect_err("stuck codex exec should time out"); + let message = format!("{error:#}"); + assert!( + message.contains("codex exec timed out after"), + "unexpected timeout error: {message}" + ); + fs::remove_dir_all(root).ok(); + } + + #[tokio::test] + async fn run_codex_timeout_cleans_child_process_group() { + let fixture = Path::new(env!("CARGO_MANIFEST_DIR")).join("fixtures/fake-codex.sh"); + let root = unique_temp_dir(); + fs::create_dir_all(&root).expect("temp dir"); + let marker = root.join("orphan-survived"); + let config = Config { + temper_url: "http://127.0.0.1:3497".to_string(), + tenant: "default".to_string(), + worker_id: "mac-mini-codex-1".to_string(), + worker_token: Some("secret".to_string()), + workspace_root: root.clone(), + repo_root: root.clone(), + codex_bin: fixture.display().to_string(), + max_concurrent_runs: 1, + enable_execution: true, + poll_on_start: true, + codex_exec_smoke: false, + codex_exec_timeout: Duration::from_millis(100), + }; + let worker_run = WorkerRunState { + id: "wr-timeout-tree".to_string(), + status: "Running".to_string(), + task: format!("PAW_FAKE_CODEX_ORPHAN:{}", marker.display()), + work_cycle_id: "wc-timeout-tree".to_string(), + worktree_path: root.display().to_string(), + branch_name: "codex/timeout-tree".to_string(), + runner_kind: "local_codex".to_string(), + allowed_worker_id: "mac-mini-codex-1".to_string(), + worker_id: "mac-mini-codex-1".to_string(), + provider_id: "local-codex".to_string(), + required_capabilities: "local_codex,repo_write".to_string(), + }; + + let result = run_codex(&config, &worker_run).await; + + let error = result.expect_err("stuck codex exec should time out"); + assert!( + format!("{error:#}").contains("codex exec timed out after"), + "unexpected timeout error: {error:#}" + ); + sleep(Duration::from_millis(1_500)).await; + assert!( + !marker.exists(), + "timed-out codex exec must not leave descendant processes running" + ); + fs::remove_dir_all(root).ok(); + } + + #[test] + fn codex_review_verdict_requires_explicit_marker() { + let approved = + parse_codex_review_verdict("SUMMARY: Looks good\nVERDICT: approve\nLIVE_E2E: passed"); + let changes = + parse_codex_review_verdict("VERDICT: request_changes\nSUMMARY: Missing E2E proof"); + let escalated = + parse_codex_review_verdict("VERDICT: escalate\nSUMMARY: Cedar risk needs human"); + let unknown = parse_codex_review_verdict("Looks fine to me"); + + assert_eq!(approved.action, ReviewDecisionAction::Approve); + assert_eq!(changes.action, ReviewDecisionAction::RequestChanges); + assert_eq!(escalated.action, ReviewDecisionAction::Escalate); + assert_eq!(unknown.action, ReviewDecisionAction::Escalate); + assert!(unknown.summary.contains("explicit VERDICT")); + } + + #[test] + fn forbidden_done_path_scan_catches_deployment_sensitive_edits() { + let status = "\ + M dd-dashboards/temperpaw-overview.json + M os-apps/paw-channels/wasm/transport_reconcile/Cargo.lock +?? .proofs/patrol.md +"; + + let violation = forbidden_done_path_violations( + status, + "os-apps/paw-agent/,os-apps/paw-channels/,crates/paw-triggers/", + ); + + assert_eq!( + violation, + vec!["os-apps/paw-channels/wasm/transport_reconcile/Cargo.lock"] + ); + } + + #[test] + fn evaluation_commands_default_to_temperpaw_foundation_check() { + let commands = evaluation_commands(None); + assert_eq!( + commands, + vec!["cargo test -p temperpaw --test paw_patrol_foundation -- --nocapture"] + ); + + let configured = + evaluation_commands(Some("cargo check -p paw-codex-worker\n git diff --check ")); + assert_eq!( + configured, + vec!["cargo check -p paw-codex-worker", "git diff --check"] + ); + } + + #[test] + fn codex_success_summary_includes_git_evidence_for_proof_packets() { + let worker_run = WorkerRunState { + id: "wr-proof".to_string(), + status: "Running".to_string(), + task: "Fix a Discord trace leak".to_string(), + work_cycle_id: "wc-proof".to_string(), + worktree_path: "/tmp/paw-worktree".to_string(), + branch_name: "codex/trace-leak".to_string(), + runner_kind: "local_codex".to_string(), + allowed_worker_id: "mac-mini-codex-1".to_string(), + worker_id: "mac-mini-codex-1".to_string(), + provider_id: "local-codex".to_string(), + required_capabilities: "local_codex,repo_write".to_string(), + }; + let evidence = WorktreeEvidence { + status_short: " M crates/temperpaw/src/discord.rs\n?? docs/proofs/trace.md\n" + .to_string(), + diff_stat: "crates/temperpaw/src/discord.rs | 12 ++++++------".to_string(), + }; + + let summary = format_codex_success_summary( + &worker_run, + Path::new("/tmp/paw-worktree"), + "implemented the fix", + &evidence, + None, + ); + + assert!(summary.contains("codex exec completed for WorkerRun wr-proof")); + assert!(summary.contains("Worktree: /tmp/paw-worktree")); + assert!(summary.contains("```git-status")); + assert!(summary.contains(" M crates/temperpaw/src/discord.rs")); + assert!(summary.contains("?? docs/proofs/trace.md")); + assert!(summary.contains("```git-diff-stat")); + assert!(summary.contains("Codex stdout")); + } + + #[tokio::test] + async fn codex_exec_bypasses_sandbox_for_assigned_worktree() { + let _guard = ENV_LOCK.lock().await; + let _datadog_mcp = + EnvOverride::set("PAW_CODEX_ENABLE_DATADOG_MCP", OsString::from("0")); + let _model = EnvOverride::remove("PAW_CODEX_MODEL"); + let args = codex_exec_args(Path::new("/tmp/paw-worktree"), "Create the proof file"); + let args = args + .iter() + .map(|arg| arg.to_string_lossy().to_string()) + .collect::>(); + + assert_eq!( + args, + vec![ + "exec", + "--ignore-user-config", + "--ephemeral", + "--dangerously-bypass-approvals-and-sandbox", + "--cd", + "/tmp/paw-worktree", + "--skip-git-repo-check", + "Create the proof file" + ] + ); + } + + #[tokio::test] + async fn config_accepts_multiple_concurrent_runs_for_process_pool_slots() { + let _guard = ENV_LOCK.lock().await; + let _temper_url = EnvOverride::set("TEMPER_URL", OsString::from("http://127.0.0.1:3000")); + let _worker_id = EnvOverride::set("WORKER_ID", OsString::from("local-codex-slot-1")); + let _max_concurrent_runs = EnvOverride::set("MAX_CONCURRENT_RUNS", OsString::from("4")); + + let config = Config::from_env().expect("worker config should parse"); + + assert_eq!(config.max_concurrent_runs, 4); + } + + #[tokio::test] + async fn worker_agent_registration_body_uses_slot_identity() { + let _guard = ENV_LOCK.lock().await; + let _provider = EnvOverride::set("WORKER_PROVIDER_ID", OsString::from("local-codex")); + let _host = EnvOverride::set("HOSTNAME", OsString::from("worker-host")); + let config = Config { + temper_url: "https://temperpaw.example.test".to_string(), + tenant: "prod".to_string(), + worker_id: "mac-mini-codex-prod-slot-02".to_string(), + worker_token: Some("secret".to_string()), + workspace_root: PathBuf::from("/Users/me/Development/temperpaw-worktrees"), + repo_root: PathBuf::from("/Users/me/Development/temperpaw"), + codex_bin: "codex".to_string(), + max_concurrent_runs: 1, + enable_execution: true, + poll_on_start: true, + codex_exec_smoke: false, + codex_exec_timeout: Duration::from_secs(90), + }; + + let body = worker_agent_registration_body(&config); + + assert_eq!(body["id"], "mac-mini-codex-prod-slot-02"); + assert_eq!(body["worker_id"], "mac-mini-codex-prod-slot-02"); + assert_eq!(body["provider_id"], "local-codex"); + assert_eq!(body["host"], "worker-host"); + assert_eq!( + body["worktree_root"], + "/Users/me/Development/temperpaw-worktrees" + ); + } + + #[tokio::test] + async fn worker_agent_registration_is_idempotent_when_slot_exists() { + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + use tokio::net::TcpListener; + + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind test listener"); + let addr = listener.local_addr().expect("listener addr"); + let requests = std::sync::Arc::new(tokio::sync::Mutex::new(Vec::::new())); + let captured = requests.clone(); + let server = tokio::spawn(async move { + loop { + let Ok((mut socket, _)) = listener.accept().await else { + break; + }; + let requests = captured.clone(); + tokio::spawn(async move { + let mut buf = [0_u8; 4096]; + let n = socket.read(&mut buf).await.expect("read request"); + let request = String::from_utf8_lossy(&buf[..n]).to_string(); + requests.lock().await.push(request.clone()); + let body = if request.starts_with( + "GET /tdata/WorkerAgents('mac-mini-codex-prod-slot-02')", + ) { + r#"{"entity_id":"mac-mini-codex-prod-slot-02","fields":{"Id":"mac-mini-codex-prod-slot-02","Status":"Registered"}}"# + } else { + r#"{"error":"unexpected request"}"# + }; + let status = if request.starts_with( + "GET /tdata/WorkerAgents('mac-mini-codex-prod-slot-02')", + ) { + "200 OK" + } else { + "500 Internal Server Error" + }; + let response = format!( + "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ); + socket + .write_all(response.as_bytes()) + .await + .expect("write response"); + }); + } + }); + let config = Config { + temper_url: format!("http://{addr}"), + tenant: "prod".to_string(), + worker_id: "mac-mini-codex-prod-slot-02".to_string(), + worker_token: Some("secret".to_string()), + workspace_root: PathBuf::from("/Users/me/Development/temperpaw-worktrees"), + repo_root: PathBuf::from("/Users/me/Development/temperpaw"), + codex_bin: "codex".to_string(), + max_concurrent_runs: 1, + enable_execution: true, + poll_on_start: true, + codex_exec_smoke: false, + codex_exec_timeout: Duration::from_secs(90), + }; + let client = reqwest::Client::new(); + + ensure_worker_agent_registered(&client, &config) + .await + .expect("existing WorkerAgent should satisfy registration"); + + server.abort(); + let requests = requests.lock().await; + assert_eq!(requests.len(), 1, "registration should not POST when the slot exists"); + assert!(requests[0].starts_with( + "GET /tdata/WorkerAgents('mac-mini-codex-prod-slot-02')" + )); + } + + #[tokio::test] + async fn worker_agent_heartbeat_falls_back_to_installed_patrol_namespace() { + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + use tokio::net::TcpListener; + + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind test listener"); + let addr = listener.local_addr().expect("listener addr"); + let requests = std::sync::Arc::new(tokio::sync::Mutex::new(Vec::::new())); + let captured = requests.clone(); + let server = tokio::spawn(async move { + loop { + let Ok((mut socket, _)) = listener.accept().await else { + break; + }; + let requests = captured.clone(); + tokio::spawn(async move { + let mut buf = [0_u8; 4096]; + let n = socket.read(&mut buf).await.expect("read request"); + let request = String::from_utf8_lossy(&buf[..n]).to_string(); + requests.lock().await.push(request.clone()); + let new_namespace = request.starts_with( + "POST /tdata/WorkerAgents('mac-mini-codex-prod-slot-02')/Temper.PawOrchestration.ReportHeartbeat", + ); + let legacy_namespace = request.starts_with( + "POST /tdata/WorkerAgents('mac-mini-codex-prod-slot-02')/TemperPaw.Patrol.ReportHeartbeat", + ); + let (status, body) = if new_namespace { + ("404 Not Found", r#"{"error":"action not installed yet"}"#) + } else if legacy_namespace { + ("200 OK", r#"{"ok":true}"#) + } else { + ("500 Internal Server Error", r#"{"error":"unexpected request"}"#) + }; + let response = format!( + "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ); + socket + .write_all(response.as_bytes()) + .await + .expect("write response"); + }); + } + }); + let config = Config { + temper_url: format!("http://{addr}"), + tenant: "prod".to_string(), + worker_id: "mac-mini-codex-prod-slot-02".to_string(), + worker_token: Some("secret".to_string()), + workspace_root: PathBuf::from("/Users/me/Development/temperpaw-worktrees"), + repo_root: PathBuf::from("/Users/me/Development/temperpaw"), + codex_bin: "codex".to_string(), + max_concurrent_runs: 1, + enable_execution: true, + poll_on_start: true, + codex_exec_smoke: false, + codex_exec_timeout: Duration::from_secs(90), + }; + let client = reqwest::Client::new(); + + report_worker_heartbeat(&client, &config) + .await + .expect("legacy installed Patrol heartbeat namespace should still work"); + + server.abort(); + let requests = requests.lock().await; + assert_eq!(requests.len(), 2, "heartbeat should retry once via Patrol"); + assert!(requests[0].starts_with( + "POST /tdata/WorkerAgents('mac-mini-codex-prod-slot-02')/Temper.PawOrchestration.ReportHeartbeat" + )); + assert!(requests[1].starts_with( + "POST /tdata/WorkerAgents('mac-mini-codex-prod-slot-02')/TemperPaw.Patrol.ReportHeartbeat" + )); + } + + #[tokio::test] + async fn codex_exec_keeps_datadog_mcp_disabled_when_env_unset() { + let _guard = ENV_LOCK.lock().await; + let _datadog_mcp = EnvOverride::remove("PAW_CODEX_ENABLE_DATADOG_MCP"); + let _datadog_url = EnvOverride::remove("PAW_CODEX_DATADOG_MCP_URL"); + let _model = EnvOverride::remove("PAW_CODEX_MODEL"); + let args = codex_exec_args(Path::new("/tmp/paw-worktree"), "Create the proof file"); + let args = args + .iter() + .map(|arg| arg.to_string_lossy().to_string()) + .collect::>(); + + assert_eq!(args[0], "exec"); + assert_eq!(args[1], "--ignore-user-config"); + assert!(!args.contains(&"-c".to_string())); + assert!(!args.contains(&"--model".to_string())); + assert!( + !args + .iter() + .any(|arg| arg.contains("mcp_servers.datadog.url")) + ); + } + + #[tokio::test] + async fn codex_exec_can_enable_datadog_mcp_without_inheriting_user_config() { + let _guard = ENV_LOCK.lock().await; + let _datadog_mcp = + EnvOverride::set("PAW_CODEX_ENABLE_DATADOG_MCP", OsString::from("1")); + let _model = EnvOverride::remove("PAW_CODEX_MODEL"); + let _datadog_url = EnvOverride::set( + "PAW_CODEX_DATADOG_MCP_URL", + OsString::from("https://mcp.datadoghq.test/mcp?toolsets=logs"), + ); + let args = codex_exec_args( + Path::new("/tmp/paw-worktree"), + "Inspect Datadog evidence for the evolution direction", + ); + let args = args + .iter() + .map(|arg| arg.to_string_lossy().to_string()) + .collect::>(); + + assert_eq!( + &args[0..4], + &[ + "exec".to_string(), + "--ignore-user-config".to_string(), + "-c".to_string(), + "mcp_servers.datadog.url=\"https://mcp.datadoghq.test/mcp?toolsets=logs\"" + .to_string(), + ] + ); + assert!(args.contains(&"--ephemeral".to_string())); + assert!(args.contains(&"--dangerously-bypass-approvals-and-sandbox".to_string())); + assert!(args.contains(&"--skip-git-repo-check".to_string())); + assert_eq!( + args.last().map(String::as_str), + Some("Inspect Datadog evidence for the evolution direction") + ); + } + + #[tokio::test] + async fn codex_exec_can_force_datadog_mcp_for_telemetry_roles() { + let _guard = ENV_LOCK.lock().await; + let _datadog_mcp = + EnvOverride::set("PAW_CODEX_ENABLE_DATADOG_MCP", OsString::from("0")); + let _model = EnvOverride::remove("PAW_CODEX_MODEL"); + let _datadog_url = EnvOverride::set( + "PAW_CODEX_DATADOG_MCP_URL", + OsString::from("https://mcp.datadoghq.test/mcp?toolsets=logs"), + ); + let args = codex_exec_args_with_datadog_mcp( + Path::new("/tmp/paw-worktree"), + "Evaluate Datadog telemetry", + ); + let args = args + .iter() + .map(|arg| arg.to_string_lossy().to_string()) + .collect::>(); + + assert_eq!( + &args[0..4], + &[ + "exec".to_string(), + "--ignore-user-config".to_string(), + "-c".to_string(), + "mcp_servers.datadog.url=\"https://mcp.datadoghq.test/mcp?toolsets=logs\"" + .to_string(), + ] + ); + assert_eq!(args.last().map(String::as_str), Some("Evaluate Datadog telemetry")); + } + + #[tokio::test] + async fn codex_exec_uses_default_datadog_mcp_url_when_enabled() { + let _guard = ENV_LOCK.lock().await; + let _datadog_mcp = + EnvOverride::set("PAW_CODEX_ENABLE_DATADOG_MCP", OsString::from("1")); + let _datadog_url = EnvOverride::remove("PAW_CODEX_DATADOG_MCP_URL"); + let _model = EnvOverride::remove("PAW_CODEX_MODEL"); + let args = codex_exec_args( + Path::new("/tmp/paw-worktree"), + "Inspect Datadog evidence for the evolution direction", + ); + let args = args + .iter() + .map(|arg| arg.to_string_lossy().to_string()) + .collect::>(); + + assert!(args.contains( + &"mcp_servers.datadog.url=\"https://mcp.datadoghq.com/api/unstable/mcp-server/mcp?toolsets=all\"" + .to_string() + )); + } + + #[tokio::test] + async fn codex_exec_can_pin_worker_model_from_env() { + let _guard = ENV_LOCK.lock().await; + let _datadog_mcp = + EnvOverride::set("PAW_CODEX_ENABLE_DATADOG_MCP", OsString::from("0")); + let _model = EnvOverride::set("PAW_CODEX_MODEL", OsString::from("gpt-5-codex")); + let args = codex_exec_args(Path::new("/tmp/paw-worktree"), "Create the proof file"); + let args = args + .iter() + .map(|arg| arg.to_string_lossy().to_string()) + .collect::>(); + + assert_eq!( + &args[0..4], + &[ + "exec".to_string(), + "--ignore-user-config".to_string(), + "--model".to_string(), + "gpt-5-codex".to_string(), + ] + ); + } + + #[test] + fn toml_basic_string_escapes_codex_config_values() { + assert_eq!( + toml_basic_string("https://mcp.example.test/a\"b\\c\n"), + "\"https://mcp.example.test/a\\\"b\\\\c\\n\"" + ); + } + + #[test] + fn worker_proof_text_does_not_call_assigned_worktree_current_checkout() { + let worker_run = WorkerRunState { + id: "wr-existing-worktree".to_string(), + status: "Running".to_string(), + task: "Inspect an existing assigned worktree".to_string(), + work_cycle_id: "wc-existing-worktree".to_string(), + worktree_path: "/tmp/paw-existing-worktree".to_string(), + branch_name: String::new(), + runner_kind: "local_codex".to_string(), + allowed_worker_id: "mac-mini-codex-1".to_string(), + worker_id: "mac-mini-codex-1".to_string(), + provider_id: "local-codex".to_string(), + required_capabilities: "local_codex,repo_write".to_string(), + }; + let evidence = WorktreeEvidence { + status_short: String::new(), + diff_stat: String::new(), + }; + + let summary = format_codex_success_summary( + &worker_run, + Path::new("/tmp/paw-existing-worktree"), + "inspected", + &evidence, + None, + ); + let review = ReviewRunState { + status: "Requested".to_string(), + worker_run_id: worker_run.id.clone(), + proof_packet_id: "proof-1".to_string(), + }; + let prompt = codex_review_prompt(&worker_run, &review); + + assert!(summary.contains("Branch: (assigned worktree without branch)")); + assert!(prompt.contains("Branch: (assigned worktree without branch)")); + assert!(!summary.contains("Branch: (current checkout)")); + assert!(!prompt.contains("Branch: (current checkout)")); + } + + #[test] + fn review_prompt_allows_pr_ready_changes_with_deployment_pending() { + let worker_run = WorkerRunState { + id: "wr-dashboard".to_string(), + status: "Done".to_string(), + task: "Update Datadog dashboard JSON but do not deploy production.".to_string(), + work_cycle_id: "wc-dashboard".to_string(), + worktree_path: "/tmp/paw-dashboard".to_string(), + branch_name: "codex/dashboard".to_string(), + runner_kind: "local_codex".to_string(), + allowed_worker_id: "mac-mini-codex-1".to_string(), + worker_id: "mac-mini-codex-1".to_string(), + provider_id: "local-codex".to_string(), + required_capabilities: "local_codex,repo_write,review".to_string(), + }; + let review = ReviewRunState { + status: "Requested".to_string(), + worker_run_id: worker_run.id.clone(), + proof_packet_id: "proof-dashboard".to_string(), + }; + + let prompt = codex_review_prompt(&worker_run, &review); + + assert!(prompt.contains("deployment-pending residual risk")); + assert!(prompt.contains("Do not require production deployment")); + } + + #[test] + fn repo_health_review_prompt_reviews_scan_contract_not_patch_contract() { + let worker_run = WorkerRunState { + id: "wr-repo-scan".to_string(), + status: "Done".to_string(), + task: "RepoGraphSnapshot: snap-1\nWorkCycle: wc-1\nRun an agent-led repo health patrol.".to_string(), + work_cycle_id: "wc-1".to_string(), + worktree_path: "/tmp/paw-repo-scan".to_string(), + branch_name: "codex/paw-repo-sweep-snap-1".to_string(), + runner_kind: "local_codex".to_string(), + allowed_worker_id: "mac-mini-codex-1".to_string(), + worker_id: "mac-mini-codex-1".to_string(), + provider_id: "local-codex".to_string(), + required_capabilities: "local_codex,repo_write,evaluation".to_string(), + }; + let review = ReviewRunState { + status: "Requested".to_string(), + worker_run_id: worker_run.id.clone(), + proof_packet_id: "proof-1".to_string(), + }; + + let prompt = codex_review_prompt(&worker_run, &review); + + assert!(prompt.contains("repo-health Patrol scan reviewer")); + assert!(prompt.contains("RepoGraphSnapshot.ScanComplete")); + assert!(prompt.contains("do not require an implementation patch")); + assert!(!prompt.contains("Inspect the git diff, changed files, tests/proofs")); + } + + #[test] + fn review_evaluation_and_work_cycle_state_read_temper_odata_fields() { + let review = review_run_from_odata_value(json!({ + "entity_id": "rev-1", + "status": "Requested", + "fields": { + "work_cycle_id": "wc-1", + "worker_run_id": "wr-1", + "proof_packet_id": "proof-1" + } + })) + .expect("review run"); + let evaluation = evaluation_run_from_odata_value(json!({ + "entity_id": "eval-1", + "status": "Queued", + "fields": { + "work_cycle_id": "wc-1", + "required_checks": "[\"repo-sweep\"]" + } + })) + .expect("evaluation run"); + let work_cycle = work_cycle_from_odata_value(json!({ + "entity_id": "wc-1", + "status": "Reviewing", + "fields": { + "implementer_worker_run_id": "wr-1", + "reviewer_run_id": "rev-1", + "evaluation_run_id": "eval-1", + "review_passed": "true" + } + })) + .expect("work cycle"); + + assert_eq!(review.worker_run_id, "wr-1"); + assert_eq!(evaluation.work_cycle_id, "wc-1"); + assert_eq!(work_cycle.status, "Reviewing"); + assert_eq!(work_cycle.implementer_worker_run_id, "wr-1"); + assert!(work_cycle.review_passed); + } + + fn unique_temp_dir() -> PathBuf { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("time") + .as_nanos(); + env::temp_dir().join(format!( + "paw-codex-worker-test-{}-{nanos}", + std::process::id() + )) + } + + struct EnvOverride { + key: &'static str, + previous: Option, + } + + impl EnvOverride { + fn set(key: &'static str, value: OsString) -> Self { + let previous = env::var_os(key); + unsafe { + env::set_var(key, value); + } + Self { key, previous } + } + + fn remove(key: &'static str) -> Self { + let previous = env::var_os(key); + unsafe { + env::remove_var(key); + } + Self { key, previous } + } + } + + impl Drop for EnvOverride { + fn drop(&mut self) { + unsafe { + if let Some(value) = &self.previous { + env::set_var(self.key, value); + } else { + env::remove_var(self.key); + } + } + } + } + + fn run_raw_git_for_test(args: &[&str]) { + let output = std::process::Command::new("git") + .args(args) + .output() + .expect("run git"); + assert!( + output.status.success(), + "git {} failed: {}{}{}", + args.join(" "), + String::from_utf8_lossy(&output.stdout), + if output.stdout.is_empty() || output.stderr.is_empty() { + "" + } else { + "\n" + }, + String::from_utf8_lossy(&output.stderr) + ); + } + + fn run_git_for_test(workdir: &Path, args: &[&str]) { + let output = std::process::Command::new("git") + .arg("-C") + .arg(workdir) + .args(args) + .output() + .expect("run git"); + assert!( + output.status.success(), + "git -C {} {} failed: {}{}{}", + workdir.display(), + args.join(" "), + String::from_utf8_lossy(&output.stdout), + if output.stdout.is_empty() || output.stderr.is_empty() { + "" + } else { + "\n" + }, + String::from_utf8_lossy(&output.stderr) + ); + } + + fn run_git_capture_for_test(workdir: &Path, args: &[&str]) -> String { + let output = std::process::Command::new("git") + .arg("-C") + .arg(workdir) + .args(args) + .output() + .expect("run git"); + assert!( + output.status.success(), + "git -C {} {} failed: {}{}{}", + workdir.display(), + args.join(" "), + String::from_utf8_lossy(&output.stdout), + if output.stdout.is_empty() || output.stderr.is_empty() { + "" + } else { + "\n" + }, + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8_lossy(&output.stdout) + .trim_end() + .to_string() + } + + #[cfg(unix)] + fn make_executable(path: &Path) { + let mut permissions = fs::metadata(path).expect("metadata").permissions(); + permissions.set_mode(0o755); + fs::set_permissions(path, permissions).expect("chmod"); + } + + #[cfg(not(unix))] + fn make_executable(_path: &Path) {} +} From 73d64512866a0a2606f9625add0b40a4e114c436 Mon Sep 17 00:00:00 2001 From: rita-aga Date: Tue, 2 Jun 2026 20:59:57 -0400 Subject: [PATCH 05/16] Add directed evolution local proof artifact --- ...rected-evolution-local-feedback-closure.md | 131 ++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 .proofs/081-directed-evolution-local-feedback-closure.md diff --git a/.proofs/081-directed-evolution-local-feedback-closure.md b/.proofs/081-directed-evolution-local-feedback-closure.md new file mode 100644 index 000000000..08035272d --- /dev/null +++ b/.proofs/081-directed-evolution-local-feedback-closure.md @@ -0,0 +1,131 @@ +# Directed Evolution Local Feedback Closure Proof + +Date: 2026-06-03 + +This proof records the strongest proof cycle available without production +Temper credentials. It is not a substitute for the required live Agent Answers +Datadog proof, but it does exercise the current Genesis-authored Directed +Evolution app, shared `paw-orchestration` worker entities, and the local +TemperPaw worker loop against a fresh local Temper runtime. + +## Runtime + +- Genesis worktree: + `/Users/seshendranalla/Development/genesis-worktrees/directed-evolution-live-backend` +- TemperPaw worktree: + `/Users/seshendranalla/Development/temperpaw-worktrees/directed-evolution-hotload-variants` +- Local proof bundle: + `/tmp/directed-evolution-local-proof-foreground.udcnRR` +- Local Temper URL: `http://127.0.0.1:3201` +- App catalog: temp symlink catalog containing: + - `directed-evolution` + - `paw-orchestration` + - `agent-answers` + - `agent-answers-evaluation` + +Startup installed the relevant app surfaces: + +```text +App 'agent-answers' installed for 'default': Answer, Question +App 'agent-answers-evaluation' installed for 'default': TrialMetricDefinition, TrialSuite, ValidatorRun +App 'directed-evolution' installed for 'default': AdaptationGoal, AutonomyPolicy, Direction, EliminationRule, Episode, EpisodeStartRequest, EvaluationStage, EvidenceArtifact, Generation, LineageEdge, Measurement, MetricDefinition, Mutation, Organism, OrganismVersion, Pressure, Promotion, ScoringRule, SelectionPressure, SelectionProtocol, Signal, SimulatedUserPlan, StageResult, Trial, Variant, ViabilityConstraint, WorkItemReceipt +App 'paw-orchestration' installed for 'default': BudgetLedger, HeartbeatRun, Organization, WorkItem, WorkerAgent, WorkerProvider, WorkerRun +``` + +Metadata checks confirmed these live entity/action surfaces: + +```text +EntitySet Name="Organisms" True +EntitySet Name="EpisodeStartRequests" True +EntitySet Name="WorkItems" True +Action Name="SubmitEpisodeStartRequest" True +``` + +## Episode Start + +The proof seeded a baseline Agent Answers organism and direction through real +OData create/action calls, then submitted `EpisodeStartRequest`: + +```text +Organisms org-agent-answers create -> 201 +OrganismVersions ov-agent-answers-local-proof create -> 201 +OrganismVersions ov-agent-answers-local-proof MarkOrganismVersionParent -> 200 +Organisms org-agent-answers ActivateOrganism -> 200 +Directions dir-agent-answers-local-proof create -> 201 +Directions dir-agent-answers-local-proof ProposeDirection -> 200 +EpisodeStartRequests episode-start-local-proof create -> 201 +EpisodeStartRequests episode-start-local-proof SubmitEpisodeStartRequest -> 200 +``` + +The inline `episode_start_requestor` path materialized: + +```text +EpisodeStartRequests: count=1 statuses={'Started': 1} +Directions: count=1 statuses={'Selected': 1} +Episodes: count=1 statuses={'Running': 1} +Generations: count=1 statuses={'Generating': 1} +WorkItems: count=3 statuses={'Queued': 3} +``` + +The started episode was +`en-019e8af2-b5b2-7200-b291-1916d528457c`, with generation +`en-019e8af2-b97d-7b22-8232-40d7c50a5dec`. + +## Worker Loop + +The local worker was started with execution disabled: + +```sh +TEMPER_URL=http://127.0.0.1:3201 \ +TEMPER_TENANT=default \ +WORKER_ID=local-proof-worker \ +PAW_CODEX_ENABLE_EXECUTION=0 \ +PAW_CODEX_POLL_ON_START=1 \ +./target/debug/paw-codex-worker run +``` + +It registered, connected to the local event stream, claimed Directed Evolution +work items, created shared `WorkerRun` rows, and routed receipts back into +Directed Evolution. Final captured state: + +```text +WorkItems: count=13 statuses={'Succeeded': 9, 'Cancelled': 4} +WorkerRuns: count=9 statuses={'Succeeded': 9} +EvidenceArtifacts: count=9 statuses={'Linked': 9} +Variants: count=3 statuses={'Eliminated': 3} +Trials: count=3 statuses={'Succeeded': 3} +StageResults: count=9 statuses={'Eliminated': 3, 'Pending': 3, 'Running': 3} +Episodes: count=1 statuses={'Failed': 1} +Generations: count=1 statuses={'Failed': 1} +``` + +The episode failed closed because dry-run worker output is not a viable real +variant/evaluation proof: + +```text +All variants were eliminated before selection. +``` + +That failure is expected for this local proof. The proof still demonstrates the +fresh feedback-closure mechanics through episode materialization, shared work +item creation, worker claiming, worker-run provenance, evidence artifact +linking, trial routing, and fail-closed elimination. + +## Remaining Live Proof Blocker + +The required production Agent Answers proof remains blocked. Production +preflight bundle +`/tmp/paw-patrol-production-preflight-20260603T001306Z-903` reported missing: + +- `TEMPER_URL` +- `WORKER_TOKEN` +- `PATROL_OPERATOR_TOKEN` +- signed webhook secrets +- launchd plist/load state +- production local Codex worker-id confirmation +- checkout ancestry against current main + +Datadog MCP access was available, and recent TemperPaw telemetry could be read, +but no fresh Directed Evolution / Agent Answers Datadog proof trace was found. +The live proof can only be closed after those production credentials and worker +activation gates are supplied. From 61ed5c722442244c3c1f60e83ffd387a73644305 Mon Sep 17 00:00:00 2001 From: rita-aga Date: Tue, 2 Jun 2026 21:38:01 -0400 Subject: [PATCH 06/16] Add Agent Answers live proof driver --- .github/workflows/ci.yml | 1 + crates/paw-codex-worker/README.md | 30 + .../paw-codex-worker/fixtures/fake-codex.sh | 8 + ...cted-evolution-agent-answers-live-proof.sh | 609 ++++++++++++++++++ .../scripts/paw-patrol-acceptance.sh | 1 + .../paw-codex-worker/src/fake_codex_tests.rs | 14 + .../temperpaw/tests/paw_patrol_foundation.rs | 81 +++ 7 files changed, 744 insertions(+) create mode 100755 crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f2f0dc1e1..b99e3b5ce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,6 +36,7 @@ jobs: bash -n crates/paw-codex-worker/scripts/webhook-intake-smoke.sh bash -n crates/paw-codex-worker/scripts/production-readiness.sh bash -n crates/paw-codex-worker/scripts/production-preflight.sh + bash -n crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh bash -n crates/paw-codex-worker/scripts/production-git-ancestry-guard.sh bash -n crates/paw-codex-worker/scripts/production-observe-only.sh bash -n crates/paw-codex-worker/scripts/production-preflight-github-smoke.sh diff --git a/crates/paw-codex-worker/README.md b/crates/paw-codex-worker/README.md index 2f7a7ce01..814b8706d 100644 --- a/crates/paw-codex-worker/README.md +++ b/crates/paw-codex-worker/README.md @@ -78,6 +78,10 @@ Script inventory: SecurityFinding, ProofPacket, and DailyBrief loop. - `production-preflight.sh`: read-only production blocker inventory for tokens, Railway, GitHub PRs, launchd, and webhook secrets. +- `directed-evolution-agent-answers-live-proof.sh`: guarded live Agent Answers + Directed Evolution proof driver that starts the semantic episode, waits for + worker/evidence closure, and requires Datadog-measured evidence plus a + successful terminal episode status. - `production-readiness.sh`: guarded doctor/build/plist path before loading the Mac mini daemon. - `production-observe-only.sh`: guarded production proof with execution off. @@ -352,6 +356,32 @@ crates/paw-codex-worker/scripts/production-preflight-diff.sh \ /tmp/current-preflight/summary.json ``` +After preflight and readiness gates are clear, run the guarded Agent Answers +Directed Evolution proof driver. It refuses to mutate production unless +`ALLOW_PRODUCTION_WRITE=1` and `CONFIRM_AGENT_ANSWERS_LIVE_PROOF=1` are set, +then calls `paw-codex-worker directed-evolution-start-episode`, waits for +`WorkItems`, `WorkerRuns`, `EvidenceArtifacts`, terminal episode state, and at +least one `datadog-measured` artifact with a Datadog URL. The script treats +attempts to disable Datadog evidence or terminal-success requirements as +blockers, not proof configuration: + +```sh +ALLOW_PRODUCTION_WRITE=1 \ +CONFIRM_AGENT_ANSWERS_LIVE_PROOF=1 \ +TEMPER_URL=https://your-railway-temperpaw.example \ +WORKER_TOKEN="$TEMPER_WORKER_TOKEN" \ +DIRECTED_EVOLUTION_ORGANISM_ID="$AGENT_ANSWERS_ORGANISM_ID" \ +DIRECTED_EVOLUTION_DIRECTION_ID="$AGENT_ANSWERS_DIRECTION_ID" \ +START_LOCAL_WORKER=0 \ +crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh +``` + +Set `START_LOCAL_WORKER=1` on the production Mac mini when launchd has not +already started the worker pool. The proof bundle is written under +`/tmp/directed-evolution-agent-answers-live-proof-*` and includes +`summary.json`, `proof.md`, `human-blockers.json` when blocked, the episode +contract, and terminal episode state. + Use `STRICT=1` when blocked gates should make the command fail, for example in a final cutover checklist: diff --git a/crates/paw-codex-worker/fixtures/fake-codex.sh b/crates/paw-codex-worker/fixtures/fake-codex.sh index 961a57160..6f63ed055 100755 --- a/crates/paw-codex-worker/fixtures/fake-codex.sh +++ b/crates/paw-codex-worker/fixtures/fake-codex.sh @@ -33,6 +33,14 @@ while [ "$#" -gt 0 ]; do fi shift ;; + -c) + shift + if [ "$#" -eq 0 ]; then + echo "-c requires a value" >&2 + exit 2 + fi + shift + ;; --cd) shift if [ "$#" -eq 0 ]; then diff --git a/crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh b/crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh new file mode 100755 index 000000000..7a07511f3 --- /dev/null +++ b/crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh @@ -0,0 +1,609 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(git rev-parse --show-toplevel)" +STAMP="$(date -u +%Y%m%dT%H%M%SZ)" +PROOF_DIR="${PROOF_DIR:-/tmp/directed-evolution-agent-answers-live-proof-${STAMP}-$$}" +SUMMARY_JSON="${PROOF_DIR}/summary.json" +PROOF_MD="${PROOF_DIR}/proof.md" +HUMAN_BLOCKERS_JSON="${PROOF_DIR}/human-blockers.json" +BLOCKERS_TSV="${PROOF_DIR}/human-blockers.tsv" +CONTRACT_JSON="${PROOF_DIR}/agent-answers-episode-contract.json" +START_JSON="${PROOF_DIR}/episode-start.json" +WORKER_LOG="${PROOF_DIR}/local-worker.log" + +TEMPER_TENANT="${TEMPER_TENANT:-default}" +WORKER_ID="${WORKER_ID:-mac-mini-codex-prod}" +WORKER_BIN="${WORKER_BIN:-${ROOT}/target/release/paw-codex-worker}" +REPO_ROOT="${REPO_ROOT:-$ROOT}" +WORKSPACE_ROOT="${WORKSPACE_ROOT:-$(dirname "$ROOT")}" +CODEX_BIN="${CODEX_BIN:-codex}" +BUILD_WORKER="${BUILD_WORKER:-1}" +START_LOCAL_WORKER="${START_LOCAL_WORKER:-0}" +PAW_CODEX_ENABLE_EXECUTION="${PAW_CODEX_ENABLE_EXECUTION:-1}" +PAW_CODEX_POLL_ON_START="${PAW_CODEX_POLL_ON_START:-1}" +MAX_CONCURRENT_RUNS="${MAX_CONCURRENT_RUNS:-3}" +READY_ATTEMPTS="${READY_ATTEMPTS:-900}" +POLL_SECONDS="${POLL_SECONDS:-2}" +REQUIRE_DATADOG_EVIDENCE="${REQUIRE_DATADOG_EVIDENCE:-1}" +REQUIRE_EPISODE_SUCCESS="${REQUIRE_EPISODE_SUCCESS:-1}" +DIRECTED_EVOLUTION_PROOF_ACTOR="${DIRECTED_EVOLUTION_PROOF_ACTOR:-codex-agent-answers-live-proof}" + +WORKER_PID="" + +log() { + printf '[directed-evolution-agent-answers-proof] %s\n' "$*" +} + +fail() { + log "$*" + exit 1 +} + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || fail "missing required command: $1" +} + +add_blocker() { + local gate="$1" + local detail="$2" + local evidence="$3" + printf '%s\t%s\t%s\n' "$gate" "$detail" "$evidence" >>"$BLOCKERS_TSV" +} + +cleanup() { + if [[ -n "$WORKER_PID" ]] && kill -0 "$WORKER_PID" >/dev/null 2>&1; then + kill "$WORKER_PID" >/dev/null 2>&1 || true + wait "$WORKER_PID" >/dev/null 2>&1 || true + fi +} +trap cleanup EXIT + +mkdir -p "$PROOF_DIR" +: >"$BLOCKERS_TSV" + +require_cmd curl +require_cmd git +require_cmd jq + +record_precondition_blockers() { + if [[ "${ALLOW_PRODUCTION_WRITE:-0}" != "1" ]]; then + add_blocker "confirm:allow_production_write" \ + "ALLOW_PRODUCTION_WRITE=1 is required because this script creates a live Directed Evolution Episode." \ + "operator confirmation required" + fi + + if [[ "${CONFIRM_AGENT_ANSWERS_LIVE_PROOF:-0}" != "1" ]]; then + add_blocker "confirm:agent_answers_live_proof" \ + "CONFIRM_AGENT_ANSWERS_LIVE_PROOF=1 is required to run a live Agent Answers Directed Evolution proof cycle." \ + "operator confirmation required" + fi + + if [[ -z "${TEMPER_URL:-}" ]]; then + add_blocker "env:temper_url" \ + "TEMPER_URL is required for the production TemperPaw control plane." \ + "human input required" + fi + + if [[ -z "${WORKER_TOKEN:-}" ]]; then + add_blocker "env:worker_token" \ + "WORKER_TOKEN is required for the episode starter and optional local worker." \ + "mint or provide a Temper worker credential" + fi + + if [[ -z "${DIRECTED_EVOLUTION_DIRECTION_ID:-}" ]]; then + add_blocker "env:directed_evolution_direction_id" \ + "DIRECTED_EVOLUTION_DIRECTION_ID is required and must point at the fresh Agent Answers direction under review." \ + "create or select the production Direction first" + fi + + if [[ -z "${DIRECTED_EVOLUTION_ORGANISM_ID:-}" ]]; then + add_blocker "env:directed_evolution_organism_id" \ + "DIRECTED_EVOLUTION_ORGANISM_ID is required and must point at the Agent Answers organism." \ + "create or select the production Organism first" + fi + + if [[ "$START_LOCAL_WORKER" == "1" && "$PAW_CODEX_ENABLE_EXECUTION" != "1" ]]; then + add_blocker "env:paw_codex_enable_execution" \ + "START_LOCAL_WORKER=1 requires PAW_CODEX_ENABLE_EXECUTION=1 so Codex can execute variant/reviewer/evaluator work." \ + "set PAW_CODEX_ENABLE_EXECUTION=1" + fi + + if [[ "$REQUIRE_DATADOG_EVIDENCE" != "1" ]]; then + add_blocker "env:require_datadog_evidence" \ + "REQUIRE_DATADOG_EVIDENCE must remain 1; Datadog-measured evidence is mandatory for this proof." \ + "do not disable the Datadog observer/evaluator gate" + fi + + if [[ "$REQUIRE_EPISODE_SUCCESS" != "1" ]]; then + add_blocker "env:require_episode_success" \ + "REQUIRE_EPISODE_SUCCESS must remain 1; failed/cancelled/abandoned episodes cannot be reported as a passed proof." \ + "do not disable the terminal-success gate" + fi +} + +write_blocked_summary_and_exit() { + if [[ ! -s "$BLOCKERS_TSV" ]]; then + return 0 + fi + + jq -Rn ' + [inputs | split("\t") | { + gate: .[0], + detail: .[1], + evidence: .[2] + }] + ' <"$BLOCKERS_TSV" >"$HUMAN_BLOCKERS_JSON" + jq -n \ + --arg status "blocked" \ + --arg proof_dir "$PROOF_DIR" \ + --argjson blockers "$(cat "$HUMAN_BLOCKERS_JSON")" \ + '{ + status: $status, + proof_dir: $proof_dir, + proof_kind: "live Agent Answers Directed Evolution proof", + human_blockers: $blockers + }' >"$SUMMARY_JSON" + cat >"$PROOF_MD" </dev/null 2>&1; then + return 0 + fi + sleep "$POLL_SECONDS" + done + fail "control plane did not become readable at ${TEMPER_URL}" +} + +field() { + local key="$1" + jq -r --arg key "$key" ' + def norm: ascii_downcase | gsub("_"; ""); + . as $root + | ($root[$key] + // $root[($key | ascii_downcase)] + // ($root | to_entries[]? | select(.key | norm == ($key | norm)) | .value) + // .fields[$key] + // .fields[($key | ascii_downcase)] + // (.fields // {} | to_entries[]? | select(.key | norm == ($key | norm)) | .value) + // "") + ' +} + +list_set() { + local set="$1" + if [[ "${SCRIPT_SELF_TEST:-0}" == "1" ]]; then + cat "${SCRIPT_SELF_TEST_FIXTURE_DIR}/${set}.json" + return 0 + fi + curl_json "${TEMPER_URL}/tdata/${set}" +} + +count_for_episode() { + local set="$1" + local episode_id="$2" + list_set "$set" | jq -r --arg episode_id "$episode_id" ' + def rows: + if type == "array" then . + elif has("value") then .value + elif has("entities") then .entities + elif has("items") then .items + else [] + end; + def f($name): + .[$name] + // .[($name | ascii_downcase)] + // .fields[$name] + // .fields[($name | ascii_downcase)] + // ""; + def present($value): + $value != null and ($value | tostring | length > 0); + def corr: + (f("CorrelationJson") | fromjson? // {}); + [rows[]? | select( + f("EpisodeId") == $episode_id + or f("episode_id") == $episode_id + or corr.episode_id == $episode_id + or corr.datadog.join_fields.episode_id == $episode_id + or corr.output.episode_id == $episode_id + )] | length + ' +} + +datadog_evidence_count() { + local episode_id="$1" + list_set EvidenceArtifacts | jq -r --arg episode_id "$episode_id" ' + def rows: + if type == "array" then . + elif has("value") then .value + elif has("entities") then .entities + elif has("items") then .items + else [] + end; + def f($name): + .[$name] + // .[($name | ascii_downcase)] + // .fields[$name] + // .fields[($name | ascii_downcase)] + // ""; + def present($value): + $value != null and ($value | tostring | length > 0); + def corr: + (f("CorrelationJson") | fromjson? // {}); + def belongs_to_episode: + f("EpisodeId") == $episode_id + or f("episode_id") == $episode_id + or corr.episode_id == $episode_id + or corr.datadog.join_fields.episode_id == $episode_id + or corr.output.episode_id == $episode_id; + def datadog_url: + if (f("Uri") | tostring | test("^https://app\\.")) + then (f("Uri") | tostring) + else ((corr.output.evidence_scope // corr.output.evidenceScope // []) + | map(.datadog_url // .datadogUrl // "") + | map(select(test("^https://app\\."))) + | .[0]) + end + // ""; + [rows[]? | select( + belongs_to_episode + and + f("EvidenceProvenance") == "datadog-measured" + and present(f("Query")) + and present(f("TimeWindow")) + and present(f("ResultCount")) + and present(f("Interpretation")) + and present(f("ZeroResultMeaning")) + and (datadog_url | test("^https://app\\.")) + )] | length + ' +} + +wait_for_episode_terminal() { + local episode_id="$1" + local body status + for _ in $(seq 1 "$READY_ATTEMPTS"); do + body="$(curl_json "$(entity_url Episodes "$episode_id")")" + status="$(field Status <<<"$body")" + case "$status" in + Completed|Complete|Succeeded|Promoted|NoPromotion|Failed|Cancelled|Abandoned) + printf '%s' "$body" + return 0 + ;; + esac + sleep "$POLL_SECONDS" + done + log "Episodes('${episode_id}') did not reach a terminal status; latest state follows" + curl_json "$(entity_url Episodes "$episode_id")" | tee "${PROOF_DIR}/episode-timeout.json" | jq . + exit 1 +} + +wait_for_min_count() { + local label="$1" + local set="$2" + local episode_id="$3" + local minimum="$4" + local count + for _ in $(seq 1 "$READY_ATTEMPTS"); do + count="$(count_for_episode "$set" "$episode_id")" + if [[ "$count" -ge "$minimum" ]]; then + printf '%s\n' "$count" + return 0 + fi + sleep "$POLL_SECONDS" + done + fail "${label} did not reach count >= ${minimum} for episode ${episode_id}" +} + +wait_for_datadog_evidence() { + local episode_id="$1" + local count + for _ in $(seq 1 "$READY_ATTEMPTS"); do + count="$(datadog_evidence_count "$episode_id")" + if [[ "$count" -ge 1 ]]; then + printf '%s\n' "$count" + return 0 + fi + sleep "$POLL_SECONDS" + done + fail "no datadog-measured EvidenceArtifacts with datadog_url/query/window/result_count/interpretation/zero_result_meaning were observed" +} + +run_self_test() { + SCRIPT_SELF_TEST_FIXTURE_DIR="${PROOF_DIR}/self-test-fixtures" + export SCRIPT_SELF_TEST_FIXTURE_DIR + mkdir -p "$SCRIPT_SELF_TEST_FIXTURE_DIR" + + cat >"${SCRIPT_SELF_TEST_FIXTURE_DIR}/EvidenceArtifacts.json" <<'JSON' +{ + "value": [ + { + "fields": { + "EpisodeId": "episode-self-test", + "EvidenceProvenance": "datadog-measured", + "Uri": "https://app.datadoghq.com/logs?query=directed-evolution", + "Query": "service:temper-platform directed_evolution.episode_id:episode-self-test", + "TimeWindow": "now-15m to now", + "ResultCount": 0, + "Interpretation": "No runtime request logs were observed for the fixture.", + "ZeroResultMeaning": "failure" + } + }, + { + "fields": { + "EpisodeId": "other-episode", + "EvidenceProvenance": "datadog-measured", + "Uri": "https://app.datadoghq.com/logs?query=other", + "Query": "other", + "TimeWindow": "now-15m to now", + "ResultCount": 7, + "Interpretation": "Different episode.", + "ZeroResultMeaning": "neutral" + } + } + ] +} +JSON + + local count + count="$(datadog_evidence_count "episode-self-test")" + if [[ "$count" != "1" ]]; then + fail "self-test expected one structured Datadog evidence artifact, got ${count}" + fi + + local failed_status + failed_status="$(field Status <<'JSON' +{"fields":{"Status":"Failed"}} +JSON +)" + case "$failed_status" in + Completed|Complete|Succeeded|Promoted|NoPromotion) + fail "self-test misclassified Failed as a successful terminal episode" + ;; + Failed) + ;; + *) + fail "self-test could not read failed terminal status" + ;; + esac + + log "self-test passed" +} + +if [[ "${SCRIPT_SELF_TEST:-0}" == "1" ]]; then + run_self_test + exit 0 +fi + +record_precondition_blockers +write_blocked_summary_and_exit + +if [[ "$BUILD_WORKER" == "1" ]]; then + log "building paw-codex-worker release binary" + cargo build -p paw-codex-worker --release +fi + +if [[ ! -x "$WORKER_BIN" ]]; then + fail "worker binary is not executable: ${WORKER_BIN}" +fi + +wait_for_metadata + +jq -n \ + --arg direction_id "$DIRECTED_EVOLUTION_DIRECTION_ID" \ + --arg organism_id "$DIRECTED_EVOLUTION_ORGANISM_ID" \ + --arg parent_version_id "${DIRECTED_EVOLUTION_PARENT_VERSION_ID:-}" \ + --arg actor "$DIRECTED_EVOLUTION_PROOF_ACTOR" \ + '{ + direction_id: $direction_id, + organism_id: $organism_id, + parent_version_id: $parent_version_id, + adaptation_goal: "Improve Agent Answers while preserving question, answer, acceptance, citation, and reviewer workflows.", + human_notes: "Fresh live Agent Answers Directed Evolution proof. Requires worker provenance, simulated-user evidence, state verification, and Datadog telemetry evidence before proof closure.", + created_by_worker_run_id: ("manual:" + $actor), + evaluator_ref: "genesis://nerdsane/agent-answers-evaluation@production", + selected_by: $actor, + selection_notes: "Live proof selection is bounded to the Agent Answers organism and pinned Genesis evaluator.", + started_by: $actor, + start_reason: "Operator-confirmed live Agent Answers Directed Evolution proof cycle." + }' >"$CONTRACT_JSON" + +log "starting semantic Agent Answers Directed Evolution episode" +TEMPER_URL="$TEMPER_URL" \ +TEMPER_TENANT="$TEMPER_TENANT" \ +WORKER_ID="$WORKER_ID" \ +WORKER_TOKEN="$WORKER_TOKEN" \ +REPO_ROOT="$REPO_ROOT" \ +WORKSPACE_ROOT="$WORKSPACE_ROOT" \ +CODEX_BIN="$CODEX_BIN" \ +PAW_CODEX_ENABLE_EXECUTION="$PAW_CODEX_ENABLE_EXECUTION" \ +PAW_CODEX_POLL_ON_START="$PAW_CODEX_POLL_ON_START" \ +"$WORKER_BIN" directed-evolution-start-episode "$CONTRACT_JSON" >"$START_JSON" + +episode_id="$(jq -r '.episode_id // ""' "$START_JSON")" +if [[ -z "$episode_id" ]]; then + fail "directed-evolution-start-episode did not return episode_id" +fi + +if [[ "$START_LOCAL_WORKER" == "1" ]]; then + log "starting local worker for live proof" + TEMPER_URL="$TEMPER_URL" \ + TEMPER_TENANT="$TEMPER_TENANT" \ + WORKER_ID="$WORKER_ID" \ + WORKER_TOKEN="$WORKER_TOKEN" \ + REPO_ROOT="$REPO_ROOT" \ + WORKSPACE_ROOT="$WORKSPACE_ROOT" \ + CODEX_BIN="$CODEX_BIN" \ + PAW_CODEX_ENABLE_EXECUTION="$PAW_CODEX_ENABLE_EXECUTION" \ + PAW_CODEX_POLL_ON_START="$PAW_CODEX_POLL_ON_START" \ + MAX_CONCURRENT_RUNS="$MAX_CONCURRENT_RUNS" \ + "$WORKER_BIN" run >"$WORKER_LOG" 2>&1 & + WORKER_PID="$!" +else + log "START_LOCAL_WORKER=0; expecting an already-running production worker pool" +fi + +work_item_count="$(wait_for_min_count WorkItems WorkItems "$episode_id" 1)" +worker_run_count="$(wait_for_min_count WorkerRuns WorkerRuns "$episode_id" 1)" +evidence_count="$(wait_for_min_count EvidenceArtifacts EvidenceArtifacts "$episode_id" 1)" +datadog_count="$(wait_for_datadog_evidence "$episode_id")" + +episode_body="$(wait_for_episode_terminal "$episode_id")" +episode_status="$(field Status <<<"$episode_body")" +jq . <<<"$episode_body" >"${PROOF_DIR}/episode-terminal.json" + +case "$episode_status" in + Completed|Complete|Succeeded|Promoted|NoPromotion) + ;; + *) + jq -n \ + --arg status "failed" \ + --arg proof_dir "$PROOF_DIR" \ + --arg episode_id "$episode_id" \ + --arg episode_status "$episode_status" \ + --argjson work_item_count "$work_item_count" \ + --argjson worker_run_count "$worker_run_count" \ + --argjson evidence_count "$evidence_count" \ + --argjson datadog_evidence_count "$datadog_count" \ + '{ + status: $status, + proof_kind: "live Agent Answers Directed Evolution proof", + proof_dir: $proof_dir, + agent_answers: { + episode_id: $episode_id, + episode_status: $episode_status + }, + evidence: { + work_items: $work_item_count, + worker_runs: $worker_run_count, + evidence_artifacts: $evidence_count, + datadog_measured_evidence_artifacts: $datadog_evidence_count + }, + failure_reason: "terminal episode status was not successful" + }' >"$SUMMARY_JSON" + fail "episode ${episode_id} reached ${episode_status}; live proof requires a successful terminal status" + ;; +esac + +summary_json="$(jq -n \ + --arg status "passed" \ + --arg proof_dir "$PROOF_DIR" \ + --arg temper_url "$TEMPER_URL" \ + --arg tenant "$TEMPER_TENANT" \ + --arg episode_id "$episode_id" \ + --arg episode_status "$episode_status" \ + --arg direction_id "$DIRECTED_EVOLUTION_DIRECTION_ID" \ + --arg organism_id "$DIRECTED_EVOLUTION_ORGANISM_ID" \ + --arg worker_id "$WORKER_ID" \ + --argjson work_item_count "$work_item_count" \ + --argjson worker_run_count "$worker_run_count" \ + --argjson evidence_count "$evidence_count" \ + --argjson datadog_evidence_count "$datadog_count" \ + '{ + status: $status, + proof_kind: "live Agent Answers Directed Evolution proof", + proof_dir: $proof_dir, + control_plane: { + temper_url: $temper_url, + tenant: $tenant + }, + agent_answers: { + organism_id: $organism_id, + direction_id: $direction_id, + episode_id: $episode_id, + episode_status: $episode_status + }, + worker: { + worker_id: $worker_id, + start_local_worker: env.START_LOCAL_WORKER, + execution_enabled: env.PAW_CODEX_ENABLE_EXECUTION + }, + evidence: { + work_items: $work_item_count, + worker_runs: $worker_run_count, + evidence_artifacts: $evidence_count, + datadog_measured_evidence_artifacts: $datadog_evidence_count, + mandatory_datadog_evidence: env.REQUIRE_DATADOG_EVIDENCE + } + }')" + +printf '%s\n' "$summary_json" >"$SUMMARY_JSON" + +cat >"$PROOF_MD" < Date: Tue, 2 Jun 2026 22:17:36 -0400 Subject: [PATCH 07/16] Harden Agent Answers Datadog proof preflight --- crates/paw-codex-worker/README.md | 11 ++- ...cted-evolution-agent-answers-live-proof.sh | 20 +++++ .../temperpaw/tests/paw_patrol_foundation.rs | 81 +++++++++++++++++++ 3 files changed, 110 insertions(+), 2 deletions(-) diff --git a/crates/paw-codex-worker/README.md b/crates/paw-codex-worker/README.md index 814b8706d..389c4ac02 100644 --- a/crates/paw-codex-worker/README.md +++ b/crates/paw-codex-worker/README.md @@ -363,11 +363,15 @@ then calls `paw-codex-worker directed-evolution-start-episode`, waits for `WorkItems`, `WorkerRuns`, `EvidenceArtifacts`, terminal episode state, and at least one `datadog-measured` artifact with a Datadog URL. The script treats attempts to disable Datadog evidence or terminal-success requirements as -blockers, not proof configuration: +blockers, not proof configuration. When `START_LOCAL_WORKER=0`, also set +`CONFIRM_PRODUCTION_DATADOG_QUERY_SECRETS=1` after verifying the production +worker/vault has `dd_api_key` and `dd_app_key`; when `START_LOCAL_WORKER=1`, +export `DD_API_KEY` or `DATADOG_API_KEY` plus `DD_APP_KEY` for the local worker: ```sh ALLOW_PRODUCTION_WRITE=1 \ CONFIRM_AGENT_ANSWERS_LIVE_PROOF=1 \ +CONFIRM_PRODUCTION_DATADOG_QUERY_SECRETS=1 \ TEMPER_URL=https://your-railway-temperpaw.example \ WORKER_TOKEN="$TEMPER_WORKER_TOKEN" \ DIRECTED_EVOLUTION_ORGANISM_ID="$AGENT_ANSWERS_ORGANISM_ID" \ @@ -377,7 +381,10 @@ crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh ``` Set `START_LOCAL_WORKER=1` on the production Mac mini when launchd has not -already started the worker pool. The proof bundle is written under +already started the worker pool. In that mode, the local worker process inherits +the Datadog query environment and the production-secret confirmation is not +accepted as a substitute for `DD_API_KEY`/`DD_APP_KEY`. The proof bundle is +written under `/tmp/directed-evolution-agent-answers-live-proof-*` and includes `summary.json`, `proof.md`, `human-blockers.json` when blocked, the episode contract, and terminal episode state. diff --git a/crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh b/crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh index 7a07511f3..eae02f74f 100755 --- a/crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh +++ b/crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh @@ -115,6 +115,26 @@ record_precondition_blockers() { "do not disable the Datadog observer/evaluator gate" fi + if [[ "$REQUIRE_DATADOG_EVIDENCE" == "1" ]]; then + if [[ "$START_LOCAL_WORKER" == "1" ]]; then + if [[ -z "${DD_API_KEY:-${DATADOG_API_KEY:-}}" ]]; then + add_blocker "env:dd_api_key" \ + "START_LOCAL_WORKER=1 requires DD_API_KEY or DATADOG_API_KEY so Datadog observer/evaluator WorkItems can query live telemetry." \ + "export the Datadog API key before starting the local proof worker" + fi + + if [[ -z "${DD_APP_KEY:-}" ]]; then + add_blocker "env:dd_app_key" \ + "START_LOCAL_WORKER=1 requires DD_APP_KEY so Datadog observer/evaluator WorkItems can query live telemetry." \ + "export the Datadog application key before starting the local proof worker" + fi + elif [[ "${CONFIRM_PRODUCTION_DATADOG_QUERY_SECRETS:-0}" != "1" ]]; then + add_blocker "confirm:production_datadog_query_secrets" \ + "START_LOCAL_WORKER=0 expects an already-running production worker pool; confirm it has dd_api_key and dd_app_key secrets before starting the live proof." \ + "set CONFIRM_PRODUCTION_DATADOG_QUERY_SECRETS=1 after verifying the production worker/vault Datadog query credentials" + fi + fi + if [[ "$REQUIRE_EPISODE_SUCCESS" != "1" ]]; then add_blocker "env:require_episode_success" \ "REQUIRE_EPISODE_SUCCESS must remain 1; failed/cancelled/abandoned episodes cannot be reported as a passed proof." \ diff --git a/crates/temperpaw/tests/paw_patrol_foundation.rs b/crates/temperpaw/tests/paw_patrol_foundation.rs index 2ed413f97..a9ad44f18 100644 --- a/crates/temperpaw/tests/paw_patrol_foundation.rs +++ b/crates/temperpaw/tests/paw_patrol_foundation.rs @@ -1376,6 +1376,9 @@ fn directed_evolution_agent_answers_live_proof_requires_datadog_evidence() { "datadog_url", "present(f(\"ResultCount\"))", "REQUIRE_DATADOG_EVIDENCE must remain 1", + "CONFIRM_PRODUCTION_DATADOG_QUERY_SECRETS", + "DD_API_KEY or DATADOG_API_KEY", + "DD_APP_KEY", "REQUIRE_EPISODE_SUCCESS must remain 1", "datadog_count=\"$(wait_for_datadog_evidence", "summary.json", @@ -1399,6 +1402,84 @@ fn directed_evolution_agent_answers_live_proof_requires_datadog_evidence() { readme.contains("directed-evolution-agent-answers-live-proof.sh"), "worker README should document the Agent Answers Directed Evolution live proof driver" ); + assert!( + readme.contains("CONFIRM_PRODUCTION_DATADOG_QUERY_SECRETS=1"), + "worker README should require production Datadog query-secret confirmation" + ); + assert!( + readme.contains("DD_API_KEY") && readme.contains("DD_APP_KEY"), + "worker README should document local Datadog credentials for local-worker proof mode" + ); +} + +#[test] +fn directed_evolution_agent_answers_live_proof_preflights_datadog_credentials() { + let root = repo_root(); + let proof_dir = env::temp_dir().join(format!( + "directed-evolution-agent-answers-live-proof-blocked-{}", + std::process::id() + )); + fs::remove_dir_all(&proof_dir).ok(); + + let output = Command::new( + root.join("crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh"), + ) + .current_dir(&root) + .env("PROOF_DIR", &proof_dir) + .env("ALLOW_PRODUCTION_WRITE", "1") + .env("CONFIRM_AGENT_ANSWERS_LIVE_PROOF", "1") + .env("TEMPER_URL", "https://temperpaw.example.invalid") + .env("WORKER_TOKEN", "worker-token") + .env("DIRECTED_EVOLUTION_DIRECTION_ID", "direction-agent-answers") + .env("DIRECTED_EVOLUTION_ORGANISM_ID", "organism-agent-answers") + .env("START_LOCAL_WORKER", "0") + .output() + .expect("run blocked Agent Answers Directed Evolution proof preflight"); + + assert!( + !output.status.success(), + "proof driver should block when production Datadog query secrets are not confirmed" + ); + let blockers = read(proof_dir.join("human-blockers.json")); + assert!( + blockers.contains("confirm:production_datadog_query_secrets"), + "blocked proof should name the missing production Datadog query-secret confirmation: {blockers}" + ); + + fs::remove_dir_all(&proof_dir).ok(); + + let local_proof_dir = env::temp_dir().join(format!( + "directed-evolution-agent-answers-live-proof-local-blocked-{}", + std::process::id() + )); + fs::remove_dir_all(&local_proof_dir).ok(); + + let local_output = Command::new( + root.join("crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh"), + ) + .current_dir(&root) + .env("PROOF_DIR", &local_proof_dir) + .env("ALLOW_PRODUCTION_WRITE", "1") + .env("CONFIRM_AGENT_ANSWERS_LIVE_PROOF", "1") + .env("TEMPER_URL", "https://temperpaw.example.invalid") + .env("WORKER_TOKEN", "worker-token") + .env("DIRECTED_EVOLUTION_DIRECTION_ID", "direction-agent-answers") + .env("DIRECTED_EVOLUTION_ORGANISM_ID", "organism-agent-answers") + .env("START_LOCAL_WORKER", "1") + .env("PAW_CODEX_ENABLE_EXECUTION", "1") + .output() + .expect("run blocked local-worker Agent Answers Directed Evolution proof preflight"); + + assert!( + !local_output.status.success(), + "local-worker proof driver should block without Datadog query credentials" + ); + let local_blockers = read(local_proof_dir.join("human-blockers.json")); + assert!( + local_blockers.contains("env:dd_api_key") && local_blockers.contains("env:dd_app_key"), + "blocked local proof should name missing Datadog query credentials: {local_blockers}" + ); + fs::remove_dir_all(&local_proof_dir).ok(); } #[test] From 3f6169c020362d65f2503ae086ee440d90354181 Mon Sep 17 00:00:00 2001 From: rita-aga Date: Tue, 2 Jun 2026 22:39:58 -0400 Subject: [PATCH 08/16] Add Agent Answers proof precheck mode --- crates/paw-codex-worker/README.md | 19 +++++ ...cted-evolution-agent-answers-live-proof.sh | 84 +++++++++++++++++++ .../temperpaw/tests/paw_patrol_foundation.rs | 61 ++++++++++++++ 3 files changed, 164 insertions(+) diff --git a/crates/paw-codex-worker/README.md b/crates/paw-codex-worker/README.md index 389c4ac02..22e75761c 100644 --- a/crates/paw-codex-worker/README.md +++ b/crates/paw-codex-worker/README.md @@ -380,6 +380,25 @@ START_LOCAL_WORKER=0 \ crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh ``` +Use `PRECHECK_ONLY=1` with the same confirmations and identifiers to generate a +ready proof bundle without mutating production. The precheck writes +`summary.json` and `proof.md`, exits before the worker build, metadata read, +episode start call, worker start, episode polling, and Datadog polling, and +marks the bundle with `no_mutation: true`: + +```sh +PRECHECK_ONLY=1 \ +ALLOW_PRODUCTION_WRITE=1 \ +CONFIRM_AGENT_ANSWERS_LIVE_PROOF=1 \ +CONFIRM_PRODUCTION_DATADOG_QUERY_SECRETS=1 \ +TEMPER_URL=https://your-railway-temperpaw.example \ +WORKER_TOKEN="$TEMPER_WORKER_TOKEN" \ +DIRECTED_EVOLUTION_ORGANISM_ID="$AGENT_ANSWERS_ORGANISM_ID" \ +DIRECTED_EVOLUTION_DIRECTION_ID="$AGENT_ANSWERS_DIRECTION_ID" \ +START_LOCAL_WORKER=0 \ +crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh +``` + Set `START_LOCAL_WORKER=1` on the production Mac mini when launchd has not already started the worker pool. In that mode, the local worker process inherits the Datadog query environment and the production-secret confirmation is not diff --git a/crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh b/crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh index eae02f74f..28d3c56c0 100755 --- a/crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh +++ b/crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh @@ -20,6 +20,7 @@ WORKSPACE_ROOT="${WORKSPACE_ROOT:-$(dirname "$ROOT")}" CODEX_BIN="${CODEX_BIN:-codex}" BUILD_WORKER="${BUILD_WORKER:-1}" START_LOCAL_WORKER="${START_LOCAL_WORKER:-0}" +PRECHECK_ONLY="${PRECHECK_ONLY:-0}" PAW_CODEX_ENABLE_EXECUTION="${PAW_CODEX_ENABLE_EXECUTION:-1}" PAW_CODEX_POLL_ON_START="${PAW_CODEX_POLL_ON_START:-1}" MAX_CONCURRENT_RUNS="${MAX_CONCURRENT_RUNS:-3}" @@ -187,6 +188,88 @@ EOF exit 1 } +write_precheck_ready_summary_and_exit() { + if [[ "$PRECHECK_ONLY" != "1" ]]; then + return 0 + fi + + local summary_json + summary_json="$(jq -n \ + --arg status "ready" \ + --arg proof_dir "$PROOF_DIR" \ + --arg temper_url "$TEMPER_URL" \ + --arg tenant "$TEMPER_TENANT" \ + --arg direction_id "$DIRECTED_EVOLUTION_DIRECTION_ID" \ + --arg organism_id "$DIRECTED_EVOLUTION_ORGANISM_ID" \ + --arg parent_version_id "${DIRECTED_EVOLUTION_PARENT_VERSION_ID:-}" \ + --arg worker_id "$WORKER_ID" \ + --arg start_local_worker "$START_LOCAL_WORKER" \ + --arg max_concurrent_runs "$MAX_CONCURRENT_RUNS" \ + --arg execution_enabled "$PAW_CODEX_ENABLE_EXECUTION" \ + --arg mandatory_datadog_evidence "$REQUIRE_DATADOG_EVIDENCE" \ + --arg mandatory_terminal_success "$REQUIRE_EPISODE_SUCCESS" \ + --arg production_datadog_query_secrets_confirmed "${CONFIRM_PRODUCTION_DATADOG_QUERY_SECRETS:-0}" \ + '{ + status: $status, + proof_kind: "live Agent Answers Directed Evolution proof precheck", + proof_dir: $proof_dir, + would_create_live_episode: true, + no_mutation: true, + control_plane: { + temper_url: $temper_url, + tenant: $tenant + }, + agent_answers: { + organism_id: $organism_id, + direction_id: $direction_id, + parent_version_id: $parent_version_id + }, + worker: { + worker_id: $worker_id, + start_local_worker: $start_local_worker, + max_concurrent_runs: ($max_concurrent_runs | tonumber), + execution_enabled: $execution_enabled + }, + evidence_requirements: { + mandatory_datadog_evidence: ($mandatory_datadog_evidence == "1"), + mandatory_terminal_success: ($mandatory_terminal_success == "1"), + production_datadog_query_secrets_confirmed: ($production_datadog_query_secrets_confirmed == "1") + }, + skipped_operations: [ + "cargo build", + "metadata read", + "directed-evolution-start-episode", + "local worker start", + "episode polling", + "Datadog evidence polling" + ] + }')" + + printf '%s\n' "$summary_json" >"$SUMMARY_JSON" + cat >"$PROOF_MD" < Date: Tue, 2 Jun 2026 22:59:22 -0400 Subject: [PATCH 09/16] Gate Agent Answers proof precheck in CI --- .github/workflows/ci.yml | 20 +++++++++++++++++++ .../temperpaw/tests/paw_patrol_foundation.rs | 6 ++++++ 2 files changed, 26 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b99e3b5ce..4fc82ed05 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,6 +48,26 @@ jobs: bash -n crates/paw-codex-worker/scripts/paw-patrol-acceptance.sh crates/paw-codex-worker/scripts/ci-actions-runtime-smoke.sh crates/paw-codex-worker/scripts/production-git-ancestry-guard-smoke.sh + - name: Agent Answers proof precheck + run: | + set -euo pipefail + PROOF_DIR="${RUNNER_TEMP}/agent-answers-live-proof-precheck" + rm -rf "$PROOF_DIR" + PRECHECK_ONLY=1 \ + PROOF_DIR="$PROOF_DIR" \ + ALLOW_PRODUCTION_WRITE=1 \ + CONFIRM_AGENT_ANSWERS_LIVE_PROOF=1 \ + CONFIRM_PRODUCTION_DATADOG_QUERY_SECRETS=1 \ + TEMPER_URL=https://temperpaw-ci.example.invalid \ + WORKER_TOKEN=ci-proof-token \ + DIRECTED_EVOLUTION_ORGANISM_ID=agent-answers-ci-organism \ + DIRECTED_EVOLUTION_DIRECTION_ID=agent-answers-ci-direction \ + START_LOCAL_WORKER=0 \ + crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh + jq -e '.status == "ready" and .no_mutation == true and .would_create_live_episode == true' "$PROOF_DIR/summary.json" + grep -F '"no_mutation": true' "$PROOF_DIR/summary.json" + test ! -e "$PROOF_DIR/episode-start.json" + test ! -e "$PROOF_DIR/agent-answers-episode-contract.json" - name: WASM build (os-apps) run: | set -e diff --git a/crates/temperpaw/tests/paw_patrol_foundation.rs b/crates/temperpaw/tests/paw_patrol_foundation.rs index 101b75c24..349078b0d 100644 --- a/crates/temperpaw/tests/paw_patrol_foundation.rs +++ b/crates/temperpaw/tests/paw_patrol_foundation.rs @@ -1402,6 +1402,12 @@ fn directed_evolution_agent_answers_live_proof_requires_datadog_evidence() { ), "CI should syntax-check the Agent Answers Directed Evolution live proof driver" ); + assert!( + ci.contains("Agent Answers proof precheck") + && ci.contains("PRECHECK_ONLY=1") + && ci.contains("\"no_mutation\": true"), + "CI should run the Agent Answers proof driver in no-mutation precheck mode" + ); assert!( readme.contains("directed-evolution-agent-answers-live-proof.sh"), "worker README should document the Agent Answers Directed Evolution live proof driver" From 913155c4ef33e6b6bd6552b5cf030a674f36e8f4 Mon Sep 17 00:00:00 2001 From: rita-aga Date: Tue, 2 Jun 2026 23:31:09 -0400 Subject: [PATCH 10/16] Require observer evaluator Datadog proof gates --- crates/paw-codex-worker/README.md | 17 ++-- ...cted-evolution-agent-answers-live-proof.sh | 93 +++++++++++++++++-- .../temperpaw/tests/paw_patrol_foundation.rs | 8 ++ 3 files changed, 104 insertions(+), 14 deletions(-) diff --git a/crates/paw-codex-worker/README.md b/crates/paw-codex-worker/README.md index 22e75761c..7ada73903 100644 --- a/crates/paw-codex-worker/README.md +++ b/crates/paw-codex-worker/README.md @@ -361,12 +361,17 @@ Directed Evolution proof driver. It refuses to mutate production unless `ALLOW_PRODUCTION_WRITE=1` and `CONFIRM_AGENT_ANSWERS_LIVE_PROOF=1` are set, then calls `paw-codex-worker directed-evolution-start-episode`, waits for `WorkItems`, `WorkerRuns`, `EvidenceArtifacts`, terminal episode state, and at -least one `datadog-measured` artifact with a Datadog URL. The script treats -attempts to disable Datadog evidence or terminal-success requirements as -blockers, not proof configuration. When `START_LOCAL_WORKER=0`, also set -`CONFIRM_PRODUCTION_DATADOG_QUERY_SECRETS=1` after verifying the production -worker/vault has `dd_api_key` and `dd_app_key`; when `START_LOCAL_WORKER=1`, -export `DD_API_KEY` or `DATADOG_API_KEY` plus `DD_APP_KEY` for the local worker: +least one `datadog-measured` artifact with a Datadog URL. It also requires +distinct `observer` and `telemetry_evaluator` `datadog-measured` +EvidenceArtifacts with structured query, time window, result count, +interpretation, and zero-result meaning fields, so the proof closes the +Datadog observer/evaluator loop instead of relying on aggregate telemetry. The +script treats attempts to disable Datadog evidence or terminal-success +requirements as blockers, not proof configuration. When `START_LOCAL_WORKER=0`, +also set `CONFIRM_PRODUCTION_DATADOG_QUERY_SECRETS=1` after verifying the +production worker/vault has `dd_api_key` and `dd_app_key`; when +`START_LOCAL_WORKER=1`, export `DD_API_KEY` or `DATADOG_API_KEY` plus +`DD_APP_KEY` for the local worker: ```sh ALLOW_PRODUCTION_WRITE=1 \ diff --git a/crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh b/crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh index 28d3c56c0..31f39a2e4 100755 --- a/crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh +++ b/crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh @@ -354,7 +354,8 @@ count_for_episode() { datadog_evidence_count() { local episode_id="$1" - list_set EvidenceArtifacts | jq -r --arg episode_id "$episode_id" ' + local role="${2:-}" + list_set EvidenceArtifacts | jq -r --arg episode_id "$episode_id" --arg role "$role" ' def rows: if type == "array" then . elif has("value") then .value @@ -372,6 +373,12 @@ datadog_evidence_count() { $value != null and ($value | tostring | length > 0); def corr: (f("CorrelationJson") | fromjson? // {}); + def evidence_scope: + (corr.output.evidence_scope // corr.output.evidenceScope // []) + | if type == "array" then . + elif type == "object" then [.] + else [] + end; def belongs_to_episode: f("EpisodeId") == $episode_id or f("episode_id") == $episode_id @@ -381,12 +388,22 @@ datadog_evidence_count() { def datadog_url: if (f("Uri") | tostring | test("^https://app\\.")) then (f("Uri") | tostring) - else ((corr.output.evidence_scope // corr.output.evidenceScope // []) + else (evidence_scope | map(.datadog_url // .datadogUrl // "") | map(select(test("^https://app\\."))) | .[0]) end // ""; + def role_matches: + $role == "" + or f("Role") == $role + or corr.role == $role + or corr.datadog.role == $role + or corr.datadog.join_fields.role == $role + or corr.output.role == $role + or corr.output.evaluator_role == $role + or corr.output.evaluatorRole == $role + or (evidence_scope | any((.role // .Role // .evaluator_role // .evaluatorRole // "") == $role)); [rows[]? | select( belongs_to_episode and @@ -397,10 +414,17 @@ datadog_evidence_count() { and present(f("Interpretation")) and present(f("ZeroResultMeaning")) and (datadog_url | test("^https://app\\.")) + and role_matches )] | length ' } +datadog_role_evidence_count() { + local episode_id="$1" + local role="$2" + datadog_evidence_count "$episode_id" "$role" +} + wait_for_episode_terminal() { local episode_id="$1" local body status @@ -451,6 +475,21 @@ wait_for_datadog_evidence() { fail "no datadog-measured EvidenceArtifacts with datadog_url/query/window/result_count/interpretation/zero_result_meaning were observed" } +wait_for_datadog_role_evidence() { + local episode_id="$1" + local role="$2" + local count + for _ in $(seq 1 "$READY_ATTEMPTS"); do + count="$(datadog_role_evidence_count "$episode_id" "$role")" + if [[ "$count" -ge 1 ]]; then + printf '%s\n' "$count" + return 0 + fi + sleep "$POLL_SECONDS" + done + fail "no datadog-measured EvidenceArtifacts for ${role} with datadog_url/query/window/result_count/interpretation/zero_result_meaning were observed" +} + run_self_test() { SCRIPT_SELF_TEST_FIXTURE_DIR="${PROOF_DIR}/self-test-fixtures" export SCRIPT_SELF_TEST_FIXTURE_DIR @@ -468,7 +507,21 @@ run_self_test() { "TimeWindow": "now-15m to now", "ResultCount": 0, "Interpretation": "No runtime request logs were observed for the fixture.", - "ZeroResultMeaning": "failure" + "ZeroResultMeaning": "failure", + "CorrelationJson": "{\"episode_id\":\"episode-self-test\",\"role\":\"observer\",\"datadog\":{\"role\":\"observer\",\"join_fields\":{\"episode_id\":\"episode-self-test\",\"role\":\"observer\"}},\"output\":{\"evidence_scope\":[{\"datadog_url\":\"https://app.datadoghq.com/logs?query=directed-evolution\",\"role\":\"observer\"}]}}" + } + }, + { + "fields": { + "EpisodeId": "episode-self-test", + "EvidenceProvenance": "datadog-measured", + "Uri": "https://app.datadoghq.com/logs?query=telemetry-evaluator", + "Query": "service:temper-platform directed_evolution.episode_id:episode-self-test role:telemetry_evaluator", + "TimeWindow": "now-15m to now", + "ResultCount": 3, + "Interpretation": "Telemetry evaluator observed proof-run events.", + "ZeroResultMeaning": "failure", + "CorrelationJson": "{\"episode_id\":\"episode-self-test\",\"datadog\":{\"role\":\"telemetry_evaluator\",\"join_fields\":{\"episode_id\":\"episode-self-test\",\"role\":\"telemetry_evaluator\"}},\"output\":{\"evaluator_role\":\"telemetry_evaluator\",\"evidence_scope\":[{\"datadog_url\":\"https://app.datadoghq.com/logs?query=telemetry-evaluator\",\"role\":\"telemetry_evaluator\"}]}}" } }, { @@ -489,8 +542,20 @@ JSON local count count="$(datadog_evidence_count "episode-self-test")" - if [[ "$count" != "1" ]]; then - fail "self-test expected one structured Datadog evidence artifact, got ${count}" + if [[ "$count" != "2" ]]; then + fail "self-test expected two structured Datadog evidence artifacts, got ${count}" + fi + + local observer_count + observer_count="$(datadog_role_evidence_count "episode-self-test" observer)" + if [[ "$observer_count" != "1" ]]; then + fail "self-test expected one observer Datadog evidence artifact, got ${observer_count}" + fi + + local telemetry_count + telemetry_count="$(datadog_role_evidence_count "episode-self-test" telemetry_evaluator)" + if [[ "$telemetry_count" != "1" ]]; then + fail "self-test expected one telemetry_evaluator Datadog evidence artifact, got ${telemetry_count}" fi local failed_status @@ -590,6 +655,8 @@ work_item_count="$(wait_for_min_count WorkItems WorkItems "$episode_id" 1)" worker_run_count="$(wait_for_min_count WorkerRuns WorkerRuns "$episode_id" 1)" evidence_count="$(wait_for_min_count EvidenceArtifacts EvidenceArtifacts "$episode_id" 1)" datadog_count="$(wait_for_datadog_evidence "$episode_id")" +datadog_observer_count="$(wait_for_datadog_role_evidence "$episode_id" observer)" +datadog_telemetry_evaluator_count="$(wait_for_datadog_role_evidence "$episode_id" telemetry_evaluator)" episode_body="$(wait_for_episode_terminal "$episode_id")" episode_status="$(field Status <<<"$episode_body")" @@ -608,6 +675,8 @@ case "$episode_status" in --argjson worker_run_count "$worker_run_count" \ --argjson evidence_count "$evidence_count" \ --argjson datadog_evidence_count "$datadog_count" \ + --argjson datadog_observer_evidence_count "$datadog_observer_count" \ + --argjson datadog_telemetry_evaluator_evidence_count "$datadog_telemetry_evaluator_count" \ '{ status: $status, proof_kind: "live Agent Answers Directed Evolution proof", @@ -620,7 +689,9 @@ case "$episode_status" in work_items: $work_item_count, worker_runs: $worker_run_count, evidence_artifacts: $evidence_count, - datadog_measured_evidence_artifacts: $datadog_evidence_count + datadog_measured_evidence_artifacts: $datadog_evidence_count, + datadog_observer_evidence_artifacts: $datadog_observer_evidence_count, + datadog_telemetry_evaluator_evidence_artifacts: $datadog_telemetry_evaluator_evidence_count }, failure_reason: "terminal episode status was not successful" }' >"$SUMMARY_JSON" @@ -642,6 +713,8 @@ summary_json="$(jq -n \ --argjson worker_run_count "$worker_run_count" \ --argjson evidence_count "$evidence_count" \ --argjson datadog_evidence_count "$datadog_count" \ + --argjson datadog_observer_evidence_count "$datadog_observer_count" \ + --argjson datadog_telemetry_evaluator_evidence_count "$datadog_telemetry_evaluator_count" \ '{ status: $status, proof_kind: "live Agent Answers Directed Evolution proof", @@ -666,6 +739,8 @@ summary_json="$(jq -n \ worker_runs: $worker_run_count, evidence_artifacts: $evidence_count, datadog_measured_evidence_artifacts: $datadog_evidence_count, + datadog_observer_evidence_artifacts: $datadog_observer_evidence_count, + datadog_telemetry_evaluator_evidence_artifacts: $datadog_telemetry_evaluator_evidence_count, mandatory_datadog_evidence: env.REQUIRE_DATADOG_EVIDENCE } }')" @@ -678,8 +753,8 @@ cat >"$PROOF_MD" < Date: Tue, 2 Jun 2026 23:59:05 -0400 Subject: [PATCH 11/16] Expose role-specific Datadog precheck gates --- .github/workflows/ci.yml | 4 +++- crates/paw-codex-worker/README.md | 5 ++++- ...irected-evolution-agent-answers-live-proof.sh | 13 ++++++++----- crates/temperpaw/tests/paw_patrol_foundation.rs | 16 ++++++++++++++-- 4 files changed, 29 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4fc82ed05..31982a975 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,8 +64,10 @@ jobs: DIRECTED_EVOLUTION_DIRECTION_ID=agent-answers-ci-direction \ START_LOCAL_WORKER=0 \ crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh - jq -e '.status == "ready" and .no_mutation == true and .would_create_live_episode == true' "$PROOF_DIR/summary.json" + jq -e '.status == "ready" and .no_mutation == true and .would_create_live_episode == true and .evidence_requirements.mandatory_datadog_observer_evidence == true and .evidence_requirements.mandatory_datadog_telemetry_evaluator_evidence == true' "$PROOF_DIR/summary.json" grep -F '"no_mutation": true' "$PROOF_DIR/summary.json" + grep -F '"mandatory_datadog_observer_evidence": true' "$PROOF_DIR/summary.json" + grep -F '"mandatory_datadog_telemetry_evaluator_evidence": true' "$PROOF_DIR/summary.json" test ! -e "$PROOF_DIR/episode-start.json" test ! -e "$PROOF_DIR/agent-answers-episode-contract.json" - name: WASM build (os-apps) diff --git a/crates/paw-codex-worker/README.md b/crates/paw-codex-worker/README.md index 7ada73903..e54f18b28 100644 --- a/crates/paw-codex-worker/README.md +++ b/crates/paw-codex-worker/README.md @@ -389,7 +389,10 @@ Use `PRECHECK_ONLY=1` with the same confirmations and identifiers to generate a ready proof bundle without mutating production. The precheck writes `summary.json` and `proof.md`, exits before the worker build, metadata read, episode start call, worker start, episode polling, and Datadog polling, and -marks the bundle with `no_mutation: true`: +marks the bundle with `no_mutation: true`. Its `evidence_requirements` section +declares `mandatory_datadog_observer_evidence` and +`mandatory_datadog_telemetry_evaluator_evidence`, making the no-mutation proof +explicit about the role-specific Datadog gates the next live run must satisfy: ```sh PRECHECK_ONLY=1 \ diff --git a/crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh b/crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh index 31f39a2e4..c74f081e8 100755 --- a/crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh +++ b/crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh @@ -232,6 +232,8 @@ write_precheck_ready_summary_and_exit() { }, evidence_requirements: { mandatory_datadog_evidence: ($mandatory_datadog_evidence == "1"), + mandatory_datadog_observer_evidence: ($mandatory_datadog_evidence == "1"), + mandatory_datadog_telemetry_evaluator_evidence: ($mandatory_datadog_evidence == "1"), mandatory_terminal_success: ($mandatory_terminal_success == "1"), production_datadog_query_secrets_confirmed: ($production_datadog_query_secrets_confirmed == "1") }, @@ -241,7 +243,7 @@ write_precheck_ready_summary_and_exit() { "directed-evolution-start-episode", "local worker start", "episode polling", - "Datadog evidence polling" + "Datadog observer/evaluator evidence polling" ] }')" @@ -251,9 +253,10 @@ write_precheck_ready_summary_and_exit() { This file is generated by \`directed-evolution-agent-answers-live-proof.sh\`. \`PRECHECK_ONLY=1\` validated all locally checkable operator confirmation, -control-plane identifier, worker mode, Datadog evidence, and terminal-success -gates. The script did not build the worker, read metadata, create a live -episode, start a local worker, poll episode entities, or query Datadog. +control-plane identifier, worker mode, Datadog observer/evaluator evidence, +and terminal-success gates. The script did not build the worker, read metadata, +create a live episode, start a local worker, poll episode entities, or query +Datadog. The next non-precheck run with the same environment would create a fresh live Directed Evolution Episode for the Agent Answers organism. @@ -607,7 +610,7 @@ jq -n \ organism_id: $organism_id, parent_version_id: $parent_version_id, adaptation_goal: "Improve Agent Answers while preserving question, answer, acceptance, citation, and reviewer workflows.", - human_notes: "Fresh live Agent Answers Directed Evolution proof. Requires worker provenance, simulated-user evidence, state verification, and Datadog telemetry evidence before proof closure.", + human_notes: "Fresh live Agent Answers Directed Evolution proof. Requires worker provenance, simulated-user evidence, state verification, and Datadog observer/evaluator evidence before proof closure.", created_by_worker_run_id: ("manual:" + $actor), evaluator_ref: "genesis://nerdsane/agent-answers-evaluation@production", selected_by: $actor, diff --git a/crates/temperpaw/tests/paw_patrol_foundation.rs b/crates/temperpaw/tests/paw_patrol_foundation.rs index 2332f2450..2e07883a7 100644 --- a/crates/temperpaw/tests/paw_patrol_foundation.rs +++ b/crates/temperpaw/tests/paw_patrol_foundation.rs @@ -1389,6 +1389,9 @@ fn directed_evolution_agent_answers_live_proof_requires_datadog_evidence() { "wait_for_datadog_role_evidence \"$episode_id\" telemetry_evaluator", "datadog_observer_evidence_artifacts", "datadog_telemetry_evaluator_evidence_artifacts", + "mandatory_datadog_observer_evidence", + "mandatory_datadog_telemetry_evaluator_evidence", + "Datadog observer/evaluator evidence polling", "summary.json", "proof.md", "human-blockers.json", @@ -1409,7 +1412,9 @@ fn directed_evolution_agent_answers_live_proof_requires_datadog_evidence() { assert!( ci.contains("Agent Answers proof precheck") && ci.contains("PRECHECK_ONLY=1") - && ci.contains("\"no_mutation\": true"), + && ci.contains("\"no_mutation\": true") + && ci.contains("\"mandatory_datadog_observer_evidence\": true") + && ci.contains("\"mandatory_datadog_telemetry_evaluator_evidence\": true"), "CI should run the Agent Answers proof driver in no-mutation precheck mode" ); assert!( @@ -1541,9 +1546,16 @@ fn directed_evolution_agent_answers_live_proof_precheck_can_pass_without_mutatio assert!( summary.contains("\"status\": \"ready\"") && summary.contains("\"would_create_live_episode\": true") - && summary.contains("\"no_mutation\": true"), + && summary.contains("\"no_mutation\": true") + && summary.contains("\"mandatory_datadog_observer_evidence\": true") + && summary.contains("\"mandatory_datadog_telemetry_evaluator_evidence\": true"), "precheck summary should expose ready no-mutation state: {summary}" ); + let proof = read(proof_dir.join("proof.md")); + assert!( + proof.contains("Datadog observer/evaluator evidence"), + "precheck proof markdown should name the role-specific Datadog evidence gate: {proof}" + ); assert!( !proof_dir.join("episode-start.json").exists(), "precheck must not call directed-evolution-start-episode" From 54d18070936595435adc6c1ad4afc7207958a99f Mon Sep 17 00:00:00 2001 From: rita-aga Date: Wed, 3 Jun 2026 00:39:08 -0400 Subject: [PATCH 12/16] Prove local Codex worker slot pool readiness --- crates/paw-codex-worker/README.md | 7 ++- .../scripts/production-readiness-smoke.sh | 56 ++++++++++++++----- .../temperpaw/tests/paw_patrol_foundation.rs | 15 +++++ 3 files changed, 62 insertions(+), 16 deletions(-) diff --git a/crates/paw-codex-worker/README.md b/crates/paw-codex-worker/README.md index e54f18b28..88fa6d435 100644 --- a/crates/paw-codex-worker/README.md +++ b/crates/paw-codex-worker/README.md @@ -445,11 +445,12 @@ crates/paw-codex-worker/scripts/production-readiness-smoke.sh ``` On success, the smoke writes `summary.json`, `proof.md`, -`production-readiness.log`, and a rendered launchd plist under +`production-readiness.log`, and a rendered launchd plist pool under `/tmp/paw-patrol-production-readiness-proof-*`. It checks OData, the event stream, fake Codex availability, a guarded `codex exec` doctor smoke, -`PAW_CODEX_ENABLE_EXECUTION=0`, plist rendering, and that the worker token was -not printed to the readiness log. +`PAW_CODEX_ENABLE_EXECUTION=0`, `WORKER_SLOT_COUNT=3` process-pool rendering, +isolated worker slot identities, and that the worker token was not printed to +the readiness log. On the production Mac mini, use the guarded bootstrap wrapper to avoid manually copying production tokens. It links the known Railway `temperpaw` production diff --git a/crates/paw-codex-worker/scripts/production-readiness-smoke.sh b/crates/paw-codex-worker/scripts/production-readiness-smoke.sh index c5f584bfc..1016d2dc9 100755 --- a/crates/paw-codex-worker/scripts/production-readiness-smoke.sh +++ b/crates/paw-codex-worker/scripts/production-readiness-smoke.sh @@ -40,6 +40,7 @@ TENANT="${TEMPER_TENANT:-patrol_production_smoke}" API_KEY="${TEMPER_API_KEY:-patrol-production-smoke}" DB_PATH="${DB_PATH:-/tmp/paw-patrol-production-smoke-${PORT}-$$.db}" WORKER_ID="${WORKER_ID:-mac-mini-codex-prod}" +WORKER_SLOT_COUNT="${WORKER_SLOT_COUNT:-3}" READY_ATTEMPTS="${READY_ATTEMPTS:-480}" PROOF_DIR="${PROOF_DIR:-/tmp/paw-patrol-production-readiness-proof-${PORT}-$$}" LAUNCHD_PLIST="${LAUNCHD_PLIST:-${PROOF_DIR}/com.temperpaw.paw-codex-worker.plist}" @@ -117,13 +118,17 @@ the non-mutating Railway, launchd, token, and webhook human blockers. - \`PAW_CODEX_DOCTOR_EXEC_SMOKE=1\` proved \`codex exec\` can start in the worker environment. - \`WRITE_LAUNCHD_PLIST=1\` rendered the plist for review. +- \`WORKER_SLOT_COUNT=${WORKER_SLOT_COUNT}\` rendered a local Codex process + pool with isolated launchd worker identities. +- \`REQUIRE_MAIN_ANCESTRY=0\` scoped the smoke to local readiness mechanics; the + real production readiness script keeps ancestry checking enabled by default. - \`INSTALL_LAUNCHD=0\` prevented launchd mutation. - The fake worker token did not appear in the readiness stdout/stderr log. ## Files - Readiness log: ${READINESS_LOG} -- Rendered plist: ${LAUNCHD_PLIST} +- Rendered plist directory: $(dirname "$LAUNCHD_PLIST") - Server log: ${SERVER_LOG} ## Machine Summary @@ -162,6 +167,7 @@ set +e TEMPER_URL="$TEMPER_URL" \ TEMPER_TENANT="$TENANT" \ WORKER_ID="$WORKER_ID" \ +WORKER_SLOT_COUNT="$WORKER_SLOT_COUNT" \ WORKER_TOKEN="$API_KEY" \ REPO_ROOT="$ROOT" \ WORKSPACE_ROOT="$(dirname "$ROOT")" \ @@ -170,6 +176,7 @@ PAW_CODEX_ENABLE_EXECUTION=0 \ PAW_CODEX_DOCTOR_EXEC_SMOKE=1 \ PAW_CODEX_POLL_ON_START=1 \ PAW_CODEX_EXEC_TIMEOUT_SECS=1200 \ +REQUIRE_MAIN_ANCESTRY=0 \ WRITE_LAUNCHD_PLIST=1 \ INSTALL_LAUNCHD=0 \ LAUNCHD_PLIST="$LAUNCHD_PLIST" \ @@ -189,25 +196,45 @@ assert_contains "$READINESS_LOG" "[pass] codex_bin:" assert_contains "$READINESS_LOG" "[pass] codex_exec_smoke:" assert_contains "$READINESS_LOG" "[pass] odata:" assert_contains "$READINESS_LOG" "[pass] event_stream:" +assert_contains "$READINESS_LOG" "worker slot count: ${WORKER_SLOT_COUNT}" +assert_contains "$READINESS_LOG" "registering worker slot identities" assert_contains "$READINESS_LOG" "production readiness check passed" assert_not_contains "$READINESS_LOG" "$API_KEY" -test -s "$LAUNCHD_PLIST" -assert_contains "$LAUNCHD_PLIST" "com.temperpaw.paw-codex-worker" -assert_contains "$LAUNCHD_PLIST" "$TEMPER_URL" -assert_contains "$LAUNCHD_PLIST" "$WORKER_ID" -assert_contains "$LAUNCHD_PLIST" "PAW_CODEX_ENABLE_EXECUTION" -assert_contains "$LAUNCHD_PLIST" "0" -assert_contains "$LAUNCHD_PLIST" "PAW_CODEX_DOCTOR_EXEC_SMOKE" -assert_contains "$LAUNCHD_PLIST" "1" -assert_contains "$LAUNCHD_PLIST" "PAW_CODEX_EXEC_TIMEOUT_SECS" -assert_contains "$LAUNCHD_PLIST" "1200" +slot_plists_json="[]" +slot_worker_ids_json="[]" +for slot in $(seq 1 "$WORKER_SLOT_COUNT"); do + slot_suffix="$(printf '%02d' "$slot")" + slot_plist="$(dirname "$LAUNCHD_PLIST")/com.temperpaw.paw-codex-worker.slot-${slot_suffix}.plist" + if [[ "$slot" == "1" ]]; then + slot_worker_id="$WORKER_ID" + else + slot_worker_id="${WORKER_ID}-slot-${slot_suffix}" + fi + + test -s "$slot_plist" + assert_contains "$slot_plist" "com.temperpaw.paw-codex-worker.slot-${slot_suffix}" + assert_contains "$slot_plist" "$TEMPER_URL" + assert_contains "$slot_plist" "$slot_worker_id" + assert_contains "$slot_plist" "PAW_CODEX_ENABLE_EXECUTION" + assert_contains "$slot_plist" "0" + assert_contains "$slot_plist" "PAW_CODEX_DOCTOR_EXEC_SMOKE" + assert_contains "$slot_plist" "1" + assert_contains "$slot_plist" "PAW_CODEX_EXEC_TIMEOUT_SECS" + assert_contains "$slot_plist" "1200" + assert_contains "$slot_plist" "MAX_CONCURRENT_RUNS" + + slot_plists_json="$(jq -c --arg path "$slot_plist" '. + [$path]' <<<"$slot_plists_json")" + slot_worker_ids_json="$(jq -c --arg worker "$slot_worker_id" '. + [$worker]' <<<"$slot_worker_ids_json")" +done summary_json="$(jq -n \ --arg temper_url "$TEMPER_URL" \ --arg worker_id "$WORKER_ID" \ + --argjson worker_slot_count "$WORKER_SLOT_COUNT" \ + --argjson worker_slot_ids "$slot_worker_ids_json" \ --arg readiness_log "$READINESS_LOG" \ - --arg launchd_plist "$LAUNCHD_PLIST" \ + --argjson launchd_plists "$slot_plists_json" \ --arg server_log "$SERVER_LOG" \ '{ status: "passed", @@ -218,6 +245,8 @@ summary_json="$(jq -n \ }, worker: { worker_id: $worker_id, + worker_slot_count: $worker_slot_count, + worker_slot_ids: $worker_slot_ids, execution_enabled: false, codex_binary: "fake-codex fixture", codex_exec_smoke: "doctor pass" @@ -225,7 +254,8 @@ summary_json="$(jq -n \ launchd: { rendered: true, installed: false, - plist: $launchd_plist + process_pool_model: "launchd worker slot per local Codex process", + plists: $launchd_plists }, logs: { production_readiness: $readiness_log, diff --git a/crates/temperpaw/tests/paw_patrol_foundation.rs b/crates/temperpaw/tests/paw_patrol_foundation.rs index 2e07883a7..71068048e 100644 --- a/crates/temperpaw/tests/paw_patrol_foundation.rs +++ b/crates/temperpaw/tests/paw_patrol_foundation.rs @@ -1710,6 +1710,10 @@ fn production_readiness_script_keeps_mac_mini_activation_checkable() { "INSTALL_LAUNCHD=1", "PAW_CODEX_ENABLE_EXECUTION=0", "PAW_CODEX_DOCTOR_EXEC_SMOKE", + "WORKER_SLOT_COUNT", + "slot_worker_id", + "slot_launchd_label", + "registering worker slot identities", ] { assert!( script.contains(needle), @@ -1784,6 +1788,14 @@ fn production_readiness_script_keeps_mac_mini_activation_checkable() { "INSTALL_LAUNCHD=0", "PAW_CODEX_ENABLE_EXECUTION=0", "PAW_CODEX_DOCTOR_EXEC_SMOKE=1", + "WORKER_SLOT_COUNT=\"${WORKER_SLOT_COUNT:-3}\"", + "worker_slot_count", + "worker_slot_ids", + "process_pool_model", + "launchd worker slot per local Codex process", + "REQUIRE_MAIN_ANCESTRY=0", + "real production readiness script keeps ancestry checking enabled by default", + "slot-${slot_suffix}.plist", "token_not_printed_to_readiness_log", "launchd_not_loaded", "production readiness smoke passed", @@ -1800,6 +1812,9 @@ fn production_readiness_script_keeps_mac_mini_activation_checkable() { "scripts/production-readiness-smoke.sh", "WRITE_LAUNCHD_PLIST=1", "INSTALL_LAUNCHD=1", + "WORKER_SLOT_COUNT=3", + "process-pool rendering", + "isolated worker slot identities", "does not print `WORKER_TOKEN`", ] { assert!( From 98b1656295b5f8f2450a529ded32863d8ae34910 Mon Sep 17 00:00:00 2001 From: rita-aga Date: Wed, 3 Jun 2026 01:03:42 -0400 Subject: [PATCH 13/16] Start Agent Answers proof worker pool --- crates/paw-codex-worker/README.md | 12 +- ...cted-evolution-agent-answers-live-proof.sh | 144 +++++++++++++++--- .../temperpaw/tests/paw_patrol_foundation.rs | 110 ++++++++++++- 3 files changed, 238 insertions(+), 28 deletions(-) diff --git a/crates/paw-codex-worker/README.md b/crates/paw-codex-worker/README.md index 88fa6d435..be33f0278 100644 --- a/crates/paw-codex-worker/README.md +++ b/crates/paw-codex-worker/README.md @@ -408,10 +408,14 @@ crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh ``` Set `START_LOCAL_WORKER=1` on the production Mac mini when launchd has not -already started the worker pool. In that mode, the local worker process inherits -the Datadog query environment and the production-secret confirmation is not -accepted as a substitute for `DD_API_KEY`/`DD_APP_KEY`. The proof bundle is -written under +already started the worker pool. In that mode, the proof driver starts a local +worker process pool and defaults `WORKER_SLOT_COUNT` and `MAX_CONCURRENT_RUNS` +to `3`, with each local slot running `MAX_CONCURRENT_RUNS=1`. Slot 1 keeps +`WORKER_ID`, later slots use identities such as `-slot-02`, and each +slot writes its own log under the proof bundle's `local-worker-logs/` +directory. Local worker processes inherit the Datadog query environment and the +production-secret confirmation is not accepted as a substitute for +`DD_API_KEY`/`DD_APP_KEY`. The proof bundle is written under `/tmp/directed-evolution-agent-answers-live-proof-*` and includes `summary.json`, `proof.md`, `human-blockers.json` when blocked, the episode contract, and terminal episode state. diff --git a/crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh b/crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh index c74f081e8..73370d62f 100755 --- a/crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh +++ b/crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh @@ -10,7 +10,7 @@ HUMAN_BLOCKERS_JSON="${PROOF_DIR}/human-blockers.json" BLOCKERS_TSV="${PROOF_DIR}/human-blockers.tsv" CONTRACT_JSON="${PROOF_DIR}/agent-answers-episode-contract.json" START_JSON="${PROOF_DIR}/episode-start.json" -WORKER_LOG="${PROOF_DIR}/local-worker.log" +WORKER_LOG_DIR="${PROOF_DIR}/local-worker-logs" TEMPER_TENANT="${TEMPER_TENANT:-default}" WORKER_ID="${WORKER_ID:-mac-mini-codex-prod}" @@ -24,13 +24,14 @@ PRECHECK_ONLY="${PRECHECK_ONLY:-0}" PAW_CODEX_ENABLE_EXECUTION="${PAW_CODEX_ENABLE_EXECUTION:-1}" PAW_CODEX_POLL_ON_START="${PAW_CODEX_POLL_ON_START:-1}" MAX_CONCURRENT_RUNS="${MAX_CONCURRENT_RUNS:-3}" +WORKER_SLOT_COUNT="${WORKER_SLOT_COUNT:-$MAX_CONCURRENT_RUNS}" READY_ATTEMPTS="${READY_ATTEMPTS:-900}" POLL_SECONDS="${POLL_SECONDS:-2}" REQUIRE_DATADOG_EVIDENCE="${REQUIRE_DATADOG_EVIDENCE:-1}" REQUIRE_EPISODE_SUCCESS="${REQUIRE_EPISODE_SUCCESS:-1}" DIRECTED_EVOLUTION_PROOF_ACTOR="${DIRECTED_EVOLUTION_PROOF_ACTOR:-codex-agent-answers-live-proof}" -WORKER_PID="" +WORKER_PIDS=() log() { printf '[directed-evolution-agent-answers-proof] %s\n' "$*" @@ -53,13 +54,48 @@ add_blocker() { } cleanup() { - if [[ -n "$WORKER_PID" ]] && kill -0 "$WORKER_PID" >/dev/null 2>&1; then - kill "$WORKER_PID" >/dev/null 2>&1 || true - wait "$WORKER_PID" >/dev/null 2>&1 || true - fi + local pid + for pid in "${WORKER_PIDS[@]:-}"; do + if [[ -n "$pid" ]] && kill -0 "$pid" >/dev/null 2>&1; then + kill "$pid" >/dev/null 2>&1 || true + wait "$pid" >/dev/null 2>&1 || true + fi + done } trap cleanup EXIT +slot_worker_id() { + local slot="$1" + if [[ "$slot" == "1" ]]; then + printf '%s\n' "$WORKER_ID" + else + printf '%s-slot-%02d\n' "$WORKER_ID" "$slot" + fi +} + +slot_worker_log() { + local slot="$1" + printf '%s/slot-%02d.log\n' "$WORKER_LOG_DIR" "$slot" +} + +worker_slot_ids_json() { + local ids_json="[]" + local slot + for slot in $(seq 1 "$WORKER_SLOT_COUNT"); do + ids_json="$(jq -c --arg worker "$(slot_worker_id "$slot")" '. + [$worker]' <<<"$ids_json")" + done + printf '%s\n' "$ids_json" +} + +worker_slot_logs_json() { + local logs_json="[]" + local slot + for slot in $(seq 1 "$WORKER_SLOT_COUNT"); do + logs_json="$(jq -c --arg log_path "$(slot_worker_log "$slot")" '. + [$log_path]' <<<"$logs_json")" + done + printf '%s\n' "$logs_json" +} + mkdir -p "$PROOF_DIR" : >"$BLOCKERS_TSV" @@ -110,6 +146,22 @@ record_precondition_blockers() { "set PAW_CODEX_ENABLE_EXECUTION=1" fi + if ! [[ "$MAX_CONCURRENT_RUNS" =~ ^[0-9]+$ ]] || [[ "$MAX_CONCURRENT_RUNS" -lt 1 ]]; then + add_blocker "env:max_concurrent_runs" \ + "MAX_CONCURRENT_RUNS must be a positive integer for the Directed Evolution proof contract." \ + "set MAX_CONCURRENT_RUNS=3 or another positive concurrency budget" + fi + + if ! [[ "$WORKER_SLOT_COUNT" =~ ^[0-9]+$ ]] || [[ "$WORKER_SLOT_COUNT" -lt 1 ]]; then + add_blocker "env:worker_slot_count" \ + "WORKER_SLOT_COUNT must be a positive integer for the Directed Evolution proof worker pool." \ + "set WORKER_SLOT_COUNT=3 or another positive pool size" + elif [[ "$START_LOCAL_WORKER" == "1" && "$WORKER_SLOT_COUNT" -lt 2 ]]; then + add_blocker "env:worker_slot_count" \ + "START_LOCAL_WORKER=1 requires WORKER_SLOT_COUNT >= 2 so the live proof exercises a local Codex worker process pool, not a single worker." \ + "set WORKER_SLOT_COUNT=3" + fi + if [[ "$REQUIRE_DATADOG_EVIDENCE" != "1" ]]; then add_blocker "env:require_datadog_evidence" \ "REQUIRE_DATADOG_EVIDENCE must remain 1; Datadog-measured evidence is mandatory for this proof." \ @@ -205,6 +257,9 @@ write_precheck_ready_summary_and_exit() { --arg worker_id "$WORKER_ID" \ --arg start_local_worker "$START_LOCAL_WORKER" \ --arg max_concurrent_runs "$MAX_CONCURRENT_RUNS" \ + --arg worker_slot_count "$WORKER_SLOT_COUNT" \ + --argjson worker_slot_ids "$(worker_slot_ids_json)" \ + --argjson worker_slot_logs "$(worker_slot_logs_json)" \ --arg execution_enabled "$PAW_CODEX_ENABLE_EXECUTION" \ --arg mandatory_datadog_evidence "$REQUIRE_DATADOG_EVIDENCE" \ --arg mandatory_terminal_success "$REQUIRE_EPISODE_SUCCESS" \ @@ -228,12 +283,17 @@ write_precheck_ready_summary_and_exit() { worker_id: $worker_id, start_local_worker: $start_local_worker, max_concurrent_runs: ($max_concurrent_runs | tonumber), + worker_slot_count: ($worker_slot_count | tonumber), + worker_slot_ids: $worker_slot_ids, + local_worker_logs: $worker_slot_logs, + process_pool_model: (if $start_local_worker == "1" then "local worker process pool" else "already-running production worker pool" end), execution_enabled: $execution_enabled }, evidence_requirements: { mandatory_datadog_evidence: ($mandatory_datadog_evidence == "1"), mandatory_datadog_observer_evidence: ($mandatory_datadog_evidence == "1"), mandatory_datadog_telemetry_evaluator_evidence: ($mandatory_datadog_evidence == "1"), + local_worker_process_pool: ($start_local_worker == "1"), mandatory_terminal_success: ($mandatory_terminal_success == "1"), production_datadog_query_secrets_confirmed: ($production_datadog_query_secrets_confirmed == "1") }, @@ -637,19 +697,25 @@ if [[ -z "$episode_id" ]]; then fi if [[ "$START_LOCAL_WORKER" == "1" ]]; then - log "starting local worker for live proof" - TEMPER_URL="$TEMPER_URL" \ - TEMPER_TENANT="$TEMPER_TENANT" \ - WORKER_ID="$WORKER_ID" \ - WORKER_TOKEN="$WORKER_TOKEN" \ - REPO_ROOT="$REPO_ROOT" \ - WORKSPACE_ROOT="$WORKSPACE_ROOT" \ - CODEX_BIN="$CODEX_BIN" \ - PAW_CODEX_ENABLE_EXECUTION="$PAW_CODEX_ENABLE_EXECUTION" \ - PAW_CODEX_POLL_ON_START="$PAW_CODEX_POLL_ON_START" \ - MAX_CONCURRENT_RUNS="$MAX_CONCURRENT_RUNS" \ - "$WORKER_BIN" run >"$WORKER_LOG" 2>&1 & - WORKER_PID="$!" + log "starting ${WORKER_SLOT_COUNT} local worker slots for live proof" + mkdir -p "$WORKER_LOG_DIR" + for slot in $(seq 1 "$WORKER_SLOT_COUNT"); do + slot_worker="$(slot_worker_id "$slot")" + slot_log="$(slot_worker_log "$slot")" + log "starting local worker slot ${slot} as ${slot_worker}" + TEMPER_URL="$TEMPER_URL" \ + TEMPER_TENANT="$TEMPER_TENANT" \ + WORKER_ID="$slot_worker" \ + WORKER_TOKEN="$WORKER_TOKEN" \ + REPO_ROOT="$REPO_ROOT" \ + WORKSPACE_ROOT="$WORKSPACE_ROOT" \ + CODEX_BIN="$CODEX_BIN" \ + PAW_CODEX_ENABLE_EXECUTION="$PAW_CODEX_ENABLE_EXECUTION" \ + PAW_CODEX_POLL_ON_START="$PAW_CODEX_POLL_ON_START" \ + MAX_CONCURRENT_RUNS="1" \ + "$WORKER_BIN" run >"$slot_log" 2>&1 & + WORKER_PIDS+=("$!") + done else log "START_LOCAL_WORKER=0; expecting an already-running production worker pool" fi @@ -674,6 +740,12 @@ case "$episode_status" in --arg proof_dir "$PROOF_DIR" \ --arg episode_id "$episode_id" \ --arg episode_status "$episode_status" \ + --arg worker_id "$WORKER_ID" \ + --arg start_local_worker "$START_LOCAL_WORKER" \ + --arg execution_enabled "$PAW_CODEX_ENABLE_EXECUTION" \ + --arg worker_slot_count "$WORKER_SLOT_COUNT" \ + --argjson worker_slot_ids "$(worker_slot_ids_json)" \ + --argjson worker_slot_logs "$(worker_slot_logs_json)" \ --argjson work_item_count "$work_item_count" \ --argjson worker_run_count "$worker_run_count" \ --argjson evidence_count "$evidence_count" \ @@ -688,6 +760,15 @@ case "$episode_status" in episode_id: $episode_id, episode_status: $episode_status }, + worker: { + worker_id: $worker_id, + start_local_worker: $start_local_worker, + execution_enabled: $execution_enabled, + worker_slot_count: ($worker_slot_count | tonumber), + worker_slot_ids: $worker_slot_ids, + local_worker_logs: $worker_slot_logs, + process_pool_model: (if $start_local_worker == "1" then "local worker process pool" else "already-running production worker pool" end) + }, evidence: { work_items: $work_item_count, worker_runs: $worker_run_count, @@ -712,6 +793,12 @@ summary_json="$(jq -n \ --arg direction_id "$DIRECTED_EVOLUTION_DIRECTION_ID" \ --arg organism_id "$DIRECTED_EVOLUTION_ORGANISM_ID" \ --arg worker_id "$WORKER_ID" \ + --arg start_local_worker "$START_LOCAL_WORKER" \ + --arg execution_enabled "$PAW_CODEX_ENABLE_EXECUTION" \ + --arg worker_slot_count "$WORKER_SLOT_COUNT" \ + --argjson worker_slot_ids "$(worker_slot_ids_json)" \ + --argjson worker_slot_logs "$(worker_slot_logs_json)" \ + --arg mandatory_datadog_evidence "$REQUIRE_DATADOG_EVIDENCE" \ --argjson work_item_count "$work_item_count" \ --argjson worker_run_count "$worker_run_count" \ --argjson evidence_count "$evidence_count" \ @@ -734,8 +821,12 @@ summary_json="$(jq -n \ }, worker: { worker_id: $worker_id, - start_local_worker: env.START_LOCAL_WORKER, - execution_enabled: env.PAW_CODEX_ENABLE_EXECUTION + start_local_worker: $start_local_worker, + execution_enabled: $execution_enabled, + worker_slot_count: ($worker_slot_count | tonumber), + worker_slot_ids: $worker_slot_ids, + local_worker_logs: $worker_slot_logs, + process_pool_model: (if $start_local_worker == "1" then "local worker process pool" else "already-running production worker pool" end) }, evidence: { work_items: $work_item_count, @@ -744,7 +835,7 @@ summary_json="$(jq -n \ datadog_measured_evidence_artifacts: $datadog_evidence_count, datadog_observer_evidence_artifacts: $datadog_observer_evidence_count, datadog_telemetry_evaluator_evidence_artifacts: $datadog_telemetry_evaluator_evidence_count, - mandatory_datadog_evidence: env.REQUIRE_DATADOG_EVIDENCE + mandatory_datadog_evidence: ($mandatory_datadog_evidence == "1") } }')" @@ -775,12 +866,19 @@ observer/evaluator evidence before declaring the live proof cycle passed. - datadog-measured observer EvidenceArtifacts: ${datadog_observer_count} - datadog-measured telemetry_evaluator EvidenceArtifacts: ${datadog_telemetry_evaluator_count} +## Worker Pool + +- Start local worker: ${START_LOCAL_WORKER} +- Worker slot count: ${WORKER_SLOT_COUNT} +- Worker slot IDs: $(worker_slot_ids_json) +- Local worker log directory: ${WORKER_LOG_DIR} + ## Files - Contract: ${CONTRACT_JSON} - Episode start response: ${START_JSON} - Terminal episode state: ${PROOF_DIR}/episode-terminal.json -- Local worker log: ${WORKER_LOG} +- Local worker log directory: ${WORKER_LOG_DIR} ## Machine Summary diff --git a/crates/temperpaw/tests/paw_patrol_foundation.rs b/crates/temperpaw/tests/paw_patrol_foundation.rs index 71068048e..798b4865f 100644 --- a/crates/temperpaw/tests/paw_patrol_foundation.rs +++ b/crates/temperpaw/tests/paw_patrol_foundation.rs @@ -1369,6 +1369,16 @@ fn directed_evolution_agent_answers_live_proof_requires_datadog_evidence() { "directed-evolution-start-episode", "PAW_CODEX_ENABLE_EXECUTION=1", "START_LOCAL_WORKER", + "WORKER_SLOT_COUNT", + "WORKER_PIDS", + "slot_worker_id", + "worker_slot_count", + "worker_slot_ids", + "local_worker_logs", + "process_pool_model", + "MAX_CONCURRENT_RUNS=\"1\"", + "local worker process pool", + "START_LOCAL_WORKER=1 requires WORKER_SLOT_COUNT >= 2", "EvidenceArtifacts", "WorkerRuns", "WorkItems", @@ -1429,6 +1439,12 @@ fn directed_evolution_agent_answers_live_proof_requires_datadog_evidence() { readme.contains("DD_API_KEY") && readme.contains("DD_APP_KEY"), "worker README should document local Datadog credentials for local-worker proof mode" ); + assert!( + readme.contains("WORKER_SLOT_COUNT") + && readme.contains("-slot-02") + && readme.contains("local-worker-logs"), + "worker README should document the local worker process pool proof mode" + ); assert!( readme.contains("observer") && readme.contains("telemetry_evaluator"), "worker README should require distinct observer and telemetry_evaluator Datadog evidence" @@ -1503,6 +1519,43 @@ fn directed_evolution_agent_answers_live_proof_preflights_datadog_credentials() "blocked local proof should name missing Datadog query credentials: {local_blockers}" ); fs::remove_dir_all(&local_proof_dir).ok(); + + let single_slot_proof_dir = env::temp_dir().join(format!( + "directed-evolution-agent-answers-live-proof-single-slot-blocked-{}", + std::process::id() + )); + fs::remove_dir_all(&single_slot_proof_dir).ok(); + + let single_slot_output = Command::new( + root.join("crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh"), + ) + .current_dir(&root) + .env("PROOF_DIR", &single_slot_proof_dir) + .env("ALLOW_PRODUCTION_WRITE", "1") + .env("CONFIRM_AGENT_ANSWERS_LIVE_PROOF", "1") + .env("TEMPER_URL", "https://temperpaw.example.invalid") + .env("WORKER_TOKEN", "worker-token") + .env("DIRECTED_EVOLUTION_DIRECTION_ID", "direction-agent-answers") + .env("DIRECTED_EVOLUTION_ORGANISM_ID", "organism-agent-answers") + .env("START_LOCAL_WORKER", "1") + .env("PAW_CODEX_ENABLE_EXECUTION", "1") + .env("WORKER_SLOT_COUNT", "1") + .env("DD_API_KEY", "dd-api") + .env("DD_APP_KEY", "dd-app") + .output() + .expect("run blocked single-slot Agent Answers Directed Evolution proof preflight"); + + assert!( + !single_slot_output.status.success(), + "local-worker proof driver should block a single-worker local proof" + ); + let single_slot_blockers = read(single_slot_proof_dir.join("human-blockers.json")); + assert!( + single_slot_blockers.contains("env:worker_slot_count") + && single_slot_blockers.contains("WORKER_SLOT_COUNT >= 2"), + "blocked local proof should require a real worker process pool: {single_slot_blockers}" + ); + fs::remove_dir_all(&single_slot_proof_dir).ok(); } #[test] @@ -1548,7 +1601,9 @@ fn directed_evolution_agent_answers_live_proof_precheck_can_pass_without_mutatio && summary.contains("\"would_create_live_episode\": true") && summary.contains("\"no_mutation\": true") && summary.contains("\"mandatory_datadog_observer_evidence\": true") - && summary.contains("\"mandatory_datadog_telemetry_evaluator_evidence\": true"), + && summary.contains("\"mandatory_datadog_telemetry_evaluator_evidence\": true") + && summary + .contains("\"process_pool_model\": \"already-running production worker pool\""), "precheck summary should expose ready no-mutation state: {summary}" ); let proof = read(proof_dir.join("proof.md")); @@ -1567,6 +1622,59 @@ fn directed_evolution_agent_answers_live_proof_precheck_can_pass_without_mutatio "precheck must not write a live episode contract" ); fs::remove_dir_all(&proof_dir).ok(); + + let local_pool_proof_dir = env::temp_dir().join(format!( + "directed-evolution-agent-answers-live-proof-local-pool-precheck-{}", + std::process::id() + )); + fs::remove_dir_all(&local_pool_proof_dir).ok(); + + let local_pool_output = Command::new( + root.join("crates/paw-codex-worker/scripts/directed-evolution-agent-answers-live-proof.sh"), + ) + .current_dir(&root) + .env("PROOF_DIR", &local_pool_proof_dir) + .env("PRECHECK_ONLY", "1") + .env("ALLOW_PRODUCTION_WRITE", "1") + .env("CONFIRM_AGENT_ANSWERS_LIVE_PROOF", "1") + .env("TEMPER_URL", "https://temperpaw.example.invalid") + .env("WORKER_TOKEN", "worker-token") + .env("DIRECTED_EVOLUTION_DIRECTION_ID", "direction-agent-answers") + .env("DIRECTED_EVOLUTION_ORGANISM_ID", "organism-agent-answers") + .env("START_LOCAL_WORKER", "1") + .env("PAW_CODEX_ENABLE_EXECUTION", "1") + .env("WORKER_SLOT_COUNT", "3") + .env("DD_API_KEY", "dd-api") + .env("DD_APP_KEY", "dd-app") + .output() + .expect("run local worker pool Agent Answers Directed Evolution proof precheck"); + + assert!( + local_pool_output.status.success(), + "local worker pool proof precheck should pass without mutating production\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&local_pool_output.stdout), + String::from_utf8_lossy(&local_pool_output.stderr) + ); + let local_pool_summary = read(local_pool_proof_dir.join("summary.json")); + assert!( + local_pool_summary.contains("\"worker_slot_count\": 3") + && local_pool_summary.contains("\"mac-mini-codex-prod-slot-02\"") + && local_pool_summary.contains("\"local_worker_logs\"") + && local_pool_summary.contains("\"process_pool_model\": \"local worker process pool\"") + && local_pool_summary.contains("\"local_worker_process_pool\": true"), + "local precheck summary should expose the local worker pool contract: {local_pool_summary}" + ); + assert!( + !local_pool_proof_dir.join("episode-start.json").exists(), + "local precheck must not call directed-evolution-start-episode" + ); + assert!( + !local_pool_proof_dir + .join("agent-answers-episode-contract.json") + .exists(), + "local precheck must not write a live episode contract" + ); + fs::remove_dir_all(&local_pool_proof_dir).ok(); } #[test] From 1d44d6d43a0e7f4b383d137bd28cfec0bf3e4bf2 Mon Sep 17 00:00:00 2001 From: rita-aga Date: Wed, 3 Jun 2026 08:45:37 -0400 Subject: [PATCH 14/16] Classify observer Datadog evidence provenance --- .../src/directed_evolution/evidence.rs | 38 +++++++++++++------ .../src/directed_evolution/tests.rs | 21 ++++++++++ 2 files changed, 47 insertions(+), 12 deletions(-) diff --git a/crates/paw-codex-worker/src/directed_evolution/evidence.rs b/crates/paw-codex-worker/src/directed_evolution/evidence.rs index c7a26427a..c85b0a9c0 100644 --- a/crates/paw-codex-worker/src/directed_evolution/evidence.rs +++ b/crates/paw-codex-worker/src/directed_evolution/evidence.rs @@ -172,6 +172,31 @@ fn directed_evolution_scope_satisfies_datadog_contract(scope: &Value) -> bool { } fn directed_evolution_evidence_provenance(role: &str, output: &Value) -> String { + let explicit = value_field_string( + output, + &[ + "provenance_kind", + "provenanceKind", + "EvidenceProvenance", + "evidence_provenance", + ], + ); + if !explicit.trim().is_empty() { + return explicit; + } + if role == "observer" + && output + .get("evidence_scope") + .or_else(|| output.get("evidenceScope")) + .and_then(Value::as_array) + .is_some_and(|items| { + items + .iter() + .any(directed_evolution_scope_satisfies_datadog_contract) + }) + { + return "datadog-measured".to_string(); + } let fallback = match role { "simulated_user" => "agent-observed", "state_verifier" => "state-verified", @@ -180,18 +205,7 @@ fn directed_evolution_evidence_provenance(role: &str, output: &Value) -> String "reviewer" | "viability_evaluator" => "brain-judged", _ => "agent-observed", }; - nonempty( - value_field_string( - output, - &[ - "provenance_kind", - "provenanceKind", - "EvidenceProvenance", - "evidence_provenance", - ], - ), - fallback.to_string(), - ) + fallback.to_string() } fn directed_evolution_evidence_uri( diff --git a/crates/paw-codex-worker/src/directed_evolution/tests.rs b/crates/paw-codex-worker/src/directed_evolution/tests.rs index 2912e7b4b..1d3221c35 100644 --- a/crates/paw-codex-worker/src/directed_evolution/tests.rs +++ b/crates/paw-codex-worker/src/directed_evolution/tests.rs @@ -913,6 +913,27 @@ mod directed_evolution_tests { ); } + #[test] + fn directed_evolution_observer_datadog_scope_records_datadog_provenance() { + let output = json!({ + "actionable": true, + "evidence_scope": [{ + "surface": "logs", + "query": "@directed_evolution.episode_id:episode-1", + "time_window": "now-24h to now", + "result_count": 206, + "interpretation": "Episode-scoped runtime logs were present.", + "zero_result_meaning": "failure", + "datadog_url": "https://app.datadoghq.com/logs?query=episode-1" + }] + }); + + assert_eq!( + directed_evolution_evidence_provenance("observer", &output), + "datadog-measured" + ); + } + #[test] fn directed_evolution_summary_uses_agent_summary_when_available() { let work_item = DirectedEvolutionWorkItemState { From c9a999040398a80983fbe988e1251924b2781cfb Mon Sep 17 00:00:00 2001 From: rita-aga Date: Mon, 8 Jun 2026 20:36:11 -0400 Subject: [PATCH 15/16] Close directed evolution observability gaps --- Cargo.lock | 86 +++++++++---------- Dockerfile | 2 +- crates/paw-codex-worker/Cargo.toml | 2 +- crates/paw-codex-worker/src/boot_watch.rs | 41 ++++++++- .../src/directed_evolution/evidence.rs | 9 ++ .../src/directed_evolution/prompt.rs | 6 +- .../src/directed_evolution/recovery.rs | 31 +++---- .../src/directed_evolution/tests.rs | 4 +- crates/paw-codex-worker/src/http_headers.rs | 18 ++++ crates/temperpaw/Cargo.toml | 16 ++-- .../tests/datadog_observability_contract.rs | 8 +- crates/temperpaw/tests/paw_fs_hot_path.rs | 2 +- ...2-directed-evolution-worker-concurrency.md | 42 +++++++++ os-apps/paw-agent/wasm/agent_reply/Cargo.toml | 2 +- .../wasm/coding_agent_runner/Cargo.toml | 2 +- .../wasm/context_compactor/Cargo.lock | 2 +- .../wasm/context_compactor/Cargo.toml | 2 +- .../wasm/context_preparer/Cargo.lock | 2 +- .../wasm/context_preparer/Cargo.toml | 2 +- .../wasm/cron_compute_next/Cargo.toml | 2 +- .../wasm/emit_ots_trajectory/Cargo.lock | 2 +- .../wasm/emit_ots_trajectory/Cargo.toml | 2 +- .../wasm/guest_observability_probe/Cargo.toml | 2 +- os-apps/paw-agent/wasm/monty_repl/Cargo.toml | 2 +- .../wasm/openai_codex_auth/Cargo.toml | 2 +- .../wasm/plan_approval_handler/Cargo.toml | 2 +- .../plan_review_feedback_handler/Cargo.toml | 2 +- .../wasm/provider_auth_gate/Cargo.toml | 2 +- .../paw-agent/wasm/provider_caller/Cargo.lock | 2 +- .../paw-agent/wasm/provider_caller/Cargo.toml | 2 +- .../wasm/provider_response_applier/Cargo.lock | 2 +- .../wasm/provider_response_applier/Cargo.toml | 2 +- .../wasm/request_approval/Cargo.toml | 2 +- .../wasm/request_plan_review/Cargo.toml | 2 +- .../wasm/sandbox_provisioner/Cargo.toml | 2 +- .../wasm/session_link_monitor/Cargo.toml | 2 +- .../wasm/session_recoverer/Cargo.toml | 2 +- .../wasm/steering_checker/Cargo.toml | 2 +- .../paw-agent/wasm/wasm-helpers/Cargo.toml | 2 +- .../wasm/workspace_provisioner/Cargo.toml | 2 +- .../wasm/workspace_restorer/Cargo.toml | 2 +- .../wasm/initialize_tournament/Cargo.lock | 2 +- .../wasm/initialize_tournament/Cargo.toml | 2 +- .../paw-autoreason/wasm/run_round/Cargo.lock | 2 +- .../paw-autoreason/wasm/run_round/Cargo.toml | 2 +- .../wasm/tally_votes/Cargo.lock | 2 +- .../wasm/tally_votes/Cargo.toml | 2 +- .../wasm/channel_connect/Cargo.toml | 2 +- .../wasm/route_message/Cargo.toml | 2 +- .../paw-channels/wasm/send_reply/Cargo.toml | 2 +- .../wasm/transport_reconcile/Cargo.lock | 2 +- .../wasm/transport_reconcile/Cargo.toml | 2 +- .../wasm/check_and_synthesize/Cargo.lock | 2 +- .../wasm/check_and_synthesize/Cargo.toml | 2 +- .../wasm/spawn_perspectives/Cargo.lock | 2 +- .../wasm/spawn_perspectives/Cargo.toml | 2 +- .../wasm/advance_step/Cargo.toml | 2 +- .../wasm/handle_convergence/Cargo.lock | 2 +- .../wasm/handle_convergence/Cargo.toml | 2 +- .../wasm/handle_probe_done/Cargo.lock | 2 +- .../wasm/handle_probe_done/Cargo.toml | 2 +- .../wasm/handle_projection_updated/Cargo.lock | 2 +- .../wasm/handle_projection_updated/Cargo.toml | 2 +- .../paw-foresight/wasm/seed_model/Cargo.toml | 2 +- .../wasm/spawn_probes/Cargo.lock | 2 +- .../wasm/spawn_probes/Cargo.toml | 2 +- .../wasm/spawn_seed_agent/Cargo.lock | 2 +- .../wasm/spawn_seed_agent/Cargo.toml | 2 +- .../wasm/artifact_batch_apply/Cargo.toml | 2 +- os-apps/paw-fs/wasm/workspace_fs/Cargo.toml | 2 +- os-apps/paw-heal/wasm/alert_opener/Cargo.toml | 2 +- .../paw-heal/wasm/alert_verifier/Cargo.toml | 2 +- .../paw-heal/wasm/cicd_initiator/Cargo.toml | 2 +- os-apps/paw-heal/wasm/cicd_merger/Cargo.toml | 2 +- .../wasm/deployment_tracker/Cargo.toml | 2 +- .../paw-heal/wasm/heal_reporter/Cargo.toml | 2 +- .../wasm/process_webhook/Cargo.toml | 2 +- .../paw-ingest/wasm/route_webhook/Cargo.toml | 2 +- .../wasm/validate_webhook/Cargo.toml | 2 +- .../wasm/event_emitter/Cargo.lock | 2 +- .../wasm/event_emitter/Cargo.toml | 2 +- .../wasm/managed_agent_updater/Cargo.lock | 2 +- .../wasm/managed_agent_updater/Cargo.toml | 2 +- .../wasm/session_orchestrator/Cargo.lock | 2 +- .../wasm/session_orchestrator/Cargo.toml | 2 +- .../wasm/session_terminator/Cargo.lock | 2 +- .../wasm/session_terminator/Cargo.toml | 2 +- .../wasm/daily_brief_lifecycle/Cargo.toml | 2 +- .../wasm/finding_lifecycle/Cargo.toml | 2 +- .../wasm/patrol_request_router/Cargo.toml | 2 +- .../wasm/patrol_run_lifecycle/Cargo.toml | 2 +- .../wasm/patrol_schedule_lifecycle/Cargo.toml | 2 +- .../wasm/repo_sweep_lifecycle/Cargo.toml | 2 +- .../wasm/review_gate_lifecycle/Cargo.toml | 2 +- .../paw-patrol/wasm/signal_router/Cargo.toml | 2 +- .../wasm/work_cycle_lifecycle/Cargo.toml | 2 +- .../wasm/worker_run_lifecycle/Cargo.toml | 2 +- .../paw-research/wasm/web_fetch/Cargo.toml | 2 +- .../paw-research/wasm/web_search/Cargo.toml | 2 +- .../wasm/skill_installer/Cargo.toml | 2 +- .../wasm/build_session_message/Cargo.lock | 2 +- .../wasm/build_session_message/Cargo.toml | 2 +- .../wasm/finalize_spawned_session/Cargo.lock | 2 +- .../wasm/finalize_spawned_session/Cargo.toml | 2 +- 104 files changed, 274 insertions(+), 175 deletions(-) create mode 100644 docs/adrs/0062-directed-evolution-worker-concurrency.md diff --git a/Cargo.lock b/Cargo.lock index e8d3c360a..fee71959f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -665,7 +665,7 @@ dependencies = [ "cap-primitives", "cap-std", "io-lifetimes", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -694,7 +694,7 @@ dependencies = [ "maybe-owned", "rustix 1.1.4", "rustix-linux-procfs", - "windows-sys 0.59.0", + "windows-sys 0.52.0", "winx", ] @@ -971,9 +971,9 @@ dependencies = [ [[package]] name = "cmov" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f88a43d011fc4a6876cb7344703e297c71dda42494fee094d5f7c76bf13f746" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" [[package]] name = "cobs" @@ -1352,9 +1352,9 @@ dependencies = [ [[package]] name = "crypto-common" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77727bb15fa921304124b128af125e7e3b968275d1b108b379190264f4423710" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" dependencies = [ "hybrid-array", ] @@ -1576,7 +1576,7 @@ checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ "block-buffer 0.12.0", "const-oid 0.10.2", - "crypto-common 0.2.1", + "crypto-common 0.2.2", "ctutils", ] @@ -1855,7 +1855,7 @@ checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" dependencies = [ "cfg-if", "rustix 1.1.4", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -1996,7 +1996,7 @@ checksum = "94e7099f6313ecacbe1256e8ff9d617b75d1bcb16a6fddef94866d225a01a14a" dependencies = [ "io-lifetimes", "rustix 1.1.4", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -2840,7 +2840,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2285ddfe3054097ef4b2fe909ef8c3bcd1ea52a8f0d274416caebeef39f04a65" dependencies = [ "io-lifetimes", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -4349,12 +4349,12 @@ dependencies = [ [[package]] name = "prost" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" dependencies = [ "bytes", - "prost-derive 0.14.3", + "prost-derive 0.14.4", ] [[package]] @@ -4385,12 +4385,12 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.12.1", "proc-macro2", "quote", "syn", @@ -4398,11 +4398,11 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" dependencies = [ - "prost 0.14.3", + "prost 0.14.4", ] [[package]] @@ -5834,9 +5834,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "symbolic-common" -version = "12.18.2" +version = "12.18.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bbc8b4f883265fb2207a0d6ce67095f4bd6cf13e5f9183eb8b3e73fa1b2466a" +checksum = "332615d90111d8eeaf86a84dc9bbe9f65d0d8c5cf11b4caccedc37754eb0dcfd" dependencies = [ "debugid", "memmap2", @@ -5846,9 +5846,9 @@ dependencies = [ [[package]] name = "symbolic-demangle" -version = "12.18.2" +version = "12.18.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eda470a26ea40eb5cafc35487718e23a2f19153d3a589affb9e8f7fa1aa73b3" +checksum = "912017718eb4d21930546245af9a3475c9dccf15675a5c215664e76621afc471" dependencies = [ "cpp_demangle", "rustc-demangle", @@ -5925,7 +5925,7 @@ dependencies = [ "fd-lock", "io-lifetimes", "rustix 0.38.44", - "windows-sys 0.59.0", + "windows-sys 0.52.0", "winx", ] @@ -5938,13 +5938,13 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" [[package]] name = "temper-actor-runtime" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "anyhow", "async-trait", "chrono", "deadpool-postgres", - "prost 0.14.3", + "prost 0.14.4", "prost-types", "serde", "serde_json", @@ -5961,7 +5961,7 @@ dependencies = [ [[package]] name = "temper-authz" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "cedar-policy", "opentelemetry", @@ -5975,7 +5975,7 @@ dependencies = [ [[package]] name = "temper-evolution" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "chrono", "serde", @@ -5992,7 +5992,7 @@ dependencies = [ [[package]] name = "temper-jit" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", @@ -6004,7 +6004,7 @@ dependencies = [ [[package]] name = "temper-observe" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "chrono", "opentelemetry", @@ -6027,7 +6027,7 @@ dependencies = [ [[package]] name = "temper-odata" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "axum 0.8.9", "chrono", @@ -6041,7 +6041,7 @@ dependencies = [ [[package]] name = "temper-optimize" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", @@ -6055,7 +6055,7 @@ dependencies = [ [[package]] name = "temper-platform" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "anyhow", "async-trait", @@ -6088,7 +6088,7 @@ dependencies = [ [[package]] name = "temper-runtime" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "anyhow", "chrono", @@ -6104,7 +6104,7 @@ dependencies = [ [[package]] name = "temper-sandbox" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "anyhow", "monty", @@ -6118,7 +6118,7 @@ dependencies = [ [[package]] name = "temper-server" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "aes-gcm", "async-stream", @@ -6171,7 +6171,7 @@ dependencies = [ [[package]] name = "temper-spec" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "quick-xml", "serde", @@ -6184,7 +6184,7 @@ dependencies = [ [[package]] name = "temper-store-postgres" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "chrono", "opentelemetry", @@ -6202,7 +6202,7 @@ dependencies = [ [[package]] name = "temper-store-redis" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "chrono", "fred", @@ -6219,7 +6219,7 @@ dependencies = [ [[package]] name = "temper-store-turso" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "libsql", "opentelemetry", @@ -6235,7 +6235,7 @@ dependencies = [ [[package]] name = "temper-verify" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "proptest", "serde", @@ -6249,7 +6249,7 @@ dependencies = [ [[package]] name = "temper-wasm" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "async-stream", "async-trait", @@ -8203,7 +8203,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f3fd376f71958b862e7afb20cfe5a22830e1963462f3a17f49d82a6c1d1f42d" dependencies = [ "bitflags 2.11.1", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] diff --git a/Dockerfile b/Dockerfile index 6d1f0deb5..e4c4e960a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,7 +24,7 @@ COPY os-apps ./os-apps # catalog discovers them. Pin the default ref to the exact Katagami commit baked # into this image; callers may still override KATAGAMI_REF with a branch or tag. ARG KATAGAMI_REF=00c4db68c8ca119bc22df98ca6417d6ecf0f8a79 -ARG TEMPER_OBSERVABILITY_REV=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c +ARG TEMPER_OBSERVABILITY_REV=9d1b39f5d92b8783f6572a20d1cf62caf4724b23 RUN apt-get update && apt-get install -y --no-install-recommends git ca-certificates && rm -rf /var/lib/apt/lists/* \ && rm -rf os-apps/katagami-curation os-apps/katagami-commons \ && git init /tmp/katagami \ diff --git a/crates/paw-codex-worker/Cargo.toml b/crates/paw-codex-worker/Cargo.toml index 21f26d3d3..969b4edf7 100644 --- a/crates/paw-codex-worker/Cargo.toml +++ b/crates/paw-codex-worker/Cargo.toml @@ -18,7 +18,7 @@ libc = "0.2" reqwest = { version = "0.12", features = ["json", "stream"] } serde = { workspace = true } serde_json = { workspace = true } -temper-observe = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-observe = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } tokio = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } diff --git a/crates/paw-codex-worker/src/boot_watch.rs b/crates/paw-codex-worker/src/boot_watch.rs index 237fc326e..3a2aa00e9 100644 --- a/crates/paw-codex-worker/src/boot_watch.rs +++ b/crates/paw-codex-worker/src/boot_watch.rs @@ -80,9 +80,46 @@ async fn claim_boot_queued_directed_evolution_work_items( warn!(%error, "Directed Evolution stage-result recovery pass failed"); } let ids = query_boot_entity_ids(client, config, "WorkItems", "Queued").await?; - for work_item_id in ids { - handle_queued_directed_evolution_work_item(client, config, &work_item_id).await?; + if ids.is_empty() { + return Ok(()); } + let concurrency = config.max_concurrent_runs.max(1); + info!( + queued_work_item_count = ids.len(), + max_concurrent_runs = concurrency, + "claiming queued Directed Evolution WorkItems" + ); + let results = futures_util::stream::iter(ids.into_iter().map(|work_item_id| { + let client = client.clone(); + let config = config.clone(); + async move { + let result = + handle_queued_directed_evolution_work_item(&client, &config, &work_item_id).await; + (work_item_id, result) + } + })) + .buffer_unordered(concurrency) + .collect::>() + .await; + + let mut failure_count = 0usize; + for (work_item_id, result) in results { + if let Err(error) = result { + failure_count += 1; + warn!( + work_item_id = %work_item_id, + %error, + "Directed Evolution WorkItem pickup failed" + ); + } + } + if failure_count > 0 { + bail!( + "{} Directed Evolution WorkItem pickup task(s) failed", + failure_count + ); + } + Ok(()) } diff --git a/crates/paw-codex-worker/src/directed_evolution/evidence.rs b/crates/paw-codex-worker/src/directed_evolution/evidence.rs index c85b0a9c0..14604d158 100644 --- a/crates/paw-codex-worker/src/directed_evolution/evidence.rs +++ b/crates/paw-codex-worker/src/directed_evolution/evidence.rs @@ -305,6 +305,10 @@ fn directed_evolution_datadog_context(work_item: &DirectedEvolutionWorkItemState } } let query = query_parts.join(" "); + let runtime_observation_query = format!( + "service:{service} env:{env_name} \"app usage:\" @observation_metadata:*{}*", + work_item.id + ); json!({ "service": service, "env": env_name, @@ -319,6 +323,11 @@ fn directed_evolution_datadog_context(work_item: &DirectedEvolutionWorkItemState "https://app.{site}/logs?query={}", encode_url_component(&query) ), + "runtime_observation_query": runtime_observation_query, + "runtime_observation_logs_url": format!( + "https://app.{site}/logs?query={}", + encode_url_component(&runtime_observation_query) + ), }) } diff --git a/crates/paw-codex-worker/src/directed_evolution/prompt.rs b/crates/paw-codex-worker/src/directed_evolution/prompt.rs index a598b9b25..eb291d009 100644 --- a/crates/paw-codex-worker/src/directed_evolution/prompt.rs +++ b/crates/paw-codex-worker/src/directed_evolution/prompt.rs @@ -19,7 +19,7 @@ fn directed_evolution_prompt(work_item: &DirectedEvolutionWorkItemState) -> Stri "Run deterministic state/spec verification against the variant. Use specs, CSDL, Cedar, runtime state, and transition checks rather than subjective judgment. Return pass/fail, provenance_kind=state-verified, metrics, decision_basis, and concrete evidence." } "telemetry_evaluator" => { - "Evaluate Datadog telemetry evidence for the variant. Datadog is the primary judging surface: query logs for service:temper-platform \"directed evolution runtime request\" scoped by directed_evolution.episode_id, directed_evolution.variant_id, and the runtime tenant parsed from RuntimeRef. Do not require directed_evolution.runtime_ref unless Datadog field discovery proves it is indexed for these logs. Return top-level provenance_kind=datadog-measured. The first evidence_scope item must include surface=logs, the exact Datadog query, time_window, result_count, interpretation, zero_result_meaning, and a Datadog URL. Zero matching runtime-request logs is failure for this stage; runtime OData probes may be supporting evidence but must not replace Datadog." + "Evaluate Datadog telemetry evidence for the variant. Datadog is the primary judging surface: query runtime app-usage logs and traces for the runtime service/tenant parsed from RuntimeRef. Scope by the generic Temper observation metadata emitted by the runtime, for example observation_metadata containing de.episode_id/de.variant_id in logs or temper.observation.de.* attributes in traces. Do not require a producer-specific field unless Datadog field discovery proves it is indexed. Return top-level provenance_kind=datadog-measured. The first evidence_scope item must include surface=logs|traces, the exact Datadog query, time_window, result_count, interpretation, zero_result_meaning, and a Datadog URL. Zero matching runtime app-usage telemetry is failure for this stage; runtime OData probes may be supporting evidence but must not replace Datadog." } "wasm_evaluator" => { "Run or inspect the pinned deterministic evaluator bundle for this stage. Do not let the variant alter the evaluator judging it. Return pass/fail, provenance_kind=wasm-computed, metrics, evaluator inputs, and evidence." @@ -144,8 +144,8 @@ fn directed_evolution_output_contract(role: &str) -> &'static str { "provenance_kind": "datadog-measured", "metrics": {"runtime_request_log_count":{"value":0,"unit":"logs","provenance_kind":"datadog-measured","interpretation":"..."}}, "decision_basis": {"why":"Datadog query result is the pass/fail source; runtime OData probes are supporting only.","tradeoffs":["Do not over-scope by runtime_ref unless field discovery shows it is indexed."]}, - "inputs": {"telemetry":["Datadog logs query scoped by directed_evolution episode, variant, and tenant fields"],"observations":[],"state":[]}, - "evidence_scope": [{"surface":"logs","query":"service:temper-platform \"directed evolution runtime request\" ...","time_window":"...","result_count":1,"interpretation":"...","zero_result_meaning":"failure","datadog_url":"https://app.datadoghq.com/logs?query=..."}], + "inputs": {"telemetry":["Datadog logs/traces query scoped by Temper observation metadata and runtime tenant"],"observations":[],"state":[]}, + "evidence_scope": [{"surface":"logs|traces","query":"service:temperpaw @tenant: \"app usage:\" @observation_metadata:*de.variant_id* ...","time_window":"...","result_count":1,"interpretation":"...","zero_result_meaning":"failure","datadog_url":"https://app.datadoghq.com/logs?query=..."}], "evidence_refs": ["..."], "reasoning_summary": "..." }"# diff --git a/crates/paw-codex-worker/src/directed_evolution/recovery.rs b/crates/paw-codex-worker/src/directed_evolution/recovery.rs index 8f368ee5b..8804cff7e 100644 --- a/crates/paw-codex-worker/src/directed_evolution/recovery.rs +++ b/crates/paw-codex-worker/src/directed_evolution/recovery.rs @@ -248,26 +248,17 @@ fn recovered_stage_prompt(args: RecoveredStagePrompt<'_>) -> String { let runtime_ref = value_field_string(args.variant_fields, &["RuntimeRef", "runtime_ref"]); let variant_summary = value_field_string(args.variant_fields, &["Summary", "summary"]); let trial_context = recovered_trial_context(args.trials, args.variant_id); - let header_block = format!( - concat!( - "x-de-episode-id: {episode_id}\n", - "x-de-generation-id: {generation_id}\n", - "x-de-variant-id: {variant_id}\n", - "x-de-stage-id: {stage_id}\n", - "x-de-stage-result-id: {stage_result_id}\n", - "x-de-work-item-id: {work_item_id}\n", - "x-de-runtime-ref: {runtime_ref}\n", - "x-de-app-ref: {app_ref}", - ), - episode_id = args.episode_id, - generation_id = args.generation_id, - variant_id = args.variant_id, - stage_id = args.stage_id, - stage_result_id = args.stage_result_id, - work_item_id = args.work_item_id, - runtime_ref = runtime_ref, - app_ref = app_ref - ); + let header_metadata = serde_json::json!({ + "de.episode_id": args.episode_id, + "de.generation_id": args.generation_id, + "de.variant_id": args.variant_id, + "de.stage_id": args.stage_id, + "de.stage_result_id": args.stage_result_id, + "de.work_item_id": args.work_item_id, + "de.runtime_ref": runtime_ref, + "de.app_ref": app_ref, + }); + let header_block = format!("X-Temper-Observe-Metadata: {header_metadata}"); if args.role == "telemetry_evaluator" { return format!( diff --git a/crates/paw-codex-worker/src/directed_evolution/tests.rs b/crates/paw-codex-worker/src/directed_evolution/tests.rs index 1d3221c35..4991c2356 100644 --- a/crates/paw-codex-worker/src/directed_evolution/tests.rs +++ b/crates/paw-codex-worker/src/directed_evolution/tests.rs @@ -93,7 +93,9 @@ mod directed_evolution_tests { let prompt = directed_evolution_prompt(&work_item); assert!(prompt.contains("Datadog is the primary judging surface")); - assert!(prompt.contains("directed_evolution.variant_id")); + assert!(prompt.contains("observation_metadata")); + assert!(prompt.contains("temper.observation.de.*")); + assert!(prompt.contains("de.variant_id")); assert!(prompt.contains("provenance_kind=datadog-measured")); assert!(prompt.contains("result_count")); assert!(prompt.contains("zero_result_meaning")); diff --git a/crates/paw-codex-worker/src/http_headers.rs b/crates/paw-codex-worker/src/http_headers.rs index f0cc6113e..c6f9443d6 100644 --- a/crates/paw-codex-worker/src/http_headers.rs +++ b/crates/paw-codex-worker/src/http_headers.rs @@ -50,6 +50,24 @@ fn event_stream_headers(config: &Config) -> Result { "x-tenant-id", HeaderValue::from_str(&config.tenant).context("invalid TEMPER_TENANT")?, ); + if let Ok(kind) = env::var("PAW_CODEX_EVENT_STREAM_PRINCIPAL_KIND") + && !kind.trim().is_empty() + { + let principal_id = env::var("PAW_CODEX_EVENT_STREAM_PRINCIPAL_ID") + .ok() + .filter(|value| !value.trim().is_empty()) + .unwrap_or_else(|| config.worker_id.clone()); + headers.insert( + "x-temper-principal-kind", + HeaderValue::from_str(kind.trim()) + .context("invalid PAW_CODEX_EVENT_STREAM_PRINCIPAL_KIND")?, + ); + headers.insert( + "x-temper-principal-id", + HeaderValue::from_str(principal_id.trim()) + .context("invalid PAW_CODEX_EVENT_STREAM_PRINCIPAL_ID")?, + ); + } if let Some(token) = &config.worker_token { headers.insert( AUTHORIZATION, diff --git a/crates/temperpaw/Cargo.toml b/crates/temperpaw/Cargo.toml index 821bc872f..a577fbded 100644 --- a/crates/temperpaw/Cargo.toml +++ b/crates/temperpaw/Cargo.toml @@ -12,14 +12,14 @@ path = "src/main.rs" [dependencies] # Temper platform engine -temper-platform = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } -temper-observe = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } -temper-server = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c", features = ["observe"] } -temper-runtime = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } -temper-jit = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } -temper-authz = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } -temper-store-postgres = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } -temper-store-turso = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-platform = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } +temper-observe = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } +temper-server = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23", features = ["observe"] } +temper-runtime = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } +temper-jit = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } +temper-authz = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } +temper-store-postgres = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } +temper-store-turso = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } # Paw transport (local) paw-transport = { path = "../paw-transport" } diff --git a/crates/temperpaw/tests/datadog_observability_contract.rs b/crates/temperpaw/tests/datadog_observability_contract.rs index ae6d17f90..3ea0783ce 100644 --- a/crates/temperpaw/tests/datadog_observability_contract.rs +++ b/crates/temperpaw/tests/datadog_observability_contract.rs @@ -59,7 +59,7 @@ fn collect_cargo_manifests(root: &Path, relative_dir: &Path, files: &mut Vec PathBuf { diff --git a/docs/adrs/0062-directed-evolution-worker-concurrency.md b/docs/adrs/0062-directed-evolution-worker-concurrency.md new file mode 100644 index 000000000..362d1d1f4 --- /dev/null +++ b/docs/adrs/0062-directed-evolution-worker-concurrency.md @@ -0,0 +1,42 @@ +# ADR-0062: Directed Evolution Worker Concurrency + +- Status: Accepted +- Date: 2026-06-08 +- Deciders: TemperPaw maintainers + +## Context + +The local `paw-codex-worker` can queue many Directed Evolution `WorkItems`, +but boot/backlog pickup processed queued work items with a sequential +`for ... await` loop. Setting `MAX_CONCURRENT_RUNS=4` therefore did not make a +single worker process run four simulated users at once; it only changed the +reported worker capacity. Operators had to start multiple worker processes to +see parallel seed journeys. + +## Decision + +Directed Evolution boot/backlog pickup will process queued `WorkItems` with a +bounded unordered async executor capped by `MAX_CONCURRENT_RUNS`. Each work item +still uses the existing claim action before execution, so concurrent workers +can race safely and losers observe the already-claimed state. + +The worker event stream remains token-only by default. Local Genesis/proxy +setups that require an observe principal for SSE may set +`PAW_CODEX_EVENT_STREAM_PRINCIPAL_KIND` and +`PAW_CODEX_EVENT_STREAM_PRINCIPAL_ID`; those headers are applied only to event +stream requests and do not change WorkItem action identity. + +## Consequences + +A single local worker process can run multiple simulated-user journeys in +parallel. Worker process pools remain useful for isolation or capacity, but are +no longer required just to satisfy the concurrency setting. + +Local workers can also connect to observe-protected streams without weakening +normal WorkItem mutation headers. + +## Non-Goals + +This does not change `WorkerRun`, `ReviewRun`, or `EvaluationRun` pickup +semantics. It also does not replace polling fallback; event-stream delivery is +handled separately by the runtime/proxy path. diff --git a/os-apps/paw-agent/wasm/agent_reply/Cargo.toml b/os-apps/paw-agent/wasm/agent_reply/Cargo.toml index 43b53ad78..348025049 100644 --- a/os-apps/paw-agent/wasm/agent_reply/Cargo.toml +++ b/os-apps/paw-agent/wasm/agent_reply/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/coding_agent_runner/Cargo.toml b/os-apps/paw-agent/wasm/coding_agent_runner/Cargo.toml index 87f8f203f..fc4997299 100644 --- a/os-apps/paw-agent/wasm/coding_agent_runner/Cargo.toml +++ b/os-apps/paw-agent/wasm/coding_agent_runner/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/context_compactor/Cargo.lock b/os-apps/paw-agent/wasm/context_compactor/Cargo.lock index f92452918..3042a92fe 100644 --- a/os-apps/paw-agent/wasm/context_compactor/Cargo.lock +++ b/os-apps/paw-agent/wasm/context_compactor/Cargo.lock @@ -113,7 +113,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-agent/wasm/context_compactor/Cargo.toml b/os-apps/paw-agent/wasm/context_compactor/Cargo.toml index bb7f28e8f..0bd00bb92 100644 --- a/os-apps/paw-agent/wasm/context_compactor/Cargo.toml +++ b/os-apps/paw-agent/wasm/context_compactor/Cargo.toml @@ -9,7 +9,7 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } session-tree-lib = { path = "../session-tree-lib" } wasm-helpers = { path = "../wasm-helpers" } openai-codex-wire = { path = "../openai-codex-wire" } diff --git a/os-apps/paw-agent/wasm/context_preparer/Cargo.lock b/os-apps/paw-agent/wasm/context_preparer/Cargo.lock index 65b400f1f..bd365b5e3 100644 --- a/os-apps/paw-agent/wasm/context_preparer/Cargo.lock +++ b/os-apps/paw-agent/wasm/context_preparer/Cargo.lock @@ -117,7 +117,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-agent/wasm/context_preparer/Cargo.toml b/os-apps/paw-agent/wasm/context_preparer/Cargo.toml index 0709e07bf..d97f08dc9 100644 --- a/os-apps/paw-agent/wasm/context_preparer/Cargo.toml +++ b/os-apps/paw-agent/wasm/context_preparer/Cargo.toml @@ -9,7 +9,7 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } session-tree-lib = { path = "../session-tree-lib" } session-turn-artifacts = { path = "../session-turn-artifacts" } tool-catalog = { path = "../tool-catalog" } diff --git a/os-apps/paw-agent/wasm/cron_compute_next/Cargo.toml b/os-apps/paw-agent/wasm/cron_compute_next/Cargo.toml index 00071ceca..d3abc541f 100644 --- a/os-apps/paw-agent/wasm/cron_compute_next/Cargo.toml +++ b/os-apps/paw-agent/wasm/cron_compute_next/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/emit_ots_trajectory/Cargo.lock b/os-apps/paw-agent/wasm/emit_ots_trajectory/Cargo.lock index 5d9fe62ac..cc93977d1 100644 --- a/os-apps/paw-agent/wasm/emit_ots_trajectory/Cargo.lock +++ b/os-apps/paw-agent/wasm/emit_ots_trajectory/Cargo.lock @@ -98,7 +98,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-agent/wasm/emit_ots_trajectory/Cargo.toml b/os-apps/paw-agent/wasm/emit_ots_trajectory/Cargo.toml index a5ab2cbf5..fd1f4523d 100644 --- a/os-apps/paw-agent/wasm/emit_ots_trajectory/Cargo.toml +++ b/os-apps/paw-agent/wasm/emit_ots_trajectory/Cargo.toml @@ -9,6 +9,6 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } wasm-helpers = { path = "../wasm-helpers" } serde_json = "1" diff --git a/os-apps/paw-agent/wasm/guest_observability_probe/Cargo.toml b/os-apps/paw-agent/wasm/guest_observability_probe/Cargo.toml index bcaa66493..31b4b366b 100644 --- a/os-apps/paw-agent/wasm/guest_observability_probe/Cargo.toml +++ b/os-apps/paw-agent/wasm/guest_observability_probe/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } serde_json = "1" diff --git a/os-apps/paw-agent/wasm/monty_repl/Cargo.toml b/os-apps/paw-agent/wasm/monty_repl/Cargo.toml index 5e86a8b4f..d56f3f0a1 100644 --- a/os-apps/paw-agent/wasm/monty_repl/Cargo.toml +++ b/os-apps/paw-agent/wasm/monty_repl/Cargo.toml @@ -9,7 +9,7 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } session-tree-lib = { path = "../session-tree-lib" } wasm-helpers = { path = "../wasm-helpers" } tool-catalog = { path = "../tool-catalog" } diff --git a/os-apps/paw-agent/wasm/openai_codex_auth/Cargo.toml b/os-apps/paw-agent/wasm/openai_codex_auth/Cargo.toml index 3f7f31d4c..6d1865b07 100644 --- a/os-apps/paw-agent/wasm/openai_codex_auth/Cargo.toml +++ b/os-apps/paw-agent/wasm/openai_codex_auth/Cargo.toml @@ -9,7 +9,7 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } wasm-helpers = { path = "../wasm-helpers" } openai-codex-wire = { path = "../openai-codex-wire" } serde_json = "1" diff --git a/os-apps/paw-agent/wasm/plan_approval_handler/Cargo.toml b/os-apps/paw-agent/wasm/plan_approval_handler/Cargo.toml index e5c44092b..c5d800fee 100644 --- a/os-apps/paw-agent/wasm/plan_approval_handler/Cargo.toml +++ b/os-apps/paw-agent/wasm/plan_approval_handler/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/plan_review_feedback_handler/Cargo.toml b/os-apps/paw-agent/wasm/plan_review_feedback_handler/Cargo.toml index 8cc3d341b..5d93a442c 100644 --- a/os-apps/paw-agent/wasm/plan_review_feedback_handler/Cargo.toml +++ b/os-apps/paw-agent/wasm/plan_review_feedback_handler/Cargo.toml @@ -11,5 +11,5 @@ crate-type = ["cdylib"] [dependencies] serde_json = "1" session-tree-lib = { path = "../session-tree-lib" } -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/provider_auth_gate/Cargo.toml b/os-apps/paw-agent/wasm/provider_auth_gate/Cargo.toml index 748bebce6..75206f3fb 100644 --- a/os-apps/paw-agent/wasm/provider_auth_gate/Cargo.toml +++ b/os-apps/paw-agent/wasm/provider_auth_gate/Cargo.toml @@ -9,6 +9,6 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } wasm-helpers = { path = "../wasm-helpers" } serde_json = "1" diff --git a/os-apps/paw-agent/wasm/provider_caller/Cargo.lock b/os-apps/paw-agent/wasm/provider_caller/Cargo.lock index 9ba1bec2c..0a96cd7db 100644 --- a/os-apps/paw-agent/wasm/provider_caller/Cargo.lock +++ b/os-apps/paw-agent/wasm/provider_caller/Cargo.lock @@ -124,7 +124,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-agent/wasm/provider_caller/Cargo.toml b/os-apps/paw-agent/wasm/provider_caller/Cargo.toml index a2c2fae27..a89ff8bd7 100644 --- a/os-apps/paw-agent/wasm/provider_caller/Cargo.toml +++ b/os-apps/paw-agent/wasm/provider_caller/Cargo.toml @@ -9,7 +9,7 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } session-tree-lib = { path = "../session-tree-lib" } session-turn-artifacts = { path = "../session-turn-artifacts" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/provider_response_applier/Cargo.lock b/os-apps/paw-agent/wasm/provider_response_applier/Cargo.lock index 9c191b43f..4784d7fc2 100644 --- a/os-apps/paw-agent/wasm/provider_response_applier/Cargo.lock +++ b/os-apps/paw-agent/wasm/provider_response_applier/Cargo.lock @@ -116,7 +116,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-agent/wasm/provider_response_applier/Cargo.toml b/os-apps/paw-agent/wasm/provider_response_applier/Cargo.toml index 590cdcd1a..e456b12be 100644 --- a/os-apps/paw-agent/wasm/provider_response_applier/Cargo.toml +++ b/os-apps/paw-agent/wasm/provider_response_applier/Cargo.toml @@ -9,7 +9,7 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } session-tree-lib = { path = "../session-tree-lib" } session-turn-artifacts = { path = "../session-turn-artifacts" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/request_approval/Cargo.toml b/os-apps/paw-agent/wasm/request_approval/Cargo.toml index 49ebf0f15..154e2262b 100644 --- a/os-apps/paw-agent/wasm/request_approval/Cargo.toml +++ b/os-apps/paw-agent/wasm/request_approval/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/request_plan_review/Cargo.toml b/os-apps/paw-agent/wasm/request_plan_review/Cargo.toml index 026030664..e2f1e03db 100644 --- a/os-apps/paw-agent/wasm/request_plan_review/Cargo.toml +++ b/os-apps/paw-agent/wasm/request_plan_review/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/sandbox_provisioner/Cargo.toml b/os-apps/paw-agent/wasm/sandbox_provisioner/Cargo.toml index 59f1e2d69..d2cfd4665 100644 --- a/os-apps/paw-agent/wasm/sandbox_provisioner/Cargo.toml +++ b/os-apps/paw-agent/wasm/sandbox_provisioner/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/session_link_monitor/Cargo.toml b/os-apps/paw-agent/wasm/session_link_monitor/Cargo.toml index f4f43d948..c5afe42bc 100644 --- a/os-apps/paw-agent/wasm/session_link_monitor/Cargo.toml +++ b/os-apps/paw-agent/wasm/session_link_monitor/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/session_recoverer/Cargo.toml b/os-apps/paw-agent/wasm/session_recoverer/Cargo.toml index c0252a3ba..0daba1b67 100644 --- a/os-apps/paw-agent/wasm/session_recoverer/Cargo.toml +++ b/os-apps/paw-agent/wasm/session_recoverer/Cargo.toml @@ -9,6 +9,6 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } session-tree-lib = { path = "../session-tree-lib" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/steering_checker/Cargo.toml b/os-apps/paw-agent/wasm/steering_checker/Cargo.toml index fd2c78a1e..e7d72f042 100644 --- a/os-apps/paw-agent/wasm/steering_checker/Cargo.toml +++ b/os-apps/paw-agent/wasm/steering_checker/Cargo.toml @@ -9,6 +9,6 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } session-tree-lib = { path = "../session-tree-lib" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/wasm-helpers/Cargo.toml b/os-apps/paw-agent/wasm/wasm-helpers/Cargo.toml index 68c135f4e..02ecc93f6 100644 --- a/os-apps/paw-agent/wasm/wasm-helpers/Cargo.toml +++ b/os-apps/paw-agent/wasm/wasm-helpers/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["rlib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } serde_json = "1" diff --git a/os-apps/paw-agent/wasm/workspace_provisioner/Cargo.toml b/os-apps/paw-agent/wasm/workspace_provisioner/Cargo.toml index cf696944d..c9204a828 100644 --- a/os-apps/paw-agent/wasm/workspace_provisioner/Cargo.toml +++ b/os-apps/paw-agent/wasm/workspace_provisioner/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/workspace_restorer/Cargo.toml b/os-apps/paw-agent/wasm/workspace_restorer/Cargo.toml index d65b372ca..89244b809 100644 --- a/os-apps/paw-agent/wasm/workspace_restorer/Cargo.toml +++ b/os-apps/paw-agent/wasm/workspace_restorer/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-autoreason/wasm/initialize_tournament/Cargo.lock b/os-apps/paw-autoreason/wasm/initialize_tournament/Cargo.lock index 74e331b04..33d304b57 100644 --- a/os-apps/paw-autoreason/wasm/initialize_tournament/Cargo.lock +++ b/os-apps/paw-autoreason/wasm/initialize_tournament/Cargo.lock @@ -97,7 +97,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-autoreason/wasm/initialize_tournament/Cargo.toml b/os-apps/paw-autoreason/wasm/initialize_tournament/Cargo.toml index 4a84547d9..bc04d4c62 100644 --- a/os-apps/paw-autoreason/wasm/initialize_tournament/Cargo.toml +++ b/os-apps/paw-autoreason/wasm/initialize_tournament/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } serde_json = "1" diff --git a/os-apps/paw-autoreason/wasm/run_round/Cargo.lock b/os-apps/paw-autoreason/wasm/run_round/Cargo.lock index ada597109..5d85b2085 100644 --- a/os-apps/paw-autoreason/wasm/run_round/Cargo.lock +++ b/os-apps/paw-autoreason/wasm/run_round/Cargo.lock @@ -97,7 +97,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-autoreason/wasm/run_round/Cargo.toml b/os-apps/paw-autoreason/wasm/run_round/Cargo.toml index 0513de596..b4b4c3623 100644 --- a/os-apps/paw-autoreason/wasm/run_round/Cargo.toml +++ b/os-apps/paw-autoreason/wasm/run_round/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } serde_json = "1" diff --git a/os-apps/paw-autoreason/wasm/tally_votes/Cargo.lock b/os-apps/paw-autoreason/wasm/tally_votes/Cargo.lock index a18200edf..d0ef590f4 100644 --- a/os-apps/paw-autoreason/wasm/tally_votes/Cargo.lock +++ b/os-apps/paw-autoreason/wasm/tally_votes/Cargo.lock @@ -97,7 +97,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-autoreason/wasm/tally_votes/Cargo.toml b/os-apps/paw-autoreason/wasm/tally_votes/Cargo.toml index 170fdb097..822edc1b2 100644 --- a/os-apps/paw-autoreason/wasm/tally_votes/Cargo.toml +++ b/os-apps/paw-autoreason/wasm/tally_votes/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } serde_json = "1" diff --git a/os-apps/paw-channels/wasm/channel_connect/Cargo.toml b/os-apps/paw-channels/wasm/channel_connect/Cargo.toml index 538c5a2c4..e1f28017c 100644 --- a/os-apps/paw-channels/wasm/channel_connect/Cargo.toml +++ b/os-apps/paw-channels/wasm/channel_connect/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-channels/wasm/route_message/Cargo.toml b/os-apps/paw-channels/wasm/route_message/Cargo.toml index 87701eed2..0ffc0dff6 100644 --- a/os-apps/paw-channels/wasm/route_message/Cargo.toml +++ b/os-apps/paw-channels/wasm/route_message/Cargo.toml @@ -11,4 +11,4 @@ crate-type = ["cdylib", "rlib"] [dependencies] session-tree-lib = { path = "../../../paw-agent/wasm/session-tree-lib" } wasm-helpers = { path = "../../../paw-agent/wasm/wasm-helpers" } -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-channels/wasm/send_reply/Cargo.toml b/os-apps/paw-channels/wasm/send_reply/Cargo.toml index 811c9828b..2ea80774b 100644 --- a/os-apps/paw-channels/wasm/send_reply/Cargo.toml +++ b/os-apps/paw-channels/wasm/send_reply/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-channels/wasm/transport_reconcile/Cargo.lock b/os-apps/paw-channels/wasm/transport_reconcile/Cargo.lock index 082050d48..ada76bd21 100644 --- a/os-apps/paw-channels/wasm/transport_reconcile/Cargo.lock +++ b/os-apps/paw-channels/wasm/transport_reconcile/Cargo.lock @@ -89,7 +89,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-channels/wasm/transport_reconcile/Cargo.toml b/os-apps/paw-channels/wasm/transport_reconcile/Cargo.toml index 1bccf687a..2f95013bc 100644 --- a/os-apps/paw-channels/wasm/transport_reconcile/Cargo.toml +++ b/os-apps/paw-channels/wasm/transport_reconcile/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-consilium/wasm/check_and_synthesize/Cargo.lock b/os-apps/paw-consilium/wasm/check_and_synthesize/Cargo.lock index 17a710afd..f612abaa4 100644 --- a/os-apps/paw-consilium/wasm/check_and_synthesize/Cargo.lock +++ b/os-apps/paw-consilium/wasm/check_and_synthesize/Cargo.lock @@ -97,7 +97,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-consilium/wasm/check_and_synthesize/Cargo.toml b/os-apps/paw-consilium/wasm/check_and_synthesize/Cargo.toml index 634148f1c..cf7271874 100644 --- a/os-apps/paw-consilium/wasm/check_and_synthesize/Cargo.toml +++ b/os-apps/paw-consilium/wasm/check_and_synthesize/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } serde_json = "1" diff --git a/os-apps/paw-consilium/wasm/spawn_perspectives/Cargo.lock b/os-apps/paw-consilium/wasm/spawn_perspectives/Cargo.lock index 8f79d8514..19e588279 100644 --- a/os-apps/paw-consilium/wasm/spawn_perspectives/Cargo.lock +++ b/os-apps/paw-consilium/wasm/spawn_perspectives/Cargo.lock @@ -97,7 +97,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-consilium/wasm/spawn_perspectives/Cargo.toml b/os-apps/paw-consilium/wasm/spawn_perspectives/Cargo.toml index d6afd6ccd..2d6e38922 100644 --- a/os-apps/paw-consilium/wasm/spawn_perspectives/Cargo.toml +++ b/os-apps/paw-consilium/wasm/spawn_perspectives/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } serde_json = "1" diff --git a/os-apps/paw-foresight/wasm/advance_step/Cargo.toml b/os-apps/paw-foresight/wasm/advance_step/Cargo.toml index 517070f5f..32ff4401f 100644 --- a/os-apps/paw-foresight/wasm/advance_step/Cargo.toml +++ b/os-apps/paw-foresight/wasm/advance_step/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-foresight/wasm/handle_convergence/Cargo.lock b/os-apps/paw-foresight/wasm/handle_convergence/Cargo.lock index b9f3a86f9..d8aa7d15e 100644 --- a/os-apps/paw-foresight/wasm/handle_convergence/Cargo.lock +++ b/os-apps/paw-foresight/wasm/handle_convergence/Cargo.lock @@ -96,7 +96,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-foresight/wasm/handle_convergence/Cargo.toml b/os-apps/paw-foresight/wasm/handle_convergence/Cargo.toml index a74303d03..7b86645fc 100644 --- a/os-apps/paw-foresight/wasm/handle_convergence/Cargo.toml +++ b/os-apps/paw-foresight/wasm/handle_convergence/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-foresight/wasm/handle_probe_done/Cargo.lock b/os-apps/paw-foresight/wasm/handle_probe_done/Cargo.lock index ce277b4ee..d2464e5f4 100644 --- a/os-apps/paw-foresight/wasm/handle_probe_done/Cargo.lock +++ b/os-apps/paw-foresight/wasm/handle_probe_done/Cargo.lock @@ -96,7 +96,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-foresight/wasm/handle_probe_done/Cargo.toml b/os-apps/paw-foresight/wasm/handle_probe_done/Cargo.toml index d44fb19c1..32229d5fc 100644 --- a/os-apps/paw-foresight/wasm/handle_probe_done/Cargo.toml +++ b/os-apps/paw-foresight/wasm/handle_probe_done/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-foresight/wasm/handle_projection_updated/Cargo.lock b/os-apps/paw-foresight/wasm/handle_projection_updated/Cargo.lock index 30c8c3662..caf3d4e56 100644 --- a/os-apps/paw-foresight/wasm/handle_projection_updated/Cargo.lock +++ b/os-apps/paw-foresight/wasm/handle_projection_updated/Cargo.lock @@ -96,7 +96,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-foresight/wasm/handle_projection_updated/Cargo.toml b/os-apps/paw-foresight/wasm/handle_projection_updated/Cargo.toml index 0fcd33320..97099ddd9 100644 --- a/os-apps/paw-foresight/wasm/handle_projection_updated/Cargo.toml +++ b/os-apps/paw-foresight/wasm/handle_projection_updated/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-foresight/wasm/seed_model/Cargo.toml b/os-apps/paw-foresight/wasm/seed_model/Cargo.toml index c605fc377..ac704a315 100644 --- a/os-apps/paw-foresight/wasm/seed_model/Cargo.toml +++ b/os-apps/paw-foresight/wasm/seed_model/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-foresight/wasm/spawn_probes/Cargo.lock b/os-apps/paw-foresight/wasm/spawn_probes/Cargo.lock index c6a57114c..9f28bb9b4 100644 --- a/os-apps/paw-foresight/wasm/spawn_probes/Cargo.lock +++ b/os-apps/paw-foresight/wasm/spawn_probes/Cargo.lock @@ -96,7 +96,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-foresight/wasm/spawn_probes/Cargo.toml b/os-apps/paw-foresight/wasm/spawn_probes/Cargo.toml index b3467ea0f..4eae3ea1e 100644 --- a/os-apps/paw-foresight/wasm/spawn_probes/Cargo.toml +++ b/os-apps/paw-foresight/wasm/spawn_probes/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-foresight/wasm/spawn_seed_agent/Cargo.lock b/os-apps/paw-foresight/wasm/spawn_seed_agent/Cargo.lock index d1a21c538..d3902b225 100644 --- a/os-apps/paw-foresight/wasm/spawn_seed_agent/Cargo.lock +++ b/os-apps/paw-foresight/wasm/spawn_seed_agent/Cargo.lock @@ -97,7 +97,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-foresight/wasm/spawn_seed_agent/Cargo.toml b/os-apps/paw-foresight/wasm/spawn_seed_agent/Cargo.toml index 2181eb5bc..7bbe2177f 100644 --- a/os-apps/paw-foresight/wasm/spawn_seed_agent/Cargo.toml +++ b/os-apps/paw-foresight/wasm/spawn_seed_agent/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } serde_json = "1" diff --git a/os-apps/paw-fs/wasm/artifact_batch_apply/Cargo.toml b/os-apps/paw-fs/wasm/artifact_batch_apply/Cargo.toml index 6666e1ce7..47c16ba3c 100644 --- a/os-apps/paw-fs/wasm/artifact_batch_apply/Cargo.toml +++ b/os-apps/paw-fs/wasm/artifact_batch_apply/Cargo.toml @@ -9,6 +9,6 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/os-apps/paw-fs/wasm/workspace_fs/Cargo.toml b/os-apps/paw-fs/wasm/workspace_fs/Cargo.toml index d989c6bbf..e91f99f68 100644 --- a/os-apps/paw-fs/wasm/workspace_fs/Cargo.toml +++ b/os-apps/paw-fs/wasm/workspace_fs/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } serde_json = "1" diff --git a/os-apps/paw-heal/wasm/alert_opener/Cargo.toml b/os-apps/paw-heal/wasm/alert_opener/Cargo.toml index 6398debdd..ce7298c4a 100644 --- a/os-apps/paw-heal/wasm/alert_opener/Cargo.toml +++ b/os-apps/paw-heal/wasm/alert_opener/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-heal/wasm/alert_verifier/Cargo.toml b/os-apps/paw-heal/wasm/alert_verifier/Cargo.toml index cbe9ed4a8..90fbf02ec 100644 --- a/os-apps/paw-heal/wasm/alert_verifier/Cargo.toml +++ b/os-apps/paw-heal/wasm/alert_verifier/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-heal/wasm/cicd_initiator/Cargo.toml b/os-apps/paw-heal/wasm/cicd_initiator/Cargo.toml index 3141e149e..a54d19f8d 100644 --- a/os-apps/paw-heal/wasm/cicd_initiator/Cargo.toml +++ b/os-apps/paw-heal/wasm/cicd_initiator/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-heal/wasm/cicd_merger/Cargo.toml b/os-apps/paw-heal/wasm/cicd_merger/Cargo.toml index 0f13205fa..1c56ce48c 100644 --- a/os-apps/paw-heal/wasm/cicd_merger/Cargo.toml +++ b/os-apps/paw-heal/wasm/cicd_merger/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-heal/wasm/deployment_tracker/Cargo.toml b/os-apps/paw-heal/wasm/deployment_tracker/Cargo.toml index f5618b681..3abbcde20 100644 --- a/os-apps/paw-heal/wasm/deployment_tracker/Cargo.toml +++ b/os-apps/paw-heal/wasm/deployment_tracker/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-heal/wasm/heal_reporter/Cargo.toml b/os-apps/paw-heal/wasm/heal_reporter/Cargo.toml index 72dfcf488..9486b5177 100644 --- a/os-apps/paw-heal/wasm/heal_reporter/Cargo.toml +++ b/os-apps/paw-heal/wasm/heal_reporter/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-ingest/wasm/process_webhook/Cargo.toml b/os-apps/paw-ingest/wasm/process_webhook/Cargo.toml index 834adc50a..9fb6abf5e 100644 --- a/os-apps/paw-ingest/wasm/process_webhook/Cargo.toml +++ b/os-apps/paw-ingest/wasm/process_webhook/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-ingest/wasm/route_webhook/Cargo.toml b/os-apps/paw-ingest/wasm/route_webhook/Cargo.toml index d193c3e9c..fe6e8379f 100644 --- a/os-apps/paw-ingest/wasm/route_webhook/Cargo.toml +++ b/os-apps/paw-ingest/wasm/route_webhook/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-ingest/wasm/validate_webhook/Cargo.toml b/os-apps/paw-ingest/wasm/validate_webhook/Cargo.toml index 2d9d00bd6..ec23197bd 100644 --- a/os-apps/paw-ingest/wasm/validate_webhook/Cargo.toml +++ b/os-apps/paw-ingest/wasm/validate_webhook/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-managed-agents/wasm/event_emitter/Cargo.lock b/os-apps/paw-managed-agents/wasm/event_emitter/Cargo.lock index 4dd8c01f6..19885fca5 100644 --- a/os-apps/paw-managed-agents/wasm/event_emitter/Cargo.lock +++ b/os-apps/paw-managed-agents/wasm/event_emitter/Cargo.lock @@ -105,7 +105,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-managed-agents/wasm/event_emitter/Cargo.toml b/os-apps/paw-managed-agents/wasm/event_emitter/Cargo.toml index bdc85e898..7418b9364 100644 --- a/os-apps/paw-managed-agents/wasm/event_emitter/Cargo.toml +++ b/os-apps/paw-managed-agents/wasm/event_emitter/Cargo.toml @@ -9,6 +9,6 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } session-tree-lib = { path = "../../../paw-agent/wasm/session-tree-lib" } wasm-helpers = { path = "../../../paw-agent/wasm/wasm-helpers" } diff --git a/os-apps/paw-managed-agents/wasm/managed_agent_updater/Cargo.lock b/os-apps/paw-managed-agents/wasm/managed_agent_updater/Cargo.lock index 4a1ae2d5d..6cf8a36d3 100644 --- a/os-apps/paw-managed-agents/wasm/managed_agent_updater/Cargo.lock +++ b/os-apps/paw-managed-agents/wasm/managed_agent_updater/Cargo.lock @@ -97,7 +97,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-managed-agents/wasm/managed_agent_updater/Cargo.toml b/os-apps/paw-managed-agents/wasm/managed_agent_updater/Cargo.toml index 5ac37e0fd..afc53186e 100644 --- a/os-apps/paw-managed-agents/wasm/managed_agent_updater/Cargo.toml +++ b/os-apps/paw-managed-agents/wasm/managed_agent_updater/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } wasm-helpers = { path = "../../../paw-agent/wasm/wasm-helpers" } diff --git a/os-apps/paw-managed-agents/wasm/session_orchestrator/Cargo.lock b/os-apps/paw-managed-agents/wasm/session_orchestrator/Cargo.lock index f9a6f5d3b..61235102a 100644 --- a/os-apps/paw-managed-agents/wasm/session_orchestrator/Cargo.lock +++ b/os-apps/paw-managed-agents/wasm/session_orchestrator/Cargo.lock @@ -97,7 +97,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-managed-agents/wasm/session_orchestrator/Cargo.toml b/os-apps/paw-managed-agents/wasm/session_orchestrator/Cargo.toml index 7edfe087e..cda22f8c4 100644 --- a/os-apps/paw-managed-agents/wasm/session_orchestrator/Cargo.toml +++ b/os-apps/paw-managed-agents/wasm/session_orchestrator/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } wasm-helpers = { path = "../../../paw-agent/wasm/wasm-helpers" } diff --git a/os-apps/paw-managed-agents/wasm/session_terminator/Cargo.lock b/os-apps/paw-managed-agents/wasm/session_terminator/Cargo.lock index 2881dc289..f197a8372 100644 --- a/os-apps/paw-managed-agents/wasm/session_terminator/Cargo.lock +++ b/os-apps/paw-managed-agents/wasm/session_terminator/Cargo.lock @@ -97,7 +97,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-managed-agents/wasm/session_terminator/Cargo.toml b/os-apps/paw-managed-agents/wasm/session_terminator/Cargo.toml index c22bc30ea..043f92500 100644 --- a/os-apps/paw-managed-agents/wasm/session_terminator/Cargo.toml +++ b/os-apps/paw-managed-agents/wasm/session_terminator/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } wasm-helpers = { path = "../../../paw-agent/wasm/wasm-helpers" } diff --git a/os-apps/paw-patrol/wasm/daily_brief_lifecycle/Cargo.toml b/os-apps/paw-patrol/wasm/daily_brief_lifecycle/Cargo.toml index 1aa7cb246..ad77e56da 100644 --- a/os-apps/paw-patrol/wasm/daily_brief_lifecycle/Cargo.toml +++ b/os-apps/paw-patrol/wasm/daily_brief_lifecycle/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-patrol/wasm/finding_lifecycle/Cargo.toml b/os-apps/paw-patrol/wasm/finding_lifecycle/Cargo.toml index 5d69b1727..732406c77 100644 --- a/os-apps/paw-patrol/wasm/finding_lifecycle/Cargo.toml +++ b/os-apps/paw-patrol/wasm/finding_lifecycle/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-patrol/wasm/patrol_request_router/Cargo.toml b/os-apps/paw-patrol/wasm/patrol_request_router/Cargo.toml index 54f2aaca3..d719f6f05 100644 --- a/os-apps/paw-patrol/wasm/patrol_request_router/Cargo.toml +++ b/os-apps/paw-patrol/wasm/patrol_request_router/Cargo.toml @@ -11,4 +11,4 @@ test = false [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-patrol/wasm/patrol_run_lifecycle/Cargo.toml b/os-apps/paw-patrol/wasm/patrol_run_lifecycle/Cargo.toml index 599773fbe..01d207631 100644 --- a/os-apps/paw-patrol/wasm/patrol_run_lifecycle/Cargo.toml +++ b/os-apps/paw-patrol/wasm/patrol_run_lifecycle/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-patrol/wasm/patrol_schedule_lifecycle/Cargo.toml b/os-apps/paw-patrol/wasm/patrol_schedule_lifecycle/Cargo.toml index 1060c5115..5e0ca2d2e 100644 --- a/os-apps/paw-patrol/wasm/patrol_schedule_lifecycle/Cargo.toml +++ b/os-apps/paw-patrol/wasm/patrol_schedule_lifecycle/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-patrol/wasm/repo_sweep_lifecycle/Cargo.toml b/os-apps/paw-patrol/wasm/repo_sweep_lifecycle/Cargo.toml index bf1811e82..b94fb942c 100644 --- a/os-apps/paw-patrol/wasm/repo_sweep_lifecycle/Cargo.toml +++ b/os-apps/paw-patrol/wasm/repo_sweep_lifecycle/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-patrol/wasm/review_gate_lifecycle/Cargo.toml b/os-apps/paw-patrol/wasm/review_gate_lifecycle/Cargo.toml index 6c4230109..62a4e8b5f 100644 --- a/os-apps/paw-patrol/wasm/review_gate_lifecycle/Cargo.toml +++ b/os-apps/paw-patrol/wasm/review_gate_lifecycle/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-patrol/wasm/signal_router/Cargo.toml b/os-apps/paw-patrol/wasm/signal_router/Cargo.toml index 8144ecd0b..8801b2573 100644 --- a/os-apps/paw-patrol/wasm/signal_router/Cargo.toml +++ b/os-apps/paw-patrol/wasm/signal_router/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-patrol/wasm/work_cycle_lifecycle/Cargo.toml b/os-apps/paw-patrol/wasm/work_cycle_lifecycle/Cargo.toml index e29bdf73a..393d69651 100644 --- a/os-apps/paw-patrol/wasm/work_cycle_lifecycle/Cargo.toml +++ b/os-apps/paw-patrol/wasm/work_cycle_lifecycle/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-patrol/wasm/worker_run_lifecycle/Cargo.toml b/os-apps/paw-patrol/wasm/worker_run_lifecycle/Cargo.toml index 873fd24d1..40d28ae0a 100644 --- a/os-apps/paw-patrol/wasm/worker_run_lifecycle/Cargo.toml +++ b/os-apps/paw-patrol/wasm/worker_run_lifecycle/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-research/wasm/web_fetch/Cargo.toml b/os-apps/paw-research/wasm/web_fetch/Cargo.toml index 4df5352a5..87a108e3e 100644 --- a/os-apps/paw-research/wasm/web_fetch/Cargo.toml +++ b/os-apps/paw-research/wasm/web_fetch/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } serde_json = "1" diff --git a/os-apps/paw-research/wasm/web_search/Cargo.toml b/os-apps/paw-research/wasm/web_search/Cargo.toml index 6a46f1f84..5e3138e32 100644 --- a/os-apps/paw-research/wasm/web_search/Cargo.toml +++ b/os-apps/paw-research/wasm/web_search/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } serde_json = "1" diff --git a/os-apps/paw-skills/wasm/skill_installer/Cargo.toml b/os-apps/paw-skills/wasm/skill_installer/Cargo.toml index d7dd2b015..60d692cc5 100644 --- a/os-apps/paw-skills/wasm/skill_installer/Cargo.toml +++ b/os-apps/paw-skills/wasm/skill_installer/Cargo.toml @@ -12,4 +12,4 @@ crate-type = ["cdylib"] hex = "0.4" serde_json = "1" sha2 = "0.10" -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } diff --git a/os-apps/paw-wiki/wasm/build_session_message/Cargo.lock b/os-apps/paw-wiki/wasm/build_session_message/Cargo.lock index 5f408ab64..2efbe1621 100644 --- a/os-apps/paw-wiki/wasm/build_session_message/Cargo.lock +++ b/os-apps/paw-wiki/wasm/build_session_message/Cargo.lock @@ -97,7 +97,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-wiki/wasm/build_session_message/Cargo.toml b/os-apps/paw-wiki/wasm/build_session_message/Cargo.toml index 0e8bc91a8..6d9f2c511 100644 --- a/os-apps/paw-wiki/wasm/build_session_message/Cargo.toml +++ b/os-apps/paw-wiki/wasm/build_session_message/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } serde_json = "1" diff --git a/os-apps/paw-wiki/wasm/finalize_spawned_session/Cargo.lock b/os-apps/paw-wiki/wasm/finalize_spawned_session/Cargo.lock index 020d95c58..448f9703a 100644 --- a/os-apps/paw-wiki/wasm/finalize_spawned_session/Cargo.lock +++ b/os-apps/paw-wiki/wasm/finalize_spawned_session/Cargo.lock @@ -97,7 +97,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=5ee4429f45d8f2bcf48f1269e377ef79b2c5544c#5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" +source = "git+https://github.com/nerdsane/temper.git?rev=9d1b39f5d92b8783f6572a20d1cf62caf4724b23#9d1b39f5d92b8783f6572a20d1cf62caf4724b23" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-wiki/wasm/finalize_spawned_session/Cargo.toml b/os-apps/paw-wiki/wasm/finalize_spawned_session/Cargo.toml index a6da2711f..39f0e95f1 100644 --- a/os-apps/paw-wiki/wasm/finalize_spawned_session/Cargo.toml +++ b/os-apps/paw-wiki/wasm/finalize_spawned_session/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "5ee4429f45d8f2bcf48f1269e377ef79b2c5544c" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "9d1b39f5d92b8783f6572a20d1cf62caf4724b23" } serde_json = "1" From 8336875d96f047170c4e08de1037cd7e00ba1d67 Mon Sep 17 00:00:00 2001 From: rita-aga Date: Mon, 8 Jun 2026 21:44:41 -0400 Subject: [PATCH 16/16] Deploy OpenPaw from source Dockerfile --- railway.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/railway.toml b/railway.toml index 2cff68d94..3be836899 100644 --- a/railway.toml +++ b/railway.toml @@ -1,6 +1,6 @@ [build] builder = "DOCKERFILE" -dockerfilePath = "Dockerfile.deploy" +dockerfilePath = "Dockerfile" [deploy] healthcheckPath = "/healthz"