From 6551b6161e6b47ba0f4ce6030b0898b3289ad67a Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 15:30:24 +0800 Subject: [PATCH 001/140] fix(cua-bench): reject invalid concurrency and step limits before scheduling --- libs/cua-bench/cua_bench/runners.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/libs/cua-bench/cua_bench/runners.py b/libs/cua-bench/cua_bench/runners.py index 9e888b0e65..ac33b50f89 100644 --- a/libs/cua-bench/cua_bench/runners.py +++ b/libs/cua-bench/cua_bench/runners.py @@ -222,6 +222,17 @@ def random_agent(screenshot: bytes, task: Task) -> Action: max_parallel=4, ) """ + if not isinstance(max_parallel, int) or isinstance(max_parallel, bool) or max_parallel < 1: + raise ValueError("max_parallel must be a positive integer") + if not isinstance(max_steps, int) or isinstance(max_steps, bool) or max_steps < 1: + raise ValueError("max_steps must be a positive integer") + if max_variants is not None and ( + not isinstance(max_variants, int) + or isinstance(max_variants, bool) + or max_variants < 1 + ): + raise ValueError("max_variants must be a positive integer when provided") + start_time = time.time() # Validate dataset path From 9fd6269a1cf51e9609df890b2261561fd6dd194b Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 15:30:42 +0800 Subject: [PATCH 002/140] test(cua-bench): prevent zero-permit benchmark hangs and invalid limits --- .../cua_bench/tests/test_run_benchmark.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/libs/cua-bench/cua_bench/tests/test_run_benchmark.py b/libs/cua-bench/cua_bench/tests/test_run_benchmark.py index 5e9f45a8fa..1d097dec6d 100644 --- a/libs/cua-bench/cua_bench/tests/test_run_benchmark.py +++ b/libs/cua-bench/cua_bench/tests/test_run_benchmark.py @@ -220,6 +220,28 @@ async def test_run_single_task_failure(self): assert result.error is not None +class TestBenchmarkParameterValidation: + """Parameter failures must surface before scheduling asynchronous workers.""" + + @pytest.mark.asyncio + @pytest.mark.parametrize("value", [0, -1, True, 1.5, "2"]) + async def test_reject_invalid_parallelism(self, tmp_path, value): + with pytest.raises(ValueError, match="max_parallel"): + await run_benchmark(tmp_path, max_parallel=value) + + @pytest.mark.asyncio + @pytest.mark.parametrize("value", [0, -1, False, 2.5]) + async def test_reject_invalid_step_limit(self, tmp_path, value): + with pytest.raises(ValueError, match="max_steps"): + await run_benchmark(tmp_path, max_steps=value) + + @pytest.mark.asyncio + @pytest.mark.parametrize("value", [0, -1, True, "3"]) + async def test_reject_invalid_variant_limit(self, tmp_path, value): + with pytest.raises(ValueError, match="max_variants"): + await run_benchmark(tmp_path, max_variants=value) + + class TestRunBenchmark: """Tests for run_benchmark function.""" From 697d179c5a184b174d53f417126d0d5d47b9b11b Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 15:31:39 +0800 Subject: [PATCH 003/140] docs(cua-bench): preserve parameter-validation defect and acceptance evidence --- libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md | 26 ++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md diff --git a/libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md b/libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md new file mode 100644 index 0000000000..6327d4ced3 --- /dev/null +++ b/libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md @@ -0,0 +1,26 @@ +# CUA Bench — 2026-10-09 engineering evidence + +Project: `hippoley/cua` (upstream `trycua/cua`), **not OpenLEADR**. + +## Baseline +- Relevant implementation: `libs/cua-bench/cua_bench/runners.py`, `run_benchmark`. +- Related tests: `libs/cua-bench/cua_bench/tests/test_run_benchmark.py`. +- Issue: `asyncio.Semaphore(max_parallel)` with zero permits never grants a worker slot when the dataset has runnable tasks, potentially hanging a benchmark indefinitely. + +## Change and cross-story audit +- Validate `max_parallel`, `max_steps`, and optional `max_variants` as positive integers at the API boundary; disallow Python booleans (subclass of int) and floats/strings. +- Add negative tests using pytest parametrization for zero, negative, boolean and wrong-type inputs. +- Cross-story impact: valid positive integer inputs keep their previous behavior. No external adapter, fixture or dataset schema modified. +- Dataset/benchmark truthfulness: this prevents the runner from silently hanging due to invalid configuration. It does **not** demonstrate improved agent task success, episode replay, sandbox compatibility, oracle correctness or runtime reproducibility. + +## Verification +- GitHub source fetch confirmed both changes on the feature branch. +- No repository-level pytest execution: execution container cannot resolve `github.com` for cloning and has no mounted CUA source. Absence of CI runs at commit inspection is not a pass. +- State: **CODE COMMITTED / TESTS NOT EXECUTED / NOT VERIFIED CLOSED**. + +## Next execution entry +```sh +cd libs/cua-bench +uv run pytest cua_bench/tests/test_run_benchmark.py -q +``` +Follow with positive real dataset runs on `cua-bench-basic`, a parallelism stress run, and evidence of termination and cleanup. Run from the branch `fix/cua-bench-validate-parallelism-20261009`. From 55999795483d944a9f14635f59d3afaad5c3554a Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 15:34:43 +0800 Subject: [PATCH 004/140] fix(cua-bench): fail closed on broken task discovery instead of inventing variants --- libs/cua-bench/cua_bench/runners.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/libs/cua-bench/cua_bench/runners.py b/libs/cua-bench/cua_bench/runners.py index ac33b50f89..204b9ed4bf 100644 --- a/libs/cua-bench/cua_bench/runners.py +++ b/libs/cua-bench/cua_bench/runners.py @@ -266,8 +266,16 @@ def random_agent(screenshot: bytes, task: Task) -> Action: variant_count = len(env.tasks_config_fn()) else: variant_count = 1 - except Exception: - variant_count = 1 + except Exception as exc: + raise ValueError( + f"Failed to load task variants for {task_path} (split={split!r}): {exc}" + ) from exc + + if variant_count < 1: + raise ValueError( + f"Task {task_path} (split={split!r}) has no variants; " + "refusing to report an incomplete benchmark" + ) if max_variants: variant_count = min(variant_count, max_variants) From ff1e8009e61b8d58d943f0bd73381c59b002d566 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 15:34:58 +0800 Subject: [PATCH 005/140] test(cua-bench): reject broken and empty task datasets before scoring --- .../cua_bench/tests/test_run_benchmark.py | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/libs/cua-bench/cua_bench/tests/test_run_benchmark.py b/libs/cua-bench/cua_bench/tests/test_run_benchmark.py index 1d097dec6d..f083acd34d 100644 --- a/libs/cua-bench/cua_bench/tests/test_run_benchmark.py +++ b/libs/cua-bench/cua_bench/tests/test_run_benchmark.py @@ -220,6 +220,36 @@ async def test_run_single_task_failure(self): assert result.error is not None +class TestBenchmarkDatasetDiscoveryIntegrity: + """Invalid task definitions must not be silently scored as one variant.""" + + @pytest.mark.asyncio + async def test_broken_task_configuration_aborts_benchmark(self, tmp_path): + task_dir = tmp_path / "broken-task" + task_dir.mkdir() + (task_dir / "main.py").write_text( + "import cua_bench as cb\n" + "@cb.tasks_config(split='train')\n" + "def get_tasks():\n" + " raise RuntimeError('broken fixture')\n" + ) + with pytest.raises(ValueError, match="Failed to load task variants"): + await run_benchmark(tmp_path) + + @pytest.mark.asyncio + async def test_zero_variants_aborts_instead_of_zero_denominator(self, tmp_path): + task_dir = tmp_path / "empty-task" + task_dir.mkdir() + (task_dir / "main.py").write_text( + "import cua_bench as cb\n" + "@cb.tasks_config(split='train')\n" + "def get_tasks():\n" + " return []\n" + ) + with pytest.raises(ValueError, match="has no variants"): + await run_benchmark(tmp_path) + + class TestBenchmarkParameterValidation: """Parameter failures must surface before scheduling asynchronous workers.""" From ee27e8c270a24e28478c74f7dcd16244fab367c1 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 15:35:16 +0800 Subject: [PATCH 006/140] docs(cua-bench): record fail-closed task discovery defect and acceptance gates --- libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md b/libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md index 6327d4ced3..43ab39a19f 100644 --- a/libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md +++ b/libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md @@ -24,3 +24,10 @@ cd libs/cua-bench uv run pytest cua_bench/tests/test_run_benchmark.py -q ``` Follow with positive real dataset runs on `cua-bench-basic`, a parallelism stress run, and evidence of termination and cleanup. Run from the branch `fix/cua-bench-validate-parallelism-20261009`. + +## Continuation: dataset discovery integrity +- **P0 failure mechanism:** `run_benchmark` previously swallowed all task-config exceptions and replaced the true variant count with `1`; an empty `tasks_config_fn()` produced zero runs and misleading empty metrics. Both violate truthful dataset denominators. +- **Code fix:** `runners.py` raises an informative ValueError containing task path and split on configuration errors and rejects zero variants, before scheduling any workers. Commit `55999795483d944a9f14635f59d3afaad5c3554a`. +- **Counterexamples authored:** `test_run_benchmark.py` creates a task whose config raises `RuntimeError` and another which returns zero tasks; both must now raise ValueError rather than fabricate results. Commit `ff1e8009e61b8d58d943f0bd73381c59b002d566`. +- **Status:** source committed and remote source verified; no pytest or end-to-end execution result witnessed in this round. Therefore NOT Verified Closed. +- **HCA:** function P, state P, integration B, security N/A (no new credentials/privileges), scale B, maintainability P, observation P, testability P, user value B, external compatibility N/A (pure benchmark orchestration). To close: run relevant pytest on actual branch, ensure valid dataset cases still pass, inspect run counts before and after, and run actual simulated provider end-to-end. From 24e815587c08a9a52014fa4bc5c5d5219db3493b Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 15:45:34 +0800 Subject: [PATCH 007/140] fix(cua-bench): reject invalid oracle scores and attribute worker failures to tasks --- libs/cua-bench/cua_bench/runners.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/libs/cua-bench/cua_bench/runners.py b/libs/cua-bench/cua_bench/runners.py index 204b9ed4bf..1a70df8e25 100644 --- a/libs/cua-bench/cua_bench/runners.py +++ b/libs/cua-bench/cua_bench/runners.py @@ -6,6 +6,7 @@ import asyncio import fnmatch +import math import time import uuid from dataclasses import dataclass @@ -143,6 +144,12 @@ def my_agent(screenshot: bytes, task: Task) -> Action: reward = float(result[0]) elif isinstance(result, dict) and "reward" in result: reward = float(result["reward"]) + else: + raise ValueError("Evaluator returned an unsupported reward format") + if not math.isfinite(reward) or not 0.0 <= reward <= 1.0: + raise ValueError(f"Evaluator reward must be finite and within [0, 1], got {reward!r}") + else: + raise ValueError("Task has no evaluator; cannot produce a verified score") return TaskResult( task_path=str(env_path), @@ -310,12 +317,12 @@ async def run_with_semaphore(task_path: Path, variant_id: int) -> TaskResult: task_results: List[Dict[str, Any]] = [] rewards: List[float] = [] - for result in results: - if isinstance(result, Exception): + for (task_path, variant_id), result in zip(task_variants, results): + if isinstance(result, BaseException): task_results.append( { - "task_path": "unknown", - "variant_id": -1, + "task_path": str(task_path), + "variant_id": variant_id, "success": False, "reward": 0.0, "steps": 0, From 558d201661f9a7dfcf92838103f8c2e35eea9567 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 15:45:48 +0800 Subject: [PATCH 008/140] test(cua-bench): challenge invalid and absent evaluator scores --- .../cua_bench/tests/test_run_benchmark.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/libs/cua-bench/cua_bench/tests/test_run_benchmark.py b/libs/cua-bench/cua_bench/tests/test_run_benchmark.py index f083acd34d..7f14c12770 100644 --- a/libs/cua-bench/cua_bench/tests/test_run_benchmark.py +++ b/libs/cua-bench/cua_bench/tests/test_run_benchmark.py @@ -220,6 +220,43 @@ async def test_run_single_task_failure(self): assert result.error is not None +class TestBenchmarkScoreIntegrity: + """Invalid evaluator output must never increase benchmark success.""" + + @pytest.mark.asyncio + @pytest.mark.parametrize("expression", ["float('nan')", "float('inf')", "-0.25", "1.25", "'not-a-score'"]) + async def test_reject_invalid_evaluator_result(self, tmp_path, expression): + task_dir = tmp_path / "invalid-reward" + task_dir.mkdir() + (task_dir / "main.py").write_text( + "import cua_bench as cb\n" + "@cb.tasks_config(split='train')\n" + "def get_tasks():\n" + " return [cb.Task(description='score test')]\n" + "@cb.evaluate_task(split='train')\n" + "async def evaluate(task, session):\n" + f" return {expression}\n" + ) + result = await run_single_task(task_dir) + assert result.success is False + assert result.reward == 0.0 + assert result.error is not None + + @pytest.mark.asyncio + async def test_missing_evaluator_never_claims_success(self, tmp_path): + task_dir = tmp_path / "missing-evaluator" + task_dir.mkdir() + (task_dir / "main.py").write_text( + "import cua_bench as cb\n" + "@cb.tasks_config(split='train')\n" + "def get_tasks():\n" + " return [cb.Task(description='no evaluator')]\n" + ) + result = await run_single_task(task_dir) + assert result.success is False + assert result.error is not None + + class TestBenchmarkDatasetDiscoveryIntegrity: """Invalid task definitions must not be silently scored as one variant.""" From edd98cfbadfdb4fc82cb736d23799a2b4691d14b Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 15:46:05 +0800 Subject: [PATCH 009/140] test(cua-bench): preserve failed worker identity and benchmark denominator --- .../cua_bench/tests/test_run_benchmark.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/libs/cua-bench/cua_bench/tests/test_run_benchmark.py b/libs/cua-bench/cua_bench/tests/test_run_benchmark.py index 7f14c12770..49496bb096 100644 --- a/libs/cua-bench/cua_bench/tests/test_run_benchmark.py +++ b/libs/cua-bench/cua_bench/tests/test_run_benchmark.py @@ -220,6 +220,40 @@ async def test_run_single_task_failure(self): assert result.error is not None +class TestBenchmarkAggregationIntegrity: + @pytest.mark.asyncio + async def test_worker_crash_retains_task_identity_and_denominator(self, tmp_path, monkeypatch): + from types import SimpleNamespace + import cua_bench.runners as runners + + for name in ("first", "second"): + task_dir = tmp_path / name + task_dir.mkdir() + (task_dir / "main.py").write_text("# discovery placeholder\n") + + monkeypatch.setattr( + runners, "make", + lambda path, split="train": SimpleNamespace(tasks_config_fn=lambda: [object()]), + ) + + async def fake_run(path, task_index=0, **kwargs): + if path.name == "first": + raise RuntimeError("worker exploded") + return runners.TaskResult(str(path), task_index, True, 1.0, 1) + + monkeypatch.setattr(runners, "run_single_task", fake_run) + result = await runners.run_benchmark(tmp_path, max_parallel=2) + + assert result.total_tasks == 2 + assert result.success_count == 1 + assert result.failed_count == 1 + assert result.avg_reward == 0.5 + assert result.task_results[0]["task_path"] == str(tmp_path / "first") + assert result.task_results[0]["variant_id"] == 0 + assert "worker exploded" in result.task_results[0]["error"] + assert result.task_results[1]["task_path"] == str(tmp_path / "second") + + class TestBenchmarkScoreIntegrity: """Invalid evaluator output must never increase benchmark success.""" From f121f92616da89d2c9fe161ff1224c82ac8ffb85 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 15:46:29 +0800 Subject: [PATCH 010/140] docs(cua-bench): record score integrity and worker provenance verification gates --- libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md b/libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md index 43ab39a19f..4bcdee2d9a 100644 --- a/libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md +++ b/libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md @@ -31,3 +31,11 @@ Follow with positive real dataset runs on `cua-bench-basic`, a parallelism stres - **Counterexamples authored:** `test_run_benchmark.py` creates a task whose config raises `RuntimeError` and another which returns zero tasks; both must now raise ValueError rather than fabricate results. Commit `ff1e8009e61b8d58d943f0bd73381c59b002d566`. - **Status:** source committed and remote source verified; no pytest or end-to-end execution result witnessed in this round. Therefore NOT Verified Closed. - **HCA:** function P, state P, integration B, security N/A (no new credentials/privileges), scale B, maintainability P, observation P, testability P, user value B, external compatibility N/A (pure benchmark orchestration). To close: run relevant pytest on actual branch, ensure valid dataset cases still pass, inspect run counts before and after, and run actual simulated provider end-to-end. + +## Score and aggregation integrity follow-up (2026-10-09) + +- **P0 score integrity:** `run_single_task` now rejects unsupported evaluator outputs, nonfinite values and rewards outside the normalized [0,1] contract. Missing evaluators no longer return a nominal score without an explicit error. Commit `24e815587c08a9a52014fa4bc5c5d5219db3493b`. +- **P0 failure provenance:** `run_benchmark` retains task path and variant ID for gathered worker exceptions, while accounting for failure in total_tasks and mean reward denominator. Commit `24e815587c08a9a52014fa4bc5c5d5219db3493b`. +- **Adversarial tests added:** malformed evaluator output (NaN, Infinity, negative, >1, nonnumeric), missing evaluator and two-task exception aggregation retaining both run identities and expected 1/2 success fraction. Commits `558d201661f9a7dfcf92838103f8c2e35eea9567`, `edd98cfbadfdb4fc82cb736d23799a2b4691d14b`. +- **Evidence status:** remote GitHub fetch confirms both code and tests. No local pytest execution or browser/sandbox E2E was witnessed; no User Story may be called Verified Closed. +- **HCA:** functional P; task/result state P; integration B; security NA (no new trust boundary); performance B; maintainability P; traceability P; testability P; end-user value B; external compatibility B. Challenge the normalized [0,1] reward assumption against each existing dataset/oracle before treating it as ecosystem-compatible. Regression suite and live sandbox tests remain mandatory. From 614e8ad561583e14c99f8f28107c4008398c1aab Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 16:34:02 +0800 Subject: [PATCH 011/140] fix(cua-bench): prevent setup-only tasks being scored as agent successes --- libs/cua-bench/cua_bench/runners.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/libs/cua-bench/cua_bench/runners.py b/libs/cua-bench/cua_bench/runners.py index 1a70df8e25..dcdf12ee79 100644 --- a/libs/cua-bench/cua_bench/runners.py +++ b/libs/cua-bench/cua_bench/runners.py @@ -135,6 +135,17 @@ def my_agent(screenshot: bytes, task: Task) -> Action: step_count += 1 done = isinstance(action, DoneAction) + # A setup-only run must never be counted as successful agent execution. + if not oracle and agent_fn is None: + return TaskResult( + task_path=str(env_path), + variant_id=task_index, + success=False, + reward=0.0, + steps=step_count, + error="No agent_fn supplied; task was not executed", + ) + # Evaluate if env.evaluate_task_fn is not None: result = await env.evaluate() From 6c3e54561474b67572f5598e410f3b008705bf42 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 16:34:19 +0800 Subject: [PATCH 012/140] test(cua-bench): reject pre-satisfied evaluator without agent execution --- .../cua_bench/tests/test_run_benchmark.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/libs/cua-bench/cua_bench/tests/test_run_benchmark.py b/libs/cua-bench/cua_bench/tests/test_run_benchmark.py index 49496bb096..9c271edceb 100644 --- a/libs/cua-bench/cua_bench/tests/test_run_benchmark.py +++ b/libs/cua-bench/cua_bench/tests/test_run_benchmark.py @@ -220,6 +220,33 @@ async def test_run_single_task_failure(self): assert result.error is not None +class TestNoAgentExecutionEvidence: + @pytest.mark.asyncio + async def test_pre_satisfied_task_without_agent_cannot_pass(self, tmp_path, monkeypatch): + import cua_bench.runners as runners + + class PresatisfiedEnvironment: + solve_task_fn = None + evaluate_task_fn = object() + max_steps = 100 + + async def reset(self, task_id=0): + return b"screenshot", object() + + async def evaluate(self): + raise AssertionError("must not invoke evaluator for unexecuted agent") + + async def close(self): + return None + + monkeypatch.setattr(runners, "make", lambda path, split="train": PresatisfiedEnvironment()) + result = await runners.run_single_task(tmp_path, oracle=False, agent_fn=None) + assert result.success is False + assert result.reward == 0.0 + assert result.steps == 0 + assert "not executed" in result.error + + class TestBenchmarkAggregationIntegrity: @pytest.mark.asyncio async def test_worker_crash_retains_task_identity_and_denominator(self, tmp_path, monkeypatch): From 455acf6e5bfdcc6d79f5776a35899ae3b8946987 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 16:34:50 +0800 Subject: [PATCH 013/140] docs(cua-bench): capture no-agent scoring counterexample and verification gap --- libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md b/libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md index 4bcdee2d9a..fd09759fa3 100644 --- a/libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md +++ b/libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md @@ -39,3 +39,10 @@ Follow with positive real dataset runs on `cua-bench-basic`, a parallelism stres - **Adversarial tests added:** malformed evaluator output (NaN, Infinity, negative, >1, nonnumeric), missing evaluator and two-task exception aggregation retaining both run identities and expected 1/2 success fraction. Commits `558d201661f9a7dfcf92838103f8c2e35eea9567`, `edd98cfbadfdb4fc82cb736d23799a2b4691d14b`. - **Evidence status:** remote GitHub fetch confirms both code and tests. No local pytest execution or browser/sandbox E2E was witnessed; no User Story may be called Verified Closed. - **HCA:** functional P; task/result state P; integration B; security NA (no new trust boundary); performance B; maintainability P; traceability P; testability P; end-user value B; external compatibility B. Challenge the normalized [0,1] reward assumption against each existing dataset/oracle before treating it as ecosystem-compatible. Regression suite and live sandbox tests remain mandatory. + +## Follow-up — no-agent false-positive guard (2026-10-09) +- **P0 finding:** `run_single_task(oracle=False, agent_fn=None)` previously reset the environment, skipped agent actions, then evaluated. A pre-satisfied environment could be reported as successful despite zero agent actions. +- **Code mitigation:** explicit failure `No agent_fn supplied; task was not executed`, with reward 0 and success False; no evaluator is invoked on this path. Commit `614e8ad561583e14c99f8f28107c4008398c1aab`. +- **Independent counterexample:** mock evaluator that raises if called, confirming absence of agent actions never earns score. Commit `6c3e54561474b67572f5598e410f3b008705bf42`. +- **Verification state:** remote code and tests present. Runtime container has Python and pytest but not full CUA repository sources at the relevant branch, so repository-level pytest and real simulated-provider tests were NOT executed. No Verified Closed. +- **Next P0:** execute `cd libs/cua-bench && uv run pytest cua_bench/tests/test_run_benchmark.py -q` on this branch; verify no existing legitimate no-agent setup-only callers rely on scoring; run an independent simulated provider case with initial evaluator reward 1.0 but no actions and assert failure. From 34772d45bfe89e133fb766c46e6e3c931eb70210 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 16:36:09 +0800 Subject: [PATCH 014/140] fix(cua-bench): validate worker result identity and score consistency before aggregation --- libs/cua-bench/cua_bench/runners.py | 33 +++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/libs/cua-bench/cua_bench/runners.py b/libs/cua-bench/cua_bench/runners.py index dcdf12ee79..77a0be3d32 100644 --- a/libs/cua-bench/cua_bench/runners.py +++ b/libs/cua-bench/cua_bench/runners.py @@ -342,10 +342,39 @@ async def run_with_semaphore(task_path: Path, variant_id: int) -> TaskResult: ) rewards.append(0.0) else: + # Validate worker provenance against the scheduled task before scoring. + invalid = ( + not isinstance(result, TaskResult) + or result.task_path != str(task_path) + or result.variant_id != variant_id + or type(result.success) is not bool + or not isinstance(result.reward, (int, float)) + or isinstance(result.reward, bool) + or not math.isfinite(result.reward) + or not 0.0 <= result.reward <= 1.0 + or not isinstance(result.steps, int) + or isinstance(result.steps, bool) + or result.steps < 0 + or (bool(result.error) and result.success) + or (not result.error and result.success != (result.reward >= 0.5)) + ) + if invalid: + task_results.append( + { + "task_path": str(task_path), + "variant_id": variant_id, + "success": False, + "reward": 0.0, + "steps": 0, + "error": "Invalid or misattributed worker result", + } + ) + rewards.append(0.0) + continue task_results.append( { - "task_path": result.task_path, - "variant_id": result.variant_id, + "task_path": str(task_path), + "variant_id": variant_id, "success": result.success, "reward": result.reward, "steps": result.steps, From fbf83e1aa44660a5bf4b7f2cecb2711dcdf74fec Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 16:36:25 +0800 Subject: [PATCH 015/140] test(cua-bench): reject fabricated worker identity and reward in benchmark metrics --- .../cua_bench/tests/test_run_benchmark.py | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/libs/cua-bench/cua_bench/tests/test_run_benchmark.py b/libs/cua-bench/cua_bench/tests/test_run_benchmark.py index 9c271edceb..32c41da5f9 100644 --- a/libs/cua-bench/cua_bench/tests/test_run_benchmark.py +++ b/libs/cua-bench/cua_bench/tests/test_run_benchmark.py @@ -247,6 +247,44 @@ async def close(self): assert "not executed" in result.error +class TestWorkerResultProvenance: + @pytest.mark.asyncio + @pytest.mark.parametrize("fault", ["wrong_path", "wrong_variant", "nan_reward", "contradictory_success"]) + async def test_forged_worker_result_cannot_inflate_score(self, tmp_path, monkeypatch, fault): + from types import SimpleNamespace + import cua_bench.runners as runners + + task_dir = tmp_path / "one" + task_dir.mkdir() + (task_dir / "main.py").write_text("# stub\n") + monkeypatch.setattr( + runners, "make", + lambda path, split="train": SimpleNamespace(tasks_config_fn=lambda: [object()]), + ) + + async def forged_result(path, task_index=0, **kwargs): + result = runners.TaskResult(str(path), task_index, True, 1.0, 1) + if fault == "wrong_path": + result.task_path = "/other/task" + elif fault == "wrong_variant": + result.variant_id = 123 + elif fault == "nan_reward": + result.reward = float("nan") + else: + result.success = False + return result + + monkeypatch.setattr(runners, "run_single_task", forged_result) + result = await runners.run_benchmark(tmp_path, max_parallel=1) + assert result.total_tasks == 1 + assert result.success_count == 0 + assert result.failed_count == 1 + assert result.avg_reward == 0.0 + assert result.task_results[0]["task_path"] == str(task_dir) + assert result.task_results[0]["variant_id"] == 0 + assert "Invalid or misattributed" in result.task_results[0]["error"] + + class TestBenchmarkAggregationIntegrity: @pytest.mark.asyncio async def test_worker_crash_retains_task_identity_and_denominator(self, tmp_path, monkeypatch): From 4b46ac851ecc0c39aa4e19c6ac0f469ab000e717 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 16:36:37 +0800 Subject: [PATCH 016/140] fix(cua-bench): prohibit failed worker rewards inflating mean scores --- libs/cua-bench/cua_bench/runners.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/cua-bench/cua_bench/runners.py b/libs/cua-bench/cua_bench/runners.py index 77a0be3d32..9bb36732b5 100644 --- a/libs/cua-bench/cua_bench/runners.py +++ b/libs/cua-bench/cua_bench/runners.py @@ -355,7 +355,7 @@ async def run_with_semaphore(task_path: Path, variant_id: int) -> TaskResult: or not isinstance(result.steps, int) or isinstance(result.steps, bool) or result.steps < 0 - or (bool(result.error) and result.success) + or (bool(result.error) and (result.success or result.reward != 0.0)) or (not result.error and result.success != (result.reward >= 0.5)) ) if invalid: From f6de5619342e6451a5a475489d4c3ed4e2138668 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 16:36:55 +0800 Subject: [PATCH 017/140] docs(cua-bench): trace worker result integrity change and acceptance limits --- libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md b/libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md index fd09759fa3..dddabf6ddb 100644 --- a/libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md +++ b/libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md @@ -46,3 +46,9 @@ Follow with positive real dataset runs on `cua-bench-basic`, a parallelism stres - **Independent counterexample:** mock evaluator that raises if called, confirming absence of agent actions never earns score. Commit `6c3e54561474b67572f5598e410f3b008705bf42`. - **Verification state:** remote code and tests present. Runtime container has Python and pytest but not full CUA repository sources at the relevant branch, so repository-level pytest and real simulated-provider tests were NOT executed. No Verified Closed. - **Next P0:** execute `cd libs/cua-bench && uv run pytest cua_bench/tests/test_run_benchmark.py -q` on this branch; verify no existing legitimate no-agent setup-only callers rely on scoring; run an independent simulated provider case with initial evaluator reward 1.0 but no actions and assert failure. + +## Follow-up: worker-output integrity and benchmark denominator (2026-10-09) +- P0: `run_benchmark` now validates each returned worker result against the scheduled task path and variant ID, requires finite normalized reward, sane step count, coherent success/reward semantics, and zero reward on worker errors. Invalid results become attributable failures with reward 0 rather than polluting success rate. Commits `34772d45bfe89e133fb766c46e6e3c931eb70210`, `4b46ac851ecc0c39aa4e19c6ac0f469ab000e717`. +- Counterexamples added for forged path, forged variant, NaN reward and contradictory success/reward: commit `fbf83e1aa44660a5bf4b7f2cecb2711dcdf74fec`. +- Source changes committed; no fresh pytest/CI execution or real sandbox E2E evidenced. Validation requires running `uv run pytest cua_bench/tests/test_run_benchmark.py -q` from `libs/cua-bench`, followed by dataset-level evaluation and independent oracle replay. +- Horizontal status: Function P, state P, integration B, security P, performance B, maintainability P, observability P, testability P, user value B, external compatibility B. No Verified Closed. From a3bfd7c23f5e1866007cb85419d8a2d00e4ef31b Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:01:47 +0800 Subject: [PATCH 018/140] ci(cua-bench): enable branch and manual regression runs for proof collection --- .github/workflows/ci-py-bench.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/ci-py-bench.yml b/.github/workflows/ci-py-bench.yml index 58c7120f1c..edbb0d7c13 100644 --- a/.github/workflows/ci-py-bench.yml +++ b/.github/workflows/ci-py-bench.yml @@ -1,6 +1,14 @@ name: "CI: cua-bench" on: + push: + branches: + - fix/cua-bench-validate-parallelism-20261009 + paths: + - "libs/cua-bench/**" + - ".github/workflows/ci-py-bench.yml" + - ".github/workflows/py-reusable-build.yml" + workflow_dispatch: pull_request: paths: - "libs/cua-bench/**" From d1fc1e039228dcc20e04e73ec55546e7dea16651 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:03:05 +0800 Subject: [PATCH 019/140] ci(cua-bench): execute runner regressions on supported Python 3.12 --- .github/workflows/ci-py-bench.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/ci-py-bench.yml b/.github/workflows/ci-py-bench.yml index edbb0d7c13..844f5f54a6 100644 --- a/.github/workflows/ci-py-bench.yml +++ b/.github/workflows/ci-py-bench.yml @@ -21,3 +21,16 @@ jobs: with: package_name: "bench" package_dir: "libs/cua-bench" + + regression: + runs-on: ubuntu-latest + timeout-minutes: 35 + steps: + - uses: actions/checkout@v4 + - name: Set up uv and Python 3.12 + uses: astral-sh/setup-uv@v6 + with: + python-version: "3.12" + - name: Run CUA benchmark regression suite + working-directory: libs/cua-bench + run: uv run --extra dev pytest cua_bench/tests/test_run_benchmark.py -q From e0e669c56fe037a3897689616bae5c14ed78f1b0 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:03:17 +0800 Subject: [PATCH 020/140] ci(cua-bench): run complete benchmark test suite rather than runner subset --- .github/workflows/ci-py-bench.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-py-bench.yml b/.github/workflows/ci-py-bench.yml index 844f5f54a6..0d46ebc4c4 100644 --- a/.github/workflows/ci-py-bench.yml +++ b/.github/workflows/ci-py-bench.yml @@ -33,4 +33,4 @@ jobs: python-version: "3.12" - name: Run CUA benchmark regression suite working-directory: libs/cua-bench - run: uv run --extra dev pytest cua_bench/tests/test_run_benchmark.py -q + run: uv run --extra dev pytest cua_bench/tests -q From 42cec9df2e3778b256febdfa866c9d28ac1f1c03 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:04:28 +0800 Subject: [PATCH 021/140] ci(cua-bench): separate core oracle regression from full RL dependency suite --- .github/workflows/ci-py-bench.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-py-bench.yml b/.github/workflows/ci-py-bench.yml index 0d46ebc4c4..2340d2c96f 100644 --- a/.github/workflows/ci-py-bench.yml +++ b/.github/workflows/ci-py-bench.yml @@ -31,6 +31,13 @@ jobs: uses: astral-sh/setup-uv@v6 with: python-version: "3.12" - - name: Run CUA benchmark regression suite + - name: Run CUA benchmark core and oracle regression suite working-directory: libs/cua-bench - run: uv run --extra dev pytest cua_bench/tests -q + run: | + uv run --extra dev pytest \ + cua_bench/tests/test_run_benchmark.py \ + cua_bench/tests/test_gym_interface.py \ + cua_bench/tests/test_actions.py -q + - name: Run full benchmark suite including RL workers + working-directory: libs/cua-bench + run: uv run --extra dev --extra rl pytest cua_bench/tests -q From 4a5e91f50733c7a175f00b375a5fb5fbb5aaaee4 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:05:46 +0800 Subject: [PATCH 022/140] ci(cua-bench): provision Chromium before simulated desktop Oracle E2E --- .github/workflows/ci-py-bench.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci-py-bench.yml b/.github/workflows/ci-py-bench.yml index 2340d2c96f..a91163e6cc 100644 --- a/.github/workflows/ci-py-bench.yml +++ b/.github/workflows/ci-py-bench.yml @@ -31,6 +31,9 @@ jobs: uses: astral-sh/setup-uv@v6 with: python-version: "3.12" + - name: Install Playwright Chromium for simulated desktop E2E + working-directory: libs/cua-bench + run: uv run --extra dev python -m playwright install --with-deps chromium - name: Run CUA benchmark core and oracle regression suite working-directory: libs/cua-bench run: | From 55203190fe26282224a417dd22960de3c077f23e Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:07:08 +0800 Subject: [PATCH 023/140] ci(cua-bench): run headed browser E2E under virtual X display --- .github/workflows/ci-py-bench.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-py-bench.yml b/.github/workflows/ci-py-bench.yml index a91163e6cc..9fd5264406 100644 --- a/.github/workflows/ci-py-bench.yml +++ b/.github/workflows/ci-py-bench.yml @@ -37,7 +37,7 @@ jobs: - name: Run CUA benchmark core and oracle regression suite working-directory: libs/cua-bench run: | - uv run --extra dev pytest \ + xvfb-run -a uv run --extra dev pytest \ cua_bench/tests/test_run_benchmark.py \ cua_bench/tests/test_gym_interface.py \ cua_bench/tests/test_actions.py -q From 394ef2c1a2995c2983455dec43dd6d0a8862d98c Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:08:10 +0800 Subject: [PATCH 024/140] docs(cua-bench): record real CI failures and honest E2E proof boundaries --- libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md b/libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md index dddabf6ddb..92e3080eb2 100644 --- a/libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md +++ b/libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md @@ -52,3 +52,11 @@ Follow with positive real dataset runs on `cua-bench-basic`, a parallelism stres - Counterexamples added for forged path, forged variant, NaN reward and contradictory success/reward: commit `fbf83e1aa44660a5bf4b7f2cecb2711dcdf74fec`. - Source changes committed; no fresh pytest/CI execution or real sandbox E2E evidenced. Validation requires running `uv run pytest cua_bench/tests/test_run_benchmark.py -q` from `libs/cua-bench`, followed by dataset-level evaluation and independent oracle replay. - Horizontal status: Function P, state P, integration B, security P, performance B, maintainability P, observability P, testability P, user value B, external compatibility B. No Verified Closed. + +## Live CI evidence 2026-10-09 +- GitHub run 37908647343 passed prior build-only workflow, not pytest. +- Run 37908806518: full-suite collection FAILED (3 errors importing optional `torch` from worker dataloader). No test pass may be inferred from collection failure. +- Run 37908930187: core suite produced **79 passed, 11 failed**; all 11 browser failures cited missing Chromium/Playwright executables. CI installed browser dependencies subsequently. +- Run 37909070828: core suite produced **89 passed, 1 failed**; only headed-browser interactive E2E failed because runner lacked X Server. Browser/Oracle noninteractive tests executed. Commit 55203190fe26282224a417dd22960de3c077f23e uses `xvfb-run -a` for headed test, with run 37909217474 queued/in progress when recorded. Do not treat it as passed until reviewed. +- Local offline CUA contract and Outcome Witness standalone bundles: **27 passed in 0.06s**; unrelated to full `cua_bench/tests` run and not interchangeable evidence. +- Still OPEN P0: full RL worker suite must run with optional `rl` dependencies and survive real execution; fault injection proving process/resource cleanup in dedicated sandbox; independent Oracle provenance and action/outcome witness against actual scene. No Verified Closed issued. From 570033a22dc5b3fc4fbdddb70335cfdd6257de76 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:13:37 +0800 Subject: [PATCH 025/140] ci(cua-bench): install worker server extra for complete regression collection --- .github/workflows/ci-py-bench.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-py-bench.yml b/.github/workflows/ci-py-bench.yml index 9fd5264406..e31c5ca6ee 100644 --- a/.github/workflows/ci-py-bench.yml +++ b/.github/workflows/ci-py-bench.yml @@ -43,4 +43,4 @@ jobs: cua_bench/tests/test_actions.py -q - name: Run full benchmark suite including RL workers working-directory: libs/cua-bench - run: uv run --extra dev --extra rl pytest cua_bench/tests -q + run: uv run --extra dev --extra rl --extra server pytest cua_bench/tests -q From 808a3bbb48d1c8e8f31bddf1be03b6b679ab33d2 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:15:10 +0800 Subject: [PATCH 026/140] docs(cua-bench): establish falsifiable witness infrastructure thesis and P0-P1 acceptance gates --- libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md | 27 ++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md b/libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md index 92e3080eb2..30425592c9 100644 --- a/libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md +++ b/libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md @@ -60,3 +60,30 @@ Follow with positive real dataset runs on `cua-bench-basic`, a parallelism stres - Run 37909070828: core suite produced **89 passed, 1 failed**; only headed-browser interactive E2E failed because runner lacked X Server. Browser/Oracle noninteractive tests executed. Commit 55203190fe26282224a417dd22960de3c077f23e uses `xvfb-run -a` for headed test, with run 37909217474 queued/in progress when recorded. Do not treat it as passed until reviewed. - Local offline CUA contract and Outcome Witness standalone bundles: **27 passed in 0.06s**; unrelated to full `cua_bench/tests` run and not interchangeable evidence. - Still OPEN P0: full RL worker suite must run with optional `rl` dependencies and survive real execution; fault injection proving process/resource cleanup in dedicated sandbox; independent Oracle provenance and action/outcome witness against actual scene. No Verified Closed issued. + +## Infrastructure thesis and stop/go gates — 2026-10-09 + +**Current hypothesis, not an adoption claim:** Build a reusable, provider-neutral CUA Execution Witness over existing agent action traces and evaluator outputs. Do not create a separate leaderboard or a competing runtime until direct users demonstrate a missing interface. OSWorld-V2.1's pinning of task/assets/website/provider images is a relevant mature precedent (https://github.com/xlang-ai/OSWorld-V2); match its provenance discipline, not copy its benchmark dataset. + +### Audit of traceable vertical-to-horizontal gaps +| Existing requirement path | P0 defect/gap | Horizontal impact | Mandatory independent receipt | +| --- | --- | --- | --- | +| Task discovery -> scheduling | exceptions, zero variants and invalid concurrency were previously silent/hanging | denominator and task-identity errors | executed pytest plus dataset manifest/hash | +| Action -> environment | no-agent scoring risk | false positive independent of actual action | action count and environment before/after evidence | +| Environment -> oracle | nonfinite scores, unversioned evaluator | score provenance cannot be independently repeated | oracle ID/version, inputs and independent rerun | +| Worker -> aggregation | worker identity mismatch and dropped exceptions | false-success and lost failure provenance | schedule/return identity matching and raw failure list | +| Runtime -> cleanup | no independent teardown inventory or crash injection receipt | state leak, nonreproducibility | force stop, live resource enumeration, cleanup=0 | +| Outcome -> third-party use | no stable witness schema, consumer SDK or external review | no downstream dependency proof | independent package consumer, issue/PR and reviewer confirmation | + +### Candidate implementation contract (not yet implemented) +`WitnessV0` should bind `run_id`, `task_id`, `variant_id`, `dataset_release`, `environment/provider/image_digest`, `agent_version`, `action_trace_digest`, `before_state_digest`, `after_state_digest`, `oracle_id/version`, `oracle_result`, `cleanup_state`, and `result_status`. Sensitive screenshots, tokens and personal data must remain optional protected artifacts, with hashes and redacted metadata sufficient to identify a replay target. An individual SHA-256 hash is integrity metadata, NOT a signature, origin attestation or proof of a true observation. + +### Prioritization and falsifiable next gates +- **Gate A, P0:** Python 3.12 benchmark core/oracle suite passes in real browser/Xvfb; full RL worker suite collects and runs with declared extras. Report number of tests, all failures and CI run/commit. +- **Gate B, P0:** intentional process interruption during a disposable CUA VM task, independent observation of remaining processes/containers, repeatable zero-resource cleanup. Not yet executed. +- **Gate C, P1:** independent evaluator reads raw action and state evidence to reproduce at least one task result; reject forged reward, edited task path, missing actions and mismatched image versions. +- **Gate D, P1:** a second repository consumes the versioned witness contract without fork-local imports and publishes a reproducible report. +- **Stop/pivot condition:** if an external adopter cannot name a contract deficiency after two genuine integrations, prefer upstream conformance contributions rather than branding another infra repo. If external feedback shows existing OSWorld tools solve the same job at lower cost, re-use them and kill redundant implementation. + +### Anti-claims +No verified external adoption, certification, real VM cleanup receipt or long-term defensibility proven yet. Evidence artifacts and test coverage are not themselves network effects. From 67515ebf362d370aefd609cb11a0f18c916e7c45 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:15:48 +0800 Subject: [PATCH 027/140] ci(cua-bench): provision Starlette testclient httpx for worker suite --- .github/workflows/ci-py-bench.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-py-bench.yml b/.github/workflows/ci-py-bench.yml index e31c5ca6ee..14b0c09c69 100644 --- a/.github/workflows/ci-py-bench.yml +++ b/.github/workflows/ci-py-bench.yml @@ -43,4 +43,4 @@ jobs: cua_bench/tests/test_actions.py -q - name: Run full benchmark suite including RL workers working-directory: libs/cua-bench - run: uv run --extra dev --extra rl --extra server pytest cua_bench/tests -q + run: uv run --extra dev --extra rl --extra server --with 'httpx>=0.27,<1' pytest cua_bench/tests -q From dc3aefab3180db39419b65f6bec2fce1375765d0 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:27:41 +0800 Subject: [PATCH 028/140] ci(cua-bench): run full RL worker regression under virtual display --- .github/workflows/ci-py-bench.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-py-bench.yml b/.github/workflows/ci-py-bench.yml index 14b0c09c69..5d105a9d65 100644 --- a/.github/workflows/ci-py-bench.yml +++ b/.github/workflows/ci-py-bench.yml @@ -43,4 +43,4 @@ jobs: cua_bench/tests/test_actions.py -q - name: Run full benchmark suite including RL workers working-directory: libs/cua-bench - run: uv run --extra dev --extra rl --extra server --with 'httpx>=0.27,<1' pytest cua_bench/tests -q + run: xvfb-run -a uv run --extra dev --extra rl --extra server --with 'httpx>=0.27,<1' pytest cua_bench/tests -q From 32c1d3828d8e04a0fb601d4bd3b92ffd5e06f9c7 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:32:36 +0800 Subject: [PATCH 029/140] ci(cua-bench): integrate MIT pytest-json-report and publish individual test evidence --- .github/workflows/ci-py-bench.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-py-bench.yml b/.github/workflows/ci-py-bench.yml index 5d105a9d65..6d0e398f76 100644 --- a/.github/workflows/ci-py-bench.yml +++ b/.github/workflows/ci-py-bench.yml @@ -37,10 +37,20 @@ jobs: - name: Run CUA benchmark core and oracle regression suite working-directory: libs/cua-bench run: | - xvfb-run -a uv run --extra dev pytest \ + xvfb-run -a uv run --extra dev --with 'pytest-json-report>=1.5,<2' pytest --json-report --json-report-file=core-report.json --json-report-omit=streams \ cua_bench/tests/test_run_benchmark.py \ cua_bench/tests/test_gym_interface.py \ cua_bench/tests/test_actions.py -q - name: Run full benchmark suite including RL workers working-directory: libs/cua-bench - run: xvfb-run -a uv run --extra dev --extra rl --extra server --with 'httpx>=0.27,<1' pytest cua_bench/tests -q + run: xvfb-run -a uv run --extra dev --extra rl --extra server --with 'httpx>=0.27,<1' --with 'pytest-json-report>=1.5,<2' pytest --json-report --json-report-file=full-report.json --json-report-omit=streams cua_bench/tests -q + - name: Preserve CUA benchmark machine-readable evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: cua-bench-pytest-evidence + path: | + libs/cua-bench/core-report.json + libs/cua-bench/full-report.json + if-no-files-found: warn + retention-days: 30 From 2b1d13d88e5ddd0d04c005591617f0520abdf53d Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:33:08 +0800 Subject: [PATCH 030/140] feat(cua-bench): fail closed on incomplete or forged pytest report structure --- .../scripts/verify_pytest_receipt.py | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 libs/cua-bench/scripts/verify_pytest_receipt.py diff --git a/libs/cua-bench/scripts/verify_pytest_receipt.py b/libs/cua-bench/scripts/verify_pytest_receipt.py new file mode 100644 index 0000000000..001aedf761 --- /dev/null +++ b/libs/cua-bench/scripts/verify_pytest_receipt.py @@ -0,0 +1,48 @@ +"""Validate pytest-json-report output without trusting a green CI label alone. + +This is a structural receipt check, not independent task-oracle attestation. +""" +import argparse +import json +from pathlib import Path + + +def verify_report(doc): + if not isinstance(doc, dict): + raise ValueError("report must be an object") + tests = doc.get("tests") + summary = doc.get("summary") + if not isinstance(tests, list) or not tests: + raise ValueError("no executed test records") + if not isinstance(summary, dict): + raise ValueError("missing summary") + if type(doc.get("exitcode")) is not int or doc["exitcode"] != 0: + raise ValueError("pytest did not exit successfully") + nodeids = [x.get("nodeid") for x in tests if isinstance(x, dict)] + if len(nodeids) != len(tests) or any(not isinstance(x, str) or not x for x in nodeids): + raise ValueError("invalid test identity") + if len(nodeids) != len(set(nodeids)): + raise ValueError("duplicate test identities") + if any(x.get("outcome") != "passed" for x in tests): + raise ValueError("not all test outcomes passed") + if any(summary.get(k, 0) for k in ("failed", "error", "skipped", "xfailed", "xpassed", "deselected")): + raise ValueError("report contains non-passing or unexecuted tests") + if summary.get("passed") != len(tests) or summary.get("total") != len(tests): + raise ValueError("reported totals disagree with test records") + if summary.get("collected", len(tests)) != len(tests): + raise ValueError("collection count disagrees with test records") + if any(x.get("outcome") == "failed" for x in doc.get("collectors", []) if isinstance(x, dict)): + raise ValueError("collection failure") + return {"passed": len(tests), "distinct_tests": len(nodeids), "report_valid": True} + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("reports", nargs="+", type=Path) + args = parser.parse_args() + for path in args.reports: + print(json.dumps({"file": str(path), **verify_report(json.loads(path.read_text()))})) + + +if __name__ == "__main__": + main() From 3db6d69c0ff3c001bd6553cb334aaa8fcf9001ff Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:33:23 +0800 Subject: [PATCH 031/140] test(cua-bench): independently reject forged duplicate and incomplete receipt reports --- .../cua_bench/tests/test_pytest_receipt.py | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 libs/cua-bench/cua_bench/tests/test_pytest_receipt.py diff --git a/libs/cua-bench/cua_bench/tests/test_pytest_receipt.py b/libs/cua-bench/cua_bench/tests/test_pytest_receipt.py new file mode 100644 index 0000000000..3889509c6e --- /dev/null +++ b/libs/cua-bench/cua_bench/tests/test_pytest_receipt.py @@ -0,0 +1,59 @@ +"""Independent negative tests for the pytest-json-report receipt verifier.""" +import importlib.util +from pathlib import Path +import unittest + +source = Path(__file__).resolve().parents[2] / "scripts" / "verify_pytest_receipt.py" +spec = importlib.util.spec_from_file_location("verify_pytest_receipt", source) +mod = importlib.util.module_from_spec(spec) +spec.loader.exec_module(mod) + + +class ReceiptTests(unittest.TestCase): + def valid(self): + return { + "exitcode": 0, + "summary": {"collected": 2, "total": 2, "passed": 2}, + "tests": [ + {"nodeid": "a.py::test_one", "outcome": "passed"}, + {"nodeid": "a.py::test_two", "outcome": "passed"}, + ], + "collectors": [], + } + + def test_valid(self): + self.assertEqual(mod.verify_report(self.valid())["passed"], 2) + + def test_no_tests(self): + doc = self.valid() + doc["tests"] = [] + with self.assertRaises(ValueError): + mod.verify_report(doc) + + def test_forged_duplicate_identity(self): + doc = self.valid() + doc["tests"][1]["nodeid"] = doc["tests"][0]["nodeid"] + with self.assertRaises(ValueError): + mod.verify_report(doc) + + def test_hidden_skips(self): + doc = self.valid() + doc["summary"]["skipped"] = 1 + with self.assertRaises(ValueError): + mod.verify_report(doc) + + def test_false_green_exit_code(self): + doc = self.valid() + doc["exitcode"] = 1 + with self.assertRaises(ValueError): + mod.verify_report(doc) + + def test_missing_case_with_false_summary(self): + doc = self.valid() + doc["tests"].pop() + with self.assertRaises(ValueError): + mod.verify_report(doc) + + +if __name__ == "__main__": + unittest.main() From 597ebf8486e2fefc235c38fc3642be6bf0f23d2d Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:33:35 +0800 Subject: [PATCH 032/140] ci(cua-bench): verify JSON receipt structure before accepting benchmark regression --- .github/workflows/ci-py-bench.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci-py-bench.yml b/.github/workflows/ci-py-bench.yml index 6d0e398f76..b7da37a373 100644 --- a/.github/workflows/ci-py-bench.yml +++ b/.github/workflows/ci-py-bench.yml @@ -44,6 +44,9 @@ jobs: - name: Run full benchmark suite including RL workers working-directory: libs/cua-bench run: xvfb-run -a uv run --extra dev --extra rl --extra server --with 'httpx>=0.27,<1' --with 'pytest-json-report>=1.5,<2' pytest --json-report --json-report-file=full-report.json --json-report-omit=streams cua_bench/tests -q + - name: Independently verify test reports + working-directory: libs/cua-bench + run: python3 scripts/verify_pytest_receipt.py core-report.json full-report.json - name: Preserve CUA benchmark machine-readable evidence if: always() uses: actions/upload-artifact@v4 From 320f725587b6f1b1117b26d15971d0a3ab906d31 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:41:49 +0800 Subject: [PATCH 033/140] feat(cua-bench): publish versioned CI receipt JSON Schema contract --- .../schemas/ci-receipt-v1.schema.json | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 libs/cua-bench/schemas/ci-receipt-v1.schema.json diff --git a/libs/cua-bench/schemas/ci-receipt-v1.schema.json b/libs/cua-bench/schemas/ci-receipt-v1.schema.json new file mode 100644 index 0000000000..78ea99c512 --- /dev/null +++ b/libs/cua-bench/schemas/ci-receipt-v1.schema.json @@ -0,0 +1,61 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/hippoley/cua/blob/fix/cua-bench-validate-parallelism-20261009/libs/cua-bench/schemas/ci-receipt-v1.schema.json", + "title": "CUA Bench CI test receipt v1", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "commit", + "reports" + ], + "properties": { + "schema_version": { + "const": "cua-bench-ci-receipt/v1" + }, + "commit": { + "type": "string", + "pattern": "^[a-f0-9]{40}$" + }, + "reports": { + "type": "array", + "minItems": 2, + "maxItems": 2, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "path", + "sha256", + "passed", + "distinct_tests" + ], + "properties": { + "kind": { + "enum": [ + "core", + "full" + ] + }, + "path": { + "type": "string", + "minLength": 1 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "passed": { + "type": "integer", + "minimum": 1 + }, + "distinct_tests": { + "type": "integer", + "minimum": 1 + } + } + } + } + } +} From 4695722fcea0ef5370268e2ccd555f5e6136d726 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:42:10 +0800 Subject: [PATCH 034/140] feat(cua-bench): integrate jsonschema for portable checksum-linked CI receipts --- libs/cua-bench/scripts/ci_receipt.py | 73 ++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 libs/cua-bench/scripts/ci_receipt.py diff --git a/libs/cua-bench/scripts/ci_receipt.py b/libs/cua-bench/scripts/ci_receipt.py new file mode 100644 index 0000000000..1a6dccb345 --- /dev/null +++ b/libs/cua-bench/scripts/ci_receipt.py @@ -0,0 +1,73 @@ +"""Build a versioned, hash-linked CI test receipt using the jsonschema project. + +This witnesses archived pytest report bytes, not actual computer-use actions. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path + +from jsonschema import Draft202012Validator + +from verify_pytest_receipt import verify_report + +SCHEMA = Path(__file__).resolve().parents[1] / "schemas" / "ci-receipt-v1.schema.json" + + +def build_receipt(core: Path, full: Path, commit: str): + reports = [] + for kind, path in (("core", core), ("full", full)): + raw = path.read_bytes() + result = verify_report(json.loads(raw)) + reports.append({ + "kind": kind, + "path": path.name, + "sha256": hashlib.sha256(raw).hexdigest(), + "passed": result["passed"], + "distinct_tests": result["distinct_tests"], + }) + receipt = {"schema_version": "cua-bench-ci-receipt/v1", "commit": commit, "reports": reports} + validator = Draft202012Validator(json.loads(SCHEMA.read_text())) + validator.validate(receipt) + if set(x["kind"] for x in reports) != {"core", "full"}: + raise ValueError("receipt must cover both core and full runs") + return receipt + + +def verify_receipt(receipt, folder: Path): + schema = json.loads(SCHEMA.read_text()) + Draft202012Validator(schema).validate(receipt) + if {r["kind"] for r in receipt["reports"]} != {"core", "full"}: + raise ValueError("receipt must contain exactly one core and full result") + for item in receipt["reports"]: + name = item["path"] + # No paths outside evidence folder or URI-based remote fetches. + if Path(name).name != name: + raise ValueError("only local report basenames are allowed") + raw = (folder / name).read_bytes() + if hashlib.sha256(raw).hexdigest() != item["sha256"]: + raise ValueError(f"report checksum mismatch: {name}") + checked = verify_report(json.loads(raw)) + if checked["passed"] != item["passed"] or checked["distinct_tests"] != item["distinct_tests"]: + raise ValueError(f"report summary mismatch: {name}") + return True + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--core", type=Path, default=Path("core-report.json")) + parser.add_argument("--full", type=Path, default=Path("full-report.json")) + parser.add_argument("--commit", default=os.environ.get("GITHUB_SHA")) + parser.add_argument("--output", type=Path, default=Path("ci-receipt-v1.json")) + args = parser.parse_args() + if not args.commit: + parser.error("a real 40-character git commit is required") + receipt = build_receipt(args.core, args.full, args.commit) + verify_receipt(receipt, args.core.parent) + args.output.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n") + print(f"CUA CI receipt checked: {args.output}") + + +if __name__ == "__main__": + main() From c0c70fba6ac2beaa9f7cd3ae96c53942a654e51d Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:42:33 +0800 Subject: [PATCH 035/140] test(cua-bench): falsify changed reports, forged commits, missing suite and unsafe receipt paths --- .../cua_bench/tests/test_ci_receipt.py | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 libs/cua-bench/cua_bench/tests/test_ci_receipt.py diff --git a/libs/cua-bench/cua_bench/tests/test_ci_receipt.py b/libs/cua-bench/cua_bench/tests/test_ci_receipt.py new file mode 100644 index 0000000000..86c9ef1d71 --- /dev/null +++ b/libs/cua-bench/cua_bench/tests/test_ci_receipt.py @@ -0,0 +1,76 @@ +"""Falsification tests for checksum-linked CUA CI receipts.""" +import importlib.util +import json +from pathlib import Path +import sys + +import pytest + +scripts = Path(__file__).resolve().parents[2] / "scripts" +sys.path.insert(0, str(scripts)) +spec = importlib.util.spec_from_file_location("ci_receipt", scripts / "ci_receipt.py") +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +def report(nodeid): + return { + "exitcode": 0, + "summary": {"collected": 1, "total": 1, "passed": 1}, + "tests": [{"nodeid": nodeid, "outcome": "passed"}], + "collectors": [], + } + + +def setup_reports(tmp_path): + a, b = tmp_path / "core-report.json", tmp_path / "full-report.json" + a.write_text(json.dumps(report("core::one"))) + b.write_text(json.dumps(report("full::one"))) + return a, b + + +def test_valid_receipt(tmp_path): + a, b = setup_reports(tmp_path) + receipt = module.build_receipt(a, b, "a" * 40) + assert module.verify_receipt(receipt, tmp_path) is True + assert receipt["reports"][0]["sha256"] != receipt["reports"][1]["sha256"] + + +def test_modified_report_is_rejected(tmp_path): + a, b = setup_reports(tmp_path) + receipt = module.build_receipt(a, b, "a" * 40) + b.write_text(json.dumps(report("full::changed"))) + with pytest.raises(ValueError, match="checksum mismatch"): + module.verify_receipt(receipt, tmp_path) + + +def test_invalid_commit_is_rejected(tmp_path): + a, b = setup_reports(tmp_path) + from jsonschema import ValidationError + with pytest.raises(ValidationError): + module.build_receipt(a, b, "not-a-real-sha") + + +def test_missing_full_is_rejected(tmp_path): + a, b = setup_reports(tmp_path) + receipt = module.build_receipt(a, b, "a" * 40) + receipt["reports"][1]["kind"] = "core" + with pytest.raises(ValueError, match="exactly one"): + module.verify_receipt(receipt, tmp_path) + + +def test_path_traversal_is_rejected(tmp_path): + a, b = setup_reports(tmp_path) + receipt = module.build_receipt(a, b, "a" * 40) + receipt["reports"][1]["path"] = "../outside.json" + with pytest.raises(ValueError, match="basenames"): + module.verify_receipt(receipt, tmp_path) + + +def test_failed_underlying_test_is_rejected(tmp_path): + a, b = setup_reports(tmp_path) + bad = report("full::bad") + bad["tests"][0]["outcome"] = "failed" + b.write_text(json.dumps(bad)) + with pytest.raises(ValueError, match="not all test outcomes passed"): + module.build_receipt(a, b, "a" * 40) From 80a64295a503ea2f91731095cfd504a7612375a7 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:42:52 +0800 Subject: [PATCH 036/140] ci(cua-bench): run jsonschema contract tests and publish verified versioned receipts --- .github/workflows/ci-py-bench.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-py-bench.yml b/.github/workflows/ci-py-bench.yml index b7da37a373..705ebc63dd 100644 --- a/.github/workflows/ci-py-bench.yml +++ b/.github/workflows/ci-py-bench.yml @@ -37,16 +37,19 @@ jobs: - name: Run CUA benchmark core and oracle regression suite working-directory: libs/cua-bench run: | - xvfb-run -a uv run --extra dev --with 'pytest-json-report>=1.5,<2' pytest --json-report --json-report-file=core-report.json --json-report-omit=streams \ + xvfb-run -a uv run --extra dev --with 'pytest-json-report>=1.5,<2' --with 'jsonschema>=4.20,<5' pytest --json-report --json-report-file=core-report.json --json-report-omit=streams \ cua_bench/tests/test_run_benchmark.py \ cua_bench/tests/test_gym_interface.py \ cua_bench/tests/test_actions.py -q - name: Run full benchmark suite including RL workers working-directory: libs/cua-bench - run: xvfb-run -a uv run --extra dev --extra rl --extra server --with 'httpx>=0.27,<1' --with 'pytest-json-report>=1.5,<2' pytest --json-report --json-report-file=full-report.json --json-report-omit=streams cua_bench/tests -q + run: xvfb-run -a uv run --extra dev --extra rl --extra server --with 'httpx>=0.27,<1' --with 'pytest-json-report>=1.5,<2' --with 'jsonschema>=4.20,<5' pytest --json-report --json-report-file=full-report.json --json-report-omit=streams cua_bench/tests -q - name: Independently verify test reports working-directory: libs/cua-bench run: python3 scripts/verify_pytest_receipt.py core-report.json full-report.json + - name: Build versioned checksum-linked receipt + working-directory: libs/cua-bench + run: uv run --extra dev --with 'jsonschema>=4.20,<5' python scripts/ci_receipt.py --commit "$GITHUB_SHA" - name: Preserve CUA benchmark machine-readable evidence if: always() uses: actions/upload-artifact@v4 @@ -55,5 +58,6 @@ jobs: path: | libs/cua-bench/core-report.json libs/cua-bench/full-report.json + libs/cua-bench/ci-receipt-v1.json if-no-files-found: warn retention-days: 30 From 0c155d537d3148c3aa83bf41e41d7880e31647af Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:43:43 +0800 Subject: [PATCH 037/140] feat(cua-bench): expose offline read-only receipt verification CLI for external consumers --- libs/cua-bench/scripts/ci_receipt.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/libs/cua-bench/scripts/ci_receipt.py b/libs/cua-bench/scripts/ci_receipt.py index 1a6dccb345..59e083b794 100644 --- a/libs/cua-bench/scripts/ci_receipt.py +++ b/libs/cua-bench/scripts/ci_receipt.py @@ -56,11 +56,18 @@ def verify_receipt(receipt, folder: Path): def main(): parser = argparse.ArgumentParser() + parser.add_argument("--verify", type=Path, help="verify an existing receipt without regenerating it") + parser.add_argument("--evidence-dir", type=Path, default=Path("."), help="folder containing the archived reports") parser.add_argument("--core", type=Path, default=Path("core-report.json")) parser.add_argument("--full", type=Path, default=Path("full-report.json")) parser.add_argument("--commit", default=os.environ.get("GITHUB_SHA")) parser.add_argument("--output", type=Path, default=Path("ci-receipt-v1.json")) args = parser.parse_args() + if args.verify is not None: + receipt = json.loads(args.verify.read_text()) + verify_receipt(receipt, args.evidence_dir) + print(f"CUA CI receipt independently verified: {args.verify}") + return if not args.commit: parser.error("a real 40-character git commit is required") receipt = build_receipt(args.core, args.full, args.commit) From e8677cfc944224cce8cf2858adb077e1b1a6498c Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:43:56 +0800 Subject: [PATCH 038/140] test(cua-bench): verify downloaded receipt through public CLI subprocess --- .../cua_bench/tests/test_ci_receipt.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/libs/cua-bench/cua_bench/tests/test_ci_receipt.py b/libs/cua-bench/cua_bench/tests/test_ci_receipt.py index 86c9ef1d71..f94b396953 100644 --- a/libs/cua-bench/cua_bench/tests/test_ci_receipt.py +++ b/libs/cua-bench/cua_bench/tests/test_ci_receipt.py @@ -74,3 +74,22 @@ def test_failed_underlying_test_is_rejected(tmp_path): b.write_text(json.dumps(bad)) with pytest.raises(ValueError, match="not all test outcomes passed"): module.build_receipt(a, b, "a" * 40) + +def test_external_consumer_verifies_downloaded_receipt(tmp_path): + import subprocess + import sys + a, b = setup_reports(tmp_path) + receipt = module.build_receipt(a, b, "b" * 40) + path = tmp_path / "ci-receipt-v1.json" + path.write_text(json.dumps(receipt)) + result = subprocess.run( + [ + sys.executable, str(scripts / "ci_receipt.py"), + "--verify", str(path), "--evidence-dir", str(tmp_path), + ], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr + assert "independently verified" in result.stdout From 73549cb5e932763e200239d4f45d29150af88064 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:44:38 +0800 Subject: [PATCH 039/140] docs(cua-bench): publish external consumption contract and honesty boundaries for v1 CI receipts --- libs/cua-bench/schemas/README.md | 38 ++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 libs/cua-bench/schemas/README.md diff --git a/libs/cua-bench/schemas/README.md b/libs/cua-bench/schemas/README.md new file mode 100644 index 0000000000..3d990f6802 --- /dev/null +++ b/libs/cua-bench/schemas/README.md @@ -0,0 +1,38 @@ +# CUA Bench CI Receipt v1 — external consumer guide + +This is a **test-result integrity envelope**. It is not an independent oracle, a signed software supply-chain attestation, or proof that an agent performed desktop actions. + +## Implemented integration +- `pytest-json-report` (MIT) produces `core-report.json` and `full-report.json` from real pytest executions. +- `jsonschema` validates the published Draft 2020-12 contract at `schemas/ci-receipt-v1.schema.json`. +- `scripts/ci_receipt.py` binds the reports to their SHA-256 checksums, passing test counts and Git commit, and refuses altered or failed reports. +- GitHub Actions publishes these three files as `cua-bench-pytest-evidence` when available. + +## Generate in CI + +```sh +cd libs/cua-bench +uv run --extra dev --with 'jsonschema>=4.20,<5' python scripts/ci_receipt.py --commit "$GITHUB_SHA" +``` + +## Independently verify downloaded artifacts + +Place the downloaded `core-report.json`, `full-report.json`, and `ci-receipt-v1.json` in the same directory, then run: + +```sh +cd libs/cua-bench +uv run --extra dev --with 'jsonschema>=4.20,<5' python scripts/ci_receipt.py \ + --verify /path/to/artifacts/ci-receipt-v1.json \ + --evidence-dir /path/to/artifacts +``` + +The verifier checks the contract, two distinct report kinds, safe local basenames, exact report checksums, passing outcomes and mutually consistent counts. It does not download external URLs or execute downloaded code. + +## Limits and outstanding User Stories +1. Commit and SHA-256 metadata are unauthenticated; consumers must establish CI origin independently (e.g. official GitHub run URL, permissions and repository owner). +2. Reported tests may use mocks. This is not a full Computer-Use Execution Witness and contains neither per-action state observations nor independent oracle replays. +3. No real VM crash/cleanup inventory is included. Real desktop VM fault injection remains a separate P0 gate. +4. Compatibility: versioned schema `cua-bench-ci-receipt/v1`; breaking changes require a new schema version. +5. Evidence retention is currently 30 days. Export elsewhere before expiry if it is used as a long-lived credential. + +Success means an **authentic test run can be inspected with the raw reports and the same verifier**, not that the underlying benchmark's real-world task success is guaranteed. From b67331bea602846424207fb9a75da25bf9c6486a Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:46:45 +0800 Subject: [PATCH 040/140] ci(cua-bench): integrate pytest-timeout to bound hung browser and RL tests --- .github/workflows/ci-py-bench.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-py-bench.yml b/.github/workflows/ci-py-bench.yml index 705ebc63dd..81e312c77f 100644 --- a/.github/workflows/ci-py-bench.yml +++ b/.github/workflows/ci-py-bench.yml @@ -37,13 +37,13 @@ jobs: - name: Run CUA benchmark core and oracle regression suite working-directory: libs/cua-bench run: | - xvfb-run -a uv run --extra dev --with 'pytest-json-report>=1.5,<2' --with 'jsonschema>=4.20,<5' pytest --json-report --json-report-file=core-report.json --json-report-omit=streams \ + xvfb-run -a uv run --extra dev --with 'pytest-json-report>=1.5,<2' --with 'jsonschema>=4.20,<5' --with 'pytest-timeout>=2.3,<3' pytest --timeout=120 --timeout-method=signal --json-report --json-report-file=core-report.json --json-report-omit=streams \ cua_bench/tests/test_run_benchmark.py \ cua_bench/tests/test_gym_interface.py \ cua_bench/tests/test_actions.py -q - name: Run full benchmark suite including RL workers working-directory: libs/cua-bench - run: xvfb-run -a uv run --extra dev --extra rl --extra server --with 'httpx>=0.27,<1' --with 'pytest-json-report>=1.5,<2' --with 'jsonschema>=4.20,<5' pytest --json-report --json-report-file=full-report.json --json-report-omit=streams cua_bench/tests -q + run: xvfb-run -a uv run --extra dev --extra rl --extra server --with 'httpx>=0.27,<1' --with 'pytest-json-report>=1.5,<2' --with 'jsonschema>=4.20,<5' --with 'pytest-timeout>=2.3,<3' pytest --timeout=120 --timeout-method=signal --json-report --json-report-file=full-report.json --json-report-omit=streams cua_bench/tests -q - name: Independently verify test reports working-directory: libs/cua-bench run: python3 scripts/verify_pytest_receipt.py core-report.json full-report.json From f7a5fea9e0f6566edb1ccdf8a65b95dd2818eb17 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:47:03 +0800 Subject: [PATCH 041/140] fix(cua-bench): block symlink escape and report aliasing in offline receipts --- libs/cua-bench/scripts/ci_receipt.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/libs/cua-bench/scripts/ci_receipt.py b/libs/cua-bench/scripts/ci_receipt.py index 59e083b794..7c0f307ca8 100644 --- a/libs/cua-bench/scripts/ci_receipt.py +++ b/libs/cua-bench/scripts/ci_receipt.py @@ -40,12 +40,18 @@ def verify_receipt(receipt, folder: Path): Draft202012Validator(schema).validate(receipt) if {r["kind"] for r in receipt["reports"]} != {"core", "full"}: raise ValueError("receipt must contain exactly one core and full result") + if len({r["path"] for r in receipt["reports"]}) != len(receipt["reports"]): + raise ValueError("core and full reports must have distinct file paths") + root = folder.resolve(strict=True) for item in receipt["reports"]: name = item["path"] # No paths outside evidence folder or URI-based remote fetches. if Path(name).name != name: raise ValueError("only local report basenames are allowed") - raw = (folder / name).read_bytes() + target = (root / name).resolve(strict=True) + if target.parent != root: + raise ValueError("report escapes evidence directory via symlink") + raw = target.read_bytes() if hashlib.sha256(raw).hexdigest() != item["sha256"]: raise ValueError(f"report checksum mismatch: {name}") checked = verify_report(json.loads(raw)) From efeb59d5a59a1c94a3fdd73495a85dd9b167038e Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:47:16 +0800 Subject: [PATCH 042/140] test(cua-bench): reject receipt path aliasing and symlink traversal --- .../cua_bench/tests/test_ci_receipt.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/libs/cua-bench/cua_bench/tests/test_ci_receipt.py b/libs/cua-bench/cua_bench/tests/test_ci_receipt.py index f94b396953..e1408a71bc 100644 --- a/libs/cua-bench/cua_bench/tests/test_ci_receipt.py +++ b/libs/cua-bench/cua_bench/tests/test_ci_receipt.py @@ -93,3 +93,23 @@ def test_external_consumer_verifies_downloaded_receipt(tmp_path): ) assert result.returncode == 0, result.stderr assert "independently verified" in result.stdout + +def test_duplicate_report_path_rejected(tmp_path): + a, b = setup_reports(tmp_path) + receipt = module.build_receipt(a, b, "a" * 40) + receipt["reports"][1]["path"] = receipt["reports"][0]["path"] + with pytest.raises(ValueError, match="distinct file paths"): + module.verify_receipt(receipt, tmp_path) + + +def test_symlink_escape_rejected(tmp_path): + root = tmp_path / "artifacts" + root.mkdir() + a, b = setup_reports(root) + receipt = module.build_receipt(a, b, "a" * 40) + outside = tmp_path / "outside.json" + outside.write_bytes(b.read_bytes()) + b.unlink() + b.symlink_to(outside) + with pytest.raises(ValueError, match="symlink"): + module.verify_receipt(receipt, root) From 473cfe18bd725f83bad620391e1fa739efff0eca Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:49:12 +0800 Subject: [PATCH 043/140] feat(cua-bench): add reproducible dataset manifest and drift verification CLI --- libs/cua-bench/scripts/dataset_manifest.py | 71 ++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 libs/cua-bench/scripts/dataset_manifest.py diff --git a/libs/cua-bench/scripts/dataset_manifest.py b/libs/cua-bench/scripts/dataset_manifest.py new file mode 100644 index 0000000000..ed5d0db8ee --- /dev/null +++ b/libs/cua-bench/scripts/dataset_manifest.py @@ -0,0 +1,71 @@ +"""Reproducible, read-only dataset manifest for CUA Bench task directories. + +Does not import or execute untrusted task main.py files. +""" +import argparse +import hashlib +import json +from pathlib import Path + +VERSION = "cua-dataset-manifest/v1" +IGNORED = {"__pycache__", ".git", ".venv", "node_modules", ".pytest_cache"} + + +def scan_dataset(root: Path): + root = root.resolve(strict=True) + if not root.is_dir(): + raise ValueError("dataset must be a directory") + tasks = [root] if (root / "main.py").is_file() else sorted( + p for p in root.iterdir() if p.is_dir() and not p.is_symlink() and (p / "main.py").is_file() + ) + if not tasks: + raise ValueError("no task directories containing main.py") + output = [] + for task in tasks: + files = [] + for path in sorted(task.rglob("*")): + relative = path.relative_to(root) + if any(part in IGNORED or part.startswith(".") for part in relative.parts): + continue + if path.is_symlink(): + raise ValueError(f"symlink in dataset task: {relative}") + if not path.is_file(): + continue + data = path.read_bytes() + files.append({"path": relative.as_posix(), "sha256": hashlib.sha256(data).hexdigest(), "size": len(data)}) + if not files: + raise ValueError(f"empty task: {task}") + output.append({"task": task.relative_to(root).as_posix(), "files": files}) + return {"schema_version": VERSION, "tasks": output} + + +def verify_dataset(root: Path, manifest: dict): + if not isinstance(manifest, dict) or manifest.get("schema_version") != VERSION: + raise ValueError("unsupported dataset manifest") + expected = manifest.get("tasks") + if not isinstance(expected, list) or not expected: + raise ValueError("manifest has no tasks") + actual = scan_dataset(root) + if actual != manifest: + raise ValueError("dataset differs from manifest (added, removed, or modified task files)") + return len(actual["tasks"]) + + +def main(): + parser = argparse.ArgumentParser(description="Create or verify CUA dataset task manifest") + parser.add_argument("dataset", type=Path) + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--output", type=Path) + group.add_argument("--verify", type=Path) + args = parser.parse_args() + if args.verify: + count = verify_dataset(args.dataset, json.loads(args.verify.read_text())) + print(f"Verified {count} tasks without executing task source") + else: + manifest = scan_dataset(args.dataset) + args.output.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") + print(f"Recorded {len(manifest['tasks'])} tasks: {args.output}") + + +if __name__ == "__main__": + main() From 9fb242ddf277d08dd3e5b8b15e14afdeed211434 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:49:32 +0800 Subject: [PATCH 044/140] test(cua-bench): cover dataset manifest drift, symlink and no-execution guarantees --- .../cua_bench/tests/test_dataset_manifest.py | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 libs/cua-bench/cua_bench/tests/test_dataset_manifest.py diff --git a/libs/cua-bench/cua_bench/tests/test_dataset_manifest.py b/libs/cua-bench/cua_bench/tests/test_dataset_manifest.py new file mode 100644 index 0000000000..d374cc87f5 --- /dev/null +++ b/libs/cua-bench/cua_bench/tests/test_dataset_manifest.py @@ -0,0 +1,78 @@ +"""Dataset manifest contract tests with independent tampering examples.""" +import importlib.util +from pathlib import Path +import sys + +import pytest + +SCRIPT = Path(__file__).resolve().parents[2] / "scripts" / "dataset_manifest.py" +spec = importlib.util.spec_from_file_location("dataset_manifest", SCRIPT) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +def dataset(tmp_path): + root = tmp_path / "dataset" + for name in ("click", "type"): + task = root / name + task.mkdir(parents=True) + (task / "main.py").write_text("# task\n") + (task / "asset.txt").write_text("asset") + return root + + +def test_manifest_is_deterministic(tmp_path): + root = dataset(tmp_path) + first = module.scan_dataset(root) + assert first == module.scan_dataset(root) + assert module.verify_dataset(root, first) == 2 + assert [x["task"] for x in first["tasks"]] == ["click", "type"] + + +@pytest.mark.parametrize("mutation", ["change", "add", "remove"]) +def test_dataset_drift_fails_closed(tmp_path, mutation): + root = dataset(tmp_path) + saved = module.scan_dataset(root) + asset = root / "click" / "asset.txt" + if mutation == "change": + asset.write_text("tampered") + elif mutation == "add": + (root / "click" / "new.txt").write_text("extra") + else: + asset.unlink() + with pytest.raises(ValueError, match="dataset differs"): + module.verify_dataset(root, saved) + + +def test_new_task_fails_closed(tmp_path): + root = dataset(tmp_path) + saved = module.scan_dataset(root) + extra = root / "new" + extra.mkdir() + (extra / "main.py").write_text("# inserted task") + with pytest.raises(ValueError, match="dataset differs"): + module.verify_dataset(root, saved) + + +def test_symlink_escape_refused(tmp_path): + root = dataset(tmp_path) + external = tmp_path / "secret.txt" + external.write_text("secret") + (root / "click" / "secret-link.txt").symlink_to(external) + with pytest.raises(ValueError, match="symlink"): + module.scan_dataset(root) + + +def test_manifest_never_executes_task_source(tmp_path): + root = dataset(tmp_path) + marker = tmp_path / "executed" + (root / "click" / "main.py").write_text(f"open({str(marker)!r}, 'w').write('unsafe')") + module.scan_dataset(root) + assert not marker.exists() + + +def test_missing_tasks_refused(tmp_path): + empty = tmp_path / "empty" + empty.mkdir() + with pytest.raises(ValueError, match="no task"): + module.scan_dataset(empty) From 27620bffbb41741352a3a87b8c5116434bd3b26b Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:49:50 +0800 Subject: [PATCH 045/140] ci(cua-bench): execute new dataset manifest feature on real basic dataset --- .github/workflows/ci-py-bench.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci-py-bench.yml b/.github/workflows/ci-py-bench.yml index 81e312c77f..a79f299d46 100644 --- a/.github/workflows/ci-py-bench.yml +++ b/.github/workflows/ci-py-bench.yml @@ -34,6 +34,11 @@ jobs: - name: Install Playwright Chromium for simulated desktop E2E working-directory: libs/cua-bench run: uv run --extra dev python -m playwright install --with-deps chromium + - name: Verify dataset manifest round trip on basic dataset + working-directory: libs/cua-bench + run: | + python3 scripts/dataset_manifest.py datasets/cua-bench-basic --output /tmp/cua-basic-manifest.json + python3 scripts/dataset_manifest.py datasets/cua-bench-basic --verify /tmp/cua-basic-manifest.json - name: Run CUA benchmark core and oracle regression suite working-directory: libs/cua-bench run: | From 98840442b3f01e52a3006183ad5fb0717364c0ab Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:50:17 +0800 Subject: [PATCH 046/140] docs(cua-bench): document reproducible dataset manifest feature and integrity boundaries --- libs/cua-bench/datasets/DATASET_MANIFEST.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 libs/cua-bench/datasets/DATASET_MANIFEST.md diff --git a/libs/cua-bench/datasets/DATASET_MANIFEST.md b/libs/cua-bench/datasets/DATASET_MANIFEST.md new file mode 100644 index 0000000000..0b9aa4b70b --- /dev/null +++ b/libs/cua-bench/datasets/DATASET_MANIFEST.md @@ -0,0 +1,15 @@ +# Dataset Manifest v1 + +A new **read-only dataset integrity feature** for CUA Bench. It inventories each task's source and static assets and detects drift before a benchmark is compared with previous runs. + +```sh +cd libs/cua-bench +python3 scripts/dataset_manifest.py datasets/cua-bench-basic --output /tmp/cua-basic-manifest.json +python3 scripts/dataset_manifest.py datasets/cua-bench-basic --verify /tmp/cua-basic-manifest.json +``` + +The manifest format is `cua-dataset-manifest/v1`: deterministic task IDs (directory names), sorted relative file paths, raw-byte SHA-256 and sizes. Verification fails if task files are added, removed or edited, or if task directories are added or removed. Hidden and generated directories are excluded. Symlinked task files are rejected. The scanner never imports `main.py` or runs task code. + +**Security limits:** SHA-256 validates file integrity against the provided manifest; the manifest itself must be authenticated by a trusted commit, source repository or signed release. This is not an Oracle execution receipt and cannot prove that a GUI action was performed. Pin the task code, data assets and environment separately for comparable results. + +**Acceptance:** negative tests cover drift, new tasks, symlinks and non-execution. CI executes a manifest round-trip against `cua-bench-basic`. Do not mark the feature verified until the relevant CI run passes. From 3c1f6a7c24de2581b612571ac1e4d608d873b90e Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:59:17 +0800 Subject: [PATCH 047/140] fix(cua-bench): include hidden task files in dataset integrity manifest --- libs/cua-bench/scripts/dataset_manifest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/cua-bench/scripts/dataset_manifest.py b/libs/cua-bench/scripts/dataset_manifest.py index ed5d0db8ee..2d222a4f3c 100644 --- a/libs/cua-bench/scripts/dataset_manifest.py +++ b/libs/cua-bench/scripts/dataset_manifest.py @@ -25,7 +25,7 @@ def scan_dataset(root: Path): files = [] for path in sorted(task.rglob("*")): relative = path.relative_to(root) - if any(part in IGNORED or part.startswith(".") for part in relative.parts): + if any(part in IGNORED for part in relative.parts): continue if path.is_symlink(): raise ValueError(f"symlink in dataset task: {relative}") From 874097fede0e407e41e3817cef525a310f3b3e9b Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:59:28 +0800 Subject: [PATCH 048/140] test(cua-bench): catch hidden environment configuration drift --- libs/cua-bench/cua_bench/tests/test_dataset_manifest.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/libs/cua-bench/cua_bench/tests/test_dataset_manifest.py b/libs/cua-bench/cua_bench/tests/test_dataset_manifest.py index d374cc87f5..f1f35a6887 100644 --- a/libs/cua-bench/cua_bench/tests/test_dataset_manifest.py +++ b/libs/cua-bench/cua_bench/tests/test_dataset_manifest.py @@ -76,3 +76,12 @@ def test_missing_tasks_refused(tmp_path): empty.mkdir() with pytest.raises(ValueError, match="no task"): module.scan_dataset(empty) + +def test_hidden_task_config_must_be_fingerprinted(tmp_path): + root = dataset(tmp_path) + config = root / "click" / ".env" + config.write_text("SCORING_MODE=strict") + manifest = module.scan_dataset(root) + config.write_text("SCORING_MODE=permissive") + with pytest.raises(ValueError, match="dataset differs"): + module.verify_dataset(root, manifest) From 94bf4cd1430e7c21d398e7c5a331935877bcb2f6 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:59:50 +0800 Subject: [PATCH 049/140] docs(cua-bench): clarify hidden configuration integrity and secret exposure --- libs/cua-bench/datasets/DATASET_MANIFEST.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/cua-bench/datasets/DATASET_MANIFEST.md b/libs/cua-bench/datasets/DATASET_MANIFEST.md index 0b9aa4b70b..b0341ed815 100644 --- a/libs/cua-bench/datasets/DATASET_MANIFEST.md +++ b/libs/cua-bench/datasets/DATASET_MANIFEST.md @@ -8,7 +8,7 @@ python3 scripts/dataset_manifest.py datasets/cua-bench-basic --output /tmp/cua-b python3 scripts/dataset_manifest.py datasets/cua-bench-basic --verify /tmp/cua-basic-manifest.json ``` -The manifest format is `cua-dataset-manifest/v1`: deterministic task IDs (directory names), sorted relative file paths, raw-byte SHA-256 and sizes. Verification fails if task files are added, removed or edited, or if task directories are added or removed. Hidden and generated directories are excluded. Symlinked task files are rejected. The scanner never imports `main.py` or runs task code. +The manifest format is `cua-dataset-manifest/v1`: deterministic task IDs (directory names), sorted relative file paths, raw-byte SHA-256 and sizes. Verification fails if task files are added, removed or edited, or if task directories are added or removed. Generated cache directories are excluded, but task-local hidden files (for example `.env` and `.config`) are included because they may affect execution. Avoid placing actual secrets inside benchmark task directories; manifests expose file names and their hashes. Symlinked task files are rejected. The scanner never imports `main.py` or runs task code. **Security limits:** SHA-256 validates file integrity against the provided manifest; the manifest itself must be authenticated by a trusted commit, source repository or signed release. This is not an Oracle execution receipt and cannot prove that a GUI action was performed. Pin the task code, data assets and environment separately for comparable results. From 917a142e6f50707cc59c093bcc5c982aafac8366 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:00:29 +0800 Subject: [PATCH 050/140] docs(cua-bench): audit ten completeness axes and distinguish upstream overlapping work --- libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md | 27 ++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md b/libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md index 30425592c9..3601b01902 100644 --- a/libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md +++ b/libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md @@ -87,3 +87,30 @@ Follow with positive real dataset runs on `cua-bench-basic`, a parallelism stres ### Anti-claims No verified external adoption, certification, real VM cleanup receipt or long-term defensibility proven yet. Evidence artifacts and test coverage are not themselves network effects. + +## Reassessment against 2026-10-09 upstream and external practice + +The official `trycua/cua` project remains active. Its open PRs include #4899 (worker stdout pipe saturation at long run), #4898 (worker lock/timeout deadlock), #4884 (reject action-truth claims inconsistent with E2E oracles), and #4883 (metadata-only runtime conformance witness). **Do not describe those as our novel discoveries or duplicate their patches.** Pursue narrow upstream-compatible changes after base comparison, not a large mixed PR from this feature branch. + +External comparison: OSWorld-V2's supported benchmark release defines pinned task/assets/web/provider image versions (https://github.com/xlang-ai/OSWorld-V2/blob/main/benchmark_releases/README.md). This fork's manifest hashes local task files only and currently does not bind runtime/provider assets. OpenTelemetry GenAI agent/tool span conventions are still in Development and are useful for trace interoperability, but spans alone are not proof of correctness (https://github.com/open-telemetry/semantic-conventions-genai/blob/main/docs/gen-ai/gen-ai-agent-spans.md). + +### 10-axis horizontal audit (evidence available at this time) + +| Axis | Status | Evidence/gap | +| --- | --- | --- | +| Function | Partial | core runner and dataset manifest exist; no product-entry one-click provenance comparison | +| State | Partial | file hashes and test-result checks; no runtime state diff or replay | +| Integration | Partial | GitHub CI builds/tests; real VM provider receipt missing | +| Security/correctness | Partial | worker identity, scoring, path/hidden-file checks; manifests not origin authenticated | +| Performance | Blocked | no cross-provider latency/throughput/long-run fixture from this branch | +| Maintainability | Partial | script CLIs and v1 JSON schema; manifest not yet stable library API | +| Observability | Partial | pytest report SHA; no per-step trace, environment image digest or teardown inventory | +| Testability | Partial | mock and Chromium E2E tests present; force-kill/recovery independent test absent | +| User value | Partial | reproducibility helper available; no independent consumer demonstrated | +| External compatibility | Partial | uses pytest-json-report/jsonschema; OSWorld/provider/OpenTelemetry adapters absent | + +### Priority decisions and falsification + +P0: finish repeatable end-to-end benchmarks and worker failure recovery; **upstream #4899/#4898** should be reviewed and reused rather than copied. P0: close dataset integrity blind spots (hidden config drift fixed on branch, subject to CI). P1: add release manifest pins for provider/image/environment and independent action/outcome snapshots only after a real provider scenario verifies the need. P1: third-party consumption of a minimal stable verifier API. P2: optional OTEL trace export; adopt current semantic conventions with careful version pinning. + +**External standing:** no official PR, merge, maintainer endorsement, external dependency or documented third-party consumption established by this audit. Fork CI success is valuable engineering evidence, not external adoption. 6–12 month credential requires upstream accepted work and/or independent consumer reproduction. 5–10-year defensibility is a hypothesis, not a promise; evaluate by recurring downstream reliance and standards participation. Pivot if two genuine external users decline adoption because an existing upstream tool already meets the same need. From 792b6c368ebccba9f4c6c17531e7af327f2a54a6 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:02:12 +0800 Subject: [PATCH 051/140] fix(cua-bench): fail closed on symlinked task directories instead of shrinking denominator --- libs/cua-bench/scripts/dataset_manifest.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/libs/cua-bench/scripts/dataset_manifest.py b/libs/cua-bench/scripts/dataset_manifest.py index 2d222a4f3c..800fe957be 100644 --- a/libs/cua-bench/scripts/dataset_manifest.py +++ b/libs/cua-bench/scripts/dataset_manifest.py @@ -15,9 +15,14 @@ def scan_dataset(root: Path): root = root.resolve(strict=True) if not root.is_dir(): raise ValueError("dataset must be a directory") - tasks = [root] if (root / "main.py").is_file() else sorted( - p for p in root.iterdir() if p.is_dir() and not p.is_symlink() and (p / "main.py").is_file() - ) + if (root / "main.py").is_file(): + tasks = [root] + else: + entries = sorted(root.iterdir()) + for entry in entries: + if entry.is_symlink() and entry.is_dir(): + raise ValueError(f"symlinked dataset task directory: {entry.name}") + tasks = [p for p in entries if p.is_dir() and (p / "main.py").is_file()] if not tasks: raise ValueError("no task directories containing main.py") output = [] From f830a107c3733db4426f40cab686213deede15af Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:02:24 +0800 Subject: [PATCH 052/140] test(cua-bench): detect linked task directory denominator corruption --- .../cua-bench/cua_bench/tests/test_dataset_manifest.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/libs/cua-bench/cua_bench/tests/test_dataset_manifest.py b/libs/cua-bench/cua_bench/tests/test_dataset_manifest.py index f1f35a6887..9e7ba2df52 100644 --- a/libs/cua-bench/cua_bench/tests/test_dataset_manifest.py +++ b/libs/cua-bench/cua_bench/tests/test_dataset_manifest.py @@ -85,3 +85,13 @@ def test_hidden_task_config_must_be_fingerprinted(tmp_path): config.write_text("SCORING_MODE=permissive") with pytest.raises(ValueError, match="dataset differs"): module.verify_dataset(root, manifest) + + +def test_symlinked_top_level_task_is_not_silently_omitted(tmp_path): + root = dataset(tmp_path) + foreign = tmp_path / "external-task" + foreign.mkdir() + (foreign / "main.py").write_text("# imported via symlink") + (root / "external").symlink_to(foreign, target_is_directory=True) + with pytest.raises(ValueError, match="symlinked dataset task directory"): + module.scan_dataset(root) From e78ce868a650404a7192535a04ba80933f724c91 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:02:41 +0800 Subject: [PATCH 053/140] refactor(cua-bench): restrict symlink task rejection to actual runnable task aliases --- libs/cua-bench/scripts/dataset_manifest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/cua-bench/scripts/dataset_manifest.py b/libs/cua-bench/scripts/dataset_manifest.py index 800fe957be..41b149c496 100644 --- a/libs/cua-bench/scripts/dataset_manifest.py +++ b/libs/cua-bench/scripts/dataset_manifest.py @@ -20,7 +20,7 @@ def scan_dataset(root: Path): else: entries = sorted(root.iterdir()) for entry in entries: - if entry.is_symlink() and entry.is_dir(): + if entry.is_symlink() and entry.is_dir() and (entry / "main.py").is_file(): raise ValueError(f"symlinked dataset task directory: {entry.name}") tasks = [p for p in entries if p.is_dir() and (p / "main.py").is_file()] if not tasks: From a01520e910d19e4d76386d91de4e20b0d059b824 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:06:56 +0800 Subject: [PATCH 054/140] fix(cua-bench): preserve completed agent step count when execution fails --- libs/cua-bench/cua_bench/runners.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libs/cua-bench/cua_bench/runners.py b/libs/cua-bench/cua_bench/runners.py index 9bb36732b5..866e86caba 100644 --- a/libs/cua-bench/cua_bench/runners.py +++ b/libs/cua-bench/cua_bench/runners.py @@ -101,6 +101,7 @@ def my_agent(screenshot: bytes, task: Task) -> Action: result = await run_single_task(Path("./task"), agent_fn=my_agent) """ env = None + step_count = 0 try: # Create environment using gym interface env = make(str(env_path), split=split) @@ -176,7 +177,7 @@ def my_agent(screenshot: bytes, task: Task) -> Action: variant_id=task_index, success=False, reward=0.0, - steps=0, + steps=step_count, error=str(e), ) finally: From 825b9eeeab9872a9f44109c9418ebbd8257543c2 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:07:12 +0800 Subject: [PATCH 055/140] test(cua-bench): independently verify failed agent step count and environment teardown --- .../cua_bench/tests/test_run_benchmark.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/libs/cua-bench/cua_bench/tests/test_run_benchmark.py b/libs/cua-bench/cua_bench/tests/test_run_benchmark.py index 32c41da5f9..eebf1b7d58 100644 --- a/libs/cua-bench/cua_bench/tests/test_run_benchmark.py +++ b/libs/cua-bench/cua_bench/tests/test_run_benchmark.py @@ -591,3 +591,54 @@ async def setup(task, session): assert task_cfg.description == "Interactive task" finally: await env.close() + + +class TestFailedTaskStepEvidence: + @pytest.mark.asyncio + async def test_agent_failure_preserves_completed_steps(self, tmp_path, monkeypatch): + from types import SimpleNamespace + import cua_bench.runners as runners + + observed = {"steps": 0, "closed": False} + + class BrokenEnvironment: + solve_task_fn = None + evaluate_task_fn = object() + + async def reset(self, task_id=0): + return b"initial", SimpleNamespace() + + async def step(self, action): + observed["steps"] += 1 + if observed["steps"] == 3: + raise RuntimeError("injected action failure") + return b"next" + + async def close(self): + observed["closed"] = True + + monkeypatch.setattr(runners, "make", lambda path, split="train": BrokenEnvironment()) + result = await runners.run_single_task( + tmp_path, agent_fn=lambda screenshot, task: object(), max_steps=10 + ) + assert result.success is False + assert result.steps == 2 + assert result.reward == 0.0 + assert "injected action failure" in result.error + assert observed["closed"] is True + + @pytest.mark.asyncio + async def test_reset_failure_reports_zero_steps(self, tmp_path, monkeypatch): + import cua_bench.runners as runners + + class BrokenReset: + async def reset(self, task_id=0): + raise RuntimeError("reset failed") + + async def close(self): + return None + + monkeypatch.setattr(runners, "make", lambda path, split="train": BrokenReset()) + result = await runners.run_single_task(tmp_path, oracle=True) + assert result.steps == 0 + assert "reset failed" in result.error From b5a61578a049c83eaae3d209eb69ab791904b303 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:10:55 +0800 Subject: [PATCH 056/140] test(cua-bench): integrate Hypothesis to fuzz oracle rewards and assert fail-closed scoring --- .../cua_bench/tests/test_reward_hypothesis.py | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 libs/cua-bench/cua_bench/tests/test_reward_hypothesis.py diff --git a/libs/cua-bench/cua_bench/tests/test_reward_hypothesis.py b/libs/cua-bench/cua_bench/tests/test_reward_hypothesis.py new file mode 100644 index 0000000000..d8d5202dc1 --- /dev/null +++ b/libs/cua-bench/cua_bench/tests/test_reward_hypothesis.py @@ -0,0 +1,53 @@ +"""Property-based oracle result validation using the external Hypothesis library.""" +import asyncio +import math +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +from hypothesis import given, settings, strategies as st + +from cua_bench import DoneAction, run_single_task + + +@settings(max_examples=120, deadline=None) +@given(st.one_of( + st.floats(allow_nan=True, allow_infinity=True, width=64), + st.integers(min_value=-10**12, max_value=10**12), +)) +def test_generated_oracle_rewards_never_inflate_success(reward): + observed = {"closed": False, "steps": 0} + + class ControlledEnvironment: + evaluate_task_fn = object() + solve_task_fn = None + + async def reset(self, task_id=0): + return b"initial", SimpleNamespace(description="property test") + + async def step(self, action): + observed["steps"] += 1 + return b"next" + + async def evaluate(self): + return reward + + async def close(self): + observed["closed"] = True + + with patch("cua_bench.runners.make", return_value=ControlledEnvironment()): + result = asyncio.run(run_single_task( + Path("property-task"), agent_fn=lambda screenshot, task: DoneAction(), max_steps=2 + )) + + assert observed["closed"] is True + assert result.steps == observed["steps"] == 1 + valid = math.isfinite(reward) and 0 <= reward <= 1 + if valid: + assert result.error is None + assert result.reward == float(reward) + assert result.success is (reward >= 0.5) + else: + assert result.success is False + assert result.reward == 0.0 + assert result.error is not None From e534a37031b53653d1d1765bef82098aa5538a88 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:11:08 +0800 Subject: [PATCH 057/140] ci(cua-bench): run external Hypothesis reward fuzz tests in core and full suites --- .github/workflows/ci-py-bench.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-py-bench.yml b/.github/workflows/ci-py-bench.yml index a79f299d46..e94736b532 100644 --- a/.github/workflows/ci-py-bench.yml +++ b/.github/workflows/ci-py-bench.yml @@ -42,13 +42,14 @@ jobs: - name: Run CUA benchmark core and oracle regression suite working-directory: libs/cua-bench run: | - xvfb-run -a uv run --extra dev --with 'pytest-json-report>=1.5,<2' --with 'jsonschema>=4.20,<5' --with 'pytest-timeout>=2.3,<3' pytest --timeout=120 --timeout-method=signal --json-report --json-report-file=core-report.json --json-report-omit=streams \ + xvfb-run -a uv run --extra dev --with 'pytest-json-report>=1.5,<2' --with 'jsonschema>=4.20,<5' --with 'pytest-timeout>=2.3,<3' --with 'hypothesis>=6.100,<7' pytest --timeout=120 --timeout-method=signal --json-report --json-report-file=core-report.json --json-report-omit=streams \ cua_bench/tests/test_run_benchmark.py \ cua_bench/tests/test_gym_interface.py \ - cua_bench/tests/test_actions.py -q + cua_bench/tests/test_actions.py \ + cua_bench/tests/test_reward_hypothesis.py -q - name: Run full benchmark suite including RL workers working-directory: libs/cua-bench - run: xvfb-run -a uv run --extra dev --extra rl --extra server --with 'httpx>=0.27,<1' --with 'pytest-json-report>=1.5,<2' --with 'jsonschema>=4.20,<5' --with 'pytest-timeout>=2.3,<3' pytest --timeout=120 --timeout-method=signal --json-report --json-report-file=full-report.json --json-report-omit=streams cua_bench/tests -q + run: xvfb-run -a uv run --extra dev --extra rl --extra server --with 'httpx>=0.27,<1' --with 'pytest-json-report>=1.5,<2' --with 'jsonschema>=4.20,<5' --with 'pytest-timeout>=2.3,<3' --with 'hypothesis>=6.100,<7' pytest --timeout=120 --timeout-method=signal --json-report --json-report-file=full-report.json --json-report-omit=streams cua_bench/tests -q - name: Independently verify test reports working-directory: libs/cua-bench run: python3 scripts/verify_pytest_receipt.py core-report.json full-report.json From 33da194f1c087c48ea9a6792a768d98c59d02182 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:43:07 +0800 Subject: [PATCH 058/140] fix(cua-bench): fail closed if environment cleanup fails after a successful task --- libs/cua-bench/cua_bench/runners.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/libs/cua-bench/cua_bench/runners.py b/libs/cua-bench/cua_bench/runners.py index 866e86caba..663847ea0a 100644 --- a/libs/cua-bench/cua_bench/runners.py +++ b/libs/cua-bench/cua_bench/runners.py @@ -102,6 +102,13 @@ def my_agent(screenshot: bytes, task: Task) -> Action: """ env = None step_count = 0 + outcome = None + + def finish(**kwargs): + nonlocal outcome + outcome = TaskResult(**kwargs) + return outcome + try: # Create environment using gym interface env = make(str(env_path), split=split) @@ -119,7 +126,7 @@ def my_agent(screenshot: bytes, task: Task) -> Action: await env.solve() step_count = env.step_count else: - return TaskResult( + return finish( task_path=str(env_path), variant_id=task_index, success=False, @@ -138,7 +145,7 @@ def my_agent(screenshot: bytes, task: Task) -> Action: # A setup-only run must never be counted as successful agent execution. if not oracle and agent_fn is None: - return TaskResult( + return finish( task_path=str(env_path), variant_id=task_index, success=False, @@ -163,7 +170,7 @@ def my_agent(screenshot: bytes, task: Task) -> Action: else: raise ValueError("Task has no evaluator; cannot produce a verified score") - return TaskResult( + return finish( task_path=str(env_path), variant_id=task_index, success=reward >= 0.5, # Common threshold @@ -172,7 +179,7 @@ def my_agent(screenshot: bytes, task: Task) -> Action: ) except Exception as e: - return TaskResult( + return finish( task_path=str(env_path), variant_id=task_index, success=False, @@ -184,8 +191,14 @@ def my_agent(screenshot: bytes, task: Task) -> Action: if env is not None: try: await env.close() - except Exception: - pass + except Exception as close_error: + # A task cannot be reported as successful if its environment was + # not cleanly released; preserve its completed execution steps. + if outcome is not None: + outcome.success = False + outcome.reward = 0.0 + detail = f"Environment cleanup failed: {close_error}" + outcome.error = f"{outcome.error}; {detail}" if outcome.error else detail async def run_benchmark( From 21853fc8ea06b77141b6de2b5837aa17124cdf32 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:43:20 +0800 Subject: [PATCH 059/140] test(cua-bench): Hypothesis fault-inject teardown failures across score boundaries --- .../cua_bench/tests/test_reward_hypothesis.py | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/libs/cua-bench/cua_bench/tests/test_reward_hypothesis.py b/libs/cua-bench/cua_bench/tests/test_reward_hypothesis.py index d8d5202dc1..e77b23af69 100644 --- a/libs/cua-bench/cua_bench/tests/test_reward_hypothesis.py +++ b/libs/cua-bench/cua_bench/tests/test_reward_hypothesis.py @@ -51,3 +51,45 @@ async def close(self): assert result.success is False assert result.reward == 0.0 assert result.error is not None + + +@settings(max_examples=64, deadline=None) +@given( + st.booleans(), + st.sampled_from([0.0, 0.25, 0.5, 0.9, 1.0]), +) +def test_cleanup_failure_cannot_produce_success(should_fail_close, reward): + closed = {"attempted": False} + + class CleanupEnvironment: + evaluate_task_fn = object() + solve_task_fn = None + + async def reset(self, task_id=0): + return b"initial", SimpleNamespace() + + async def step(self, action): + return b"next" + + async def evaluate(self): + return reward + + async def close(self): + closed["attempted"] = True + if should_fail_close: + raise RuntimeError("injected teardown failure") + + with patch("cua_bench.runners.make", return_value=CleanupEnvironment()): + result = asyncio.run(run_single_task( + Path("cleanup-task"), agent_fn=lambda screenshot, task: DoneAction(), max_steps=2 + )) + assert closed["attempted"] + assert result.steps == 1 + if should_fail_close: + assert result.success is False + assert result.reward == 0.0 + assert "Environment cleanup failed" in result.error + else: + assert result.error is None + assert result.reward == reward + assert result.success is (reward >= 0.5) From 19a4efa6ee42cf5d321f00ac029fac2a5cc3a549 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 09:11:59 +0800 Subject: [PATCH 060/140] fix(cua-bench): reject boolean and string evaluator rewards instead of coercing to scores --- libs/cua-bench/cua_bench/runners.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/libs/cua-bench/cua_bench/runners.py b/libs/cua-bench/cua_bench/runners.py index 663847ea0a..514a817b32 100644 --- a/libs/cua-bench/cua_bench/runners.py +++ b/libs/cua-bench/cua_bench/runners.py @@ -157,14 +157,17 @@ def finish(**kwargs): # Evaluate if env.evaluate_task_fn is not None: result = await env.evaluate() - if isinstance(result, (int, float)): - reward = float(result) + if isinstance(result, (int, float)) and not isinstance(result, bool): + raw_reward = result elif isinstance(result, list) and len(result) > 0: - reward = float(result[0]) + raw_reward = result[0] elif isinstance(result, dict) and "reward" in result: - reward = float(result["reward"]) + raw_reward = result["reward"] else: raise ValueError("Evaluator returned an unsupported reward format") + if isinstance(raw_reward, bool) or not isinstance(raw_reward, (int, float)): + raise ValueError("Evaluator reward must be a numeric value, not a boolean or string") + reward = float(raw_reward) if not math.isfinite(reward) or not 0.0 <= reward <= 1.0: raise ValueError(f"Evaluator reward must be finite and within [0, 1], got {reward!r}") else: From 3e69d2b5e36461889ea484c199dba3c240e0ae4c Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 09:12:10 +0800 Subject: [PATCH 061/140] test(cua-bench): fuzz malformed scalar, list and dict evaluator rewards via Hypothesis --- .../cua_bench/tests/test_reward_hypothesis.py | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/libs/cua-bench/cua_bench/tests/test_reward_hypothesis.py b/libs/cua-bench/cua_bench/tests/test_reward_hypothesis.py index e77b23af69..408f7f860d 100644 --- a/libs/cua-bench/cua_bench/tests/test_reward_hypothesis.py +++ b/libs/cua-bench/cua_bench/tests/test_reward_hypothesis.py @@ -93,3 +93,42 @@ async def close(self): assert result.error is None assert result.reward == reward assert result.success is (reward >= 0.5) + + +@settings(max_examples=80, deadline=None) +@given( + st.one_of(st.booleans(), st.text(max_size=16), st.none()), + st.sampled_from(["scalar", "list", "dict"]), +) +def test_evaluator_rejects_non_numeric_reward_payloads(value, shape): + observed = {"closed": False} + + class InvalidRewardEnvironment: + evaluate_task_fn = object() + solve_task_fn = None + + async def reset(self, task_id=0): + return b"initial", SimpleNamespace() + + async def step(self, action): + return b"next" + + async def evaluate(self): + if shape == "list": + return [value] + if shape == "dict": + return {"reward": value} + return value + + async def close(self): + observed["closed"] = True + + with patch("cua_bench.runners.make", return_value=InvalidRewardEnvironment()): + outcome = asyncio.run(run_single_task( + Path("invalid-evaluator"), agent_fn=lambda screenshot, task: DoneAction(), max_steps=2 + )) + assert observed["closed"] + assert outcome.steps == 1 + assert outcome.success is False + assert outcome.reward == 0.0 + assert outcome.error is not None From fce50607d176bb4a87f903ed99ff451136270f24 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 09:20:58 +0800 Subject: [PATCH 062/140] feat(cua-bench): expose privacy-minimal deterministic action trace digest per task --- libs/cua-bench/cua_bench/runners.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/libs/cua-bench/cua_bench/runners.py b/libs/cua-bench/cua_bench/runners.py index 514a817b32..c62b35fca3 100644 --- a/libs/cua-bench/cua_bench/runners.py +++ b/libs/cua-bench/cua_bench/runners.py @@ -6,6 +6,8 @@ import asyncio import fnmatch +import hashlib +import json import math import time import uuid @@ -62,6 +64,7 @@ class TaskResult: reward: float steps: int error: Optional[str] = None + action_trace_digest: Optional[str] = None async def run_single_task( @@ -102,11 +105,15 @@ def my_agent(screenshot: bytes, task: Task) -> Action: """ env = None step_count = 0 + action_trace = [] outcome = None def finish(**kwargs): nonlocal outcome outcome = TaskResult(**kwargs) + if action_trace: + serialized = json.dumps(action_trace, sort_keys=True, separators=(",", ":")) + outcome.action_trace_digest = hashlib.sha256(serialized.encode("utf-8")).hexdigest() return outcome try: @@ -141,6 +148,9 @@ def finish(**kwargs): action = agent_fn(screenshot, task_cfg) screenshot = await env.step(action) step_count += 1 + # Privacy-minimal event chain: never persist screenshots, action + # arguments, typed text, passwords or arbitrary agent payloads. + action_trace.append({"step": step_count, "action_type": type(action).__name__}) done = isinstance(action, DoneAction) # A setup-only run must never be counted as successful agent execution. From 775da2badf8c0b9d834f7d8c4370de25abf932c6 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 09:21:15 +0800 Subject: [PATCH 063/140] test(cua-bench): prove deterministic redacted action digest and exclude failed actions --- .../cua_bench/tests/test_run_benchmark.py | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/libs/cua-bench/cua_bench/tests/test_run_benchmark.py b/libs/cua-bench/cua_bench/tests/test_run_benchmark.py index eebf1b7d58..2adffb1d07 100644 --- a/libs/cua-bench/cua_bench/tests/test_run_benchmark.py +++ b/libs/cua-bench/cua_bench/tests/test_run_benchmark.py @@ -642,3 +642,53 @@ async def close(self): result = await runners.run_single_task(tmp_path, oracle=True) assert result.steps == 0 assert "reset failed" in result.error + + +class TestActionTraceDigest: + @pytest.mark.asyncio + async def test_trace_digest_is_stable_and_redacts_payloads(self, tmp_path, monkeypatch): + import hashlib + import json + from types import SimpleNamespace + import cua_bench.runners as runners + + class Env: + evaluate_task_fn = object() + solve_task_fn = None + async def reset(self, task_id=0): + return b"private screenshot", SimpleNamespace() + async def step(self, action): + return b"private screenshot after action" + async def evaluate(self): + return 1.0 + async def close(self): + pass + + monkeypatch.setattr(runners, "make", lambda path, split="train": Env()) + one = await runners.run_single_task(tmp_path, agent_fn=lambda *_: DoneAction()) + two = await runners.run_single_task(tmp_path, agent_fn=lambda *_: DoneAction()) + expected = hashlib.sha256(json.dumps( + [{"step": 1, "action_type": "DoneAction"}], + sort_keys=True, separators=(",", ":") + ).encode()).hexdigest() + assert one.action_trace_digest == two.action_trace_digest == expected + assert one.success is True + + @pytest.mark.asyncio + async def test_failed_step_cannot_be_recorded_as_completed_action(self, tmp_path, monkeypatch): + from types import SimpleNamespace + import cua_bench.runners as runners + + class Env: + async def reset(self, task_id=0): + return b"screen", SimpleNamespace() + async def step(self, action): + raise RuntimeError("injected step failure") + async def close(self): + pass + + monkeypatch.setattr(runners, "make", lambda path, split="train": Env()) + result = await runners.run_single_task(tmp_path, agent_fn=lambda *_: DoneAction()) + assert result.action_trace_digest is None + assert result.steps == 0 + assert result.success is False From 825c9bd1d58c050e6264aec30a8bb1d5c32a038c Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 09:21:28 +0800 Subject: [PATCH 064/140] feat(cua-bench): propagate action trace digest into aggregate benchmark results --- libs/cua-bench/cua_bench/runners.py | 1 + 1 file changed, 1 insertion(+) diff --git a/libs/cua-bench/cua_bench/runners.py b/libs/cua-bench/cua_bench/runners.py index c62b35fca3..bb978510a8 100644 --- a/libs/cua-bench/cua_bench/runners.py +++ b/libs/cua-bench/cua_bench/runners.py @@ -406,6 +406,7 @@ async def run_with_semaphore(task_path: Path, variant_id: int) -> TaskResult: "reward": result.reward, "steps": result.steps, "error": result.error, + "action_trace_digest": result.action_trace_digest, } ) rewards.append(result.reward) From 6fc5bacd50cf27e8cdb498219e3ccb588018b738 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 09:22:47 +0800 Subject: [PATCH 065/140] fix(cua-bench): validate action trace digest before aggregate scoring --- libs/cua-bench/cua_bench/runners.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/libs/cua-bench/cua_bench/runners.py b/libs/cua-bench/cua_bench/runners.py index bb978510a8..301f244341 100644 --- a/libs/cua-bench/cua_bench/runners.py +++ b/libs/cua-bench/cua_bench/runners.py @@ -382,6 +382,15 @@ async def run_with_semaphore(task_path: Path, variant_id: int) -> TaskResult: or not isinstance(result.steps, int) or isinstance(result.steps, bool) or result.steps < 0 + or ( + result.action_trace_digest is not None + and ( + not isinstance(result.action_trace_digest, str) + or len(result.action_trace_digest) != 64 + or any(c not in "0123456789abcdef" for c in result.action_trace_digest) + or result.steps == 0 + ) + ) or (bool(result.error) and (result.success or result.reward != 0.0)) or (not result.error and result.success != (result.reward >= 0.5)) ) From 9602daf07b1e7a0122113412036f4aab29e45b86 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 09:23:08 +0800 Subject: [PATCH 066/140] test(cua-bench): reject forged and impossible action trace digests in aggregate worker reports --- .../cua_bench/tests/test_run_benchmark.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/libs/cua-bench/cua_bench/tests/test_run_benchmark.py b/libs/cua-bench/cua_bench/tests/test_run_benchmark.py index 2adffb1d07..a27e4f1e6c 100644 --- a/libs/cua-bench/cua_bench/tests/test_run_benchmark.py +++ b/libs/cua-bench/cua_bench/tests/test_run_benchmark.py @@ -692,3 +692,32 @@ async def close(self): assert result.action_trace_digest is None assert result.steps == 0 assert result.success is False + + +class TestActionTraceAggregateAttestation: + @pytest.mark.asyncio + @pytest.mark.parametrize("digest", ["broken", "g" * 64, "A" * 64, 123, "0" * 64]) + async def test_aggregator_rejects_bad_or_impossible_worker_digest(self, tmp_path, monkeypatch, digest): + import cua_bench.runners as runners + from types import SimpleNamespace + + task = tmp_path / "case" + task.mkdir() + (task / "main.py").write_text("# discovery stub") + + monkeypatch.setattr(runners, "make", lambda *args, **kwargs: SimpleNamespace( + tasks_config_fn=lambda: [object()] + )) + + async def forged(path, task_index=0, **kwargs): + return TaskResult( + task_path=str(path), variant_id=task_index, + success=True, reward=1.0, steps=0 if digest == "0" * 64 else 1, + action_trace_digest=digest, + ) + + monkeypatch.setattr(runners, "run_single_task", forged) + report = await runners.run_benchmark(tmp_path, agent_fn=lambda *_: DoneAction()) + assert report.success_count == 0 + assert report.avg_reward == 0.0 + assert "Invalid or misattributed" in report.task_results[0]["error"] From 626c4f9c99301c338d147030a9fd6b1ab564aa8b Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 09:42:15 +0800 Subject: [PATCH 067/140] feat(cua-bench): expose redacted action events and independently recompute worker trace digest --- libs/cua-bench/cua_bench/runners.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/libs/cua-bench/cua_bench/runners.py b/libs/cua-bench/cua_bench/runners.py index 301f244341..520ad8ace0 100644 --- a/libs/cua-bench/cua_bench/runners.py +++ b/libs/cua-bench/cua_bench/runners.py @@ -65,6 +65,7 @@ class TaskResult: steps: int error: Optional[str] = None action_trace_digest: Optional[str] = None + action_trace_events: Optional[List[Dict[str, Any]]] = None async def run_single_task( @@ -114,6 +115,7 @@ def finish(**kwargs): if action_trace: serialized = json.dumps(action_trace, sort_keys=True, separators=(",", ":")) outcome.action_trace_digest = hashlib.sha256(serialized.encode("utf-8")).hexdigest() + outcome.action_trace_events = list(action_trace) return outcome try: @@ -391,6 +393,26 @@ async def run_with_semaphore(task_path: Path, variant_id: int) -> TaskResult: or result.steps == 0 ) ) + or ( + result.action_trace_digest is not None + and ( + not isinstance(result.action_trace_events, list) + or len(result.action_trace_events) != result.steps + or any( + not isinstance(event, dict) + or set(event) != {"step", "action_type"} + or type(event["step"]) is not int + or event["step"] != index + or not isinstance(event["action_type"], str) + or not event["action_type"].isidentifier() + for index, event in enumerate(result.action_trace_events, start=1) + ) + or hashlib.sha256( + json.dumps(result.action_trace_events, sort_keys=True, separators=(",", ":")).encode("utf-8") + ).hexdigest() != result.action_trace_digest + ) + ) + or (result.action_trace_digest is None and result.action_trace_events is not None) or (bool(result.error) and (result.success or result.reward != 0.0)) or (not result.error and result.success != (result.reward >= 0.5)) ) @@ -416,6 +438,7 @@ async def run_with_semaphore(task_path: Path, variant_id: int) -> TaskResult: "steps": result.steps, "error": result.error, "action_trace_digest": result.action_trace_digest, + "action_trace_events": result.action_trace_events, } ) rewards.append(result.reward) From b804f9f4ba8cd6ec1292789c4b519809ac72487f Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 09:42:34 +0800 Subject: [PATCH 068/140] test(cua-bench): recompute action trace receipts and reject tampered events --- .../cua_bench/tests/test_run_benchmark.py | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/libs/cua-bench/cua_bench/tests/test_run_benchmark.py b/libs/cua-bench/cua_bench/tests/test_run_benchmark.py index a27e4f1e6c..2f347cd123 100644 --- a/libs/cua-bench/cua_bench/tests/test_run_benchmark.py +++ b/libs/cua-bench/cua_bench/tests/test_run_benchmark.py @@ -721,3 +721,66 @@ async def forged(path, task_index=0, **kwargs): assert report.success_count == 0 assert report.avg_reward == 0.0 assert "Invalid or misattributed" in report.task_results[0]["error"] + + +class TestIndependentlyRecomputableActionTrace: + @pytest.mark.asyncio + async def test_aggregator_accepts_only_matching_redacted_events(self, tmp_path, monkeypatch): + import hashlib + import json + from types import SimpleNamespace + import cua_bench.runners as runners + + task = tmp_path / "case" + task.mkdir() + (task / "main.py").write_text("# discovery stub") + monkeypatch.setattr(runners, "make", lambda *a, **k: SimpleNamespace( + tasks_config_fn=lambda: [object()] + )) + events = [{"step": 1, "action_type": "DoneAction"}] + digest = hashlib.sha256(json.dumps( + events, sort_keys=True, separators=(",", ":") + ).encode()).hexdigest() + + async def worker(path, task_index=0, **kwargs): + return TaskResult( + task_path=str(path), variant_id=task_index, success=True, + reward=1.0, steps=1, action_trace_digest=digest, + action_trace_events=[dict(event) for event in events], + ) + + monkeypatch.setattr(runners, "run_single_task", worker) + good = await runners.run_benchmark(tmp_path, agent_fn=lambda *_: DoneAction()) + assert good.success_count == 1 + assert good.task_results[0]["action_trace_digest"] == digest + assert good.task_results[0]["action_trace_events"] == events + + events[0]["action_type"] = "ClickAction" + tampered = await runners.run_benchmark(tmp_path, agent_fn=lambda *_: DoneAction()) + assert tampered.success_count == 0 + assert "Invalid or misattributed" in tampered.task_results[0]["error"] + + + @pytest.mark.asyncio + async def test_digest_without_events_cannot_claim_success(self, tmp_path, monkeypatch): + from types import SimpleNamespace + import cua_bench.runners as runners + + task = tmp_path / "case" + task.mkdir() + (task / "main.py").write_text("# discovery stub") + monkeypatch.setattr(runners, "make", lambda *a, **k: SimpleNamespace( + tasks_config_fn=lambda: [object()] + )) + + async def worker(path, task_index=0, **kwargs): + return TaskResult( + task_path=str(path), variant_id=task_index, + success=True, reward=1.0, steps=1, + action_trace_digest="a" * 64, + ) + + monkeypatch.setattr(runners, "run_single_task", worker) + report = await runners.run_benchmark(tmp_path, agent_fn=lambda *_: DoneAction()) + assert report.success_count == 0 + assert report.avg_reward == 0.0 From 37d3087c1e28d9e2233aa5c2d9b832f6943ea2d4 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:18:49 +0800 Subject: [PATCH 069/140] feat(cua-bench): make dataset fingerprinting importable in installed runtime --- libs/cua-bench/cua_bench/dataset_manifest.py | 59 ++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 libs/cua-bench/cua_bench/dataset_manifest.py diff --git a/libs/cua-bench/cua_bench/dataset_manifest.py b/libs/cua-bench/cua_bench/dataset_manifest.py new file mode 100644 index 0000000000..8a9b39f11c --- /dev/null +++ b/libs/cua-bench/cua_bench/dataset_manifest.py @@ -0,0 +1,59 @@ +"""Dataset identity and preflight verification for executable CUA Bench runs. + +The manifest is read without importing untrusted task Python code. +""" +import hashlib +from pathlib import Path + +VERSION = "cua-dataset-manifest/v1" +IGNORED = {"__pycache__", ".git", ".venv", "node_modules", ".pytest_cache"} + + +def scan_dataset(root: Path) -> dict: + root = Path(root).resolve(strict=True) + if not root.is_dir(): + raise ValueError("dataset must be a directory") + if (root / "main.py").is_file(): + tasks = [root] + else: + entries = sorted(root.iterdir()) + for entry in entries: + if entry.is_symlink() and entry.is_dir() and (entry / "main.py").is_file(): + raise ValueError(f"symlinked dataset task directory: {entry.name}") + tasks = [p for p in entries if p.is_dir() and (p / "main.py").is_file()] + if not tasks: + raise ValueError("no task directories containing main.py") + output = [] + for task in tasks: + files = [] + for path in sorted(task.rglob("*")): + relative = path.relative_to(root) + if any(part in IGNORED for part in relative.parts): + continue + if path.is_symlink(): + raise ValueError(f"symlink in dataset task: {relative}") + if not path.is_file(): + continue + digest = hashlib.sha256() + size = 0 + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + size += len(chunk) + files.append({"path": relative.as_posix(), "sha256": digest.hexdigest(), "size": size}) + if not files: + raise ValueError(f"empty task: {task}") + output.append({"task": task.relative_to(root).as_posix(), "files": files}) + return {"schema_version": VERSION, "tasks": output} + + +def verify_dataset(root: Path, manifest: dict) -> int: + if not isinstance(manifest, dict) or manifest.get("schema_version") != VERSION: + raise ValueError("unsupported dataset manifest") + expected = manifest.get("tasks") + if not isinstance(expected, list) or not expected: + raise ValueError("manifest has no tasks") + actual = scan_dataset(root) + if actual != manifest: + raise ValueError("dataset differs from manifest (added, removed, or modified task files)") + return len(actual["tasks"]) From b1c8e5c51b5ac831d9d9cea3c51cc7c2fd112731 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:19:06 +0800 Subject: [PATCH 070/140] feat(cua-bench): enforce pinned dataset manifest before benchmark task execution --- libs/cua-bench/cua_bench/runners.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/libs/cua-bench/cua_bench/runners.py b/libs/cua-bench/cua_bench/runners.py index 520ad8ace0..cd4178e5f8 100644 --- a/libs/cua-bench/cua_bench/runners.py +++ b/libs/cua-bench/cua_bench/runners.py @@ -16,6 +16,7 @@ from typing import Any, Callable, Dict, List, Optional, Tuple from .core import Task, make +from .dataset_manifest import verify_dataset from .environment import Environment from .types import Action, DoneAction @@ -43,6 +44,7 @@ class BenchmarkResult: avg_reward: float duration_seconds: float output_dir: Optional[str] = None + dataset_manifest_sha256: Optional[str] = None @dataclass @@ -226,6 +228,7 @@ async def run_benchmark( max_variants: Optional[int] = None, task_filter: Optional[str] = None, split: str = "train", + dataset_manifest: Optional[Path] = None, ) -> BenchmarkResult: """Run a benchmark on a dataset using the gym interface. @@ -242,6 +245,7 @@ async def run_benchmark( max_variants: Maximum variants per task (optional) task_filter: Glob pattern to filter tasks (optional) split: Dataset split (default: "train") + dataset_manifest: Optional manifest JSON to verify before importing or running tasks. Returns: BenchmarkResult with run statistics and task results @@ -286,6 +290,16 @@ def random_agent(screenshot: bytes, task: Task) -> Action: if not dataset_path.exists(): raise FileNotFoundError(f"Dataset not found: {dataset_path}") + # Verify pinned dataset bytes before importing task definitions or launching workers. + manifest_sha256 = None + if dataset_manifest is not None: + manifest_bytes = Path(dataset_manifest).read_bytes() + manifest = json.loads(manifest_bytes) + verify_dataset(dataset_path, manifest) + manifest_sha256 = hashlib.sha256( + json.dumps(manifest, sort_keys=True, separators=(",", ":")).encode("utf-8") + ).hexdigest() + # Discover tasks in dataset tasks: List[Path] = [] if (dataset_path / "main.py").exists(): @@ -457,6 +471,7 @@ async def run_with_semaphore(task_path: Path, variant_id: int) -> TaskResult: failed_count=failed_count, avg_reward=avg_reward, duration_seconds=duration_seconds, + dataset_manifest_sha256=manifest_sha256, ) From 798e5feba98c86f0de9605a35ff672e710dde0fc Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:19:30 +0800 Subject: [PATCH 071/140] test(cua-bench): prove pinned dataset blocks task loading and binds completed run --- .../cua_bench/tests/test_run_benchmark.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/libs/cua-bench/cua_bench/tests/test_run_benchmark.py b/libs/cua-bench/cua_bench/tests/test_run_benchmark.py index 2f347cd123..de3b6d825a 100644 --- a/libs/cua-bench/cua_bench/tests/test_run_benchmark.py +++ b/libs/cua-bench/cua_bench/tests/test_run_benchmark.py @@ -784,3 +784,54 @@ async def worker(path, task_index=0, **kwargs): report = await runners.run_benchmark(tmp_path, agent_fn=lambda *_: DoneAction()) assert report.success_count == 0 assert report.avg_reward == 0.0 + + +class TestPinnedDatasetRuntimeIntegration: + @pytest.mark.asyncio + async def test_modified_dataset_aborts_before_task_import_or_worker(self, tmp_path, monkeypatch): + import json + from cua_bench.dataset_manifest import scan_dataset + import cua_bench.runners as runners + + task = tmp_path / "case" + task.mkdir() + (task / "main.py").write_text("fixture = 1\n") + manifest_path = tmp_path.parent / (tmp_path.name + "-manifest.json") + manifest_path.write_text(json.dumps(scan_dataset(tmp_path))) + (task / "main.py").write_text("fixture = 2\n") + + def no_import(*args, **kwargs): + raise AssertionError("task discovery must not execute when manifest differs") + + monkeypatch.setattr(runners, "make", no_import) + with pytest.raises(ValueError, match="dataset differs from manifest"): + await runners.run_benchmark(tmp_path, dataset_manifest=manifest_path, oracle=True) + + @pytest.mark.asyncio + async def test_matching_manifest_is_bound_to_benchmark_result(self, tmp_path, monkeypatch): + import hashlib + import json + from types import SimpleNamespace + from cua_bench.dataset_manifest import scan_dataset + import cua_bench.runners as runners + + task = tmp_path / "case" + task.mkdir() + (task / "main.py").write_text("fixture = 1\n") + manifest = scan_dataset(tmp_path) + pinned = tmp_path.parent / (tmp_path.name + "-pinned.json") + pinned.write_text(json.dumps(manifest)) + monkeypatch.setattr(runners, "make", lambda *args, **kwargs: SimpleNamespace( + tasks_config_fn=lambda: [object()] + )) + + async def worker(path, task_index=0, **kwargs): + return TaskResult(task_path=str(path), variant_id=task_index, + success=True, reward=1.0, steps=1) + monkeypatch.setattr(runners, "run_single_task", worker) + outcome = await runners.run_benchmark(tmp_path, dataset_manifest=pinned, oracle=True) + expected = hashlib.sha256(json.dumps( + manifest, sort_keys=True, separators=(",", ":") + ).encode()).hexdigest() + assert outcome.success_count == 1 + assert outcome.dataset_manifest_sha256 == expected From 4b8da36d7bbaf88a729b9265d213d784a147ffe6 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:21:41 +0800 Subject: [PATCH 072/140] feat(cua-bench): expose pinned dataset verification on actual cb run dataset CLI --- libs/cua-bench/cua_bench/cli/main.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/libs/cua-bench/cua_bench/cli/main.py b/libs/cua-bench/cua_bench/cli/main.py index 426a72982c..e098c0d1ba 100644 --- a/libs/cua-bench/cua_bench/cli/main.py +++ b/libs/cua-bench/cua_bench/cli/main.py @@ -181,6 +181,11 @@ def add_common_run_args(parser): run_dataset_parser.add_argument( "dataset_path", help="Path to dataset directory, or dataset name from registry" ) + run_dataset_parser.add_argument( + "--dataset-manifest", + dest="dataset_manifest", + help="Pin dataset files to a JSON manifest; reject changes before launching sessions", + ) run_dataset_parser.add_argument( "--max-parallel", dest="max_parallel", From 132d3bf6a7cbed3ed231e46231a0ecab544c93a1 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:21:55 +0800 Subject: [PATCH 073/140] feat(cua-bench): enforce manifest preflight in real parallel two-container CLI --- libs/cua-bench/cua_bench/cli/commands/run.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/libs/cua-bench/cua_bench/cli/commands/run.py b/libs/cua-bench/cua_bench/cli/commands/run.py index 431c8a2f94..1c43103d14 100644 --- a/libs/cua-bench/cua_bench/cli/commands/run.py +++ b/libs/cua-bench/cua_bench/cli/commands/run.py @@ -1409,6 +1409,20 @@ async def _cmd_run_dataset_async(args) -> int: print(f"{RED}Error: Dataset not found: {args.dataset_path}{RESET}") return 1 + # Validate the pinned dataset before provider detection, task imports, + # session creation or asynchronous subprocess launch. + manifest_path = getattr(args, "dataset_manifest", None) + if manifest_path: + import json + from cua_bench.dataset_manifest import verify_dataset + try: + manifest = json.loads(Path(manifest_path).read_text(encoding="utf-8")) + task_count = verify_dataset(dataset_path, manifest) + except (OSError, ValueError, TypeError) as exc: + print(f"{RED}Error: Dataset manifest verification failed: {exc}{RESET}") + return 1 + print(f"{GREEN}Verified pinned dataset: {task_count} tasks{RESET}") + # Discover tasks in dataset tasks = [] From cb3eda15692a97790a30031fcf8d2de91d184774 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:22:20 +0800 Subject: [PATCH 074/140] feat(cua-bench): provide manifest generation through installed cb CLI --- libs/cua-bench/cua_bench/cli/main.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/libs/cua-bench/cua_bench/cli/main.py b/libs/cua-bench/cua_bench/cli/main.py index e098c0d1ba..2810e543ec 100644 --- a/libs/cua-bench/cua_bench/cli/main.py +++ b/libs/cua-bench/cua_bench/cli/main.py @@ -213,6 +213,13 @@ def add_common_run_args(parser): ) add_common_run_args(run_dataset_parser) + # cb run manifest --output manifest.json + manifest_parser = run_subparsers.add_parser( + "manifest", help="Generate a pinned dataset manifest before launching a run" + ) + manifest_parser.add_argument("dataset_path", help="Local dataset directory") + manifest_parser.add_argument("--output", required=True, help="JSON manifest output path") + # cb run list run_list_parser = run_subparsers.add_parser("list", help="List all runs with status") run_list_parser.add_argument( From e0696a1af8d2e1bf597c436d848177def4ca55ff Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:22:33 +0800 Subject: [PATCH 075/140] feat(cua-bench): generate pinned manifests from actual cb run manifest command --- libs/cua-bench/cua_bench/cli/commands/run.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/libs/cua-bench/cua_bench/cli/commands/run.py b/libs/cua-bench/cua_bench/cli/commands/run.py index 1c43103d14..c333d1d461 100644 --- a/libs/cua-bench/cua_bench/cli/commands/run.py +++ b/libs/cua-bench/cua_bench/cli/commands/run.py @@ -1960,6 +1960,20 @@ def execute(args): return cmd_run_task(args) elif run_command == "dataset": return cmd_run_dataset(args) + elif run_command == "manifest": + import json + from cua_bench.dataset_manifest import scan_dataset + try: + manifest = scan_dataset(Path(args.dataset_path)) + output = Path(args.output) + if output.resolve().is_relative_to(Path(args.dataset_path).resolve()): + raise ValueError("Manifest output must be outside the dataset to avoid self-inclusion") + output.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\\n", encoding="utf-8") + except (OSError, ValueError) as exc: + print(f"{RED}Error: Cannot create manifest: {exc}{RESET}") + return 1 + print(f"{GREEN}Recorded {len(manifest['tasks'])} tasks in {output}{RESET}") + return 0 elif run_command == "list": return cmd_list(args) elif run_command == "watch": From 413220f5168f55b0aa1f1631568617e77aa76b13 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:22:56 +0800 Subject: [PATCH 076/140] fix(cua-bench): terminate CLI-generated manifest with a real newline --- libs/cua-bench/cua_bench/cli/commands/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/cua-bench/cua_bench/cli/commands/run.py b/libs/cua-bench/cua_bench/cli/commands/run.py index c333d1d461..8268c52def 100644 --- a/libs/cua-bench/cua_bench/cli/commands/run.py +++ b/libs/cua-bench/cua_bench/cli/commands/run.py @@ -1968,7 +1968,7 @@ def execute(args): output = Path(args.output) if output.resolve().is_relative_to(Path(args.dataset_path).resolve()): raise ValueError("Manifest output must be outside the dataset to avoid self-inclusion") - output.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\\n", encoding="utf-8") + output.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") except (OSError, ValueError) as exc: print(f"{RED}Error: Cannot create manifest: {exc}{RESET}") return 1 From 2f4ae4182d5801a1a244df367d394c8f5ce5f992 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:23:31 +0800 Subject: [PATCH 077/140] ci(cua-bench): exercise production manifest CLI before core regression --- .github/workflows/ci-py-bench.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci-py-bench.yml b/.github/workflows/ci-py-bench.yml index e94736b532..adc1b3771b 100644 --- a/.github/workflows/ci-py-bench.yml +++ b/.github/workflows/ci-py-bench.yml @@ -39,6 +39,11 @@ jobs: run: | python3 scripts/dataset_manifest.py datasets/cua-bench-basic --output /tmp/cua-basic-manifest.json python3 scripts/dataset_manifest.py datasets/cua-bench-basic --verify /tmp/cua-basic-manifest.json + - name: Exercise installed manifest CLI + working-directory: libs/cua-bench + run: | + uv run --extra dev python -m cua_bench.cli.main run manifest datasets/cua-bench-basic --output /tmp/cua-cli-pinned.json + uv run --extra dev python -c "import json; from pathlib import Path; from cua_bench.dataset_manifest import verify_dataset; verify_dataset(Path('datasets/cua-bench-basic'), json.loads(Path('/tmp/cua-cli-pinned.json').read_text()))" - name: Run CUA benchmark core and oracle regression suite working-directory: libs/cua-bench run: | From 9b5086edc0fdb3e0761702c213273777291dea84 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:28:51 +0800 Subject: [PATCH 078/140] fix(cua-bench): propagate pinned manifest into actual detached runner and persist dataset identity --- libs/cua-bench/cua_bench/cli/commands/run.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/libs/cua-bench/cua_bench/cli/commands/run.py b/libs/cua-bench/cua_bench/cli/commands/run.py index 8268c52def..0c2066ea35 100644 --- a/libs/cua-bench/cua_bench/cli/commands/run.py +++ b/libs/cua-bench/cua_bench/cli/commands/run.py @@ -1498,6 +1498,19 @@ async def _cmd_run_dataset_async(args) -> int: output_dir.mkdir(parents=True, exist_ok=True) + if manifest_path: + import hashlib + import json + # Record the exact manifest identity used at dispatch time. + canonical_manifest = json.dumps(manifest, sort_keys=True, separators=(",", ":")) + identity = hashlib.sha256(canonical_manifest.encode("utf-8")).hexdigest() + (output_dir / "dataset-identity.json").write_text( + json.dumps({"schema_version": "cua-dataset-identity/v1", + "dataset_manifest_sha256": identity, + "manifest_path": str(Path(manifest_path).resolve())}, + indent=2) + "\n", encoding="utf-8" + ) + # Detect env type from first task first_task = task_variants[0][0] if task_variants else None if first_task: @@ -1798,6 +1811,10 @@ async def run_with_semaphore(task_path, variant_id, task_num): cmd.extend(["--task-filter", task_filter]) if user_output_dir: cmd.extend(["--output-dir", str(user_output_dir)]) + if manifest_path: + # Re-verify in the actual background worker; the dataset can change + # after the parent preflight and before subprocess startup. + cmd.extend(["--dataset-manifest", str(Path(manifest_path).resolve())]) # Set UTF-8 encoding env_vars = os.environ.copy() From 816a9331154ed1cc010905d9d075b824dba4e662 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:29:07 +0800 Subject: [PATCH 079/140] feat(cua-bench): persist real parallel TaskRunner outcomes in machine-readable run receipt --- libs/cua-bench/cua_bench/cli/commands/run.py | 28 ++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/libs/cua-bench/cua_bench/cli/commands/run.py b/libs/cua-bench/cua_bench/cli/commands/run.py index 0c2066ea35..44978c2eaf 100644 --- a/libs/cua-bench/cua_bench/cli/commands/run.py +++ b/libs/cua-bench/cua_bench/cli/commands/run.py @@ -1736,6 +1736,34 @@ async def run_with_semaphore(task_path, variant_id, task_num): log_print(summary) + # Persist actual TaskRunner outcomes for downstream consumers rather + # than forcing them to parse colored console messages. + import json + receipt_rows = [] + for (task_path, variant_id), result in zip(task_variants, results): + valid_result = result is not None and not isinstance(result, BaseException) + receipt_rows.append({ + "task": task_path.name, + "variant_id": variant_id, + "success": bool(result.success) if valid_result else False, + "error": (str(result) if isinstance(result, BaseException) + else ("task failed without result" if result is None else None)), + }) + receipt = { + "schema_version": "cua-bench-dataset-run/v1", + "run_id": run_id, + "total_tasks": len(results), + "success_count": success_count, + "failed_count": failed_count, + "tasks": receipt_rows, + } + identity_path = output_dir / "dataset-identity.json" + if identity_path.is_file(): + receipt["dataset_identity"] = json.loads(identity_path.read_text(encoding="utf-8")) + (output_dir / "run-result.json").write_text( + json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + # Summary model_name = getattr(args, "model", None) or "default" print("\n") From 82c23dbab0d2e0abea51f56fb2ddc9a823b6824c Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:29:30 +0800 Subject: [PATCH 080/140] feat(cua-bench): persist actual dual-container task outcomes as versioned run-result JSON --- libs/cua-bench/cua_bench/cli/commands/run.py | 46 ++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/libs/cua-bench/cua_bench/cli/commands/run.py b/libs/cua-bench/cua_bench/cli/commands/run.py index 44978c2eaf..19cb72b78a 100644 --- a/libs/cua-bench/cua_bench/cli/commands/run.py +++ b/libs/cua-bench/cua_bench/cli/commands/run.py @@ -1409,6 +1409,8 @@ async def _cmd_run_dataset_async(args) -> int: print(f"{RED}Error: Dataset not found: {args.dataset_path}{RESET}") return 1 + verified_manifest_digest = None + # Validate the pinned dataset before provider detection, task imports, # session creation or asynchronous subprocess launch. manifest_path = getattr(args, "dataset_manifest", None) @@ -1418,6 +1420,10 @@ async def _cmd_run_dataset_async(args) -> int: try: manifest = json.loads(Path(manifest_path).read_text(encoding="utf-8")) task_count = verify_dataset(dataset_path, manifest) + import hashlib + verified_manifest_digest = hashlib.sha256( + json.dumps(manifest, sort_keys=True, separators=(",", ":")).encode("utf-8") + ).hexdigest() except (OSError, ValueError, TypeError) as exc: print(f"{RED}Error: Dataset manifest verification failed: {exc}{RESET}") return 1 @@ -1723,6 +1729,46 @@ async def run_with_semaphore(task_path, variant_id, task_num): # Run all tasks (semaphore controls parallelism) results = await asyncio.gather(*coroutines, return_exceptions=True) + # Persist the actual two-container TaskRunner outcomes, not synthetic + # single-process runner values or parsed log-based success claims. + import json + import os + + task_outcomes = [] + for (task_path, variant_id), outcome in zip(task_variants, results): + passed = ( + not isinstance(outcome, BaseException) + and outcome is not None + and getattr(outcome, "success", False) is True + ) + task_outcomes.append({ + "task": task_path.name, + "variant_id": variant_id, + "success": passed, + "failure_type": ( + type(outcome).__name__ if isinstance(outcome, BaseException) + else "MissingResult" if outcome is None + else None if passed + else "TaskFailed" + ), + }) + + receipt = { + "schema_version": "cua-bench-run-result/v1", + "run_id": run_id, + "dataset_name": dataset_path.name, + "dataset_manifest_sha256": verified_manifest_digest, + "total_tasks": len(task_outcomes), + "success_count": sum(item["success"] for item in task_outcomes), + "failed_count": sum(not item["success"] for item in task_outcomes), + "tasks": task_outcomes, + } + receipt_path = output_dir / "run-result.json" + pending = output_dir / "run-result.json.tmp" + pending.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8") + os.replace(pending, receipt_path) + log_print(f"Machine-readable run result: {receipt_path}") + # Summarize results success_count = sum(1 for r in results if r and hasattr(r, "success") and r.success) failed_count = len(results) - success_count From 45500791cb49728ed6018ab8452eef8be02b3340 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:29:44 +0800 Subject: [PATCH 081/140] fix(cua-bench): use persisted task outcomes as single source of summary truth --- libs/cua-bench/cua_bench/cli/commands/run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/cua-bench/cua_bench/cli/commands/run.py b/libs/cua-bench/cua_bench/cli/commands/run.py index 19cb72b78a..48148709a0 100644 --- a/libs/cua-bench/cua_bench/cli/commands/run.py +++ b/libs/cua-bench/cua_bench/cli/commands/run.py @@ -1770,8 +1770,8 @@ async def run_with_semaphore(task_path, variant_id, task_num): log_print(f"Machine-readable run result: {receipt_path}") # Summarize results - success_count = sum(1 for r in results if r and hasattr(r, "success") and r.success) - failed_count = len(results) - success_count + success_count = receipt["success_count"] + failed_count = receipt["failed_count"] summary = f"\n{CYAN}{'=' * 60}{RESET}\n" summary += f"{CYAN}Dataset Complete: {dataset_path.name}{RESET}\n" From 20d714b3d9db77d107dfda2414cb7cd7c968a0cf Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:52:12 +0800 Subject: [PATCH 082/140] fix(cua-bench): preserve canonical dataset receipt and index real task artifacts --- libs/cua-bench/cua_bench/cli/commands/run.py | 57 ++++++++++---------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/libs/cua-bench/cua_bench/cli/commands/run.py b/libs/cua-bench/cua_bench/cli/commands/run.py index 48148709a0..a993cf6a2f 100644 --- a/libs/cua-bench/cua_bench/cli/commands/run.py +++ b/libs/cua-bench/cua_bench/cli/commands/run.py @@ -1763,6 +1763,36 @@ async def run_with_semaphore(task_path, variant_id, task_num): "failed_count": sum(not item["success"] for item in task_outcomes), "tasks": task_outcomes, } + # Index the artifacts actually emitted by each agent container. Hashes + # let external consumers verify bytes without trusting printed logs. + # An artifact hash is NOT proof that an action was executed or an + # oracle passed; those require separately recorded execution events. + import hashlib + + for row in task_outcomes: + task_dir = output_dir / f"{row['task']}_v{row['variant_id']}" + artifacts = [] + if task_dir.is_dir(): + for artifact in sorted(task_dir.rglob("*")): + if not artifact.is_file() or artifact.is_symlink(): + continue + relative_path = artifact.relative_to(task_dir).as_posix() + digest = hashlib.sha256() + with artifact.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + artifacts.append({ + "path": relative_path, + "size_bytes": artifact.stat().st_size, + "sha256": digest.hexdigest(), + }) + row["artifacts"] = artifacts + row["evidence_status"] = "artifacts_indexed" if artifacts else "no_artifacts" + + identity_path = output_dir / "dataset-identity.json" + if identity_path.is_file(): + receipt["dataset_identity"] = json.loads(identity_path.read_text(encoding="utf-8")) + receipt_path = output_dir / "run-result.json" pending = output_dir / "run-result.json.tmp" pending.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8") @@ -1782,33 +1812,6 @@ async def run_with_semaphore(task_path, variant_id, task_num): log_print(summary) - # Persist actual TaskRunner outcomes for downstream consumers rather - # than forcing them to parse colored console messages. - import json - receipt_rows = [] - for (task_path, variant_id), result in zip(task_variants, results): - valid_result = result is not None and not isinstance(result, BaseException) - receipt_rows.append({ - "task": task_path.name, - "variant_id": variant_id, - "success": bool(result.success) if valid_result else False, - "error": (str(result) if isinstance(result, BaseException) - else ("task failed without result" if result is None else None)), - }) - receipt = { - "schema_version": "cua-bench-dataset-run/v1", - "run_id": run_id, - "total_tasks": len(results), - "success_count": success_count, - "failed_count": failed_count, - "tasks": receipt_rows, - } - identity_path = output_dir / "dataset-identity.json" - if identity_path.is_file(): - receipt["dataset_identity"] = json.loads(identity_path.read_text(encoding="utf-8")) - (output_dir / "run-result.json").write_text( - json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8" - ) # Summary model_name = getattr(args, "model", None) or "default" From 569ed46dc51265d6af0534cfa400a78579d3724c Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:52:29 +0800 Subject: [PATCH 083/140] feat(cua-bench): bind real worker exit codes and errors to canonical task receipts --- libs/cua-bench/cua_bench/cli/commands/run.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/libs/cua-bench/cua_bench/cli/commands/run.py b/libs/cua-bench/cua_bench/cli/commands/run.py index a993cf6a2f..e8f82e62c7 100644 --- a/libs/cua-bench/cua_bench/cli/commands/run.py +++ b/libs/cua-bench/cua_bench/cli/commands/run.py @@ -1745,6 +1745,16 @@ async def run_with_semaphore(task_path, variant_id, task_num): "task": task_path.name, "variant_id": variant_id, "success": passed, + "exit_code": ( + getattr(outcome, "exit_code", None) + if outcome is not None and not isinstance(outcome, BaseException) + else None + ), + "error": ( + str(outcome) if isinstance(outcome, BaseException) + else "task failed without result" if outcome is None + else getattr(outcome, "error", None) + ), "failure_type": ( type(outcome).__name__ if isinstance(outcome, BaseException) else "MissingResult" if outcome is None From 54cc7367cb70dc9b8240079b85d1d6fbb50a02a3 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:38:31 +0800 Subject: [PATCH 084/140] feat(cua-bench): record executed desktop actions and before-after observation digests --- libs/cua-bench/cua_bench/agents/cua_agent.py | 69 +++++++++++++++++--- 1 file changed, 59 insertions(+), 10 deletions(-) diff --git a/libs/cua-bench/cua_bench/agents/cua_agent.py b/libs/cua-bench/cua_bench/agents/cua_agent.py index 01b28d1c14..3eb5ff9955 100644 --- a/libs/cua-bench/cua_bench/agents/cua_agent.py +++ b/libs/cua-bench/cua_bench/agents/cua_agent.py @@ -2,6 +2,9 @@ import asyncio import base64 +import hashlib +import json +from datetime import datetime, timezone import sys from pathlib import Path from typing import TYPE_CHECKING, Any, Dict, List @@ -56,7 +59,7 @@ def __init__(self, **kwargs): def name() -> str: return "cua-agent" - def _create_custom_computer(self, session: "DesktopSession") -> Dict[str, Any]: + def _create_custom_computer(self, session: "DesktopSession", logging_dir: Path | None = None) -> Dict[str, Any]: """Create a custom computer dict from a DesktopSession. Args: @@ -79,6 +82,52 @@ def _create_custom_computer(self, session: "DesktopSession") -> Dict[str, Any]: WaitAction, ) + # Evidence originates at the actual desktop execution boundary, not + # from model output or a post-hoc interpretation of console logs. + evidence_path = None + if logging_dir is not None: + logging_dir.mkdir(parents=True, exist_ok=True) + evidence_path = logging_dir / "executed-actions.jsonl" + + async def execute_and_record(action): + from ..actions import action_to_dict + + action_data = action_to_dict(action) + # Typed text may contain secrets; store its length, not content. + if "text" in action_data: + action_data["text_length"] = len(action_data.pop("text")) + action_data["text_redacted"] = True + + async def observe(): + try: + screenshot_bytes = await session.screenshot() + return {"sha256": hashlib.sha256(screenshot_bytes).hexdigest(), + "size_bytes": len(screenshot_bytes)} + except Exception as exc: + return {"observation_error": type(exc).__name__} + + before = await observe() if evidence_path else None + error = None + try: + return await session.execute_action(action) + except Exception as exc: + error = type(exc).__name__ + raise + finally: + if evidence_path: + after = await observe() + event = { + "schema_version": "cua-bench-executed-action/v1", + "timestamp": datetime.now(timezone.utc).isoformat(), + "action": action_data, + "before": before, + "after": after, + "execution_status": "failed" if error else "returned", + "error_type": error, + } + with evidence_path.open("a", encoding="utf-8") as stream: + stream.write(json.dumps(event, sort_keys=True) + "\\n") + # Screenshot function (required) async def screenshot(): """Take a screenshot and return as base64 string.""" @@ -96,19 +145,19 @@ async def click(x: int, y: int, button: str = "left"): action = MiddleClickAction(x=x, y=y) else: raise ValueError(f"Unknown button type: {button}") - await session.execute_action(action) + await execute_and_record(action) # Double click function async def double_click(x: int, y: int): """Double click at coordinates.""" action = DoubleClickAction(x=x, y=y) - await session.execute_action(action) + await execute_and_record(action) # Type function async def type_text(text: str): """Type text.""" action = TypeAction(text=text) - await session.execute_action(action) + await execute_and_record(action) # Keypress function async def keypress(keys): @@ -117,13 +166,13 @@ async def keypress(keys): action = KeyAction(key=keys) else: action = HotkeyAction(keys=list(keys)) - await session.execute_action(action) + await execute_and_record(action) # Move function async def move(x: int, y: int): """Move cursor to coordinates.""" action = MoveToAction(x=x, y=y) - await session.execute_action(action) + await execute_and_record(action) # Scroll function async def scroll(x: int, y: int, scroll_x: int, scroll_y: int): @@ -135,7 +184,7 @@ async def scroll(x: int, y: int, scroll_x: int, scroll_y: int): direction = "down" amount = abs(scroll_y) action = ScrollAction(direction=direction, amount=amount) - await session.execute_action(action) + await execute_and_record(action) # Drag function async def drag(path: List[Dict[str, int]]): @@ -145,13 +194,13 @@ async def drag(path: List[Dict[str, int]]): start = path[0] end = path[-1] action = DragAction(from_x=start["x"], from_y=start["y"], to_x=end["x"], to_y=end["y"]) - await session.execute_action(action) + await execute_and_record(action) # Wait function async def wait(ms: int = 1000): """Wait for specified milliseconds.""" action = WaitAction(seconds=ms / 1000.0) - await session.execute_action(action) + await execute_and_record(action) # Get dimensions async def get_dimensions(): @@ -219,7 +268,7 @@ async def perform_task( trajectory_dir.mkdir(parents=True, exist_ok=True) # Create custom computer dict from session - custom_computer = self._create_custom_computer(session) + custom_computer = self._create_custom_computer(session, logging_dir=logging_dir) # Create agent with custom computer agent = ComputerAgent( From 623c89ea798817976c245647588c5cfc3019f67d Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:38:52 +0800 Subject: [PATCH 085/140] fix(cua-bench): delimit execution evidence as real JSONL records --- libs/cua-bench/cua_bench/agents/cua_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/cua-bench/cua_bench/agents/cua_agent.py b/libs/cua-bench/cua_bench/agents/cua_agent.py index 3eb5ff9955..48e69cf030 100644 --- a/libs/cua-bench/cua_bench/agents/cua_agent.py +++ b/libs/cua-bench/cua_bench/agents/cua_agent.py @@ -126,7 +126,7 @@ async def observe(): "error_type": error, } with evidence_path.open("a", encoding="utf-8") as stream: - stream.write(json.dumps(event, sort_keys=True) + "\\n") + stream.write(json.dumps(event, sort_keys=True) + "\n") # Screenshot function (required) async def screenshot(): From ba93a321dbf46eec5a667c6d161deacdc9d665a9 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:39:17 +0800 Subject: [PATCH 086/140] feat(cua-bench): surface actual action evidence presence in dataset receipts --- libs/cua-bench/cua_bench/cli/commands/run.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/libs/cua-bench/cua_bench/cli/commands/run.py b/libs/cua-bench/cua_bench/cli/commands/run.py index e8f82e62c7..1cc7d06095 100644 --- a/libs/cua-bench/cua_bench/cli/commands/run.py +++ b/libs/cua-bench/cua_bench/cli/commands/run.py @@ -1797,7 +1797,13 @@ async def run_with_semaphore(task_path, variant_id, task_num): "sha256": digest.hexdigest(), }) row["artifacts"] = artifacts - row["evidence_status"] = "artifacts_indexed" if artifacts else "no_artifacts" + action_files = [item for item in artifacts if item["path"].endswith("/executed-actions.jsonl")] + row["action_evidence_files"] = [item["path"] for item in action_files] + row["evidence_status"] = ( + "action_records_present" if action_files + else "artifacts_only" if artifacts + else "no_artifacts" + ) identity_path = output_dir / "dataset-identity.json" if identity_path.is_file(): From 637e2a0257e6c473302d8355d4714297009793c2 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:46:04 +0800 Subject: [PATCH 087/140] feat(cua-bench): fail closed on real task oracle and persist verdict in worker output --- libs/cua-bench/cua_bench/batch/solver.py | 40 ++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/libs/cua-bench/cua_bench/batch/solver.py b/libs/cua-bench/cua_bench/batch/solver.py index a17152df17..dd9c9d6b03 100644 --- a/libs/cua-bench/cua_bench/batch/solver.py +++ b/libs/cua-bench/cua_bench/batch/solver.py @@ -358,10 +358,33 @@ async def main(): else: print("ℹ Dump mode: skipping solver") - # Evaluate + # Evaluate against the real environment. A successfully exited agent + # is not evidence of task completion; only the task oracle can decide. + import json + import math + + evaluation_receipt = { + "schema_version": "cua-bench-oracle-result/v1", + "task_index": task_index, + "evaluated": False, + "success": False, + "rewards": None, + } if env.evaluate_task_fn: result = await env.evaluate_task_fn(task_cfg, session) print(f"✓ Evaluation result: {result}") + # Task evaluators return lists of numeric rewards (e.g. [1.0]). + # Reject missing, malformed, non-finite or partial outcomes. + rewards = result if isinstance(result, (list, tuple)) else [] + valid = bool(rewards) and all( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(value) + for value in rewards + ) + evaluation_receipt["evaluated"] = valid + evaluation_receipt["rewards"] = list(rewards) if valid else None + evaluation_receipt["success"] = valid and all(value >= 0.5 for value in rewards) # Record evaluation event try: @@ -371,6 +394,16 @@ async def main(): ) except Exception as e: print(f"Warning: Failed to record evaluate event: {e}") + else: + print("✗ Task has no evaluation function; cannot claim success") + + receipt_file = output_dir / "oracle-result.json" + pending_file = output_dir / "oracle-result.json.tmp" + pending_file.write_text( + json.dumps(evaluation_receipt, sort_keys=True, indent=2) + "\n", + encoding="utf-8", + ) + pending_file.replace(receipt_file) # Save trace (non-fatal - task success doesn't depend on trace saving) trace_dir = output_dir / f"task_{task_index}_trace" @@ -394,7 +427,10 @@ async def main(): await env.close() else: await session.close() - print(f"\n✓ Task {task_index} completed successfully!") + if not evaluation_receipt["success"]: + print(f"✗ Task {task_index} did not pass its environment oracle") + sys.exit(1) + print(f"\n✓ Task {task_index} passed its environment oracle!") except Exception as e: print(f"Error running task {task_index}: {e}") From fcbc92fea4f4ad0df723e740da12f1cb3b23a6a6 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:46:24 +0800 Subject: [PATCH 088/140] feat(cua-bench): require real oracle receipt before TaskRunner reports success --- .../cua-bench/cua_bench/runner/task_runner.py | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/libs/cua-bench/cua_bench/runner/task_runner.py b/libs/cua-bench/cua_bench/runner/task_runner.py index 6ca704343e..12ac587590 100644 --- a/libs/cua-bench/cua_bench/runner/task_runner.py +++ b/libs/cua-bench/cua_bench/runner/task_runner.py @@ -5,6 +5,7 @@ """ import asyncio +import json import os import shutil from dataclasses import dataclass @@ -104,6 +105,7 @@ class TaskResult: env_logs: str output_dir: Optional[str] = None error: Optional[str] = None + oracle_result: Optional[dict] = None # ============================================================================= @@ -356,12 +358,40 @@ async def run_task( agent_logs = await get_container_logs(agent_container_name, tail=500) env_logs = await get_container_logs(env_container_name, tail=100) + oracle_result = None + oracle_error = None + oracle_path = Path(output_dir) / "oracle-result.json" if output_dir else None + if oracle_path and oracle_path.is_file(): + try: + oracle_result = json.loads(oracle_path.read_text(encoding="utf-8")) + if ( + oracle_result.get("schema_version") != "cua-bench-oracle-result/v1" + or oracle_result.get("task_index") != task_index + or oracle_result.get("evaluated") is not True + or not isinstance(oracle_result.get("success"), bool) + ): + oracle_error = "Invalid oracle result receipt" + except (OSError, ValueError, TypeError): + oracle_error = "Unreadable oracle result receipt" + else: + oracle_error = "Missing oracle result receipt" + + oracle_passed = ( + oracle_error is None + and oracle_result is not None + and oracle_result["success"] is True + ) return TaskResult( - success=exit_code == 0, + success=exit_code == 0 and oracle_passed, exit_code=exit_code, agent_logs=agent_logs, env_logs=env_logs, output_dir=output_dir, + error=oracle_error or ( + "Oracle rejected task" if oracle_result and not oracle_passed + else None + ), + oracle_result=oracle_result, ) except Exception as e: From b5f94730cd741cfcd4fc20d377f63a4c1692661e Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:46:33 +0800 Subject: [PATCH 089/140] feat(cua-bench): attach real environment oracle verdict to dataset run receipt --- libs/cua-bench/cua_bench/cli/commands/run.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/libs/cua-bench/cua_bench/cli/commands/run.py b/libs/cua-bench/cua_bench/cli/commands/run.py index 1cc7d06095..e023c176f7 100644 --- a/libs/cua-bench/cua_bench/cli/commands/run.py +++ b/libs/cua-bench/cua_bench/cli/commands/run.py @@ -1745,6 +1745,11 @@ async def run_with_semaphore(task_path, variant_id, task_num): "task": task_path.name, "variant_id": variant_id, "success": passed, + "oracle_result": ( + getattr(outcome, "oracle_result", None) + if outcome is not None and not isinstance(outcome, BaseException) + else None + ), "exit_code": ( getattr(outcome, "exit_code", None) if outcome is not None and not isinstance(outcome, BaseException) From 7438e5c8fcda259d67603ba3156682bdf49f5bc4 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:48:21 +0800 Subject: [PATCH 090/140] feat(cua-bench): inspect task Docker resources and persist cleanup receipt --- .../cua-bench/cua_bench/runner/task_runner.py | 47 ++++++++++++++++++- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/libs/cua-bench/cua_bench/runner/task_runner.py b/libs/cua-bench/cua_bench/runner/task_runner.py index 12ac587590..2dcbe22af5 100644 --- a/libs/cua-bench/cua_bench/runner/task_runner.py +++ b/libs/cua-bench/cua_bench/runner/task_runner.py @@ -425,8 +425,12 @@ async def run_task( except Exception: pass - # 7. Cleanup - await self._cleanup_task(task_id) + # 7. Cleanup and verify actual Docker resource absence. + try: + await self._cleanup_task(task_id) + finally: + if output_dir: + await self._write_cleanup_receipt(task_id, output_dir) async def run_task_interactively( self, @@ -985,6 +989,45 @@ async def _start_agent_container( remove_on_exit=False, # Don't auto-remove, we need logs ) + async def _resource_absent(self, kind: str, name: str) -> bool: + """Ask Docker whether a task resource still exists after cleanup.""" + process = await asyncio.create_subprocess_exec( + "docker", kind, "inspect", name, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + await process.communicate() + return process.returncode == 1 + + async def _write_cleanup_receipt(self, task_id: str, output_dir: str) -> None: + """Persist resource inspection results even when cleanup failed.""" + path = Path(output_dir) + path.mkdir(parents=True, exist_ok=True) + resources = { + "agent_container": ("container", f"cua-agent-{task_id}"), + "env_container": ("container", f"cua-env-{task_id}"), + "network": ("network", f"cua-task-{task_id}"), + } + observations = {} + for label, (kind, name) in resources.items(): + try: + absent = await self._resource_absent(kind, name) + observations[label] = {"name": name, "absent": absent} + except Exception as exc: + observations[label] = { + "name": name, "absent": False, "error": type(exc).__name__ + } + # Simulated tasks do not create a separate environment container. + receipt = { + "schema_version": "cua-bench-cleanup/v1", + "task_id": task_id, + "resources": observations, + "verified": all(item["absent"] for item in observations.values()), + } + temporary = path / "cleanup-result.json.tmp" + temporary.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8") + temporary.replace(path / "cleanup-result.json") + async def _cleanup_task(self, task_id: str) -> None: """Clean up task resources (containers, network, overlay, optionally images). From a70db82e3bcfc64d8d7ecbda3fd34378e2bf9cb0 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:48:40 +0800 Subject: [PATCH 091/140] fix(cua-bench): fail completed tasks when post-run Docker cleanup cannot be verified --- libs/cua-bench/cua_bench/runner/task_runner.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/libs/cua-bench/cua_bench/runner/task_runner.py b/libs/cua-bench/cua_bench/runner/task_runner.py index 2dcbe22af5..c4752a2623 100644 --- a/libs/cua-bench/cua_bench/runner/task_runner.py +++ b/libs/cua-bench/cua_bench/runner/task_runner.py @@ -289,6 +289,7 @@ async def run_task( # Track log streaming process for cleanup log_stream_process = None + completed_result: Optional[TaskResult] = None try: # 1. Create network @@ -381,7 +382,7 @@ async def run_task( and oracle_result is not None and oracle_result["success"] is True ) - return TaskResult( + completed_result = TaskResult( success=exit_code == 0 and oracle_passed, exit_code=exit_code, agent_logs=agent_logs, @@ -393,6 +394,7 @@ async def run_task( ), oracle_result=oracle_result, ) + return completed_result except Exception as e: # Collect logs on error @@ -430,7 +432,18 @@ async def run_task( await self._cleanup_task(task_id) finally: if output_dir: - await self._write_cleanup_receipt(task_id, output_dir) + try: + await self._write_cleanup_receipt(task_id, output_dir) + cleanup = json.loads( + (Path(output_dir) / "cleanup-result.json").read_text(encoding="utf-8") + ) + if completed_result is not None and cleanup.get("verified") is not True: + completed_result.success = False + completed_result.error = "Task Docker resources not fully cleaned up" + except Exception as exc: + if completed_result is not None: + completed_result.success = False + completed_result.error = f"Unable to verify Docker cleanup: {type(exc).__name__}" async def run_task_interactively( self, From fe7dcad0b1011061f1dddb83054e1e66927528d3 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:52:12 +0800 Subject: [PATCH 092/140] fix(cua-bench): verify QEMU overlay cleanup and distinguish Docker daemon failures --- .../cua-bench/cua_bench/runner/task_runner.py | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/libs/cua-bench/cua_bench/runner/task_runner.py b/libs/cua-bench/cua_bench/runner/task_runner.py index c4752a2623..a58a5054d9 100644 --- a/libs/cua-bench/cua_bench/runner/task_runner.py +++ b/libs/cua-bench/cua_bench/runner/task_runner.py @@ -1009,8 +1009,16 @@ async def _resource_absent(self, kind: str, name: str) -> bool: stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) - await process.communicate() - return process.returncode == 1 + stdout, stderr = await process.communicate() + # Docker inspect uses 1 for a missing object, but also for daemon + # failures. A nonzero exit must not be mistaken for verified cleanup. + if process.returncode == 0: + return False + message = stderr.decode("utf-8", errors="replace").lower() + missing = ("no such object", "no such container", "no such network") + if process.returncode == 1 and any(token in message for token in missing): + return True + raise RuntimeError(f"Docker {kind} inspection failed: {message[:300]}") async def _write_cleanup_receipt(self, task_id: str, output_dir: str) -> None: """Persist resource inspection results even when cleanup failed.""" @@ -1030,7 +1038,13 @@ async def _write_cleanup_receipt(self, task_id: str, output_dir: str) -> None: observations[label] = { "name": name, "absent": False, "error": type(exc).__name__ } - # Simulated tasks do not create a separate environment container. + # Check the task's isolated QEMU overlay as well as Docker objects. + # Never inspect or delete the shared golden disk here. + overlay = get_overlays_path() / task_id + observations["overlay"] = { + "name": str(overlay), + "absent": not overlay.exists(), + } receipt = { "schema_version": "cua-bench-cleanup/v1", "task_id": task_id, From 8c7654019fe5a00ca971bac122180bbdba5fed17 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:52:25 +0800 Subject: [PATCH 093/140] fix(cua-bench): recognize Docker network not-found response during cleanup verification --- libs/cua-bench/cua_bench/runner/task_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/cua-bench/cua_bench/runner/task_runner.py b/libs/cua-bench/cua_bench/runner/task_runner.py index a58a5054d9..2d6d64821c 100644 --- a/libs/cua-bench/cua_bench/runner/task_runner.py +++ b/libs/cua-bench/cua_bench/runner/task_runner.py @@ -1015,7 +1015,7 @@ async def _resource_absent(self, kind: str, name: str) -> bool: if process.returncode == 0: return False message = stderr.decode("utf-8", errors="replace").lower() - missing = ("no such object", "no such container", "no such network") + missing = ("no such object", "no such container", "no such network", "not found") if process.returncode == 1 and any(token in message for token in missing): return True raise RuntimeError(f"Docker {kind} inspection failed: {message[:300]}") From 50136a3f24a864f343c2d25cb6a467fd2cc5edcf Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:58:50 +0800 Subject: [PATCH 094/140] fix(cua-bench): reject invalid task variants and honor per-task Oracle success thresholds --- libs/cua-bench/cua_bench/batch/solver.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/libs/cua-bench/cua_bench/batch/solver.py b/libs/cua-bench/cua_bench/batch/solver.py index dd9c9d6b03..def8669527 100644 --- a/libs/cua-bench/cua_bench/batch/solver.py +++ b/libs/cua-bench/cua_bench/batch/solver.py @@ -237,9 +237,9 @@ async def main(): tasks = env.tasks_config_fn() - if task_index >= len(tasks): - print(f"Task index {task_index} >= number of tasks {len(tasks)}, skipping") - sys.exit(0) + if not 0 <= task_index < len(tasks): + print(f"Error: Task index {task_index} out of range for {len(tasks)} tasks") + sys.exit(1) task_cfg = tasks[task_index] print(f"Running task {task_index}: {task_cfg.description}") @@ -382,9 +382,22 @@ async def main(): and math.isfinite(value) for value in rewards ) + # A 0.5 reward is not automatically a completed task. Default to + # full reward; a task can explicitly define its own success bar. + metadata = getattr(task_cfg, "metadata", None) or {} + threshold = metadata.get("success_threshold", 1.0) + threshold_valid = ( + isinstance(threshold, (int, float)) + and not isinstance(threshold, bool) + and math.isfinite(threshold) + and 0.0 <= threshold <= 1.0 + ) evaluation_receipt["evaluated"] = valid evaluation_receipt["rewards"] = list(rewards) if valid else None - evaluation_receipt["success"] = valid and all(value >= 0.5 for value in rewards) + evaluation_receipt["success_threshold"] = threshold if threshold_valid else None + evaluation_receipt["success"] = ( + valid and threshold_valid and all(value >= threshold for value in rewards) + ) # Record evaluation event try: From 5c5c92c582bb57d07633fc8ff2246b409432a50a Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:59:23 +0800 Subject: [PATCH 095/140] fix(cua-bench): bind worker oracle receipt to unique TaskRunner execution ID --- libs/cua-bench/cua_bench/runner/task_runner.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libs/cua-bench/cua_bench/runner/task_runner.py b/libs/cua-bench/cua_bench/runner/task_runner.py index 2d6d64821c..c37cb71a23 100644 --- a/libs/cua-bench/cua_bench/runner/task_runner.py +++ b/libs/cua-bench/cua_bench/runner/task_runner.py @@ -368,6 +368,7 @@ async def run_task( if ( oracle_result.get("schema_version") != "cua-bench-oracle-result/v1" or oracle_result.get("task_index") != task_index + or oracle_result.get("execution_id") != task_id or oracle_result.get("evaluated") is not True or not isinstance(oracle_result.get("success"), bool) ): @@ -886,6 +887,7 @@ async def _start_agent_container( "CUA_PROVIDER": "simulated" if is_simulated else "remote", "CUA_TASK_PATH": "/app/env", "CUA_TASK_INDEX": str(task_index), + "CUA_EXECUTION_ID": task_id, "BATCH_TASK_INDEX": str(task_index), # Legacy compat "BATCH_TASK_COUNT": "1", } From 7e681d41b17f60380c70db7d8b9a2ce37f1381f7 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:59:33 +0800 Subject: [PATCH 096/140] feat(cua-bench): stamp worker oracle verdict with the current execution ID --- libs/cua-bench/cua_bench/batch/solver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/libs/cua-bench/cua_bench/batch/solver.py b/libs/cua-bench/cua_bench/batch/solver.py index def8669527..8cc7193cb5 100644 --- a/libs/cua-bench/cua_bench/batch/solver.py +++ b/libs/cua-bench/cua_bench/batch/solver.py @@ -366,6 +366,7 @@ async def main(): evaluation_receipt = { "schema_version": "cua-bench-oracle-result/v1", "task_index": task_index, + "execution_id": os.environ.get("CUA_EXECUTION_ID"), "evaluated": False, "success": False, "rewards": None, From 83558e6905f1bd6226739564589ed03bad6863bd Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:20:45 +0800 Subject: [PATCH 097/140] feat(cua-bench): add shared desktop session action evidence adapter --- .../cua_bench/batch/action_evidence.py | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 libs/cua-bench/cua_bench/batch/action_evidence.py diff --git a/libs/cua-bench/cua_bench/batch/action_evidence.py b/libs/cua-bench/cua_bench/batch/action_evidence.py new file mode 100644 index 0000000000..d34dfa54cd --- /dev/null +++ b/libs/cua-bench/cua_bench/batch/action_evidence.py @@ -0,0 +1,56 @@ +"""Common desktop action evidence boundary for benchmark agents. + +Wraps the session supplied to any agent without changing the concrete provider. +""" +import hashlib +import json +from datetime import datetime, timezone + + +class EvidenceSession: + def __init__(self, session, output_dir, execution_id): + self._session = session + self._path = output_dir / "executed-actions.jsonl" + self._execution_id = execution_id + self._sequence = 0 + + def __getattr__(self, name): + return getattr(self._session, name) + + async def _observation(self): + try: + data = await self._session.screenshot() + return {"sha256": hashlib.sha256(data).hexdigest(), "size_bytes": len(data)} + except Exception as exc: + return {"error_type": type(exc).__name__} + + async def execute_action(self, action): + from cua_bench.actions import action_to_dict + + self._sequence += 1 + before = await self._observation() + payload = action_to_dict(action) + if "text" in payload: + payload["text_length"] = len(payload.pop("text")) + payload["text_redacted"] = True + error = None + try: + return await self._session.execute_action(action) + except Exception as exc: + error = type(exc).__name__ + raise + finally: + after = await self._observation() + event = { + "schema_version": "cua-bench-executed-action/v1", + "execution_id": self._execution_id, + "sequence": self._sequence, + "timestamp": datetime.now(timezone.utc).isoformat(), + "action": payload, + "before": before, + "after": after, + "execution_status": "failed" if error else "returned", + "error_type": error, + } + with self._path.open("a", encoding="utf-8") as stream: + stream.write(json.dumps(event, sort_keys=True) + "\n") From 3657ec6b03e5b079c0f825dbd3acc853e9b6349c Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:20:57 +0800 Subject: [PATCH 098/140] feat(cua-bench): route every built-in and imported agent through common action evidence session --- libs/cua-bench/cua_bench/batch/solver.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/libs/cua-bench/cua_bench/batch/solver.py b/libs/cua-bench/cua_bench/batch/solver.py index 8cc7193cb5..9ea5fdddd4 100644 --- a/libs/cua-bench/cua_bench/batch/solver.py +++ b/libs/cua-bench/cua_bench/batch/solver.py @@ -313,10 +313,17 @@ async def main(): logging_dir.mkdir(exist_ok=True, parents=True) print(f"Running agent: {agent.name()}") + # Every agent now crosses the same execution evidence boundary. + # This observes actual desktop calls, not the agent's text output. + from cua_bench.batch.action_evidence import EvidenceSession + + evidence_session = EvidenceSession( + session, logging_dir, os.environ.get("CUA_EXECUTION_ID") + ) try: agent_result = await agent.perform_task( task_description=task_cfg.description, - session=session, + session=evidence_session, logging_dir=logging_dir, tracer=env.tracing, ) From f41f4c7e295ed246be992e8d3ac06c61dcae25e3 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:21:09 +0800 Subject: [PATCH 099/140] refactor(cua-bench): remove duplicate agent-level evidence writer in favor of shared session --- libs/cua-bench/cua_bench/agents/cua_agent.py | 65 +++----------------- 1 file changed, 8 insertions(+), 57 deletions(-) diff --git a/libs/cua-bench/cua_bench/agents/cua_agent.py b/libs/cua-bench/cua_bench/agents/cua_agent.py index 48e69cf030..20b5c3cdb1 100644 --- a/libs/cua-bench/cua_bench/agents/cua_agent.py +++ b/libs/cua-bench/cua_bench/agents/cua_agent.py @@ -2,9 +2,6 @@ import asyncio import base64 -import hashlib -import json -from datetime import datetime, timezone import sys from pathlib import Path from typing import TYPE_CHECKING, Any, Dict, List @@ -82,52 +79,6 @@ def _create_custom_computer(self, session: "DesktopSession", logging_dir: Path | WaitAction, ) - # Evidence originates at the actual desktop execution boundary, not - # from model output or a post-hoc interpretation of console logs. - evidence_path = None - if logging_dir is not None: - logging_dir.mkdir(parents=True, exist_ok=True) - evidence_path = logging_dir / "executed-actions.jsonl" - - async def execute_and_record(action): - from ..actions import action_to_dict - - action_data = action_to_dict(action) - # Typed text may contain secrets; store its length, not content. - if "text" in action_data: - action_data["text_length"] = len(action_data.pop("text")) - action_data["text_redacted"] = True - - async def observe(): - try: - screenshot_bytes = await session.screenshot() - return {"sha256": hashlib.sha256(screenshot_bytes).hexdigest(), - "size_bytes": len(screenshot_bytes)} - except Exception as exc: - return {"observation_error": type(exc).__name__} - - before = await observe() if evidence_path else None - error = None - try: - return await session.execute_action(action) - except Exception as exc: - error = type(exc).__name__ - raise - finally: - if evidence_path: - after = await observe() - event = { - "schema_version": "cua-bench-executed-action/v1", - "timestamp": datetime.now(timezone.utc).isoformat(), - "action": action_data, - "before": before, - "after": after, - "execution_status": "failed" if error else "returned", - "error_type": error, - } - with evidence_path.open("a", encoding="utf-8") as stream: - stream.write(json.dumps(event, sort_keys=True) + "\n") - # Screenshot function (required) async def screenshot(): """Take a screenshot and return as base64 string.""" @@ -145,19 +96,19 @@ async def click(x: int, y: int, button: str = "left"): action = MiddleClickAction(x=x, y=y) else: raise ValueError(f"Unknown button type: {button}") - await execute_and_record(action) + await session.execute_action(action) # Double click function async def double_click(x: int, y: int): """Double click at coordinates.""" action = DoubleClickAction(x=x, y=y) - await execute_and_record(action) + await session.execute_action(action) # Type function async def type_text(text: str): """Type text.""" action = TypeAction(text=text) - await execute_and_record(action) + await session.execute_action(action) # Keypress function async def keypress(keys): @@ -166,13 +117,13 @@ async def keypress(keys): action = KeyAction(key=keys) else: action = HotkeyAction(keys=list(keys)) - await execute_and_record(action) + await session.execute_action(action) # Move function async def move(x: int, y: int): """Move cursor to coordinates.""" action = MoveToAction(x=x, y=y) - await execute_and_record(action) + await session.execute_action(action) # Scroll function async def scroll(x: int, y: int, scroll_x: int, scroll_y: int): @@ -184,7 +135,7 @@ async def scroll(x: int, y: int, scroll_x: int, scroll_y: int): direction = "down" amount = abs(scroll_y) action = ScrollAction(direction=direction, amount=amount) - await execute_and_record(action) + await session.execute_action(action) # Drag function async def drag(path: List[Dict[str, int]]): @@ -194,13 +145,13 @@ async def drag(path: List[Dict[str, int]]): start = path[0] end = path[-1] action = DragAction(from_x=start["x"], from_y=start["y"], to_x=end["x"], to_y=end["y"]) - await execute_and_record(action) + await session.execute_action(action) # Wait function async def wait(ms: int = 1000): """Wait for specified milliseconds.""" action = WaitAction(seconds=ms / 1000.0) - await execute_and_record(action) + await session.execute_action(action) # Get dimensions async def get_dimensions(): From e677c991adc56361d08314d3373a6465d85866d2 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:21:23 +0800 Subject: [PATCH 100/140] fix(cua-bench): invalidate stale task receipts before launching new execution --- libs/cua-bench/cua_bench/runner/task_runner.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/libs/cua-bench/cua_bench/runner/task_runner.py b/libs/cua-bench/cua_bench/runner/task_runner.py index c37cb71a23..c896168105 100644 --- a/libs/cua-bench/cua_bench/runner/task_runner.py +++ b/libs/cua-bench/cua_bench/runner/task_runner.py @@ -287,6 +287,14 @@ async def run_task( "is_simulated": is_simulated, } + # Do not let artifacts from an earlier run masquerade as this execution. + # Leave unrelated user files alone; only clear known generated receipts. + if output_dir: + task_output = Path(output_dir) + task_output.mkdir(parents=True, exist_ok=True) + for receipt_name in ("oracle-result.json", "cleanup-result.json"): + (task_output / receipt_name).unlink(missing_ok=True) + # Track log streaming process for cleanup log_stream_process = None completed_result: Optional[TaskResult] = None From 91bacbd9a43e057a63bdefdd3fe2bd014892dad8 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:28:50 +0800 Subject: [PATCH 101/140] ci: sync trusted upstream release attribution configuration --- .github/release-attribution-config.json | 49 +++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 .github/release-attribution-config.json diff --git a/.github/release-attribution-config.json b/.github/release-attribution-config.json new file mode 100644 index 0000000000..c444b06728 --- /dev/null +++ b/.github/release-attribution-config.json @@ -0,0 +1,49 @@ +{ + "bots": [ + "dependabot[bot]", + "github-actions[bot]", + "claude", + "copilot", + "release-please[bot]", + "renovate[bot]", + "trycua-release[bot]" + ], + "ignoredCoauthorEmails": [ + "claude@anthropic.com", + "coding-sub-agent@trycua.com", + "codex@openai.com", + "cua@cua.localdomain", + "hermes@hermes.home.arpa", + "noreply@anthropic.com" + ], + "coauthorOverrides": { + "hippoley ": "hippoley" + }, + "identityOverrides": { + "antoine.bonnet@hcompany.ai": "abonneth", + "anvit.mail@icloud.com": "AnvitDevadiga", + "ddupont@mit.edu": "ddupont808", + "dillon@trycua.com": "ddupont808", + "egnichtel@crosscode.com": "ngnichtel", + "f@trycua.com": "f-trycua", + "fbonacci@fbonacci-xfce-vm.b40fumexvs1ujlod2ppcyfh5qa.bx.internal.cloudapp.net": "f-trycua", + "i@jyunko.cn": "HsiangNianian", + "jf-mac-mini@jf-mac-mini-4.local": "0xjohnnydev", + "klymenko@adobe.com": "pavelklymenko", + "m.fuechtenkoetter@posteo.de": "ai-ag2026", + "manfred@ubuntu26.04-vm": "ai-ag2026", + "marko.civonafets@gmail.com": "civonafets", + "me@madhavajay.com": "madhavajay", + "robert@trycua.com": "enchanted-koala", + "rwendt1337@gmail.com": "r33drichards", + "rsyuzyov@gmail.com": "rsyuzyov", + "sarinajin.li@gmail.com": "sarinali", + "steipete@gmail.com": "steipete", + "tshtark@intelligo.ai": "tshtark", + "umtcgnd@gmail.com": "c8dhjp4tyv-bit", + "zane.chee.2023@scis.smu.edu.sg": "injaneity", + "zongxin_yang@hms.harvard.edu": "z-x-yang" + }, + "internalHandles": ["ddupont808", "enchanted-koala", "f-trycua"], + "optOutHandles": [] +} From f80e5b588ab1341829b21e6bce484c4642a4b10b Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:31:01 +0800 Subject: [PATCH 102/140] feat(cua-bench): capture direct desktop shortcuts and element clicks through shared evidence boundary --- .../cua_bench/batch/action_evidence.py | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/libs/cua-bench/cua_bench/batch/action_evidence.py b/libs/cua-bench/cua_bench/batch/action_evidence.py index d34dfa54cd..1cc29e169d 100644 --- a/libs/cua-bench/cua_bench/batch/action_evidence.py +++ b/libs/cua-bench/cua_bench/batch/action_evidence.py @@ -54,3 +54,81 @@ async def execute_action(self, action): } with self._path.open("a", encoding="utf-8") as stream: stream.write(json.dumps(event, sort_keys=True) + "\n") + + # High-level DesktopSession shortcuts bypass execute_action on the wrapped + # session; route them through this adapter to avoid silently missing actions. + async def step(self, action): + return await self.execute_action(action) + + async def click(self, x, y): + from cua_bench.types import ClickAction + return await self.execute_action(ClickAction(x=x, y=y)) + + async def right_click(self, x, y): + from cua_bench.types import RightClickAction + return await self.execute_action(RightClickAction(x=x, y=y)) + + async def double_click(self, x, y): + from cua_bench.types import DoubleClickAction + return await self.execute_action(DoubleClickAction(x=x, y=y)) + + async def type(self, text): + from cua_bench.types import TypeAction + return await self.execute_action(TypeAction(text=text)) + + async def key(self, key): + from cua_bench.types import KeyAction + return await self.execute_action(KeyAction(key=key)) + + async def hotkey(self, keys): + from cua_bench.types import HotkeyAction + return await self.execute_action(HotkeyAction(keys=keys)) + + async def scroll(self, direction="down", amount=300): + from cua_bench.types import ScrollAction + return await self.execute_action(ScrollAction(direction=direction, amount=amount)) + + async def move_to(self, x, y): + from cua_bench.types import MoveToAction + return await self.execute_action(MoveToAction(x=x, y=y)) + + async def drag(self, from_x, from_y, to_x, to_y): + from cua_bench.types import DragAction + return await self.execute_action(DragAction( + from_x=from_x, from_y=from_y, to_x=to_x, to_y=to_y + )) + + async def _record_direct(self, method, *args): + # Element-based interactions do not map to pixel Actions, but they + # still mutate the real desktop and must be included in the evidence. + self._sequence += 1 + before = await self._observation() + error = None + try: + return await getattr(self._session, method)(*args) + except Exception as exc: + error = type(exc).__name__ + raise + finally: + after = await self._observation() + event = { + "schema_version": "cua-bench-executed-action/v1", + "execution_id": self._execution_id, + "sequence": self._sequence, + "timestamp": datetime.now(timezone.utc).isoformat(), + "action": {"type": method, "selector_sha256": hashlib.sha256( + str(args[1]).encode("utf-8") + ).hexdigest()}, + "before": before, + "after": after, + "execution_status": "failed" if error else "returned", + "error_type": error, + } + with self._path.open("a", encoding="utf-8") as stream: + stream.write(json.dumps(event, sort_keys=True) + "\n") + + async def click_element(self, pid, selector): + return await self._record_direct("click_element", pid, selector) + + async def right_click_element(self, pid, selector): + return await self._record_direct("right_click_element", pid, selector) From c88bb5fc132dc8e365b764a28d527d951bd9f902 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:31:13 +0800 Subject: [PATCH 103/140] fix(cua-bench): isolate action evidence between reused task executions --- libs/cua-bench/cua_bench/batch/action_evidence.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libs/cua-bench/cua_bench/batch/action_evidence.py b/libs/cua-bench/cua_bench/batch/action_evidence.py index 1cc29e169d..4d9521f456 100644 --- a/libs/cua-bench/cua_bench/batch/action_evidence.py +++ b/libs/cua-bench/cua_bench/batch/action_evidence.py @@ -11,6 +11,8 @@ class EvidenceSession: def __init__(self, session, output_dir, execution_id): self._session = session self._path = output_dir / "executed-actions.jsonl" + # A new agent execution must never append to a previous run's evidence. + self._path.unlink(missing_ok=True) self._execution_id = execution_id self._sequence = 0 From eb8b6d1094dabc27c7fbe421cc15ae634ae7c6fc Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:33:14 +0800 Subject: [PATCH 104/140] fix(cua-bench): preserve cleanup verification on timeout and exception outcomes --- libs/cua-bench/cua_bench/runner/task_runner.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/libs/cua-bench/cua_bench/runner/task_runner.py b/libs/cua-bench/cua_bench/runner/task_runner.py index c896168105..7da9c34679 100644 --- a/libs/cua-bench/cua_bench/runner/task_runner.py +++ b/libs/cua-bench/cua_bench/runner/task_runner.py @@ -354,7 +354,7 @@ async def run_task( timeout=timeout, ) except asyncio.TimeoutError: - return TaskResult( + completed_result = TaskResult( success=False, exit_code=-1, agent_logs=await get_container_logs(agent_container_name, tail=100), @@ -362,6 +362,7 @@ async def run_task( output_dir=output_dir, error="Task timed out", ) + return completed_result # 5. Collect logs agent_logs = await get_container_logs(agent_container_name, tail=500) @@ -418,7 +419,7 @@ async def run_task( except Exception: pass - return TaskResult( + completed_result = TaskResult( success=False, exit_code=-1, agent_logs=agent_logs, @@ -426,6 +427,7 @@ async def run_task( output_dir=output_dir, error=str(e), ) + return completed_result finally: # 6. Stop log streaming if active From 8d84d5561c1f78b300cab244157b3e920053b70f Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:33:55 +0800 Subject: [PATCH 105/140] fix(cua-bench): independently recompute worker oracle verdict from rewards --- .../cua-bench/cua_bench/runner/task_runner.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/libs/cua-bench/cua_bench/runner/task_runner.py b/libs/cua-bench/cua_bench/runner/task_runner.py index 7da9c34679..b1b1ac0b5d 100644 --- a/libs/cua-bench/cua_bench/runner/task_runner.py +++ b/libs/cua-bench/cua_bench/runner/task_runner.py @@ -6,6 +6,7 @@ import asyncio import json +import math import os import shutil from dataclasses import dataclass @@ -382,6 +383,32 @@ async def run_task( or not isinstance(oracle_result.get("success"), bool) ): oracle_error = "Invalid oracle result receipt" + else: + # Recalculate the verdict from the numeric rewards. + # Do not trust a self-asserted success boolean. + rewards = oracle_result.get("rewards") + threshold = oracle_result.get("success_threshold") + valid_rewards = ( + isinstance(rewards, list) and bool(rewards) + and all( + isinstance(v, (int, float)) and not isinstance(v, bool) + and math.isfinite(v) and 0.0 <= v <= 1.0 + for v in rewards + ) + ) + valid_threshold = ( + isinstance(threshold, (int, float)) + and not isinstance(threshold, bool) + and math.isfinite(threshold) and 0.0 <= threshold <= 1.0 + ) + calculated = ( + valid_rewards and valid_threshold + and all(v >= threshold for v in rewards) + ) + if not valid_rewards or not valid_threshold or ( + oracle_result["success"] is not calculated + ): + oracle_error = "Inconsistent oracle reward and success verdict" except (OSError, ValueError, TypeError): oracle_error = "Unreadable oracle result receipt" else: From a32db6f4f3900b30edcb50a741a40b88fde5305b Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:47:00 +0800 Subject: [PATCH 106/140] feat(cua-bench): validate same-execution desktop action evidence before task success --- .../cua-bench/cua_bench/runner/task_runner.py | 35 +++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/libs/cua-bench/cua_bench/runner/task_runner.py b/libs/cua-bench/cua_bench/runner/task_runner.py index b1b1ac0b5d..877d5ff4ee 100644 --- a/libs/cua-bench/cua_bench/runner/task_runner.py +++ b/libs/cua-bench/cua_bench/runner/task_runner.py @@ -419,13 +419,44 @@ async def run_task( and oracle_result is not None and oracle_result["success"] is True ) + # Validate records emitted at the live desktop action boundary. + # In oracle-solver mode, actions may legitimately bypass an agent; + # never fabricate agent evidence for that path. + evidence_error = None + if not oracle and (agent or agent_import_path): + evidence_dir = Path(output_dir) / f"task_{task_index}_agent_logs" if output_dir else None + evidence_file = evidence_dir / "executed-actions.jsonl" if evidence_dir else None + if evidence_file is None or not evidence_file.is_file(): + evidence_error = "Missing agent action evidence" + else: + try: + sequence = 0 + with evidence_file.open(encoding="utf-8") as records: + for raw in records: + record = json.loads(raw) + sequence += 1 + if ( + record.get("schema_version") != "cua-bench-executed-action/v1" + or record.get("execution_id") != task_id + or record.get("sequence") != sequence + or record.get("execution_status") not in ("returned", "failed") + or not isinstance(record.get("action"), dict) + or not isinstance(record.get("before"), dict) + or not isinstance(record.get("after"), dict) + ): + raise ValueError("Invalid action evidence record") + if sequence == 0: + evidence_error = "Empty agent action evidence" + except (OSError, ValueError, TypeError, AttributeError): + evidence_error = "Invalid agent action evidence" + completed_result = TaskResult( - success=exit_code == 0 and oracle_passed, + success=exit_code == 0 and oracle_passed and evidence_error is None, exit_code=exit_code, agent_logs=agent_logs, env_logs=env_logs, output_dir=output_dir, - error=oracle_error or ( + error=evidence_error or oracle_error or ( "Oracle rejected task" if oracle_result and not oracle_passed else None ), From 4b7c64cc8205ef542a825497e950aea069473de7 Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:47:26 +0800 Subject: [PATCH 107/140] feat(cua-bench): expose validated action event count on TaskResult --- libs/cua-bench/cua_bench/runner/task_runner.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/libs/cua-bench/cua_bench/runner/task_runner.py b/libs/cua-bench/cua_bench/runner/task_runner.py index 877d5ff4ee..58f9fa73e1 100644 --- a/libs/cua-bench/cua_bench/runner/task_runner.py +++ b/libs/cua-bench/cua_bench/runner/task_runner.py @@ -107,6 +107,7 @@ class TaskResult: output_dir: Optional[str] = None error: Optional[str] = None oracle_result: Optional[dict] = None + verified_action_count: Optional[int] = None # ============================================================================= @@ -423,6 +424,7 @@ async def run_task( # In oracle-solver mode, actions may legitimately bypass an agent; # never fabricate agent evidence for that path. evidence_error = None + verified_action_count = None if not oracle and (agent or agent_import_path): evidence_dir = Path(output_dir) / f"task_{task_index}_agent_logs" if output_dir else None evidence_file = evidence_dir / "executed-actions.jsonl" if evidence_dir else None @@ -447,6 +449,8 @@ async def run_task( raise ValueError("Invalid action evidence record") if sequence == 0: evidence_error = "Empty agent action evidence" + else: + verified_action_count = sequence except (OSError, ValueError, TypeError, AttributeError): evidence_error = "Invalid agent action evidence" @@ -461,6 +465,7 @@ async def run_task( else None ), oracle_result=oracle_result, + verified_action_count=verified_action_count, ) return completed_result From 96384f71ccff69d8f850834e77f619e0fca162dc Mon Sep 17 00:00:00 2001 From: hippo <135493401+hippoley@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:47:36 +0800 Subject: [PATCH 108/140] feat(cua-bench): publish verified action count in dataset run-result receipt --- libs/cua-bench/cua_bench/cli/commands/run.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/libs/cua-bench/cua_bench/cli/commands/run.py b/libs/cua-bench/cua_bench/cli/commands/run.py index e023c176f7..ebafafc973 100644 --- a/libs/cua-bench/cua_bench/cli/commands/run.py +++ b/libs/cua-bench/cua_bench/cli/commands/run.py @@ -1745,6 +1745,11 @@ async def run_with_semaphore(task_path, variant_id, task_num): "task": task_path.name, "variant_id": variant_id, "success": passed, + "verified_action_count": ( + getattr(outcome, "verified_action_count", None) + if outcome is not None and not isinstance(outcome, BaseException) + else None + ), "oracle_result": ( getattr(outcome, "oracle_result", None) if outcome is not None and not isinstance(outcome, BaseException) From 4647f6849c3f3e4a56eb51072daf23b1791728be Mon Sep 17 00:00:00 2001 From: Noah Ellis <135493401+hippoley@users.noreply.github.com> Date: Mon, 12 Oct 2026 14:15:29 +0800 Subject: [PATCH 109/140] feat(cua-bench): add independent offline fail-closed execution receipt verifier --- libs/cua-bench/cua_bench/verify_execution.py | 126 +++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 libs/cua-bench/cua_bench/verify_execution.py diff --git a/libs/cua-bench/cua_bench/verify_execution.py b/libs/cua-bench/cua_bench/verify_execution.py new file mode 100644 index 0000000000..a94e29d64b --- /dev/null +++ b/libs/cua-bench/cua_bench/verify_execution.py @@ -0,0 +1,126 @@ +"""Offline, read-only verifier for cua-bench execution artifacts. + +Checks internal consistency, not the truth of screenshots or the remote oracle. +Usage: python -m cua_bench.verify_execution OUTPUT_DIR --task-index 0 --execution-id ID +""" +import argparse +import json +import math +from pathlib import Path + + +def _json(path): + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError(f"{path.name}: expected object") + return value + + +def verify_execution(root: Path, task_index: int, execution_id: str, *, require_actions=True): + """Return a machine-readable fail-closed independent verification report.""" + errors = [] + checks = {} + root = Path(root) + oracle_path = root / "oracle-result.json" + cleanup_path = root / "cleanup-result.json" + action_path = root / f"task_{task_index}_agent_logs" / "executed-actions.jsonl" + + try: + oracle = _json(oracle_path) + rewards = oracle.get("rewards") + threshold = oracle.get("success_threshold") + valid = ( + oracle.get("schema_version") == "cua-bench-oracle-result/v1" + and oracle.get("task_index") == task_index + and oracle.get("execution_id") == execution_id + and oracle.get("evaluated") is True + and isinstance(rewards, list) and len(rewards) > 0 + and all(type(v) in (float, int) and math.isfinite(v) and 0 <= v <= 1 for v in rewards) + and type(threshold) in (float, int) and math.isfinite(threshold) and 0 <= threshold <= 1 + and type(oracle.get("success")) is bool + and oracle["success"] == all(v >= threshold for v in rewards) + ) + checks["oracle"] = bool(valid and oracle["success"]) + if not checks["oracle"]: + errors.append("oracle_invalid_or_unsuccessful") + except (OSError, ValueError, TypeError, KeyError) as exc: + checks["oracle"] = False + errors.append(f"oracle_unreadable:{type(exc).__name__}") + + try: + cleanup = _json(cleanup_path) + resources = cleanup.get("resources") + valid = ( + cleanup.get("schema_version") == "cua-bench-cleanup/v1" + and cleanup.get("task_id") == execution_id + and isinstance(resources, dict) and len(resources) > 0 + and all(isinstance(v, dict) and v.get("absent") is True for v in resources.values()) + and cleanup.get("verified") is True + ) + checks["cleanup"] = valid + if not valid: + errors.append("cleanup_unverified") + except (OSError, ValueError, TypeError) as exc: + checks["cleanup"] = False + errors.append(f"cleanup_unreadable:{type(exc).__name__}") + + action_count = 0 + if require_actions: + try: + with action_path.open(encoding="utf-8") as stream: + for line in stream: + record = json.loads(line) + action_count += 1 + if ( + not isinstance(record, dict) + or record.get("schema_version") != "cua-bench-executed-action/v1" + or record.get("execution_id") != execution_id + or record.get("sequence") != action_count + or record.get("execution_status") not in ("returned", "failed") + or not isinstance(record.get("action"), dict) + or not isinstance(record.get("before"), dict) + or not isinstance(record.get("after"), dict) + ): + raise ValueError("Invalid action event") + checks["actions"] = action_count > 0 + if not checks["actions"]: + errors.append("actions_empty") + except (OSError, ValueError, TypeError) as exc: + checks["actions"] = False + errors.append(f"actions_unreadable_or_invalid:{type(exc).__name__}") + else: + checks["actions"] = None + + return { + "schema_version": "cua-bench-verification/v1", + "execution_id": execution_id, + "task_index": task_index, + "valid": not errors, + "checks": checks, + "verified_action_count": action_count, + "errors": errors, + "limitations": [ + "Verifies internal artifact consistency only, not screenshot authenticity", + "Does not independently re-execute or trustlessly validate the environment oracle", + "Cannot establish that the recording intercepted all device actions", + ], + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("output_dir", type=Path) + parser.add_argument("--task-index", required=True, type=int) + parser.add_argument("--execution-id", required=True) + parser.add_argument("--no-agent", action="store_true", help="Oracle-only task; no agent action stream") + args = parser.parse_args() + result = verify_execution( + args.output_dir, args.task_index, args.execution_id, + require_actions=not args.no_agent, + ) + print(json.dumps(result, indent=2, sort_keys=True)) + raise SystemExit(0 if result["valid"] else 1) + + +if __name__ == "__main__": + main() From 795b1567a7dbfe374cb5c92ba7e4797cc30a7316 Mon Sep 17 00:00:00 2001 From: Noah Ellis <135493401+hippoley@users.noreply.github.com> Date: Mon, 12 Oct 2026 14:15:51 +0800 Subject: [PATCH 110/140] test(cua-bench): validate offline verifier with tampering and missing evidence cases --- .../cua_bench/tests/test_verify_execution.py | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 libs/cua-bench/cua_bench/tests/test_verify_execution.py diff --git a/libs/cua-bench/cua_bench/tests/test_verify_execution.py b/libs/cua-bench/cua_bench/tests/test_verify_execution.py new file mode 100644 index 0000000000..a45a858abd --- /dev/null +++ b/libs/cua-bench/cua_bench/tests/test_verify_execution.py @@ -0,0 +1,78 @@ +"""Offline verifier contract tests. No Docker or model credentials required.""" +import json + +from cua_bench.verify_execution import verify_execution + + +def write_json(path, obj): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(obj) + "\n", encoding="utf-8") + + +def make_receipts(root, execution_id="run-1"): + write_json(root / "oracle-result.json", { + "schema_version": "cua-bench-oracle-result/v1", + "execution_id": execution_id, + "task_index": 0, + "evaluated": True, "success": True, + "rewards": [1.0], "success_threshold": 1.0, + }) + write_json(root / "cleanup-result.json", { + "schema_version": "cua-bench-cleanup/v1", + "task_id": execution_id, + "verified": True, + "resources": {"agent_container": {"name": "a", "absent": True}}, + }) + logs = root / "task_0_agent_logs" / "executed-actions.jsonl" + write_json(logs, { + "schema_version": "cua-bench-executed-action/v1", + "execution_id": execution_id, "sequence": 1, + "execution_status": "returned", "action": {"type": "click"}, + "before": {"sha256": "a"}, "after": {"sha256": "b"}, + }) + + +def test_complete_receipts_pass_internal_consistency(tmp_path): + make_receipts(tmp_path) + result = verify_execution(tmp_path, 0, "run-1") + assert result["valid"] is True + assert result["verified_action_count"] == 1 + + +def test_wrong_execution_id_fails_closed(tmp_path): + make_receipts(tmp_path) + result = verify_execution(tmp_path, 0, "run-2") + assert result["valid"] is False + assert not result["checks"]["oracle"] + + +def test_missing_action_evidence_fails_closed(tmp_path): + make_receipts(tmp_path) + (tmp_path / "task_0_agent_logs" / "executed-actions.jsonl").unlink() + assert verify_execution(tmp_path, 0, "run-1")["valid"] is False + + +def test_forged_oracle_success_is_detected(tmp_path): + make_receipts(tmp_path) + p = tmp_path / "oracle-result.json" + value = json.loads(p.read_text()) + value["rewards"] = [0.0] + write_json(p, value) + assert verify_execution(tmp_path, 0, "run-1")["checks"]["oracle"] is False + + +def test_cleanup_residual_is_rejected(tmp_path): + make_receipts(tmp_path) + p = tmp_path / "cleanup-result.json" + value = json.loads(p.read_text()) + value["resources"]["agent_container"]["absent"] = False + write_json(p, value) + assert verify_execution(tmp_path, 0, "run-1")["checks"]["cleanup"] is False + + +def test_oracle_only_mode_allows_no_agent_stream(tmp_path): + make_receipts(tmp_path) + (tmp_path / "task_0_agent_logs" / "executed-actions.jsonl").unlink() + result = verify_execution(tmp_path, 0, "run-1", require_actions=False) + assert result["valid"] is True + assert result["checks"]["actions"] is None From 25c475274449d3d5985d8fd8c215641685244ae8 Mon Sep 17 00:00:00 2001 From: Noah Ellis <135493401+hippoley@users.noreply.github.com> Date: Mon, 12 Oct 2026 15:22:48 +0800 Subject: [PATCH 111/140] fix(cua-bench): align offline oracle reward semantics and reject failed action receipts --- libs/cua-bench/cua_bench/verify_execution.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/cua-bench/cua_bench/verify_execution.py b/libs/cua-bench/cua_bench/verify_execution.py index a94e29d64b..d390663d87 100644 --- a/libs/cua-bench/cua_bench/verify_execution.py +++ b/libs/cua-bench/cua_bench/verify_execution.py @@ -35,7 +35,7 @@ def verify_execution(root: Path, task_index: int, execution_id: str, *, require_ and oracle.get("execution_id") == execution_id and oracle.get("evaluated") is True and isinstance(rewards, list) and len(rewards) > 0 - and all(type(v) in (float, int) and math.isfinite(v) and 0 <= v <= 1 for v in rewards) + and all(type(v) in (float, int) and math.isfinite(v) for v in rewards) and type(threshold) in (float, int) and math.isfinite(threshold) and 0 <= threshold <= 1 and type(oracle.get("success")) is bool and oracle["success"] == all(v >= threshold for v in rewards) @@ -76,7 +76,7 @@ def verify_execution(root: Path, task_index: int, execution_id: str, *, require_ or record.get("schema_version") != "cua-bench-executed-action/v1" or record.get("execution_id") != execution_id or record.get("sequence") != action_count - or record.get("execution_status") not in ("returned", "failed") + or record.get("execution_status") != "returned" or not isinstance(record.get("action"), dict) or not isinstance(record.get("before"), dict) or not isinstance(record.get("after"), dict) From c1b0aad61c8d27378f502e276b1a7b33ff02687b Mon Sep 17 00:00:00 2001 From: Noah Ellis <135493401+hippoley@users.noreply.github.com> Date: Mon, 12 Oct 2026 15:23:03 +0800 Subject: [PATCH 112/140] test(cua-bench): reject failed action and exercise oracle threshold consistency --- .../cua_bench/tests/test_verify_execution.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/libs/cua-bench/cua_bench/tests/test_verify_execution.py b/libs/cua-bench/cua_bench/tests/test_verify_execution.py index a45a858abd..4196c5a8a9 100644 --- a/libs/cua-bench/cua_bench/tests/test_verify_execution.py +++ b/libs/cua-bench/cua_bench/tests/test_verify_execution.py @@ -76,3 +76,27 @@ def test_oracle_only_mode_allows_no_agent_stream(tmp_path): result = verify_execution(tmp_path, 0, "run-1", require_actions=False) assert result["valid"] is True assert result["checks"]["actions"] is None + + +def test_finite_negative_reward_uses_explicit_threshold(tmp_path): + make_receipts(tmp_path) + p = tmp_path / "oracle-result.json" + value = json.loads(p.read_text()) + value["rewards"] = [-0.25] + value["success_threshold"] = 0.0 + value["success"] = False + write_json(p, value) + result = verify_execution(tmp_path, 0, "run-1") + assert result["checks"]["oracle"] is False + assert "oracle_invalid_or_unsuccessful" in result["errors"] + + +def test_failed_action_cannot_prove_successful_execution(tmp_path): + make_receipts(tmp_path) + p = tmp_path / "task_0_agent_logs" / "executed-actions.jsonl" + value = json.loads(p.read_text()) + value["execution_status"] = "failed" + write_json(p, value) + result = verify_execution(tmp_path, 0, "run-1") + assert result["valid"] is False + assert result["checks"]["actions"] is False From f267ae7db4ef258bf271ff8ccf2e3f16c919d6eb Mon Sep 17 00:00:00 2001 From: Noah Ellis <135493401+hippoley@users.noreply.github.com> Date: Mon, 12 Oct 2026 15:24:12 +0800 Subject: [PATCH 113/140] fix(cua-bench): require screenshot observations for each verified action --- libs/cua-bench/cua_bench/verify_execution.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/libs/cua-bench/cua_bench/verify_execution.py b/libs/cua-bench/cua_bench/verify_execution.py index d390663d87..4d0a649b83 100644 --- a/libs/cua-bench/cua_bench/verify_execution.py +++ b/libs/cua-bench/cua_bench/verify_execution.py @@ -6,6 +6,7 @@ import argparse import json import math +import re from pathlib import Path @@ -16,6 +17,16 @@ def _json(path): return value +def _valid_observation(value): + return ( + isinstance(value, dict) + and isinstance(value.get("sha256"), str) + and re.fullmatch(r"[0-9a-f]{64}", value["sha256"]) is not None + and type(value.get("size_bytes")) is int + and value["size_bytes"] >= 0 + ) + + def verify_execution(root: Path, task_index: int, execution_id: str, *, require_actions=True): """Return a machine-readable fail-closed independent verification report.""" errors = [] @@ -78,8 +89,8 @@ def verify_execution(root: Path, task_index: int, execution_id: str, *, require_ or record.get("sequence") != action_count or record.get("execution_status") != "returned" or not isinstance(record.get("action"), dict) - or not isinstance(record.get("before"), dict) - or not isinstance(record.get("after"), dict) + or not _valid_observation(record.get("before")) + or not _valid_observation(record.get("after")) ): raise ValueError("Invalid action event") checks["actions"] = action_count > 0 From 469ad71e32c0dc1a1c5190b346786ebf43b4b72b Mon Sep 17 00:00:00 2001 From: Noah Ellis <135493401+hippoley@users.noreply.github.com> Date: Mon, 12 Oct 2026 15:24:25 +0800 Subject: [PATCH 114/140] test(cua-bench): require complete screenshot hashes in action receipts --- .../cua_bench/tests/test_verify_execution.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/libs/cua-bench/cua_bench/tests/test_verify_execution.py b/libs/cua-bench/cua_bench/tests/test_verify_execution.py index 4196c5a8a9..aba16f6f8d 100644 --- a/libs/cua-bench/cua_bench/tests/test_verify_execution.py +++ b/libs/cua-bench/cua_bench/tests/test_verify_execution.py @@ -28,7 +28,8 @@ def make_receipts(root, execution_id="run-1"): "schema_version": "cua-bench-executed-action/v1", "execution_id": execution_id, "sequence": 1, "execution_status": "returned", "action": {"type": "click"}, - "before": {"sha256": "a"}, "after": {"sha256": "b"}, + "before": {"sha256": "a" * 64, "size_bytes": 1}, + "after": {"sha256": "b" * 64, "size_bytes": 1}, }) @@ -100,3 +101,14 @@ def test_failed_action_cannot_prove_successful_execution(tmp_path): result = verify_execution(tmp_path, 0, "run-1") assert result["valid"] is False assert result["checks"]["actions"] is False + + +def test_missing_screenshot_observation_fails_closed(tmp_path): + make_receipts(tmp_path) + p = tmp_path / "task_0_agent_logs" / "executed-actions.jsonl" + value = json.loads(p.read_text()) + value["before"] = {"error_type": "TimeoutError"} + write_json(p, value) + result = verify_execution(tmp_path, 0, "run-1") + assert result["valid"] is False + assert result["checks"]["actions"] is False From 9399814492cf203ef41c122e88f967595c97bd2f Mon Sep 17 00:00:00 2001 From: Noah Ellis <135493401+hippoley@users.noreply.github.com> Date: Mon, 12 Oct 2026 15:34:34 +0800 Subject: [PATCH 115/140] fix(cua-bench): distinguish evidence consistency, task outcome and accepted run --- libs/cua-bench/cua_bench/verify_execution.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/libs/cua-bench/cua_bench/verify_execution.py b/libs/cua-bench/cua_bench/verify_execution.py index 4d0a649b83..4029a3865c 100644 --- a/libs/cua-bench/cua_bench/verify_execution.py +++ b/libs/cua-bench/cua_bench/verify_execution.py @@ -31,6 +31,8 @@ def verify_execution(root: Path, task_index: int, execution_id: str, *, require_ """Return a machine-readable fail-closed independent verification report.""" errors = [] checks = {} + task_success = False + failed_action_count = 0 root = Path(root) oracle_path = root / "oracle-result.json" cleanup_path = root / "cleanup-result.json" @@ -51,9 +53,10 @@ def verify_execution(root: Path, task_index: int, execution_id: str, *, require_ and type(oracle.get("success")) is bool and oracle["success"] == all(v >= threshold for v in rewards) ) - checks["oracle"] = bool(valid and oracle["success"]) + checks["oracle"] = bool(valid) + task_success = bool(valid and oracle["success"]) if not checks["oracle"]: - errors.append("oracle_invalid_or_unsuccessful") + errors.append("oracle_invalid") except (OSError, ValueError, TypeError, KeyError) as exc: checks["oracle"] = False errors.append(f"oracle_unreadable:{type(exc).__name__}") @@ -87,12 +90,13 @@ def verify_execution(root: Path, task_index: int, execution_id: str, *, require_ or record.get("schema_version") != "cua-bench-executed-action/v1" or record.get("execution_id") != execution_id or record.get("sequence") != action_count - or record.get("execution_status") != "returned" + or record.get("execution_status") not in ("returned", "failed") or not isinstance(record.get("action"), dict) or not _valid_observation(record.get("before")) or not _valid_observation(record.get("after")) ): raise ValueError("Invalid action event") + failed_action_count += record["execution_status"] == "failed" checks["actions"] = action_count > 0 if not checks["actions"]: errors.append("actions_empty") @@ -102,11 +106,17 @@ def verify_execution(root: Path, task_index: int, execution_id: str, *, require_ else: checks["actions"] = None + evidence_consistent = not errors + run_accepted = evidence_consistent and task_success and failed_action_count == 0 return { "schema_version": "cua-bench-verification/v1", "execution_id": execution_id, "task_index": task_index, - "valid": not errors, + "valid": run_accepted, + "evidence_consistent": evidence_consistent, + "task_success": task_success, + "run_accepted": run_accepted, + "failed_action_count": failed_action_count, "checks": checks, "verified_action_count": action_count, "errors": errors, From b2f11e0560a2bfc8d04a66c2e984f5576a372bc3 Mon Sep 17 00:00:00 2001 From: Noah Ellis <135493401+hippoley@users.noreply.github.com> Date: Mon, 12 Oct 2026 15:34:50 +0800 Subject: [PATCH 116/140] test(cua-bench): distinguish verifiable failure from successful execution --- .../cua_bench/tests/test_verify_execution.py | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/libs/cua-bench/cua_bench/tests/test_verify_execution.py b/libs/cua-bench/cua_bench/tests/test_verify_execution.py index aba16f6f8d..64d6baf5f5 100644 --- a/libs/cua-bench/cua_bench/tests/test_verify_execution.py +++ b/libs/cua-bench/cua_bench/tests/test_verify_execution.py @@ -89,7 +89,9 @@ def test_finite_negative_reward_uses_explicit_threshold(tmp_path): write_json(p, value) result = verify_execution(tmp_path, 0, "run-1") assert result["checks"]["oracle"] is False - assert "oracle_invalid_or_unsuccessful" in result["errors"] + assert result["evidence_consistent"] is True + assert result["task_success"] is False + assert result["run_accepted"] is False def test_failed_action_cannot_prove_successful_execution(tmp_path): @@ -100,7 +102,10 @@ def test_failed_action_cannot_prove_successful_execution(tmp_path): write_json(p, value) result = verify_execution(tmp_path, 0, "run-1") assert result["valid"] is False - assert result["checks"]["actions"] is False + assert result["checks"]["actions"] is True + assert result["evidence_consistent"] is True + assert result["failed_action_count"] == 1 + assert result["run_accepted"] is False def test_missing_screenshot_observation_fails_closed(tmp_path): @@ -112,3 +117,18 @@ def test_missing_screenshot_observation_fails_closed(tmp_path): result = verify_execution(tmp_path, 0, "run-1") assert result["valid"] is False assert result["checks"]["actions"] is False + + +def test_failed_task_can_have_consistent_evidence(tmp_path): + make_receipts(tmp_path) + p = tmp_path / "oracle-result.json" + value = json.loads(p.read_text()) + value["rewards"] = [0.0] + value["success"] = False + write_json(p, value) + result = verify_execution(tmp_path, 0, "run-1") + assert result["checks"]["oracle"] is True + assert result["evidence_consistent"] is True + assert result["task_success"] is False + assert result["run_accepted"] is False + assert result["valid"] is False From 34662aa8affb7f95cf37b393112dfeb0c98a23b7 Mon Sep 17 00:00:00 2001 From: Noah Ellis <135493401+hippoley@users.noreply.github.com> Date: Mon, 12 Oct 2026 15:40:29 +0800 Subject: [PATCH 117/140] test(cua-bench): align negative-reward assertion with oracle consistency contract --- libs/cua-bench/cua_bench/tests/test_verify_execution.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/cua-bench/cua_bench/tests/test_verify_execution.py b/libs/cua-bench/cua_bench/tests/test_verify_execution.py index 64d6baf5f5..2a7ad16c1b 100644 --- a/libs/cua-bench/cua_bench/tests/test_verify_execution.py +++ b/libs/cua-bench/cua_bench/tests/test_verify_execution.py @@ -88,7 +88,7 @@ def test_finite_negative_reward_uses_explicit_threshold(tmp_path): value["success"] = False write_json(p, value) result = verify_execution(tmp_path, 0, "run-1") - assert result["checks"]["oracle"] is False + assert result["checks"]["oracle"] is True assert result["evidence_consistent"] is True assert result["task_success"] is False assert result["run_accepted"] is False From 397c2cb90f19ef29488795bb8bc03ad3082c86bc Mon Sep 17 00:00:00 2001 From: Noah Ellis <135493401+hippoley@users.noreply.github.com> Date: Mon, 12 Oct 2026 15:53:27 +0800 Subject: [PATCH 118/140] fix(cua-bench): align live runner evidence admission with offline verifier --- libs/cua-bench/cua_bench/runner/task_runner.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/libs/cua-bench/cua_bench/runner/task_runner.py b/libs/cua-bench/cua_bench/runner/task_runner.py index 58f9fa73e1..6cc5da163f 100644 --- a/libs/cua-bench/cua_bench/runner/task_runner.py +++ b/libs/cua-bench/cua_bench/runner/task_runner.py @@ -8,6 +8,7 @@ import json import math import os +import re import shutil from dataclasses import dataclass from pathlib import Path @@ -393,7 +394,7 @@ async def run_task( isinstance(rewards, list) and bool(rewards) and all( isinstance(v, (int, float)) and not isinstance(v, bool) - and math.isfinite(v) and 0.0 <= v <= 1.0 + and math.isfinite(v) for v in rewards ) ) @@ -443,10 +444,18 @@ async def run_task( or record.get("sequence") != sequence or record.get("execution_status") not in ("returned", "failed") or not isinstance(record.get("action"), dict) - or not isinstance(record.get("before"), dict) - or not isinstance(record.get("after"), dict) + or not all( + isinstance(record.get(phase), dict) + and isinstance(record[phase].get("sha256"), str) + and re.fullmatch(r"[0-9a-f]{64}", record[phase]["sha256"]) + and type(record[phase].get("size_bytes")) is int + and record[phase]["size_bytes"] >= 0 + for phase in ("before", "after") + ) ): raise ValueError("Invalid action evidence record") + if record["execution_status"] == "failed": + raise ValueError("Failed agent action cannot establish a successful run") if sequence == 0: evidence_error = "Empty agent action evidence" else: From 2513228f3add5314290cfdceb0f6138d5c37ae58 Mon Sep 17 00:00:00 2001 From: Noah Ellis <135493401+hippoley@users.noreply.github.com> Date: Mon, 12 Oct 2026 15:53:51 +0800 Subject: [PATCH 119/140] test(cua-bench): cover finite reward above one in live and offline contract --- .../cua_bench/tests/test_verify_execution.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/libs/cua-bench/cua_bench/tests/test_verify_execution.py b/libs/cua-bench/cua_bench/tests/test_verify_execution.py index 2a7ad16c1b..f55b127009 100644 --- a/libs/cua-bench/cua_bench/tests/test_verify_execution.py +++ b/libs/cua-bench/cua_bench/tests/test_verify_execution.py @@ -132,3 +132,15 @@ def test_failed_task_can_have_consistent_evidence(tmp_path): assert result["task_success"] is False assert result["run_accepted"] is False assert result["valid"] is False + + +def test_finite_reward_above_one_is_consistent_with_solver_contract(tmp_path): + make_receipts(tmp_path) + p = tmp_path / "oracle-result.json" + value = json.loads(p.read_text()) + value["rewards"] = [2.0] + write_json(p, value) + result = verify_execution(tmp_path, 0, "run-1") + assert result["checks"]["oracle"] is True + assert result["task_success"] is True + assert result["run_accepted"] is True From fe83fe796de399fb118d5e338736a1b2c5ae1b98 Mon Sep 17 00:00:00 2001 From: Noah Ellis <135493401+hippoley@users.noreply.github.com> Date: Mon, 12 Oct 2026 16:01:48 +0800 Subject: [PATCH 120/140] refactor(cua-bench): centralize shared oracle and action receipt validation --- libs/cua-bench/cua_bench/evidence_contract.py | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 libs/cua-bench/cua_bench/evidence_contract.py diff --git a/libs/cua-bench/cua_bench/evidence_contract.py b/libs/cua-bench/cua_bench/evidence_contract.py new file mode 100644 index 0000000000..0b96cfbb94 --- /dev/null +++ b/libs/cua-bench/cua_bench/evidence_contract.py @@ -0,0 +1,52 @@ +"""Shared, dependency-free receipt contract for live and offline cua-bench verification. + +This validates internally consistent claims, NOT the authenticity of the source. +""" +import math +import re + + +def valid_observation(value): + return ( + isinstance(value, dict) + and isinstance(value.get("sha256"), str) + and re.fullmatch(r"[0-9a-f]{64}", value["sha256"]) is not None + and type(value.get("size_bytes")) is int + and value["size_bytes"] >= 0 + ) + + +def oracle_receipt_valid(receipt, task_index, execution_id): + if not isinstance(receipt, dict): + return False + rewards = receipt.get("rewards") + threshold = receipt.get("success_threshold") + if not ( + receipt.get("schema_version") == "cua-bench-oracle-result/v1" + and receipt.get("task_index") == task_index + and receipt.get("execution_id") == execution_id + and receipt.get("evaluated") is True + and type(receipt.get("success")) is bool + and isinstance(rewards, list) + and bool(rewards) + and all(type(v) in (int, float) and math.isfinite(v) for v in rewards) + and type(threshold) in (int, float) + and math.isfinite(threshold) + and 0 <= threshold <= 1 + ): + return False + return receipt["success"] == all(v >= threshold for v in rewards) + + +def action_event_valid(record, sequence, execution_id): + return ( + isinstance(record, dict) + and record.get("schema_version") == "cua-bench-executed-action/v1" + and record.get("execution_id") == execution_id + and type(record.get("sequence")) is int + and record["sequence"] == sequence + and record.get("execution_status") in ("returned", "failed") + and isinstance(record.get("action"), dict) + and valid_observation(record.get("before")) + and valid_observation(record.get("after")) + ) From 6e18ab479898f6292bd7058228d4c609945e2f09 Mon Sep 17 00:00:00 2001 From: Noah Ellis <135493401+hippoley@users.noreply.github.com> Date: Mon, 12 Oct 2026 16:02:01 +0800 Subject: [PATCH 121/140] refactor(cua-bench): use shared contract in offline verification --- libs/cua-bench/cua_bench/verify_execution.py | 36 ++------------------ 1 file changed, 3 insertions(+), 33 deletions(-) diff --git a/libs/cua-bench/cua_bench/verify_execution.py b/libs/cua-bench/cua_bench/verify_execution.py index 4029a3865c..8477f0d9c7 100644 --- a/libs/cua-bench/cua_bench/verify_execution.py +++ b/libs/cua-bench/cua_bench/verify_execution.py @@ -5,8 +5,7 @@ """ import argparse import json -import math -import re +from .evidence_contract import action_event_valid, oracle_receipt_valid from pathlib import Path @@ -17,16 +16,6 @@ def _json(path): return value -def _valid_observation(value): - return ( - isinstance(value, dict) - and isinstance(value.get("sha256"), str) - and re.fullmatch(r"[0-9a-f]{64}", value["sha256"]) is not None - and type(value.get("size_bytes")) is int - and value["size_bytes"] >= 0 - ) - - def verify_execution(root: Path, task_index: int, execution_id: str, *, require_actions=True): """Return a machine-readable fail-closed independent verification report.""" errors = [] @@ -42,17 +31,7 @@ def verify_execution(root: Path, task_index: int, execution_id: str, *, require_ oracle = _json(oracle_path) rewards = oracle.get("rewards") threshold = oracle.get("success_threshold") - valid = ( - oracle.get("schema_version") == "cua-bench-oracle-result/v1" - and oracle.get("task_index") == task_index - and oracle.get("execution_id") == execution_id - and oracle.get("evaluated") is True - and isinstance(rewards, list) and len(rewards) > 0 - and all(type(v) in (float, int) and math.isfinite(v) for v in rewards) - and type(threshold) in (float, int) and math.isfinite(threshold) and 0 <= threshold <= 1 - and type(oracle.get("success")) is bool - and oracle["success"] == all(v >= threshold for v in rewards) - ) + valid = oracle_receipt_valid(oracle, task_index, execution_id) checks["oracle"] = bool(valid) task_success = bool(valid and oracle["success"]) if not checks["oracle"]: @@ -85,16 +64,7 @@ def verify_execution(root: Path, task_index: int, execution_id: str, *, require_ for line in stream: record = json.loads(line) action_count += 1 - if ( - not isinstance(record, dict) - or record.get("schema_version") != "cua-bench-executed-action/v1" - or record.get("execution_id") != execution_id - or record.get("sequence") != action_count - or record.get("execution_status") not in ("returned", "failed") - or not isinstance(record.get("action"), dict) - or not _valid_observation(record.get("before")) - or not _valid_observation(record.get("after")) - ): + if not action_event_valid(record, action_count, execution_id): raise ValueError("Invalid action event") failed_action_count += record["execution_status"] == "failed" checks["actions"] = action_count > 0 From 8296e8e023912fc85d8cbaf154337475f481579d Mon Sep 17 00:00:00 2001 From: Noah Ellis <135493401+hippoley@users.noreply.github.com> Date: Mon, 12 Oct 2026 16:02:16 +0800 Subject: [PATCH 122/140] refactor(cua-bench): use shared receipt contract in live task runner --- .../cua-bench/cua_bench/runner/task_runner.py | 54 ++----------------- 1 file changed, 4 insertions(+), 50 deletions(-) diff --git a/libs/cua-bench/cua_bench/runner/task_runner.py b/libs/cua-bench/cua_bench/runner/task_runner.py index 6cc5da163f..90f70c1b37 100644 --- a/libs/cua-bench/cua_bench/runner/task_runner.py +++ b/libs/cua-bench/cua_bench/runner/task_runner.py @@ -8,12 +8,12 @@ import json import math import os -import re import shutil from dataclasses import dataclass from pathlib import Path from typing import Dict, List, Optional +from ..evidence_contract import action_event_valid, oracle_receipt_valid from ..sessions.providers.local_environment import check_image_exists, pull_image from .docker_utils import ( ContainerInfo, @@ -377,40 +377,8 @@ async def run_task( if oracle_path and oracle_path.is_file(): try: oracle_result = json.loads(oracle_path.read_text(encoding="utf-8")) - if ( - oracle_result.get("schema_version") != "cua-bench-oracle-result/v1" - or oracle_result.get("task_index") != task_index - or oracle_result.get("execution_id") != task_id - or oracle_result.get("evaluated") is not True - or not isinstance(oracle_result.get("success"), bool) - ): - oracle_error = "Invalid oracle result receipt" - else: - # Recalculate the verdict from the numeric rewards. - # Do not trust a self-asserted success boolean. - rewards = oracle_result.get("rewards") - threshold = oracle_result.get("success_threshold") - valid_rewards = ( - isinstance(rewards, list) and bool(rewards) - and all( - isinstance(v, (int, float)) and not isinstance(v, bool) - and math.isfinite(v) - for v in rewards - ) - ) - valid_threshold = ( - isinstance(threshold, (int, float)) - and not isinstance(threshold, bool) - and math.isfinite(threshold) and 0.0 <= threshold <= 1.0 - ) - calculated = ( - valid_rewards and valid_threshold - and all(v >= threshold for v in rewards) - ) - if not valid_rewards or not valid_threshold or ( - oracle_result["success"] is not calculated - ): - oracle_error = "Inconsistent oracle reward and success verdict" + if not oracle_receipt_valid(oracle_result, task_index, task_id): + oracle_error = "Invalid or inconsistent oracle result receipt" except (OSError, ValueError, TypeError): oracle_error = "Unreadable oracle result receipt" else: @@ -438,21 +406,7 @@ async def run_task( for raw in records: record = json.loads(raw) sequence += 1 - if ( - record.get("schema_version") != "cua-bench-executed-action/v1" - or record.get("execution_id") != task_id - or record.get("sequence") != sequence - or record.get("execution_status") not in ("returned", "failed") - or not isinstance(record.get("action"), dict) - or not all( - isinstance(record.get(phase), dict) - and isinstance(record[phase].get("sha256"), str) - and re.fullmatch(r"[0-9a-f]{64}", record[phase]["sha256"]) - and type(record[phase].get("size_bytes")) is int - and record[phase]["size_bytes"] >= 0 - for phase in ("before", "after") - ) - ): + if not action_event_valid(record, sequence, task_id): raise ValueError("Invalid action evidence record") if record["execution_status"] == "failed": raise ValueError("Failed agent action cannot establish a successful run") From e07aea43f7e476f9cd694bc2a2b860af422782c1 Mon Sep 17 00:00:00 2001 From: Noah Ellis <135493401+hippoley@users.noreply.github.com> Date: Mon, 12 Oct 2026 16:02:37 +0800 Subject: [PATCH 123/140] test(cua-bench): add shared contract conformance and mutation regressions --- .../cua_bench/tests/test_evidence_contract.py | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 libs/cua-bench/cua_bench/tests/test_evidence_contract.py diff --git a/libs/cua-bench/cua_bench/tests/test_evidence_contract.py b/libs/cua-bench/cua_bench/tests/test_evidence_contract.py new file mode 100644 index 0000000000..72de99564b --- /dev/null +++ b/libs/cua-bench/cua_bench/tests/test_evidence_contract.py @@ -0,0 +1,49 @@ +"""Shared contract validation: independent of Docker and Agent SDK.""" +from cua_bench.evidence_contract import action_event_valid, oracle_receipt_valid + + +def oracle(): + return dict(schema_version="cua-bench-oracle-result/v1", task_index=0, + execution_id="task-1", evaluated=True, rewards=[1.0], + success_threshold=1.0, success=True) + + +def action(): + obs = {"sha256": "a" * 64, "size_bytes": 1} + return dict(schema_version="cua-bench-executed-action/v1", + execution_id="task-1", sequence=1, execution_status="returned", + action={"type": "click"}, before=obs, after=obs) + + +def test_valid_shared_oracle(): + assert oracle_receipt_valid(oracle(), 0, "task-1") + + +def test_reject_boolean_sequence(): + event = action() + event["sequence"] = True + assert not action_event_valid(event, 1, "task-1") + + +def test_reject_missing_observation_hash(): + event = action() + event["before"] = {"error_type": "TimeoutError"} + assert not action_event_valid(event, 1, "task-1") + + +def test_reject_forged_oracle_verdict(): + receipt = oracle() + receipt["success"] = False + assert not oracle_receipt_valid(receipt, 0, "task-1") + + +def test_finite_reward_above_one_is_valid_per_solver(): + receipt = oracle() + receipt["rewards"] = [2.0] + assert oracle_receipt_valid(receipt, 0, "task-1") + + +def test_failed_event_is_valid_evidence_but_not_success(): + event = action() + event["execution_status"] = "failed" + assert action_event_valid(event, 1, "task-1") From ef9754490dd07d95680945e31607c97054839740 Mon Sep 17 00:00:00 2001 From: Noah Ellis <135493401+hippoley@users.noreply.github.com> Date: Mon, 12 Oct 2026 16:10:36 +0800 Subject: [PATCH 124/140] fix(cua-bench): preserve TaskResult when cleanup raises and still record cleanup receipt --- libs/cua-bench/cua_bench/runner/task_runner.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/libs/cua-bench/cua_bench/runner/task_runner.py b/libs/cua-bench/cua_bench/runner/task_runner.py index 90f70c1b37..df094345d8 100644 --- a/libs/cua-bench/cua_bench/runner/task_runner.py +++ b/libs/cua-bench/cua_bench/runner/task_runner.py @@ -467,6 +467,18 @@ async def run_task( # 7. Cleanup and verify actual Docker resource absence. try: await self._cleanup_task(task_id) + except Exception as exc: + # A cleanup error must not replace the task result before the + # independent resource inspection writes its receipt. + if completed_result is not None: + completed_result.success = False + detail = f"Task cleanup failed: {type(exc).__name__}" + completed_result.error = ( + f"{completed_result.error}; {detail}" + if completed_result.error else detail + ) + else: + raise finally: if output_dir: try: From 92dbd1b7529b0adb92d53538c9e45d87dcc2edcf Mon Sep 17 00:00:00 2001 From: Noah Ellis <135493401+hippoley@users.noreply.github.com> Date: Mon, 12 Oct 2026 16:10:54 +0800 Subject: [PATCH 125/140] fix(cua-bench): retain original failure reason when cleanup verification also fails --- libs/cua-bench/cua_bench/runner/task_runner.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libs/cua-bench/cua_bench/runner/task_runner.py b/libs/cua-bench/cua_bench/runner/task_runner.py index df094345d8..dfdf49cb94 100644 --- a/libs/cua-bench/cua_bench/runner/task_runner.py +++ b/libs/cua-bench/cua_bench/runner/task_runner.py @@ -488,7 +488,10 @@ async def run_task( ) if completed_result is not None and cleanup.get("verified") is not True: completed_result.success = False - completed_result.error = "Task Docker resources not fully cleaned up" + completed_result.error = ( + f"{completed_result.error}; Task Docker resources not fully cleaned up" + if completed_result.error else "Task Docker resources not fully cleaned up" + ) except Exception as exc: if completed_result is not None: completed_result.success = False From 26eb2a2f540c9dfc3c6555b8121f1ce64a003d4a Mon Sep 17 00:00:00 2001 From: Noah Ellis <135493401+hippoley@users.noreply.github.com> Date: Mon, 12 Oct 2026 16:11:48 +0800 Subject: [PATCH 126/140] test(cua-bench): preserve timeout result and cleanup receipt when cleanup throws --- .../tests/test_task_cleanup_result.py | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 libs/cua-bench/cua_bench/tests/test_task_cleanup_result.py diff --git a/libs/cua-bench/cua_bench/tests/test_task_cleanup_result.py b/libs/cua-bench/cua_bench/tests/test_task_cleanup_result.py new file mode 100644 index 0000000000..8cedf9f25d --- /dev/null +++ b/libs/cua-bench/cua_bench/tests/test_task_cleanup_result.py @@ -0,0 +1,52 @@ +"""Regression for preserving failed task results through cleanup errors.""" +import asyncio +import json +from pathlib import Path + +import pytest + +from cua_bench.runner import task_runner as module + + +@pytest.mark.asyncio +async def test_cleanup_exception_preserves_timeout_result_and_receipt(tmp_path, monkeypatch): + runner = module.TaskRunner() + events = [] + + async def create_network(*args, **kwargs): + return None + + async def start_agent(**kwargs): + return None + + async def timeout(*args, **kwargs): + raise asyncio.TimeoutError() + + async def logs(*args, **kwargs): + return "logs" + + async def fail_cleanup(task_id): + events.append("cleanup") + raise RuntimeError("injected image removal failure") + + async def receipt(task_id, output_dir): + events.append("receipt") + (Path(output_dir) / "cleanup-result.json").write_text( + json.dumps({"verified": True}), encoding="utf-8" + ) + + monkeypatch.setattr(module, "create_network", create_network) + monkeypatch.setattr(module, "wait_for_container", timeout) + monkeypatch.setattr(module, "get_container_logs", logs) + monkeypatch.setattr(runner, "_start_agent_container", start_agent) + monkeypatch.setattr(runner, "_cleanup_task", fail_cleanup) + monkeypatch.setattr(runner, "_write_cleanup_receipt", receipt) + + result = await runner.run_task( + env_path=tmp_path, task_index=0, env_type="linux-docker", + provider_type="simulated", cleanup_before=False, output_dir=str(tmp_path), + ) + assert result.success is False + assert "Task timed out" in result.error + assert "Task cleanup failed: RuntimeError" in result.error + assert events == ["cleanup", "receipt"] From 51e7493e283f55ac4b0121f023b0998a04a56cee Mon Sep 17 00:00:00 2001 From: Noah Ellis <135493401+hippoley@users.noreply.github.com> Date: Mon, 12 Oct 2026 16:36:42 +0800 Subject: [PATCH 127/140] fix(cua-bench): retain primary task failure when cleanup receipt writing fails --- libs/cua-bench/cua_bench/runner/task_runner.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libs/cua-bench/cua_bench/runner/task_runner.py b/libs/cua-bench/cua_bench/runner/task_runner.py index dfdf49cb94..b1714feb5a 100644 --- a/libs/cua-bench/cua_bench/runner/task_runner.py +++ b/libs/cua-bench/cua_bench/runner/task_runner.py @@ -495,7 +495,11 @@ async def run_task( except Exception as exc: if completed_result is not None: completed_result.success = False - completed_result.error = f"Unable to verify Docker cleanup: {type(exc).__name__}" + detail = f"Unable to verify Docker cleanup: {type(exc).__name__}" + completed_result.error = ( + f"{completed_result.error}; {detail}" + if completed_result.error else detail + ) async def run_task_interactively( self, From 8e24666dff7e08a1bd64bf95be6901b2e5917593 Mon Sep 17 00:00:00 2001 From: Noah Ellis <135493401+hippoley@users.noreply.github.com> Date: Mon, 12 Oct 2026 16:36:58 +0800 Subject: [PATCH 128/140] test(cua-bench): preserve timeout diagnostic on receipt IO failure --- .../tests/test_task_cleanup_result.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/libs/cua-bench/cua_bench/tests/test_task_cleanup_result.py b/libs/cua-bench/cua_bench/tests/test_task_cleanup_result.py index 8cedf9f25d..2ccb264a87 100644 --- a/libs/cua-bench/cua_bench/tests/test_task_cleanup_result.py +++ b/libs/cua-bench/cua_bench/tests/test_task_cleanup_result.py @@ -50,3 +50,35 @@ async def receipt(task_id, output_dir): assert "Task timed out" in result.error assert "Task cleanup failed: RuntimeError" in result.error assert events == ["cleanup", "receipt"] + + +@pytest.mark.asyncio +async def test_receipt_write_failure_keeps_original_timeout_reason(tmp_path, monkeypatch): + runner = module.TaskRunner() + + async def noop(*args, **kwargs): + return None + + async def timeout(*args, **kwargs): + raise asyncio.TimeoutError() + + async def logs(*args, **kwargs): + return "logs" + + async def broken_receipt(*args, **kwargs): + raise OSError("injected disk full") + + monkeypatch.setattr(module, "create_network", noop) + monkeypatch.setattr(module, "wait_for_container", timeout) + monkeypatch.setattr(module, "get_container_logs", logs) + monkeypatch.setattr(runner, "_start_agent_container", noop) + monkeypatch.setattr(runner, "_cleanup_task", noop) + monkeypatch.setattr(runner, "_write_cleanup_receipt", broken_receipt) + + result = await runner.run_task( + env_path=tmp_path, task_index=0, env_type="linux-docker", + provider_type="simulated", cleanup_before=False, output_dir=str(tmp_path), + ) + assert result.success is False + assert "Task timed out" in result.error + assert "Unable to verify Docker cleanup: OSError" in result.error From af36c74a4ebdf1286def7b0682a0388087715dae Mon Sep 17 00:00:00 2001 From: Noah Ellis <135493401+hippoley@users.noreply.github.com> Date: Mon, 12 Oct 2026 17:58:44 +0800 Subject: [PATCH 129/140] fix(cua-bench): encompass overlay and receipt setup in task cleanup lifecycle --- .../cua-bench/cua_bench/runner/task_runner.py | 55 ++++++++++--------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/libs/cua-bench/cua_bench/runner/task_runner.py b/libs/cua-bench/cua_bench/runner/task_runner.py index b1714feb5a..4ecc631700 100644 --- a/libs/cua-bench/cua_bench/runner/task_runner.py +++ b/libs/cua-bench/cua_bench/runner/task_runner.py @@ -273,36 +273,37 @@ async def run_task( env_container_name = f"cua-env-{task_id}" agent_container_name = f"cua-agent-{task_id}" - # Create task overlay for QEMU types (protects golden image) - overlay_path: Optional[Path] = None - if not is_simulated and config.get("use_overlays"): - overlay_path = self._create_task_overlay(task_id) - - # Track task for cleanup - self._running_tasks[task_id] = { - "network": network_name, - "env_container": env_container_name if not is_simulated else None, - "agent_container": agent_container_name, - "env_image": config.get("image") if not is_simulated else None, - "agent_image": self.agent_image, - "remove_images": remove_images_after, - "overlay_path": overlay_path, - "is_simulated": is_simulated, - } - - # Do not let artifacts from an earlier run masquerade as this execution. - # Leave unrelated user files alone; only clear known generated receipts. - if output_dir: - task_output = Path(output_dir) - task_output.mkdir(parents=True, exist_ok=True) - for receipt_name in ("oracle-result.json", "cleanup-result.json"): - (task_output / receipt_name).unlink(missing_ok=True) - - # Track log streaming process for cleanup + # Include overlay creation and receipt initialization in lifecycle cleanup. log_stream_process = None completed_result: Optional[TaskResult] = None - + overlay_path: Optional[Path] = None try: + # Create task overlay for QEMU types (protects golden image) + if not is_simulated and config.get("use_overlays"): + overlay_path = self._create_task_overlay(task_id) + + # Track task for cleanup + self._running_tasks[task_id] = { + "network": network_name, + "env_container": env_container_name if not is_simulated else None, + "agent_container": agent_container_name, + "env_image": config.get("image") if not is_simulated else None, + "agent_image": self.agent_image, + "remove_images": remove_images_after, + "overlay_path": overlay_path, + "is_simulated": is_simulated, + } + + # Do not let artifacts from an earlier run masquerade as this execution. + # Leave unrelated user files alone; only clear known generated receipts. + if output_dir: + task_output = Path(output_dir) + task_output.mkdir(parents=True, exist_ok=True) + for receipt_name in ("oracle-result.json", "cleanup-result.json"): + (task_output / receipt_name).unlink(missing_ok=True) + + # Track log streaming process for cleanup + # 1. Create network await create_network(network_name) From a2a85c5803b4a131b1bbeadded2a34dd3e660ee0 Mon Sep 17 00:00:00 2001 From: Noah Ellis <135493401+hippoley@users.noreply.github.com> Date: Mon, 12 Oct 2026 17:59:10 +0800 Subject: [PATCH 130/140] fix(cua-bench): register partial QEMU overlay before creation for rollback --- libs/cua-bench/cua_bench/runner/task_runner.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/libs/cua-bench/cua_bench/runner/task_runner.py b/libs/cua-bench/cua_bench/runner/task_runner.py index 4ecc631700..50fc39d44d 100644 --- a/libs/cua-bench/cua_bench/runner/task_runner.py +++ b/libs/cua-bench/cua_bench/runner/task_runner.py @@ -278,11 +278,7 @@ async def run_task( completed_result: Optional[TaskResult] = None overlay_path: Optional[Path] = None try: - # Create task overlay for QEMU types (protects golden image) - if not is_simulated and config.get("use_overlays"): - overlay_path = self._create_task_overlay(task_id) - - # Track task for cleanup + # Register resources before creation, including partial overlay failures. self._running_tasks[task_id] = { "network": network_name, "env_container": env_container_name if not is_simulated else None, @@ -294,6 +290,11 @@ async def run_task( "is_simulated": is_simulated, } + # Register the overlay before creation to clean up partial failures. + if not is_simulated and config.get("use_overlays"): + self._running_tasks[task_id]["overlay_path"] = get_overlays_path() / task_id + overlay_path = self._create_task_overlay(task_id) + # Do not let artifacts from an earlier run masquerade as this execution. # Leave unrelated user files alone; only clear known generated receipts. if output_dir: @@ -302,8 +303,6 @@ async def run_task( for receipt_name in ("oracle-result.json", "cleanup-result.json"): (task_output / receipt_name).unlink(missing_ok=True) - # Track log streaming process for cleanup - # 1. Create network await create_network(network_name) From 5da7a2410392a205105036e6083515db1d9f8ebc Mon Sep 17 00:00:00 2001 From: Noah Ellis <135493401+hippoley@users.noreply.github.com> Date: Mon, 12 Oct 2026 17:59:29 +0800 Subject: [PATCH 131/140] test(cua-bench): roll back partially created overlay when initialization throws --- .../tests/test_task_cleanup_result.py | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/libs/cua-bench/cua_bench/tests/test_task_cleanup_result.py b/libs/cua-bench/cua_bench/tests/test_task_cleanup_result.py index 2ccb264a87..f027135158 100644 --- a/libs/cua-bench/cua_bench/tests/test_task_cleanup_result.py +++ b/libs/cua-bench/cua_bench/tests/test_task_cleanup_result.py @@ -82,3 +82,36 @@ async def broken_receipt(*args, **kwargs): assert result.success is False assert "Task timed out" in result.error assert "Unable to verify Docker cleanup: OSError" in result.error + + +@pytest.mark.asyncio +async def test_partial_overlay_failure_triggers_registered_cleanup(tmp_path, monkeypatch): + runner = module.TaskRunner() + seen = [] + monkeypatch.setattr(module, "generate_task_id", lambda: "overlay-test") + monkeypatch.setitem(module.ENV_CONFIGS, "overlay-test-env", {"use_overlays": True}) + monkeypatch.setattr(module, "get_overlays_path", lambda: tmp_path) + + def partial_overlay(task_id): + (tmp_path / task_id).mkdir() + raise OSError("injected overlay creation failure") + + async def cleanup(task_id): + tracked = runner._running_tasks.pop(task_id) + seen.append(tracked["overlay_path"]) + tracked["overlay_path"].rmdir() + + async def receipt(task_id, output_dir): + seen.append("receipt") + + monkeypatch.setattr(runner, "_create_task_overlay", partial_overlay) + monkeypatch.setattr(runner, "_cleanup_task", cleanup) + monkeypatch.setattr(runner, "_write_cleanup_receipt", receipt) + + with pytest.raises(OSError, match="injected overlay creation failure"): + await runner.run_task( + env_path=tmp_path, task_index=0, env_type="overlay-test-env", + cleanup_before=False, output_dir=str(tmp_path), + ) + assert seen == [tmp_path / "overlay-test", "receipt"] + assert not (tmp_path / "overlay-test").exists() From b90c58e627dd7cdc71eb6de5b7cfbbe14c502f9b Mon Sep 17 00:00:00 2001 From: Noah Ellis <135493401+hippoley@users.noreply.github.com> Date: Mon, 12 Oct 2026 18:23:43 +0800 Subject: [PATCH 132/140] test(cua-bench): assert failure TaskResult rather than exception on overlay setup error --- libs/cua-bench/cua_bench/tests/test_task_cleanup_result.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libs/cua-bench/cua_bench/tests/test_task_cleanup_result.py b/libs/cua-bench/cua_bench/tests/test_task_cleanup_result.py index f027135158..ea593806fe 100644 --- a/libs/cua-bench/cua_bench/tests/test_task_cleanup_result.py +++ b/libs/cua-bench/cua_bench/tests/test_task_cleanup_result.py @@ -108,10 +108,11 @@ async def receipt(task_id, output_dir): monkeypatch.setattr(runner, "_cleanup_task", cleanup) monkeypatch.setattr(runner, "_write_cleanup_receipt", receipt) - with pytest.raises(OSError, match="injected overlay creation failure"): - await runner.run_task( + result = await runner.run_task( env_path=tmp_path, task_index=0, env_type="overlay-test-env", cleanup_before=False, output_dir=str(tmp_path), ) + assert result.success is False + assert "injected overlay creation failure" in result.error assert seen == [tmp_path / "overlay-test", "receipt"] assert not (tmp_path / "overlay-test").exists() From 4a1770b82e87545e2f4475a90a29c9d1ac12697f Mon Sep 17 00:00:00 2001 From: Noah Ellis <135493401+hippoley@users.noreply.github.com> Date: Tue, 13 Oct 2026 10:13:47 +0800 Subject: [PATCH 133/140] test(cua-bench): integrate live cleanup receipt writer with offline verification --- .../tests/test_cleanup_receipt_integration.py | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 libs/cua-bench/cua_bench/tests/test_cleanup_receipt_integration.py diff --git a/libs/cua-bench/cua_bench/tests/test_cleanup_receipt_integration.py b/libs/cua-bench/cua_bench/tests/test_cleanup_receipt_integration.py new file mode 100644 index 0000000000..8022d5eb67 --- /dev/null +++ b/libs/cua-bench/cua_bench/tests/test_cleanup_receipt_integration.py @@ -0,0 +1,59 @@ +"""Filesystem integration of production cleanup receipts and offline verifier. + +Resource probes are stubbed; these tests do NOT constitute Docker/QEMU E2E. +""" +import json + +import pytest + +from cua_bench.runner import task_runner as module +from cua_bench.verify_execution import verify_execution + + +def _write_json(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value) + "\n", encoding="utf-8") + + +def _other_receipts(root, task_id): + _write_json(root / "oracle-result.json", { + "schema_version": "cua-bench-oracle-result/v1", + "execution_id": task_id, "task_index": 0, "evaluated": True, + "success": True, "rewards": [1.0], "success_threshold": 1.0, + }) + _write_json(root / "task_0_agent_logs" / "executed-actions.jsonl", { + "schema_version": "cua-bench-executed-action/v1", + "execution_id": task_id, "sequence": 1, + "execution_status": "returned", "action": {"type": "click"}, + "before": {"sha256": "a" * 64, "size_bytes": 1}, + "after": {"sha256": "b" * 64, "size_bytes": 1}, + }) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("residual", [False, True]) +async def test_production_cleanup_receipt_consumed_by_offline_verifier( + tmp_path, monkeypatch, residual, +): + task_id = "integration-1" + runner = module.TaskRunner() + monkeypatch.setattr(module, "get_overlays_path", lambda: tmp_path / "overlays") + + async def inspect(kind, name): + return not (residual and name == f"cua-agent-{task_id}") + + monkeypatch.setattr(runner, "_resource_absent", inspect) + _other_receipts(tmp_path, task_id) + await runner._write_cleanup_receipt(task_id, str(tmp_path)) + + receipt = json.loads((tmp_path / "cleanup-result.json").read_text()) + assert set(receipt["resources"]) == { + "agent_container", "env_container", "network", "overlay", + } + assert receipt["verified"] is not residual + + report = verify_execution(tmp_path, 0, task_id) + assert report["checks"]["oracle"] is True + assert report["checks"]["actions"] is True + assert report["checks"]["cleanup"] is not residual + assert report["run_accepted"] is not residual From 114f9aa7f4c22625504b5593b66ae1bcdb2ca0ca Mon Sep 17 00:00:00 2001 From: Noah Ellis <135493401+hippoley@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:56:30 +0800 Subject: [PATCH 134/140] test(cua-bench): verify cleanup receipt against a real Docker network lifecycle --- .../tests/test_cleanup_docker_integration.py | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 libs/cua-bench/cua_bench/tests/test_cleanup_docker_integration.py diff --git a/libs/cua-bench/cua_bench/tests/test_cleanup_docker_integration.py b/libs/cua-bench/cua_bench/tests/test_cleanup_docker_integration.py new file mode 100644 index 0000000000..dba9575f9e --- /dev/null +++ b/libs/cua-bench/cua_bench/tests/test_cleanup_docker_integration.py @@ -0,0 +1,62 @@ +"""Docker-backed integration: real daemon network -> production receipt -> verifier. + +Requires a working Docker daemon. In CI, absence of Docker is a hard failure. +""" +import asyncio +import json +import os +import shutil +import uuid + +import pytest + +from cua_bench.runner import task_runner as module +from cua_bench.verify_execution import verify_execution + + +async def docker(*args): + proc = await asyncio.create_subprocess_exec( + "docker", *args, stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await proc.communicate() + if proc.returncode: + raise AssertionError(f"docker {args[0]} failed: {stderr.decode(errors='replace')}") + return stdout.decode().strip() + + +@pytest.mark.asyncio +async def test_real_docker_network_cleanup_receipt_and_offline_verification(tmp_path, monkeypatch): + available = shutil.which("docker") is not None + if not available: + if os.environ.get("CI"): + pytest.fail("CI must have a real Docker CLI and daemon for this acceptance test") + pytest.skip("Local Docker not installed") + await docker("info", "--format", "{{.ServerVersion}}") + + task_id = "integration-" + uuid.uuid4().hex[:12] + network = "cua-task-" + task_id + runner = module.TaskRunner() + monkeypatch.setattr(module, "get_overlays_path", lambda: tmp_path / "overlays") + await docker("network", "create", network) + try: + await runner._write_cleanup_receipt(task_id, str(tmp_path)) + first = json.loads((tmp_path / "cleanup-result.json").read_text()) + assert first["resources"]["network"]["absent"] is False + assert first["verified"] is False + assert verify_execution(tmp_path, 0, task_id)["checks"]["cleanup"] is False + + await docker("network", "rm", network) + await runner._write_cleanup_receipt(task_id, str(tmp_path)) + second = json.loads((tmp_path / "cleanup-result.json").read_text()) + assert second["resources"]["network"]["absent"] is True + assert second["verified"] is True + assert verify_execution(tmp_path, 0, task_id)["checks"]["cleanup"] is True + finally: + # Never leave test-created Docker resources on the runner. + proc = await asyncio.create_subprocess_exec( + "docker", "network", "rm", network, + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.DEVNULL, + ) + await proc.wait() From 436cc06213c7ae8fbdad5169f913b68b78231d9a Mon Sep 17 00:00:00 2001 From: Noah Ellis <135493401+hippoley@users.noreply.github.com> Date: Tue, 13 Oct 2026 15:14:24 +0800 Subject: [PATCH 135/140] fix(cua-bench): reject incomplete Docker cleanup resource receipts --- libs/cua-bench/cua_bench/verify_execution.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libs/cua-bench/cua_bench/verify_execution.py b/libs/cua-bench/cua_bench/verify_execution.py index 8477f0d9c7..ce7ca0b473 100644 --- a/libs/cua-bench/cua_bench/verify_execution.py +++ b/libs/cua-bench/cua_bench/verify_execution.py @@ -46,7 +46,8 @@ def verify_execution(root: Path, task_index: int, execution_id: str, *, require_ valid = ( cleanup.get("schema_version") == "cua-bench-cleanup/v1" and cleanup.get("task_id") == execution_id - and isinstance(resources, dict) and len(resources) > 0 + and isinstance(resources, dict) + and {"agent_container", "env_container", "network", "overlay"} <= resources.keys() and all(isinstance(v, dict) and v.get("absent") is True for v in resources.values()) and cleanup.get("verified") is True ) From 8a7c9487a2c4b5ce1eb1171492ea611bb4856a64 Mon Sep 17 00:00:00 2001 From: Noah Ellis <135493401+hippoley@users.noreply.github.com> Date: Tue, 13 Oct 2026 15:14:35 +0800 Subject: [PATCH 136/140] test(cua-bench): require complete four-resource cleanup receipts --- .../cua_bench/tests/test_verify_execution.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/libs/cua-bench/cua_bench/tests/test_verify_execution.py b/libs/cua-bench/cua_bench/tests/test_verify_execution.py index f55b127009..5a201beb22 100644 --- a/libs/cua-bench/cua_bench/tests/test_verify_execution.py +++ b/libs/cua-bench/cua_bench/tests/test_verify_execution.py @@ -21,7 +21,8 @@ def make_receipts(root, execution_id="run-1"): "schema_version": "cua-bench-cleanup/v1", "task_id": execution_id, "verified": True, - "resources": {"agent_container": {"name": "a", "absent": True}}, + "resources": {key: {"name": key, "absent": True} for key in + ("agent_container", "env_container", "network", "overlay")}, }) logs = root / "task_0_agent_logs" / "executed-actions.jsonl" write_json(logs, { @@ -144,3 +145,14 @@ def test_finite_reward_above_one_is_consistent_with_solver_contract(tmp_path): assert result["checks"]["oracle"] is True assert result["task_success"] is True assert result["run_accepted"] is True + + +def test_missing_cleanup_resource_fails_closed(tmp_path): + make_receipts(tmp_path) + path = tmp_path / "cleanup-result.json" + receipt = json.loads(path.read_text()) + del receipt["resources"]["overlay"] + write_json(path, receipt) + report = verify_execution(tmp_path, 0, "run-1") + assert report["checks"]["cleanup"] is False + assert report["run_accepted"] is False From d19784e7351ca55168f1c21a96800f66d41bfb86 Mon Sep 17 00:00:00 2001 From: Noah Ellis <135493401+hippoley@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:58:25 +0800 Subject: [PATCH 137/140] fix(cua-bench): bind cleanup resource names to execution identity --- libs/cua-bench/cua_bench/verify_execution.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/libs/cua-bench/cua_bench/verify_execution.py b/libs/cua-bench/cua_bench/verify_execution.py index ce7ca0b473..81bb2675ec 100644 --- a/libs/cua-bench/cua_bench/verify_execution.py +++ b/libs/cua-bench/cua_bench/verify_execution.py @@ -48,6 +48,11 @@ def verify_execution(root: Path, task_index: int, execution_id: str, *, require_ and cleanup.get("task_id") == execution_id and isinstance(resources, dict) and {"agent_container", "env_container", "network", "overlay"} <= resources.keys() + and resources["agent_container"].get("name") == f"cua-agent-{execution_id}" + and resources["env_container"].get("name") == f"cua-env-{execution_id}" + and resources["network"].get("name") == f"cua-task-{execution_id}" + and isinstance(resources["overlay"].get("name"), str) + and Path(resources["overlay"]["name"]).name == execution_id and all(isinstance(v, dict) and v.get("absent") is True for v in resources.values()) and cleanup.get("verified") is True ) From fd6f289de09330b628d1d0336e45ec61e0263b0e Mon Sep 17 00:00:00 2001 From: Noah Ellis <135493401+hippoley@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:58:36 +0800 Subject: [PATCH 138/140] test(cua-bench): reject cleanup receipts naming another execution's resources --- .../cua_bench/tests/test_verify_execution.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/libs/cua-bench/cua_bench/tests/test_verify_execution.py b/libs/cua-bench/cua_bench/tests/test_verify_execution.py index 5a201beb22..523d11611c 100644 --- a/libs/cua-bench/cua_bench/tests/test_verify_execution.py +++ b/libs/cua-bench/cua_bench/tests/test_verify_execution.py @@ -21,8 +21,12 @@ def make_receipts(root, execution_id="run-1"): "schema_version": "cua-bench-cleanup/v1", "task_id": execution_id, "verified": True, - "resources": {key: {"name": key, "absent": True} for key in - ("agent_container", "env_container", "network", "overlay")}, + "resources": { + "agent_container": {"name": f"cua-agent-{execution_id}", "absent": True}, + "env_container": {"name": f"cua-env-{execution_id}", "absent": True}, + "network": {"name": f"cua-task-{execution_id}", "absent": True}, + "overlay": {"name": f"/tmp/overlays/{execution_id}", "absent": True}, + }, }) logs = root / "task_0_agent_logs" / "executed-actions.jsonl" write_json(logs, { @@ -156,3 +160,14 @@ def test_missing_cleanup_resource_fails_closed(tmp_path): report = verify_execution(tmp_path, 0, "run-1") assert report["checks"]["cleanup"] is False assert report["run_accepted"] is False + + +def test_wrong_cleanup_resource_identity_is_rejected(tmp_path): + make_receipts(tmp_path) + path = tmp_path / "cleanup-result.json" + receipt = json.loads(path.read_text()) + receipt["resources"]["network"]["name"] = "cua-task-some-other-run" + write_json(path, receipt) + result = verify_execution(tmp_path, 0, "run-1") + assert result["checks"]["cleanup"] is False + assert result["run_accepted"] is False From 1cb7c368a3241a17cc47eeb200c128ced2f95baa Mon Sep 17 00:00:00 2001 From: Noah Ellis <135493401+hippoley@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:58:51 +0800 Subject: [PATCH 139/140] fix(cua-bench): fail closed on malformed cleanup resource entries --- libs/cua-bench/cua_bench/verify_execution.py | 1 + 1 file changed, 1 insertion(+) diff --git a/libs/cua-bench/cua_bench/verify_execution.py b/libs/cua-bench/cua_bench/verify_execution.py index 81bb2675ec..3f3c4170b0 100644 --- a/libs/cua-bench/cua_bench/verify_execution.py +++ b/libs/cua-bench/cua_bench/verify_execution.py @@ -48,6 +48,7 @@ def verify_execution(root: Path, task_index: int, execution_id: str, *, require_ and cleanup.get("task_id") == execution_id and isinstance(resources, dict) and {"agent_container", "env_container", "network", "overlay"} <= resources.keys() + and all(isinstance(v, dict) for v in resources.values()) and resources["agent_container"].get("name") == f"cua-agent-{execution_id}" and resources["env_container"].get("name") == f"cua-env-{execution_id}" and resources["network"].get("name") == f"cua-task-{execution_id}" From 61800d74e68f459dc8428b052660cd422019cc0a Mon Sep 17 00:00:00 2001 From: Noah Ellis <135493401+hippoley@users.noreply.github.com> Date: Wed, 14 Oct 2026 14:33:29 +0800 Subject: [PATCH 140/140] test(cua-bench): reject tampered identity in real Docker cleanup receipts --- .../cua_bench/tests/test_cleanup_docker_integration.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/libs/cua-bench/cua_bench/tests/test_cleanup_docker_integration.py b/libs/cua-bench/cua_bench/tests/test_cleanup_docker_integration.py index dba9575f9e..45ebdb0abd 100644 --- a/libs/cua-bench/cua_bench/tests/test_cleanup_docker_integration.py +++ b/libs/cua-bench/cua_bench/tests/test_cleanup_docker_integration.py @@ -52,6 +52,14 @@ async def test_real_docker_network_cleanup_receipt_and_offline_verification(tmp_ assert second["resources"]["network"]["absent"] is True assert second["verified"] is True assert verify_execution(tmp_path, 0, task_id)["checks"]["cleanup"] is True + # A genuine daemon observation must not validate another run identity. + second["resources"]["network"]["name"] = "cua-task-different-execution" + (tmp_path / "cleanup-result.json").write_text( + json.dumps(second) + "\n", encoding="utf-8" + ) + forged = verify_execution(tmp_path, 0, task_id) + assert forged["checks"]["cleanup"] is False + assert forged["run_accepted"] is False finally: # Never leave test-created Docker resources on the runner. proc = await asyncio.create_subprocess_exec(