diff --git a/.github/release-attribution-config.json b/.github/release-attribution-config.json new file mode 100644 index 0000000000..c444b06728 --- /dev/null +++ b/.github/release-attribution-config.json @@ -0,0 +1,49 @@ +{ + "bots": [ + "dependabot[bot]", + "github-actions[bot]", + "claude", + "copilot", + "release-please[bot]", + "renovate[bot]", + "trycua-release[bot]" + ], + "ignoredCoauthorEmails": [ + "claude@anthropic.com", + "coding-sub-agent@trycua.com", + "codex@openai.com", + "cua@cua.localdomain", + "hermes@hermes.home.arpa", + "noreply@anthropic.com" + ], + "coauthorOverrides": { + "hippoley ": "hippoley" + }, + "identityOverrides": { + "antoine.bonnet@hcompany.ai": "abonneth", + "anvit.mail@icloud.com": "AnvitDevadiga", + "ddupont@mit.edu": "ddupont808", + "dillon@trycua.com": "ddupont808", + "egnichtel@crosscode.com": "ngnichtel", + "f@trycua.com": "f-trycua", + "fbonacci@fbonacci-xfce-vm.b40fumexvs1ujlod2ppcyfh5qa.bx.internal.cloudapp.net": "f-trycua", + "i@jyunko.cn": "HsiangNianian", + "jf-mac-mini@jf-mac-mini-4.local": "0xjohnnydev", + "klymenko@adobe.com": "pavelklymenko", + "m.fuechtenkoetter@posteo.de": "ai-ag2026", + "manfred@ubuntu26.04-vm": "ai-ag2026", + "marko.civonafets@gmail.com": "civonafets", + "me@madhavajay.com": "madhavajay", + "robert@trycua.com": "enchanted-koala", + "rwendt1337@gmail.com": "r33drichards", + "rsyuzyov@gmail.com": "rsyuzyov", + "sarinajin.li@gmail.com": "sarinali", + "steipete@gmail.com": "steipete", + "tshtark@intelligo.ai": "tshtark", + "umtcgnd@gmail.com": "c8dhjp4tyv-bit", + "zane.chee.2023@scis.smu.edu.sg": "injaneity", + "zongxin_yang@hms.harvard.edu": "z-x-yang" + }, + "internalHandles": ["ddupont808", "enchanted-koala", "f-trycua"], + "optOutHandles": [] +} diff --git a/.github/workflows/ci-py-bench.yml b/.github/workflows/ci-py-bench.yml index 58c7120f1c..adc1b3771b 100644 --- a/.github/workflows/ci-py-bench.yml +++ b/.github/workflows/ci-py-bench.yml @@ -1,6 +1,14 @@ name: "CI: cua-bench" on: + push: + branches: + - fix/cua-bench-validate-parallelism-20261009 + paths: + - "libs/cua-bench/**" + - ".github/workflows/ci-py-bench.yml" + - ".github/workflows/py-reusable-build.yml" + workflow_dispatch: pull_request: paths: - "libs/cua-bench/**" @@ -13,3 +21,54 @@ jobs: with: package_name: "bench" package_dir: "libs/cua-bench" + + regression: + runs-on: ubuntu-latest + timeout-minutes: 35 + steps: + - uses: actions/checkout@v4 + - name: Set up uv and Python 3.12 + uses: astral-sh/setup-uv@v6 + with: + python-version: "3.12" + - name: Install Playwright Chromium for simulated desktop E2E + working-directory: libs/cua-bench + run: uv run --extra dev python -m playwright install --with-deps chromium + - name: Verify dataset manifest round trip on basic dataset + working-directory: libs/cua-bench + run: | + python3 scripts/dataset_manifest.py datasets/cua-bench-basic --output /tmp/cua-basic-manifest.json + python3 scripts/dataset_manifest.py datasets/cua-bench-basic --verify /tmp/cua-basic-manifest.json + - name: Exercise installed manifest CLI + working-directory: libs/cua-bench + run: | + uv run --extra dev python -m cua_bench.cli.main run manifest datasets/cua-bench-basic --output /tmp/cua-cli-pinned.json + uv run --extra dev python -c "import json; from pathlib import Path; from cua_bench.dataset_manifest import verify_dataset; verify_dataset(Path('datasets/cua-bench-basic'), json.loads(Path('/tmp/cua-cli-pinned.json').read_text()))" + - name: Run CUA benchmark core and oracle regression suite + working-directory: libs/cua-bench + run: | + xvfb-run -a uv run --extra dev --with 'pytest-json-report>=1.5,<2' --with 'jsonschema>=4.20,<5' --with 'pytest-timeout>=2.3,<3' --with 'hypothesis>=6.100,<7' pytest --timeout=120 --timeout-method=signal --json-report --json-report-file=core-report.json --json-report-omit=streams \ + cua_bench/tests/test_run_benchmark.py \ + cua_bench/tests/test_gym_interface.py \ + cua_bench/tests/test_actions.py \ + cua_bench/tests/test_reward_hypothesis.py -q + - name: Run full benchmark suite including RL workers + working-directory: libs/cua-bench + run: xvfb-run -a uv run --extra dev --extra rl --extra server --with 'httpx>=0.27,<1' --with 'pytest-json-report>=1.5,<2' --with 'jsonschema>=4.20,<5' --with 'pytest-timeout>=2.3,<3' --with 'hypothesis>=6.100,<7' pytest --timeout=120 --timeout-method=signal --json-report --json-report-file=full-report.json --json-report-omit=streams cua_bench/tests -q + - name: Independently verify test reports + working-directory: libs/cua-bench + run: python3 scripts/verify_pytest_receipt.py core-report.json full-report.json + - name: Build versioned checksum-linked receipt + working-directory: libs/cua-bench + run: uv run --extra dev --with 'jsonschema>=4.20,<5' python scripts/ci_receipt.py --commit "$GITHUB_SHA" + - name: Preserve CUA benchmark machine-readable evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: cua-bench-pytest-evidence + path: | + libs/cua-bench/core-report.json + libs/cua-bench/full-report.json + libs/cua-bench/ci-receipt-v1.json + if-no-files-found: warn + retention-days: 30 diff --git a/libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md b/libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md new file mode 100644 index 0000000000..3601b01902 --- /dev/null +++ b/libs/cua-bench/BENCHMARK_AUDIT_2026-10-09.md @@ -0,0 +1,116 @@ +# CUA Bench — 2026-10-09 engineering evidence + +Project: `hippoley/cua` (upstream `trycua/cua`), **not OpenLEADR**. + +## Baseline +- Relevant implementation: `libs/cua-bench/cua_bench/runners.py`, `run_benchmark`. +- Related tests: `libs/cua-bench/cua_bench/tests/test_run_benchmark.py`. +- Issue: `asyncio.Semaphore(max_parallel)` with zero permits never grants a worker slot when the dataset has runnable tasks, potentially hanging a benchmark indefinitely. + +## Change and cross-story audit +- Validate `max_parallel`, `max_steps`, and optional `max_variants` as positive integers at the API boundary; disallow Python booleans (subclass of int) and floats/strings. +- Add negative tests using pytest parametrization for zero, negative, boolean and wrong-type inputs. +- Cross-story impact: valid positive integer inputs keep their previous behavior. No external adapter, fixture or dataset schema modified. +- Dataset/benchmark truthfulness: this prevents the runner from silently hanging due to invalid configuration. It does **not** demonstrate improved agent task success, episode replay, sandbox compatibility, oracle correctness or runtime reproducibility. + +## Verification +- GitHub source fetch confirmed both changes on the feature branch. +- No repository-level pytest execution: execution container cannot resolve `github.com` for cloning and has no mounted CUA source. Absence of CI runs at commit inspection is not a pass. +- State: **CODE COMMITTED / TESTS NOT EXECUTED / NOT VERIFIED CLOSED**. + +## Next execution entry +```sh +cd libs/cua-bench +uv run pytest cua_bench/tests/test_run_benchmark.py -q +``` +Follow with positive real dataset runs on `cua-bench-basic`, a parallelism stress run, and evidence of termination and cleanup. Run from the branch `fix/cua-bench-validate-parallelism-20261009`. + +## Continuation: dataset discovery integrity +- **P0 failure mechanism:** `run_benchmark` previously swallowed all task-config exceptions and replaced the true variant count with `1`; an empty `tasks_config_fn()` produced zero runs and misleading empty metrics. Both violate truthful dataset denominators. +- **Code fix:** `runners.py` raises an informative ValueError containing task path and split on configuration errors and rejects zero variants, before scheduling any workers. Commit `55999795483d944a9f14635f59d3afaad5c3554a`. +- **Counterexamples authored:** `test_run_benchmark.py` creates a task whose config raises `RuntimeError` and another which returns zero tasks; both must now raise ValueError rather than fabricate results. Commit `ff1e8009e61b8d58d943f0bd73381c59b002d566`. +- **Status:** source committed and remote source verified; no pytest or end-to-end execution result witnessed in this round. Therefore NOT Verified Closed. +- **HCA:** function P, state P, integration B, security N/A (no new credentials/privileges), scale B, maintainability P, observation P, testability P, user value B, external compatibility N/A (pure benchmark orchestration). To close: run relevant pytest on actual branch, ensure valid dataset cases still pass, inspect run counts before and after, and run actual simulated provider end-to-end. + +## Score and aggregation integrity follow-up (2026-10-09) + +- **P0 score integrity:** `run_single_task` now rejects unsupported evaluator outputs, nonfinite values and rewards outside the normalized [0,1] contract. Missing evaluators no longer return a nominal score without an explicit error. Commit `24e815587c08a9a52014fa4bc5c5d5219db3493b`. +- **P0 failure provenance:** `run_benchmark` retains task path and variant ID for gathered worker exceptions, while accounting for failure in total_tasks and mean reward denominator. Commit `24e815587c08a9a52014fa4bc5c5d5219db3493b`. +- **Adversarial tests added:** malformed evaluator output (NaN, Infinity, negative, >1, nonnumeric), missing evaluator and two-task exception aggregation retaining both run identities and expected 1/2 success fraction. Commits `558d201661f9a7dfcf92838103f8c2e35eea9567`, `edd98cfbadfdb4fc82cb736d23799a2b4691d14b`. +- **Evidence status:** remote GitHub fetch confirms both code and tests. No local pytest execution or browser/sandbox E2E was witnessed; no User Story may be called Verified Closed. +- **HCA:** functional P; task/result state P; integration B; security NA (no new trust boundary); performance B; maintainability P; traceability P; testability P; end-user value B; external compatibility B. Challenge the normalized [0,1] reward assumption against each existing dataset/oracle before treating it as ecosystem-compatible. Regression suite and live sandbox tests remain mandatory. + +## Follow-up — no-agent false-positive guard (2026-10-09) +- **P0 finding:** `run_single_task(oracle=False, agent_fn=None)` previously reset the environment, skipped agent actions, then evaluated. A pre-satisfied environment could be reported as successful despite zero agent actions. +- **Code mitigation:** explicit failure `No agent_fn supplied; task was not executed`, with reward 0 and success False; no evaluator is invoked on this path. Commit `614e8ad561583e14c99f8f28107c4008398c1aab`. +- **Independent counterexample:** mock evaluator that raises if called, confirming absence of agent actions never earns score. Commit `6c3e54561474b67572f5598e410f3b008705bf42`. +- **Verification state:** remote code and tests present. Runtime container has Python and pytest but not full CUA repository sources at the relevant branch, so repository-level pytest and real simulated-provider tests were NOT executed. No Verified Closed. +- **Next P0:** execute `cd libs/cua-bench && uv run pytest cua_bench/tests/test_run_benchmark.py -q` on this branch; verify no existing legitimate no-agent setup-only callers rely on scoring; run an independent simulated provider case with initial evaluator reward 1.0 but no actions and assert failure. + +## Follow-up: worker-output integrity and benchmark denominator (2026-10-09) +- P0: `run_benchmark` now validates each returned worker result against the scheduled task path and variant ID, requires finite normalized reward, sane step count, coherent success/reward semantics, and zero reward on worker errors. Invalid results become attributable failures with reward 0 rather than polluting success rate. Commits `34772d45bfe89e133fb766c46e6e3c931eb70210`, `4b46ac851ecc0c39aa4e19c6ac0f469ab000e717`. +- Counterexamples added for forged path, forged variant, NaN reward and contradictory success/reward: commit `fbf83e1aa44660a5bf4b7f2cecb2711dcdf74fec`. +- Source changes committed; no fresh pytest/CI execution or real sandbox E2E evidenced. Validation requires running `uv run pytest cua_bench/tests/test_run_benchmark.py -q` from `libs/cua-bench`, followed by dataset-level evaluation and independent oracle replay. +- Horizontal status: Function P, state P, integration B, security P, performance B, maintainability P, observability P, testability P, user value B, external compatibility B. No Verified Closed. + +## Live CI evidence 2026-10-09 +- GitHub run 37908647343 passed prior build-only workflow, not pytest. +- Run 37908806518: full-suite collection FAILED (3 errors importing optional `torch` from worker dataloader). No test pass may be inferred from collection failure. +- Run 37908930187: core suite produced **79 passed, 11 failed**; all 11 browser failures cited missing Chromium/Playwright executables. CI installed browser dependencies subsequently. +- Run 37909070828: core suite produced **89 passed, 1 failed**; only headed-browser interactive E2E failed because runner lacked X Server. Browser/Oracle noninteractive tests executed. Commit 55203190fe26282224a417dd22960de3c077f23e uses `xvfb-run -a` for headed test, with run 37909217474 queued/in progress when recorded. Do not treat it as passed until reviewed. +- Local offline CUA contract and Outcome Witness standalone bundles: **27 passed in 0.06s**; unrelated to full `cua_bench/tests` run and not interchangeable evidence. +- Still OPEN P0: full RL worker suite must run with optional `rl` dependencies and survive real execution; fault injection proving process/resource cleanup in dedicated sandbox; independent Oracle provenance and action/outcome witness against actual scene. No Verified Closed issued. + +## Infrastructure thesis and stop/go gates — 2026-10-09 + +**Current hypothesis, not an adoption claim:** Build a reusable, provider-neutral CUA Execution Witness over existing agent action traces and evaluator outputs. Do not create a separate leaderboard or a competing runtime until direct users demonstrate a missing interface. OSWorld-V2.1's pinning of task/assets/website/provider images is a relevant mature precedent (https://github.com/xlang-ai/OSWorld-V2); match its provenance discipline, not copy its benchmark dataset. + +### Audit of traceable vertical-to-horizontal gaps +| Existing requirement path | P0 defect/gap | Horizontal impact | Mandatory independent receipt | +| --- | --- | --- | --- | +| Task discovery -> scheduling | exceptions, zero variants and invalid concurrency were previously silent/hanging | denominator and task-identity errors | executed pytest plus dataset manifest/hash | +| Action -> environment | no-agent scoring risk | false positive independent of actual action | action count and environment before/after evidence | +| Environment -> oracle | nonfinite scores, unversioned evaluator | score provenance cannot be independently repeated | oracle ID/version, inputs and independent rerun | +| Worker -> aggregation | worker identity mismatch and dropped exceptions | false-success and lost failure provenance | schedule/return identity matching and raw failure list | +| Runtime -> cleanup | no independent teardown inventory or crash injection receipt | state leak, nonreproducibility | force stop, live resource enumeration, cleanup=0 | +| Outcome -> third-party use | no stable witness schema, consumer SDK or external review | no downstream dependency proof | independent package consumer, issue/PR and reviewer confirmation | + +### Candidate implementation contract (not yet implemented) +`WitnessV0` should bind `run_id`, `task_id`, `variant_id`, `dataset_release`, `environment/provider/image_digest`, `agent_version`, `action_trace_digest`, `before_state_digest`, `after_state_digest`, `oracle_id/version`, `oracle_result`, `cleanup_state`, and `result_status`. Sensitive screenshots, tokens and personal data must remain optional protected artifacts, with hashes and redacted metadata sufficient to identify a replay target. An individual SHA-256 hash is integrity metadata, NOT a signature, origin attestation or proof of a true observation. + +### Prioritization and falsifiable next gates +- **Gate A, P0:** Python 3.12 benchmark core/oracle suite passes in real browser/Xvfb; full RL worker suite collects and runs with declared extras. Report number of tests, all failures and CI run/commit. +- **Gate B, P0:** intentional process interruption during a disposable CUA VM task, independent observation of remaining processes/containers, repeatable zero-resource cleanup. Not yet executed. +- **Gate C, P1:** independent evaluator reads raw action and state evidence to reproduce at least one task result; reject forged reward, edited task path, missing actions and mismatched image versions. +- **Gate D, P1:** a second repository consumes the versioned witness contract without fork-local imports and publishes a reproducible report. +- **Stop/pivot condition:** if an external adopter cannot name a contract deficiency after two genuine integrations, prefer upstream conformance contributions rather than branding another infra repo. If external feedback shows existing OSWorld tools solve the same job at lower cost, re-use them and kill redundant implementation. + +### Anti-claims +No verified external adoption, certification, real VM cleanup receipt or long-term defensibility proven yet. Evidence artifacts and test coverage are not themselves network effects. + +## Reassessment against 2026-10-09 upstream and external practice + +The official `trycua/cua` project remains active. Its open PRs include #4899 (worker stdout pipe saturation at long run), #4898 (worker lock/timeout deadlock), #4884 (reject action-truth claims inconsistent with E2E oracles), and #4883 (metadata-only runtime conformance witness). **Do not describe those as our novel discoveries or duplicate their patches.** Pursue narrow upstream-compatible changes after base comparison, not a large mixed PR from this feature branch. + +External comparison: OSWorld-V2's supported benchmark release defines pinned task/assets/web/provider image versions (https://github.com/xlang-ai/OSWorld-V2/blob/main/benchmark_releases/README.md). This fork's manifest hashes local task files only and currently does not bind runtime/provider assets. OpenTelemetry GenAI agent/tool span conventions are still in Development and are useful for trace interoperability, but spans alone are not proof of correctness (https://github.com/open-telemetry/semantic-conventions-genai/blob/main/docs/gen-ai/gen-ai-agent-spans.md). + +### 10-axis horizontal audit (evidence available at this time) + +| Axis | Status | Evidence/gap | +| --- | --- | --- | +| Function | Partial | core runner and dataset manifest exist; no product-entry one-click provenance comparison | +| State | Partial | file hashes and test-result checks; no runtime state diff or replay | +| Integration | Partial | GitHub CI builds/tests; real VM provider receipt missing | +| Security/correctness | Partial | worker identity, scoring, path/hidden-file checks; manifests not origin authenticated | +| Performance | Blocked | no cross-provider latency/throughput/long-run fixture from this branch | +| Maintainability | Partial | script CLIs and v1 JSON schema; manifest not yet stable library API | +| Observability | Partial | pytest report SHA; no per-step trace, environment image digest or teardown inventory | +| Testability | Partial | mock and Chromium E2E tests present; force-kill/recovery independent test absent | +| User value | Partial | reproducibility helper available; no independent consumer demonstrated | +| External compatibility | Partial | uses pytest-json-report/jsonschema; OSWorld/provider/OpenTelemetry adapters absent | + +### Priority decisions and falsification + +P0: finish repeatable end-to-end benchmarks and worker failure recovery; **upstream #4899/#4898** should be reviewed and reused rather than copied. P0: close dataset integrity blind spots (hidden config drift fixed on branch, subject to CI). P1: add release manifest pins for provider/image/environment and independent action/outcome snapshots only after a real provider scenario verifies the need. P1: third-party consumption of a minimal stable verifier API. P2: optional OTEL trace export; adopt current semantic conventions with careful version pinning. + +**External standing:** no official PR, merge, maintainer endorsement, external dependency or documented third-party consumption established by this audit. Fork CI success is valuable engineering evidence, not external adoption. 6–12 month credential requires upstream accepted work and/or independent consumer reproduction. 5–10-year defensibility is a hypothesis, not a promise; evaluate by recurring downstream reliance and standards participation. Pivot if two genuine external users decline adoption because an existing upstream tool already meets the same need. diff --git a/libs/cua-bench/cua_bench/agents/cua_agent.py b/libs/cua-bench/cua_bench/agents/cua_agent.py index 01b28d1c14..20b5c3cdb1 100644 --- a/libs/cua-bench/cua_bench/agents/cua_agent.py +++ b/libs/cua-bench/cua_bench/agents/cua_agent.py @@ -56,7 +56,7 @@ def __init__(self, **kwargs): def name() -> str: return "cua-agent" - def _create_custom_computer(self, session: "DesktopSession") -> Dict[str, Any]: + def _create_custom_computer(self, session: "DesktopSession", logging_dir: Path | None = None) -> Dict[str, Any]: """Create a custom computer dict from a DesktopSession. Args: @@ -219,7 +219,7 @@ async def perform_task( trajectory_dir.mkdir(parents=True, exist_ok=True) # Create custom computer dict from session - custom_computer = self._create_custom_computer(session) + custom_computer = self._create_custom_computer(session, logging_dir=logging_dir) # Create agent with custom computer agent = ComputerAgent( diff --git a/libs/cua-bench/cua_bench/batch/action_evidence.py b/libs/cua-bench/cua_bench/batch/action_evidence.py new file mode 100644 index 0000000000..4d9521f456 --- /dev/null +++ b/libs/cua-bench/cua_bench/batch/action_evidence.py @@ -0,0 +1,136 @@ +"""Common desktop action evidence boundary for benchmark agents. + +Wraps the session supplied to any agent without changing the concrete provider. +""" +import hashlib +import json +from datetime import datetime, timezone + + +class EvidenceSession: + def __init__(self, session, output_dir, execution_id): + self._session = session + self._path = output_dir / "executed-actions.jsonl" + # A new agent execution must never append to a previous run's evidence. + self._path.unlink(missing_ok=True) + self._execution_id = execution_id + self._sequence = 0 + + def __getattr__(self, name): + return getattr(self._session, name) + + async def _observation(self): + try: + data = await self._session.screenshot() + return {"sha256": hashlib.sha256(data).hexdigest(), "size_bytes": len(data)} + except Exception as exc: + return {"error_type": type(exc).__name__} + + async def execute_action(self, action): + from cua_bench.actions import action_to_dict + + self._sequence += 1 + before = await self._observation() + payload = action_to_dict(action) + if "text" in payload: + payload["text_length"] = len(payload.pop("text")) + payload["text_redacted"] = True + error = None + try: + return await self._session.execute_action(action) + except Exception as exc: + error = type(exc).__name__ + raise + finally: + after = await self._observation() + event = { + "schema_version": "cua-bench-executed-action/v1", + "execution_id": self._execution_id, + "sequence": self._sequence, + "timestamp": datetime.now(timezone.utc).isoformat(), + "action": payload, + "before": before, + "after": after, + "execution_status": "failed" if error else "returned", + "error_type": error, + } + with self._path.open("a", encoding="utf-8") as stream: + stream.write(json.dumps(event, sort_keys=True) + "\n") + + # High-level DesktopSession shortcuts bypass execute_action on the wrapped + # session; route them through this adapter to avoid silently missing actions. + async def step(self, action): + return await self.execute_action(action) + + async def click(self, x, y): + from cua_bench.types import ClickAction + return await self.execute_action(ClickAction(x=x, y=y)) + + async def right_click(self, x, y): + from cua_bench.types import RightClickAction + return await self.execute_action(RightClickAction(x=x, y=y)) + + async def double_click(self, x, y): + from cua_bench.types import DoubleClickAction + return await self.execute_action(DoubleClickAction(x=x, y=y)) + + async def type(self, text): + from cua_bench.types import TypeAction + return await self.execute_action(TypeAction(text=text)) + + async def key(self, key): + from cua_bench.types import KeyAction + return await self.execute_action(KeyAction(key=key)) + + async def hotkey(self, keys): + from cua_bench.types import HotkeyAction + return await self.execute_action(HotkeyAction(keys=keys)) + + async def scroll(self, direction="down", amount=300): + from cua_bench.types import ScrollAction + return await self.execute_action(ScrollAction(direction=direction, amount=amount)) + + async def move_to(self, x, y): + from cua_bench.types import MoveToAction + return await self.execute_action(MoveToAction(x=x, y=y)) + + async def drag(self, from_x, from_y, to_x, to_y): + from cua_bench.types import DragAction + return await self.execute_action(DragAction( + from_x=from_x, from_y=from_y, to_x=to_x, to_y=to_y + )) + + async def _record_direct(self, method, *args): + # Element-based interactions do not map to pixel Actions, but they + # still mutate the real desktop and must be included in the evidence. + self._sequence += 1 + before = await self._observation() + error = None + try: + return await getattr(self._session, method)(*args) + except Exception as exc: + error = type(exc).__name__ + raise + finally: + after = await self._observation() + event = { + "schema_version": "cua-bench-executed-action/v1", + "execution_id": self._execution_id, + "sequence": self._sequence, + "timestamp": datetime.now(timezone.utc).isoformat(), + "action": {"type": method, "selector_sha256": hashlib.sha256( + str(args[1]).encode("utf-8") + ).hexdigest()}, + "before": before, + "after": after, + "execution_status": "failed" if error else "returned", + "error_type": error, + } + with self._path.open("a", encoding="utf-8") as stream: + stream.write(json.dumps(event, sort_keys=True) + "\n") + + async def click_element(self, pid, selector): + return await self._record_direct("click_element", pid, selector) + + async def right_click_element(self, pid, selector): + return await self._record_direct("right_click_element", pid, selector) diff --git a/libs/cua-bench/cua_bench/batch/solver.py b/libs/cua-bench/cua_bench/batch/solver.py index a17152df17..9ea5fdddd4 100644 --- a/libs/cua-bench/cua_bench/batch/solver.py +++ b/libs/cua-bench/cua_bench/batch/solver.py @@ -237,9 +237,9 @@ async def main(): tasks = env.tasks_config_fn() - if task_index >= len(tasks): - print(f"Task index {task_index} >= number of tasks {len(tasks)}, skipping") - sys.exit(0) + if not 0 <= task_index < len(tasks): + print(f"Error: Task index {task_index} out of range for {len(tasks)} tasks") + sys.exit(1) task_cfg = tasks[task_index] print(f"Running task {task_index}: {task_cfg.description}") @@ -313,10 +313,17 @@ async def main(): logging_dir.mkdir(exist_ok=True, parents=True) print(f"Running agent: {agent.name()}") + # Every agent now crosses the same execution evidence boundary. + # This observes actual desktop calls, not the agent's text output. + from cua_bench.batch.action_evidence import EvidenceSession + + evidence_session = EvidenceSession( + session, logging_dir, os.environ.get("CUA_EXECUTION_ID") + ) try: agent_result = await agent.perform_task( task_description=task_cfg.description, - session=session, + session=evidence_session, logging_dir=logging_dir, tracer=env.tracing, ) @@ -358,10 +365,47 @@ async def main(): else: print("ℹ Dump mode: skipping solver") - # Evaluate + # Evaluate against the real environment. A successfully exited agent + # is not evidence of task completion; only the task oracle can decide. + import json + import math + + evaluation_receipt = { + "schema_version": "cua-bench-oracle-result/v1", + "task_index": task_index, + "execution_id": os.environ.get("CUA_EXECUTION_ID"), + "evaluated": False, + "success": False, + "rewards": None, + } if env.evaluate_task_fn: result = await env.evaluate_task_fn(task_cfg, session) print(f"✓ Evaluation result: {result}") + # Task evaluators return lists of numeric rewards (e.g. [1.0]). + # Reject missing, malformed, non-finite or partial outcomes. + rewards = result if isinstance(result, (list, tuple)) else [] + valid = bool(rewards) and all( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(value) + for value in rewards + ) + # A 0.5 reward is not automatically a completed task. Default to + # full reward; a task can explicitly define its own success bar. + metadata = getattr(task_cfg, "metadata", None) or {} + threshold = metadata.get("success_threshold", 1.0) + threshold_valid = ( + isinstance(threshold, (int, float)) + and not isinstance(threshold, bool) + and math.isfinite(threshold) + and 0.0 <= threshold <= 1.0 + ) + evaluation_receipt["evaluated"] = valid + evaluation_receipt["rewards"] = list(rewards) if valid else None + evaluation_receipt["success_threshold"] = threshold if threshold_valid else None + evaluation_receipt["success"] = ( + valid and threshold_valid and all(value >= threshold for value in rewards) + ) # Record evaluation event try: @@ -371,6 +415,16 @@ async def main(): ) except Exception as e: print(f"Warning: Failed to record evaluate event: {e}") + else: + print("✗ Task has no evaluation function; cannot claim success") + + receipt_file = output_dir / "oracle-result.json" + pending_file = output_dir / "oracle-result.json.tmp" + pending_file.write_text( + json.dumps(evaluation_receipt, sort_keys=True, indent=2) + "\n", + encoding="utf-8", + ) + pending_file.replace(receipt_file) # Save trace (non-fatal - task success doesn't depend on trace saving) trace_dir = output_dir / f"task_{task_index}_trace" @@ -394,7 +448,10 @@ async def main(): await env.close() else: await session.close() - print(f"\n✓ Task {task_index} completed successfully!") + if not evaluation_receipt["success"]: + print(f"✗ Task {task_index} did not pass its environment oracle") + sys.exit(1) + print(f"\n✓ Task {task_index} passed its environment oracle!") except Exception as e: print(f"Error running task {task_index}: {e}") diff --git a/libs/cua-bench/cua_bench/cli/commands/run.py b/libs/cua-bench/cua_bench/cli/commands/run.py index 431c8a2f94..ebafafc973 100644 --- a/libs/cua-bench/cua_bench/cli/commands/run.py +++ b/libs/cua-bench/cua_bench/cli/commands/run.py @@ -1409,6 +1409,26 @@ async def _cmd_run_dataset_async(args) -> int: print(f"{RED}Error: Dataset not found: {args.dataset_path}{RESET}") return 1 + verified_manifest_digest = None + + # Validate the pinned dataset before provider detection, task imports, + # session creation or asynchronous subprocess launch. + manifest_path = getattr(args, "dataset_manifest", None) + if manifest_path: + import json + from cua_bench.dataset_manifest import verify_dataset + try: + manifest = json.loads(Path(manifest_path).read_text(encoding="utf-8")) + task_count = verify_dataset(dataset_path, manifest) + import hashlib + verified_manifest_digest = hashlib.sha256( + json.dumps(manifest, sort_keys=True, separators=(",", ":")).encode("utf-8") + ).hexdigest() + except (OSError, ValueError, TypeError) as exc: + print(f"{RED}Error: Dataset manifest verification failed: {exc}{RESET}") + return 1 + print(f"{GREEN}Verified pinned dataset: {task_count} tasks{RESET}") + # Discover tasks in dataset tasks = [] @@ -1484,6 +1504,19 @@ async def _cmd_run_dataset_async(args) -> int: output_dir.mkdir(parents=True, exist_ok=True) + if manifest_path: + import hashlib + import json + # Record the exact manifest identity used at dispatch time. + canonical_manifest = json.dumps(manifest, sort_keys=True, separators=(",", ":")) + identity = hashlib.sha256(canonical_manifest.encode("utf-8")).hexdigest() + (output_dir / "dataset-identity.json").write_text( + json.dumps({"schema_version": "cua-dataset-identity/v1", + "dataset_manifest_sha256": identity, + "manifest_path": str(Path(manifest_path).resolve())}, + indent=2) + "\n", encoding="utf-8" + ) + # Detect env type from first task first_task = task_variants[0][0] if task_variants else None if first_task: @@ -1696,9 +1729,105 @@ async def run_with_semaphore(task_path, variant_id, task_num): # Run all tasks (semaphore controls parallelism) results = await asyncio.gather(*coroutines, return_exceptions=True) + # Persist the actual two-container TaskRunner outcomes, not synthetic + # single-process runner values or parsed log-based success claims. + import json + import os + + task_outcomes = [] + for (task_path, variant_id), outcome in zip(task_variants, results): + passed = ( + not isinstance(outcome, BaseException) + and outcome is not None + and getattr(outcome, "success", False) is True + ) + task_outcomes.append({ + "task": task_path.name, + "variant_id": variant_id, + "success": passed, + "verified_action_count": ( + getattr(outcome, "verified_action_count", None) + if outcome is not None and not isinstance(outcome, BaseException) + else None + ), + "oracle_result": ( + getattr(outcome, "oracle_result", None) + if outcome is not None and not isinstance(outcome, BaseException) + else None + ), + "exit_code": ( + getattr(outcome, "exit_code", None) + if outcome is not None and not isinstance(outcome, BaseException) + else None + ), + "error": ( + str(outcome) if isinstance(outcome, BaseException) + else "task failed without result" if outcome is None + else getattr(outcome, "error", None) + ), + "failure_type": ( + type(outcome).__name__ if isinstance(outcome, BaseException) + else "MissingResult" if outcome is None + else None if passed + else "TaskFailed" + ), + }) + + receipt = { + "schema_version": "cua-bench-run-result/v1", + "run_id": run_id, + "dataset_name": dataset_path.name, + "dataset_manifest_sha256": verified_manifest_digest, + "total_tasks": len(task_outcomes), + "success_count": sum(item["success"] for item in task_outcomes), + "failed_count": sum(not item["success"] for item in task_outcomes), + "tasks": task_outcomes, + } + # Index the artifacts actually emitted by each agent container. Hashes + # let external consumers verify bytes without trusting printed logs. + # An artifact hash is NOT proof that an action was executed or an + # oracle passed; those require separately recorded execution events. + import hashlib + + for row in task_outcomes: + task_dir = output_dir / f"{row['task']}_v{row['variant_id']}" + artifacts = [] + if task_dir.is_dir(): + for artifact in sorted(task_dir.rglob("*")): + if not artifact.is_file() or artifact.is_symlink(): + continue + relative_path = artifact.relative_to(task_dir).as_posix() + digest = hashlib.sha256() + with artifact.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + artifacts.append({ + "path": relative_path, + "size_bytes": artifact.stat().st_size, + "sha256": digest.hexdigest(), + }) + row["artifacts"] = artifacts + action_files = [item for item in artifacts if item["path"].endswith("/executed-actions.jsonl")] + row["action_evidence_files"] = [item["path"] for item in action_files] + row["evidence_status"] = ( + "action_records_present" if action_files + else "artifacts_only" if artifacts + else "no_artifacts" + ) + + identity_path = output_dir / "dataset-identity.json" + if identity_path.is_file(): + receipt["dataset_identity"] = json.loads(identity_path.read_text(encoding="utf-8")) + + receipt_path = output_dir / "run-result.json" + pending = output_dir / "run-result.json.tmp" + pending.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8") + os.replace(pending, receipt_path) + log_print(f"Machine-readable run result: {receipt_path}") + # Summarize results - success_count = sum(1 for r in results if r and hasattr(r, "success") and r.success) - failed_count = len(results) - success_count + success_count = receipt["success_count"] + failed_count = receipt["failed_count"] summary = f"\n{CYAN}{'=' * 60}{RESET}\n" summary += f"{CYAN}Dataset Complete: {dataset_path.name}{RESET}\n" @@ -1709,6 +1838,7 @@ async def run_with_semaphore(task_path, variant_id, task_num): log_print(summary) + # Summary model_name = getattr(args, "model", None) or "default" print("\n") @@ -1784,6 +1914,10 @@ async def run_with_semaphore(task_path, variant_id, task_num): cmd.extend(["--task-filter", task_filter]) if user_output_dir: cmd.extend(["--output-dir", str(user_output_dir)]) + if manifest_path: + # Re-verify in the actual background worker; the dataset can change + # after the parent preflight and before subprocess startup. + cmd.extend(["--dataset-manifest", str(Path(manifest_path).resolve())]) # Set UTF-8 encoding env_vars = os.environ.copy() @@ -1946,6 +2080,20 @@ def execute(args): return cmd_run_task(args) elif run_command == "dataset": return cmd_run_dataset(args) + elif run_command == "manifest": + import json + from cua_bench.dataset_manifest import scan_dataset + try: + manifest = scan_dataset(Path(args.dataset_path)) + output = Path(args.output) + if output.resolve().is_relative_to(Path(args.dataset_path).resolve()): + raise ValueError("Manifest output must be outside the dataset to avoid self-inclusion") + output.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + except (OSError, ValueError) as exc: + print(f"{RED}Error: Cannot create manifest: {exc}{RESET}") + return 1 + print(f"{GREEN}Recorded {len(manifest['tasks'])} tasks in {output}{RESET}") + return 0 elif run_command == "list": return cmd_list(args) elif run_command == "watch": diff --git a/libs/cua-bench/cua_bench/cli/main.py b/libs/cua-bench/cua_bench/cli/main.py index 426a72982c..2810e543ec 100644 --- a/libs/cua-bench/cua_bench/cli/main.py +++ b/libs/cua-bench/cua_bench/cli/main.py @@ -181,6 +181,11 @@ def add_common_run_args(parser): run_dataset_parser.add_argument( "dataset_path", help="Path to dataset directory, or dataset name from registry" ) + run_dataset_parser.add_argument( + "--dataset-manifest", + dest="dataset_manifest", + help="Pin dataset files to a JSON manifest; reject changes before launching sessions", + ) run_dataset_parser.add_argument( "--max-parallel", dest="max_parallel", @@ -208,6 +213,13 @@ def add_common_run_args(parser): ) add_common_run_args(run_dataset_parser) + # cb run manifest --output manifest.json + manifest_parser = run_subparsers.add_parser( + "manifest", help="Generate a pinned dataset manifest before launching a run" + ) + manifest_parser.add_argument("dataset_path", help="Local dataset directory") + manifest_parser.add_argument("--output", required=True, help="JSON manifest output path") + # cb run list run_list_parser = run_subparsers.add_parser("list", help="List all runs with status") run_list_parser.add_argument( diff --git a/libs/cua-bench/cua_bench/dataset_manifest.py b/libs/cua-bench/cua_bench/dataset_manifest.py new file mode 100644 index 0000000000..8a9b39f11c --- /dev/null +++ b/libs/cua-bench/cua_bench/dataset_manifest.py @@ -0,0 +1,59 @@ +"""Dataset identity and preflight verification for executable CUA Bench runs. + +The manifest is read without importing untrusted task Python code. +""" +import hashlib +from pathlib import Path + +VERSION = "cua-dataset-manifest/v1" +IGNORED = {"__pycache__", ".git", ".venv", "node_modules", ".pytest_cache"} + + +def scan_dataset(root: Path) -> dict: + root = Path(root).resolve(strict=True) + if not root.is_dir(): + raise ValueError("dataset must be a directory") + if (root / "main.py").is_file(): + tasks = [root] + else: + entries = sorted(root.iterdir()) + for entry in entries: + if entry.is_symlink() and entry.is_dir() and (entry / "main.py").is_file(): + raise ValueError(f"symlinked dataset task directory: {entry.name}") + tasks = [p for p in entries if p.is_dir() and (p / "main.py").is_file()] + if not tasks: + raise ValueError("no task directories containing main.py") + output = [] + for task in tasks: + files = [] + for path in sorted(task.rglob("*")): + relative = path.relative_to(root) + if any(part in IGNORED for part in relative.parts): + continue + if path.is_symlink(): + raise ValueError(f"symlink in dataset task: {relative}") + if not path.is_file(): + continue + digest = hashlib.sha256() + size = 0 + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + size += len(chunk) + files.append({"path": relative.as_posix(), "sha256": digest.hexdigest(), "size": size}) + if not files: + raise ValueError(f"empty task: {task}") + output.append({"task": task.relative_to(root).as_posix(), "files": files}) + return {"schema_version": VERSION, "tasks": output} + + +def verify_dataset(root: Path, manifest: dict) -> int: + if not isinstance(manifest, dict) or manifest.get("schema_version") != VERSION: + raise ValueError("unsupported dataset manifest") + expected = manifest.get("tasks") + if not isinstance(expected, list) or not expected: + raise ValueError("manifest has no tasks") + actual = scan_dataset(root) + if actual != manifest: + raise ValueError("dataset differs from manifest (added, removed, or modified task files)") + return len(actual["tasks"]) diff --git a/libs/cua-bench/cua_bench/evidence_contract.py b/libs/cua-bench/cua_bench/evidence_contract.py new file mode 100644 index 0000000000..0b96cfbb94 --- /dev/null +++ b/libs/cua-bench/cua_bench/evidence_contract.py @@ -0,0 +1,52 @@ +"""Shared, dependency-free receipt contract for live and offline cua-bench verification. + +This validates internally consistent claims, NOT the authenticity of the source. +""" +import math +import re + + +def valid_observation(value): + return ( + isinstance(value, dict) + and isinstance(value.get("sha256"), str) + and re.fullmatch(r"[0-9a-f]{64}", value["sha256"]) is not None + and type(value.get("size_bytes")) is int + and value["size_bytes"] >= 0 + ) + + +def oracle_receipt_valid(receipt, task_index, execution_id): + if not isinstance(receipt, dict): + return False + rewards = receipt.get("rewards") + threshold = receipt.get("success_threshold") + if not ( + receipt.get("schema_version") == "cua-bench-oracle-result/v1" + and receipt.get("task_index") == task_index + and receipt.get("execution_id") == execution_id + and receipt.get("evaluated") is True + and type(receipt.get("success")) is bool + and isinstance(rewards, list) + and bool(rewards) + and all(type(v) in (int, float) and math.isfinite(v) for v in rewards) + and type(threshold) in (int, float) + and math.isfinite(threshold) + and 0 <= threshold <= 1 + ): + return False + return receipt["success"] == all(v >= threshold for v in rewards) + + +def action_event_valid(record, sequence, execution_id): + return ( + isinstance(record, dict) + and record.get("schema_version") == "cua-bench-executed-action/v1" + and record.get("execution_id") == execution_id + and type(record.get("sequence")) is int + and record["sequence"] == sequence + and record.get("execution_status") in ("returned", "failed") + and isinstance(record.get("action"), dict) + and valid_observation(record.get("before")) + and valid_observation(record.get("after")) + ) diff --git a/libs/cua-bench/cua_bench/runner/task_runner.py b/libs/cua-bench/cua_bench/runner/task_runner.py index 6ca704343e..50fc39d44d 100644 --- a/libs/cua-bench/cua_bench/runner/task_runner.py +++ b/libs/cua-bench/cua_bench/runner/task_runner.py @@ -5,12 +5,15 @@ """ import asyncio +import json +import math import os import shutil from dataclasses import dataclass from pathlib import Path from typing import Dict, List, Optional +from ..evidence_contract import action_event_valid, oracle_receipt_valid from ..sessions.providers.local_environment import check_image_exists, pull_image from .docker_utils import ( ContainerInfo, @@ -104,6 +107,8 @@ class TaskResult: env_logs: str output_dir: Optional[str] = None error: Optional[str] = None + oracle_result: Optional[dict] = None + verified_action_count: Optional[int] = None # ============================================================================= @@ -268,27 +273,36 @@ async def run_task( env_container_name = f"cua-env-{task_id}" agent_container_name = f"cua-agent-{task_id}" - # Create task overlay for QEMU types (protects golden image) - overlay_path: Optional[Path] = None - if not is_simulated and config.get("use_overlays"): - overlay_path = self._create_task_overlay(task_id) - - # Track task for cleanup - self._running_tasks[task_id] = { - "network": network_name, - "env_container": env_container_name if not is_simulated else None, - "agent_container": agent_container_name, - "env_image": config.get("image") if not is_simulated else None, - "agent_image": self.agent_image, - "remove_images": remove_images_after, - "overlay_path": overlay_path, - "is_simulated": is_simulated, - } - - # Track log streaming process for cleanup + # Include overlay creation and receipt initialization in lifecycle cleanup. log_stream_process = None - + completed_result: Optional[TaskResult] = None + overlay_path: Optional[Path] = None try: + # Register resources before creation, including partial overlay failures. + self._running_tasks[task_id] = { + "network": network_name, + "env_container": env_container_name if not is_simulated else None, + "agent_container": agent_container_name, + "env_image": config.get("image") if not is_simulated else None, + "agent_image": self.agent_image, + "remove_images": remove_images_after, + "overlay_path": overlay_path, + "is_simulated": is_simulated, + } + + # Register the overlay before creation to clean up partial failures. + if not is_simulated and config.get("use_overlays"): + self._running_tasks[task_id]["overlay_path"] = get_overlays_path() / task_id + overlay_path = self._create_task_overlay(task_id) + + # Do not let artifacts from an earlier run masquerade as this execution. + # Leave unrelated user files alone; only clear known generated receipts. + if output_dir: + task_output = Path(output_dir) + task_output.mkdir(parents=True, exist_ok=True) + for receipt_name in ("oracle-result.json", "cleanup-result.json"): + (task_output / receipt_name).unlink(missing_ok=True) + # 1. Create network await create_network(network_name) @@ -343,7 +357,7 @@ async def run_task( timeout=timeout, ) except asyncio.TimeoutError: - return TaskResult( + completed_result = TaskResult( success=False, exit_code=-1, agent_logs=await get_container_logs(agent_container_name, tail=100), @@ -351,18 +365,72 @@ async def run_task( output_dir=output_dir, error="Task timed out", ) + return completed_result # 5. Collect logs agent_logs = await get_container_logs(agent_container_name, tail=500) env_logs = await get_container_logs(env_container_name, tail=100) - return TaskResult( - success=exit_code == 0, + oracle_result = None + oracle_error = None + oracle_path = Path(output_dir) / "oracle-result.json" if output_dir else None + if oracle_path and oracle_path.is_file(): + try: + oracle_result = json.loads(oracle_path.read_text(encoding="utf-8")) + if not oracle_receipt_valid(oracle_result, task_index, task_id): + oracle_error = "Invalid or inconsistent oracle result receipt" + except (OSError, ValueError, TypeError): + oracle_error = "Unreadable oracle result receipt" + else: + oracle_error = "Missing oracle result receipt" + + oracle_passed = ( + oracle_error is None + and oracle_result is not None + and oracle_result["success"] is True + ) + # Validate records emitted at the live desktop action boundary. + # In oracle-solver mode, actions may legitimately bypass an agent; + # never fabricate agent evidence for that path. + evidence_error = None + verified_action_count = None + if not oracle and (agent or agent_import_path): + evidence_dir = Path(output_dir) / f"task_{task_index}_agent_logs" if output_dir else None + evidence_file = evidence_dir / "executed-actions.jsonl" if evidence_dir else None + if evidence_file is None or not evidence_file.is_file(): + evidence_error = "Missing agent action evidence" + else: + try: + sequence = 0 + with evidence_file.open(encoding="utf-8") as records: + for raw in records: + record = json.loads(raw) + sequence += 1 + if not action_event_valid(record, sequence, task_id): + raise ValueError("Invalid action evidence record") + if record["execution_status"] == "failed": + raise ValueError("Failed agent action cannot establish a successful run") + if sequence == 0: + evidence_error = "Empty agent action evidence" + else: + verified_action_count = sequence + except (OSError, ValueError, TypeError, AttributeError): + evidence_error = "Invalid agent action evidence" + + completed_result = TaskResult( + success=exit_code == 0 and oracle_passed and evidence_error is None, exit_code=exit_code, agent_logs=agent_logs, env_logs=env_logs, output_dir=output_dir, + error=evidence_error or oracle_error or ( + "Oracle rejected task" if oracle_result and not oracle_passed + else None + ), + oracle_result=oracle_result, + verified_action_count=verified_action_count, ) + return completed_result except Exception as e: # Collect logs on error @@ -377,7 +445,7 @@ async def run_task( except Exception: pass - return TaskResult( + completed_result = TaskResult( success=False, exit_code=-1, agent_logs=agent_logs, @@ -385,6 +453,7 @@ async def run_task( output_dir=output_dir, error=str(e), ) + return completed_result finally: # 6. Stop log streaming if active @@ -395,8 +464,42 @@ async def run_task( except Exception: pass - # 7. Cleanup - await self._cleanup_task(task_id) + # 7. Cleanup and verify actual Docker resource absence. + try: + await self._cleanup_task(task_id) + except Exception as exc: + # A cleanup error must not replace the task result before the + # independent resource inspection writes its receipt. + if completed_result is not None: + completed_result.success = False + detail = f"Task cleanup failed: {type(exc).__name__}" + completed_result.error = ( + f"{completed_result.error}; {detail}" + if completed_result.error else detail + ) + else: + raise + finally: + if output_dir: + try: + await self._write_cleanup_receipt(task_id, output_dir) + cleanup = json.loads( + (Path(output_dir) / "cleanup-result.json").read_text(encoding="utf-8") + ) + if completed_result is not None and cleanup.get("verified") is not True: + completed_result.success = False + completed_result.error = ( + f"{completed_result.error}; Task Docker resources not fully cleaned up" + if completed_result.error else "Task Docker resources not fully cleaned up" + ) + except Exception as exc: + if completed_result is not None: + completed_result.success = False + detail = f"Unable to verify Docker cleanup: {type(exc).__name__}" + completed_result.error = ( + f"{completed_result.error}; {detail}" + if completed_result.error else detail + ) async def run_task_interactively( self, @@ -839,6 +942,7 @@ async def _start_agent_container( "CUA_PROVIDER": "simulated" if is_simulated else "remote", "CUA_TASK_PATH": "/app/env", "CUA_TASK_INDEX": str(task_index), + "CUA_EXECUTION_ID": task_id, "BATCH_TASK_INDEX": str(task_index), # Legacy compat "BATCH_TASK_COUNT": "1", } @@ -955,6 +1059,59 @@ async def _start_agent_container( remove_on_exit=False, # Don't auto-remove, we need logs ) + async def _resource_absent(self, kind: str, name: str) -> bool: + """Ask Docker whether a task resource still exists after cleanup.""" + process = await asyncio.create_subprocess_exec( + "docker", kind, "inspect", name, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await process.communicate() + # Docker inspect uses 1 for a missing object, but also for daemon + # failures. A nonzero exit must not be mistaken for verified cleanup. + if process.returncode == 0: + return False + message = stderr.decode("utf-8", errors="replace").lower() + missing = ("no such object", "no such container", "no such network", "not found") + if process.returncode == 1 and any(token in message for token in missing): + return True + raise RuntimeError(f"Docker {kind} inspection failed: {message[:300]}") + + async def _write_cleanup_receipt(self, task_id: str, output_dir: str) -> None: + """Persist resource inspection results even when cleanup failed.""" + path = Path(output_dir) + path.mkdir(parents=True, exist_ok=True) + resources = { + "agent_container": ("container", f"cua-agent-{task_id}"), + "env_container": ("container", f"cua-env-{task_id}"), + "network": ("network", f"cua-task-{task_id}"), + } + observations = {} + for label, (kind, name) in resources.items(): + try: + absent = await self._resource_absent(kind, name) + observations[label] = {"name": name, "absent": absent} + except Exception as exc: + observations[label] = { + "name": name, "absent": False, "error": type(exc).__name__ + } + # Check the task's isolated QEMU overlay as well as Docker objects. + # Never inspect or delete the shared golden disk here. + overlay = get_overlays_path() / task_id + observations["overlay"] = { + "name": str(overlay), + "absent": not overlay.exists(), + } + receipt = { + "schema_version": "cua-bench-cleanup/v1", + "task_id": task_id, + "resources": observations, + "verified": all(item["absent"] for item in observations.values()), + } + temporary = path / "cleanup-result.json.tmp" + temporary.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8") + temporary.replace(path / "cleanup-result.json") + async def _cleanup_task(self, task_id: str) -> None: """Clean up task resources (containers, network, overlay, optionally images). diff --git a/libs/cua-bench/cua_bench/runners.py b/libs/cua-bench/cua_bench/runners.py index 9e888b0e65..cd4178e5f8 100644 --- a/libs/cua-bench/cua_bench/runners.py +++ b/libs/cua-bench/cua_bench/runners.py @@ -6,6 +6,9 @@ import asyncio import fnmatch +import hashlib +import json +import math import time import uuid from dataclasses import dataclass @@ -13,6 +16,7 @@ from typing import Any, Callable, Dict, List, Optional, Tuple from .core import Task, make +from .dataset_manifest import verify_dataset from .environment import Environment from .types import Action, DoneAction @@ -40,6 +44,7 @@ class BenchmarkResult: avg_reward: float duration_seconds: float output_dir: Optional[str] = None + dataset_manifest_sha256: Optional[str] = None @dataclass @@ -61,6 +66,8 @@ class TaskResult: reward: float steps: int error: Optional[str] = None + action_trace_digest: Optional[str] = None + action_trace_events: Optional[List[Dict[str, Any]]] = None async def run_single_task( @@ -100,6 +107,19 @@ def my_agent(screenshot: bytes, task: Task) -> Action: result = await run_single_task(Path("./task"), agent_fn=my_agent) """ env = None + step_count = 0 + action_trace = [] + outcome = None + + def finish(**kwargs): + nonlocal outcome + outcome = TaskResult(**kwargs) + if action_trace: + serialized = json.dumps(action_trace, sort_keys=True, separators=(",", ":")) + outcome.action_trace_digest = hashlib.sha256(serialized.encode("utf-8")).hexdigest() + outcome.action_trace_events = list(action_trace) + return outcome + try: # Create environment using gym interface env = make(str(env_path), split=split) @@ -117,7 +137,7 @@ def my_agent(screenshot: bytes, task: Task) -> Action: await env.solve() step_count = env.step_count else: - return TaskResult( + return finish( task_path=str(env_path), variant_id=task_index, success=False, @@ -132,19 +152,42 @@ def my_agent(screenshot: bytes, task: Task) -> Action: action = agent_fn(screenshot, task_cfg) screenshot = await env.step(action) step_count += 1 + # Privacy-minimal event chain: never persist screenshots, action + # arguments, typed text, passwords or arbitrary agent payloads. + action_trace.append({"step": step_count, "action_type": type(action).__name__}) done = isinstance(action, DoneAction) + # A setup-only run must never be counted as successful agent execution. + if not oracle and agent_fn is None: + return finish( + task_path=str(env_path), + variant_id=task_index, + success=False, + reward=0.0, + steps=step_count, + error="No agent_fn supplied; task was not executed", + ) + # Evaluate if env.evaluate_task_fn is not None: result = await env.evaluate() - if isinstance(result, (int, float)): - reward = float(result) + if isinstance(result, (int, float)) and not isinstance(result, bool): + raw_reward = result elif isinstance(result, list) and len(result) > 0: - reward = float(result[0]) + raw_reward = result[0] elif isinstance(result, dict) and "reward" in result: - reward = float(result["reward"]) + raw_reward = result["reward"] + else: + raise ValueError("Evaluator returned an unsupported reward format") + if isinstance(raw_reward, bool) or not isinstance(raw_reward, (int, float)): + raise ValueError("Evaluator reward must be a numeric value, not a boolean or string") + reward = float(raw_reward) + if not math.isfinite(reward) or not 0.0 <= reward <= 1.0: + raise ValueError(f"Evaluator reward must be finite and within [0, 1], got {reward!r}") + else: + raise ValueError("Task has no evaluator; cannot produce a verified score") - return TaskResult( + return finish( task_path=str(env_path), variant_id=task_index, success=reward >= 0.5, # Common threshold @@ -153,20 +196,26 @@ def my_agent(screenshot: bytes, task: Task) -> Action: ) except Exception as e: - return TaskResult( + return finish( task_path=str(env_path), variant_id=task_index, success=False, reward=0.0, - steps=0, + steps=step_count, error=str(e), ) finally: if env is not None: try: await env.close() - except Exception: - pass + except Exception as close_error: + # A task cannot be reported as successful if its environment was + # not cleanly released; preserve its completed execution steps. + if outcome is not None: + outcome.success = False + outcome.reward = 0.0 + detail = f"Environment cleanup failed: {close_error}" + outcome.error = f"{outcome.error}; {detail}" if outcome.error else detail async def run_benchmark( @@ -179,6 +228,7 @@ async def run_benchmark( max_variants: Optional[int] = None, task_filter: Optional[str] = None, split: str = "train", + dataset_manifest: Optional[Path] = None, ) -> BenchmarkResult: """Run a benchmark on a dataset using the gym interface. @@ -195,6 +245,7 @@ async def run_benchmark( max_variants: Maximum variants per task (optional) task_filter: Glob pattern to filter tasks (optional) split: Dataset split (default: "train") + dataset_manifest: Optional manifest JSON to verify before importing or running tasks. Returns: BenchmarkResult with run statistics and task results @@ -222,12 +273,33 @@ def random_agent(screenshot: bytes, task: Task) -> Action: max_parallel=4, ) """ + if not isinstance(max_parallel, int) or isinstance(max_parallel, bool) or max_parallel < 1: + raise ValueError("max_parallel must be a positive integer") + if not isinstance(max_steps, int) or isinstance(max_steps, bool) or max_steps < 1: + raise ValueError("max_steps must be a positive integer") + if max_variants is not None and ( + not isinstance(max_variants, int) + or isinstance(max_variants, bool) + or max_variants < 1 + ): + raise ValueError("max_variants must be a positive integer when provided") + start_time = time.time() # Validate dataset path if not dataset_path.exists(): raise FileNotFoundError(f"Dataset not found: {dataset_path}") + # Verify pinned dataset bytes before importing task definitions or launching workers. + manifest_sha256 = None + if dataset_manifest is not None: + manifest_bytes = Path(dataset_manifest).read_bytes() + manifest = json.loads(manifest_bytes) + verify_dataset(dataset_path, manifest) + manifest_sha256 = hashlib.sha256( + json.dumps(manifest, sort_keys=True, separators=(",", ":")).encode("utf-8") + ).hexdigest() + # Discover tasks in dataset tasks: List[Path] = [] if (dataset_path / "main.py").exists(): @@ -255,8 +327,16 @@ def random_agent(screenshot: bytes, task: Task) -> Action: variant_count = len(env.tasks_config_fn()) else: variant_count = 1 - except Exception: - variant_count = 1 + except Exception as exc: + raise ValueError( + f"Failed to load task variants for {task_path} (split={split!r}): {exc}" + ) from exc + + if variant_count < 1: + raise ValueError( + f"Task {task_path} (split={split!r}) has no variants; " + "refusing to report an incomplete benchmark" + ) if max_variants: variant_count = min(variant_count, max_variants) @@ -291,12 +371,12 @@ async def run_with_semaphore(task_path: Path, variant_id: int) -> TaskResult: task_results: List[Dict[str, Any]] = [] rewards: List[float] = [] - for result in results: - if isinstance(result, Exception): + for (task_path, variant_id), result in zip(task_variants, results): + if isinstance(result, BaseException): task_results.append( { - "task_path": "unknown", - "variant_id": -1, + "task_path": str(task_path), + "variant_id": variant_id, "success": False, "reward": 0.0, "steps": 0, @@ -305,14 +385,74 @@ async def run_with_semaphore(task_path: Path, variant_id: int) -> TaskResult: ) rewards.append(0.0) else: + # Validate worker provenance against the scheduled task before scoring. + invalid = ( + not isinstance(result, TaskResult) + or result.task_path != str(task_path) + or result.variant_id != variant_id + or type(result.success) is not bool + or not isinstance(result.reward, (int, float)) + or isinstance(result.reward, bool) + or not math.isfinite(result.reward) + or not 0.0 <= result.reward <= 1.0 + or not isinstance(result.steps, int) + or isinstance(result.steps, bool) + or result.steps < 0 + or ( + result.action_trace_digest is not None + and ( + not isinstance(result.action_trace_digest, str) + or len(result.action_trace_digest) != 64 + or any(c not in "0123456789abcdef" for c in result.action_trace_digest) + or result.steps == 0 + ) + ) + or ( + result.action_trace_digest is not None + and ( + not isinstance(result.action_trace_events, list) + or len(result.action_trace_events) != result.steps + or any( + not isinstance(event, dict) + or set(event) != {"step", "action_type"} + or type(event["step"]) is not int + or event["step"] != index + or not isinstance(event["action_type"], str) + or not event["action_type"].isidentifier() + for index, event in enumerate(result.action_trace_events, start=1) + ) + or hashlib.sha256( + json.dumps(result.action_trace_events, sort_keys=True, separators=(",", ":")).encode("utf-8") + ).hexdigest() != result.action_trace_digest + ) + ) + or (result.action_trace_digest is None and result.action_trace_events is not None) + or (bool(result.error) and (result.success or result.reward != 0.0)) + or (not result.error and result.success != (result.reward >= 0.5)) + ) + if invalid: + task_results.append( + { + "task_path": str(task_path), + "variant_id": variant_id, + "success": False, + "reward": 0.0, + "steps": 0, + "error": "Invalid or misattributed worker result", + } + ) + rewards.append(0.0) + continue task_results.append( { - "task_path": result.task_path, - "variant_id": result.variant_id, + "task_path": str(task_path), + "variant_id": variant_id, "success": result.success, "reward": result.reward, "steps": result.steps, "error": result.error, + "action_trace_digest": result.action_trace_digest, + "action_trace_events": result.action_trace_events, } ) rewards.append(result.reward) @@ -331,6 +471,7 @@ async def run_with_semaphore(task_path: Path, variant_id: int) -> TaskResult: failed_count=failed_count, avg_reward=avg_reward, duration_seconds=duration_seconds, + dataset_manifest_sha256=manifest_sha256, ) diff --git a/libs/cua-bench/cua_bench/tests/test_ci_receipt.py b/libs/cua-bench/cua_bench/tests/test_ci_receipt.py new file mode 100644 index 0000000000..e1408a71bc --- /dev/null +++ b/libs/cua-bench/cua_bench/tests/test_ci_receipt.py @@ -0,0 +1,115 @@ +"""Falsification tests for checksum-linked CUA CI receipts.""" +import importlib.util +import json +from pathlib import Path +import sys + +import pytest + +scripts = Path(__file__).resolve().parents[2] / "scripts" +sys.path.insert(0, str(scripts)) +spec = importlib.util.spec_from_file_location("ci_receipt", scripts / "ci_receipt.py") +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +def report(nodeid): + return { + "exitcode": 0, + "summary": {"collected": 1, "total": 1, "passed": 1}, + "tests": [{"nodeid": nodeid, "outcome": "passed"}], + "collectors": [], + } + + +def setup_reports(tmp_path): + a, b = tmp_path / "core-report.json", tmp_path / "full-report.json" + a.write_text(json.dumps(report("core::one"))) + b.write_text(json.dumps(report("full::one"))) + return a, b + + +def test_valid_receipt(tmp_path): + a, b = setup_reports(tmp_path) + receipt = module.build_receipt(a, b, "a" * 40) + assert module.verify_receipt(receipt, tmp_path) is True + assert receipt["reports"][0]["sha256"] != receipt["reports"][1]["sha256"] + + +def test_modified_report_is_rejected(tmp_path): + a, b = setup_reports(tmp_path) + receipt = module.build_receipt(a, b, "a" * 40) + b.write_text(json.dumps(report("full::changed"))) + with pytest.raises(ValueError, match="checksum mismatch"): + module.verify_receipt(receipt, tmp_path) + + +def test_invalid_commit_is_rejected(tmp_path): + a, b = setup_reports(tmp_path) + from jsonschema import ValidationError + with pytest.raises(ValidationError): + module.build_receipt(a, b, "not-a-real-sha") + + +def test_missing_full_is_rejected(tmp_path): + a, b = setup_reports(tmp_path) + receipt = module.build_receipt(a, b, "a" * 40) + receipt["reports"][1]["kind"] = "core" + with pytest.raises(ValueError, match="exactly one"): + module.verify_receipt(receipt, tmp_path) + + +def test_path_traversal_is_rejected(tmp_path): + a, b = setup_reports(tmp_path) + receipt = module.build_receipt(a, b, "a" * 40) + receipt["reports"][1]["path"] = "../outside.json" + with pytest.raises(ValueError, match="basenames"): + module.verify_receipt(receipt, tmp_path) + + +def test_failed_underlying_test_is_rejected(tmp_path): + a, b = setup_reports(tmp_path) + bad = report("full::bad") + bad["tests"][0]["outcome"] = "failed" + b.write_text(json.dumps(bad)) + with pytest.raises(ValueError, match="not all test outcomes passed"): + module.build_receipt(a, b, "a" * 40) + +def test_external_consumer_verifies_downloaded_receipt(tmp_path): + import subprocess + import sys + a, b = setup_reports(tmp_path) + receipt = module.build_receipt(a, b, "b" * 40) + path = tmp_path / "ci-receipt-v1.json" + path.write_text(json.dumps(receipt)) + result = subprocess.run( + [ + sys.executable, str(scripts / "ci_receipt.py"), + "--verify", str(path), "--evidence-dir", str(tmp_path), + ], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr + assert "independently verified" in result.stdout + +def test_duplicate_report_path_rejected(tmp_path): + a, b = setup_reports(tmp_path) + receipt = module.build_receipt(a, b, "a" * 40) + receipt["reports"][1]["path"] = receipt["reports"][0]["path"] + with pytest.raises(ValueError, match="distinct file paths"): + module.verify_receipt(receipt, tmp_path) + + +def test_symlink_escape_rejected(tmp_path): + root = tmp_path / "artifacts" + root.mkdir() + a, b = setup_reports(root) + receipt = module.build_receipt(a, b, "a" * 40) + outside = tmp_path / "outside.json" + outside.write_bytes(b.read_bytes()) + b.unlink() + b.symlink_to(outside) + with pytest.raises(ValueError, match="symlink"): + module.verify_receipt(receipt, root) diff --git a/libs/cua-bench/cua_bench/tests/test_cleanup_docker_integration.py b/libs/cua-bench/cua_bench/tests/test_cleanup_docker_integration.py new file mode 100644 index 0000000000..45ebdb0abd --- /dev/null +++ b/libs/cua-bench/cua_bench/tests/test_cleanup_docker_integration.py @@ -0,0 +1,70 @@ +"""Docker-backed integration: real daemon network -> production receipt -> verifier. + +Requires a working Docker daemon. In CI, absence of Docker is a hard failure. +""" +import asyncio +import json +import os +import shutil +import uuid + +import pytest + +from cua_bench.runner import task_runner as module +from cua_bench.verify_execution import verify_execution + + +async def docker(*args): + proc = await asyncio.create_subprocess_exec( + "docker", *args, stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await proc.communicate() + if proc.returncode: + raise AssertionError(f"docker {args[0]} failed: {stderr.decode(errors='replace')}") + return stdout.decode().strip() + + +@pytest.mark.asyncio +async def test_real_docker_network_cleanup_receipt_and_offline_verification(tmp_path, monkeypatch): + available = shutil.which("docker") is not None + if not available: + if os.environ.get("CI"): + pytest.fail("CI must have a real Docker CLI and daemon for this acceptance test") + pytest.skip("Local Docker not installed") + await docker("info", "--format", "{{.ServerVersion}}") + + task_id = "integration-" + uuid.uuid4().hex[:12] + network = "cua-task-" + task_id + runner = module.TaskRunner() + monkeypatch.setattr(module, "get_overlays_path", lambda: tmp_path / "overlays") + await docker("network", "create", network) + try: + await runner._write_cleanup_receipt(task_id, str(tmp_path)) + first = json.loads((tmp_path / "cleanup-result.json").read_text()) + assert first["resources"]["network"]["absent"] is False + assert first["verified"] is False + assert verify_execution(tmp_path, 0, task_id)["checks"]["cleanup"] is False + + await docker("network", "rm", network) + await runner._write_cleanup_receipt(task_id, str(tmp_path)) + second = json.loads((tmp_path / "cleanup-result.json").read_text()) + assert second["resources"]["network"]["absent"] is True + assert second["verified"] is True + assert verify_execution(tmp_path, 0, task_id)["checks"]["cleanup"] is True + # A genuine daemon observation must not validate another run identity. + second["resources"]["network"]["name"] = "cua-task-different-execution" + (tmp_path / "cleanup-result.json").write_text( + json.dumps(second) + "\n", encoding="utf-8" + ) + forged = verify_execution(tmp_path, 0, task_id) + assert forged["checks"]["cleanup"] is False + assert forged["run_accepted"] is False + finally: + # Never leave test-created Docker resources on the runner. + proc = await asyncio.create_subprocess_exec( + "docker", "network", "rm", network, + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.DEVNULL, + ) + await proc.wait() diff --git a/libs/cua-bench/cua_bench/tests/test_cleanup_receipt_integration.py b/libs/cua-bench/cua_bench/tests/test_cleanup_receipt_integration.py new file mode 100644 index 0000000000..8022d5eb67 --- /dev/null +++ b/libs/cua-bench/cua_bench/tests/test_cleanup_receipt_integration.py @@ -0,0 +1,59 @@ +"""Filesystem integration of production cleanup receipts and offline verifier. + +Resource probes are stubbed; these tests do NOT constitute Docker/QEMU E2E. +""" +import json + +import pytest + +from cua_bench.runner import task_runner as module +from cua_bench.verify_execution import verify_execution + + +def _write_json(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value) + "\n", encoding="utf-8") + + +def _other_receipts(root, task_id): + _write_json(root / "oracle-result.json", { + "schema_version": "cua-bench-oracle-result/v1", + "execution_id": task_id, "task_index": 0, "evaluated": True, + "success": True, "rewards": [1.0], "success_threshold": 1.0, + }) + _write_json(root / "task_0_agent_logs" / "executed-actions.jsonl", { + "schema_version": "cua-bench-executed-action/v1", + "execution_id": task_id, "sequence": 1, + "execution_status": "returned", "action": {"type": "click"}, + "before": {"sha256": "a" * 64, "size_bytes": 1}, + "after": {"sha256": "b" * 64, "size_bytes": 1}, + }) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("residual", [False, True]) +async def test_production_cleanup_receipt_consumed_by_offline_verifier( + tmp_path, monkeypatch, residual, +): + task_id = "integration-1" + runner = module.TaskRunner() + monkeypatch.setattr(module, "get_overlays_path", lambda: tmp_path / "overlays") + + async def inspect(kind, name): + return not (residual and name == f"cua-agent-{task_id}") + + monkeypatch.setattr(runner, "_resource_absent", inspect) + _other_receipts(tmp_path, task_id) + await runner._write_cleanup_receipt(task_id, str(tmp_path)) + + receipt = json.loads((tmp_path / "cleanup-result.json").read_text()) + assert set(receipt["resources"]) == { + "agent_container", "env_container", "network", "overlay", + } + assert receipt["verified"] is not residual + + report = verify_execution(tmp_path, 0, task_id) + assert report["checks"]["oracle"] is True + assert report["checks"]["actions"] is True + assert report["checks"]["cleanup"] is not residual + assert report["run_accepted"] is not residual diff --git a/libs/cua-bench/cua_bench/tests/test_dataset_manifest.py b/libs/cua-bench/cua_bench/tests/test_dataset_manifest.py new file mode 100644 index 0000000000..9e7ba2df52 --- /dev/null +++ b/libs/cua-bench/cua_bench/tests/test_dataset_manifest.py @@ -0,0 +1,97 @@ +"""Dataset manifest contract tests with independent tampering examples.""" +import importlib.util +from pathlib import Path +import sys + +import pytest + +SCRIPT = Path(__file__).resolve().parents[2] / "scripts" / "dataset_manifest.py" +spec = importlib.util.spec_from_file_location("dataset_manifest", SCRIPT) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +def dataset(tmp_path): + root = tmp_path / "dataset" + for name in ("click", "type"): + task = root / name + task.mkdir(parents=True) + (task / "main.py").write_text("# task\n") + (task / "asset.txt").write_text("asset") + return root + + +def test_manifest_is_deterministic(tmp_path): + root = dataset(tmp_path) + first = module.scan_dataset(root) + assert first == module.scan_dataset(root) + assert module.verify_dataset(root, first) == 2 + assert [x["task"] for x in first["tasks"]] == ["click", "type"] + + +@pytest.mark.parametrize("mutation", ["change", "add", "remove"]) +def test_dataset_drift_fails_closed(tmp_path, mutation): + root = dataset(tmp_path) + saved = module.scan_dataset(root) + asset = root / "click" / "asset.txt" + if mutation == "change": + asset.write_text("tampered") + elif mutation == "add": + (root / "click" / "new.txt").write_text("extra") + else: + asset.unlink() + with pytest.raises(ValueError, match="dataset differs"): + module.verify_dataset(root, saved) + + +def test_new_task_fails_closed(tmp_path): + root = dataset(tmp_path) + saved = module.scan_dataset(root) + extra = root / "new" + extra.mkdir() + (extra / "main.py").write_text("# inserted task") + with pytest.raises(ValueError, match="dataset differs"): + module.verify_dataset(root, saved) + + +def test_symlink_escape_refused(tmp_path): + root = dataset(tmp_path) + external = tmp_path / "secret.txt" + external.write_text("secret") + (root / "click" / "secret-link.txt").symlink_to(external) + with pytest.raises(ValueError, match="symlink"): + module.scan_dataset(root) + + +def test_manifest_never_executes_task_source(tmp_path): + root = dataset(tmp_path) + marker = tmp_path / "executed" + (root / "click" / "main.py").write_text(f"open({str(marker)!r}, 'w').write('unsafe')") + module.scan_dataset(root) + assert not marker.exists() + + +def test_missing_tasks_refused(tmp_path): + empty = tmp_path / "empty" + empty.mkdir() + with pytest.raises(ValueError, match="no task"): + module.scan_dataset(empty) + +def test_hidden_task_config_must_be_fingerprinted(tmp_path): + root = dataset(tmp_path) + config = root / "click" / ".env" + config.write_text("SCORING_MODE=strict") + manifest = module.scan_dataset(root) + config.write_text("SCORING_MODE=permissive") + with pytest.raises(ValueError, match="dataset differs"): + module.verify_dataset(root, manifest) + + +def test_symlinked_top_level_task_is_not_silently_omitted(tmp_path): + root = dataset(tmp_path) + foreign = tmp_path / "external-task" + foreign.mkdir() + (foreign / "main.py").write_text("# imported via symlink") + (root / "external").symlink_to(foreign, target_is_directory=True) + with pytest.raises(ValueError, match="symlinked dataset task directory"): + module.scan_dataset(root) diff --git a/libs/cua-bench/cua_bench/tests/test_evidence_contract.py b/libs/cua-bench/cua_bench/tests/test_evidence_contract.py new file mode 100644 index 0000000000..72de99564b --- /dev/null +++ b/libs/cua-bench/cua_bench/tests/test_evidence_contract.py @@ -0,0 +1,49 @@ +"""Shared contract validation: independent of Docker and Agent SDK.""" +from cua_bench.evidence_contract import action_event_valid, oracle_receipt_valid + + +def oracle(): + return dict(schema_version="cua-bench-oracle-result/v1", task_index=0, + execution_id="task-1", evaluated=True, rewards=[1.0], + success_threshold=1.0, success=True) + + +def action(): + obs = {"sha256": "a" * 64, "size_bytes": 1} + return dict(schema_version="cua-bench-executed-action/v1", + execution_id="task-1", sequence=1, execution_status="returned", + action={"type": "click"}, before=obs, after=obs) + + +def test_valid_shared_oracle(): + assert oracle_receipt_valid(oracle(), 0, "task-1") + + +def test_reject_boolean_sequence(): + event = action() + event["sequence"] = True + assert not action_event_valid(event, 1, "task-1") + + +def test_reject_missing_observation_hash(): + event = action() + event["before"] = {"error_type": "TimeoutError"} + assert not action_event_valid(event, 1, "task-1") + + +def test_reject_forged_oracle_verdict(): + receipt = oracle() + receipt["success"] = False + assert not oracle_receipt_valid(receipt, 0, "task-1") + + +def test_finite_reward_above_one_is_valid_per_solver(): + receipt = oracle() + receipt["rewards"] = [2.0] + assert oracle_receipt_valid(receipt, 0, "task-1") + + +def test_failed_event_is_valid_evidence_but_not_success(): + event = action() + event["execution_status"] = "failed" + assert action_event_valid(event, 1, "task-1") diff --git a/libs/cua-bench/cua_bench/tests/test_pytest_receipt.py b/libs/cua-bench/cua_bench/tests/test_pytest_receipt.py new file mode 100644 index 0000000000..3889509c6e --- /dev/null +++ b/libs/cua-bench/cua_bench/tests/test_pytest_receipt.py @@ -0,0 +1,59 @@ +"""Independent negative tests for the pytest-json-report receipt verifier.""" +import importlib.util +from pathlib import Path +import unittest + +source = Path(__file__).resolve().parents[2] / "scripts" / "verify_pytest_receipt.py" +spec = importlib.util.spec_from_file_location("verify_pytest_receipt", source) +mod = importlib.util.module_from_spec(spec) +spec.loader.exec_module(mod) + + +class ReceiptTests(unittest.TestCase): + def valid(self): + return { + "exitcode": 0, + "summary": {"collected": 2, "total": 2, "passed": 2}, + "tests": [ + {"nodeid": "a.py::test_one", "outcome": "passed"}, + {"nodeid": "a.py::test_two", "outcome": "passed"}, + ], + "collectors": [], + } + + def test_valid(self): + self.assertEqual(mod.verify_report(self.valid())["passed"], 2) + + def test_no_tests(self): + doc = self.valid() + doc["tests"] = [] + with self.assertRaises(ValueError): + mod.verify_report(doc) + + def test_forged_duplicate_identity(self): + doc = self.valid() + doc["tests"][1]["nodeid"] = doc["tests"][0]["nodeid"] + with self.assertRaises(ValueError): + mod.verify_report(doc) + + def test_hidden_skips(self): + doc = self.valid() + doc["summary"]["skipped"] = 1 + with self.assertRaises(ValueError): + mod.verify_report(doc) + + def test_false_green_exit_code(self): + doc = self.valid() + doc["exitcode"] = 1 + with self.assertRaises(ValueError): + mod.verify_report(doc) + + def test_missing_case_with_false_summary(self): + doc = self.valid() + doc["tests"].pop() + with self.assertRaises(ValueError): + mod.verify_report(doc) + + +if __name__ == "__main__": + unittest.main() diff --git a/libs/cua-bench/cua_bench/tests/test_reward_hypothesis.py b/libs/cua-bench/cua_bench/tests/test_reward_hypothesis.py new file mode 100644 index 0000000000..408f7f860d --- /dev/null +++ b/libs/cua-bench/cua_bench/tests/test_reward_hypothesis.py @@ -0,0 +1,134 @@ +"""Property-based oracle result validation using the external Hypothesis library.""" +import asyncio +import math +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +from hypothesis import given, settings, strategies as st + +from cua_bench import DoneAction, run_single_task + + +@settings(max_examples=120, deadline=None) +@given(st.one_of( + st.floats(allow_nan=True, allow_infinity=True, width=64), + st.integers(min_value=-10**12, max_value=10**12), +)) +def test_generated_oracle_rewards_never_inflate_success(reward): + observed = {"closed": False, "steps": 0} + + class ControlledEnvironment: + evaluate_task_fn = object() + solve_task_fn = None + + async def reset(self, task_id=0): + return b"initial", SimpleNamespace(description="property test") + + async def step(self, action): + observed["steps"] += 1 + return b"next" + + async def evaluate(self): + return reward + + async def close(self): + observed["closed"] = True + + with patch("cua_bench.runners.make", return_value=ControlledEnvironment()): + result = asyncio.run(run_single_task( + Path("property-task"), agent_fn=lambda screenshot, task: DoneAction(), max_steps=2 + )) + + assert observed["closed"] is True + assert result.steps == observed["steps"] == 1 + valid = math.isfinite(reward) and 0 <= reward <= 1 + if valid: + assert result.error is None + assert result.reward == float(reward) + assert result.success is (reward >= 0.5) + else: + assert result.success is False + assert result.reward == 0.0 + assert result.error is not None + + +@settings(max_examples=64, deadline=None) +@given( + st.booleans(), + st.sampled_from([0.0, 0.25, 0.5, 0.9, 1.0]), +) +def test_cleanup_failure_cannot_produce_success(should_fail_close, reward): + closed = {"attempted": False} + + class CleanupEnvironment: + evaluate_task_fn = object() + solve_task_fn = None + + async def reset(self, task_id=0): + return b"initial", SimpleNamespace() + + async def step(self, action): + return b"next" + + async def evaluate(self): + return reward + + async def close(self): + closed["attempted"] = True + if should_fail_close: + raise RuntimeError("injected teardown failure") + + with patch("cua_bench.runners.make", return_value=CleanupEnvironment()): + result = asyncio.run(run_single_task( + Path("cleanup-task"), agent_fn=lambda screenshot, task: DoneAction(), max_steps=2 + )) + assert closed["attempted"] + assert result.steps == 1 + if should_fail_close: + assert result.success is False + assert result.reward == 0.0 + assert "Environment cleanup failed" in result.error + else: + assert result.error is None + assert result.reward == reward + assert result.success is (reward >= 0.5) + + +@settings(max_examples=80, deadline=None) +@given( + st.one_of(st.booleans(), st.text(max_size=16), st.none()), + st.sampled_from(["scalar", "list", "dict"]), +) +def test_evaluator_rejects_non_numeric_reward_payloads(value, shape): + observed = {"closed": False} + + class InvalidRewardEnvironment: + evaluate_task_fn = object() + solve_task_fn = None + + async def reset(self, task_id=0): + return b"initial", SimpleNamespace() + + async def step(self, action): + return b"next" + + async def evaluate(self): + if shape == "list": + return [value] + if shape == "dict": + return {"reward": value} + return value + + async def close(self): + observed["closed"] = True + + with patch("cua_bench.runners.make", return_value=InvalidRewardEnvironment()): + outcome = asyncio.run(run_single_task( + Path("invalid-evaluator"), agent_fn=lambda screenshot, task: DoneAction(), max_steps=2 + )) + assert observed["closed"] + assert outcome.steps == 1 + assert outcome.success is False + assert outcome.reward == 0.0 + assert outcome.error is not None diff --git a/libs/cua-bench/cua_bench/tests/test_run_benchmark.py b/libs/cua-bench/cua_bench/tests/test_run_benchmark.py index 5e9f45a8fa..de3b6d825a 100644 --- a/libs/cua-bench/cua_bench/tests/test_run_benchmark.py +++ b/libs/cua-bench/cua_bench/tests/test_run_benchmark.py @@ -220,6 +220,194 @@ async def test_run_single_task_failure(self): assert result.error is not None +class TestNoAgentExecutionEvidence: + @pytest.mark.asyncio + async def test_pre_satisfied_task_without_agent_cannot_pass(self, tmp_path, monkeypatch): + import cua_bench.runners as runners + + class PresatisfiedEnvironment: + solve_task_fn = None + evaluate_task_fn = object() + max_steps = 100 + + async def reset(self, task_id=0): + return b"screenshot", object() + + async def evaluate(self): + raise AssertionError("must not invoke evaluator for unexecuted agent") + + async def close(self): + return None + + monkeypatch.setattr(runners, "make", lambda path, split="train": PresatisfiedEnvironment()) + result = await runners.run_single_task(tmp_path, oracle=False, agent_fn=None) + assert result.success is False + assert result.reward == 0.0 + assert result.steps == 0 + assert "not executed" in result.error + + +class TestWorkerResultProvenance: + @pytest.mark.asyncio + @pytest.mark.parametrize("fault", ["wrong_path", "wrong_variant", "nan_reward", "contradictory_success"]) + async def test_forged_worker_result_cannot_inflate_score(self, tmp_path, monkeypatch, fault): + from types import SimpleNamespace + import cua_bench.runners as runners + + task_dir = tmp_path / "one" + task_dir.mkdir() + (task_dir / "main.py").write_text("# stub\n") + monkeypatch.setattr( + runners, "make", + lambda path, split="train": SimpleNamespace(tasks_config_fn=lambda: [object()]), + ) + + async def forged_result(path, task_index=0, **kwargs): + result = runners.TaskResult(str(path), task_index, True, 1.0, 1) + if fault == "wrong_path": + result.task_path = "/other/task" + elif fault == "wrong_variant": + result.variant_id = 123 + elif fault == "nan_reward": + result.reward = float("nan") + else: + result.success = False + return result + + monkeypatch.setattr(runners, "run_single_task", forged_result) + result = await runners.run_benchmark(tmp_path, max_parallel=1) + assert result.total_tasks == 1 + assert result.success_count == 0 + assert result.failed_count == 1 + assert result.avg_reward == 0.0 + assert result.task_results[0]["task_path"] == str(task_dir) + assert result.task_results[0]["variant_id"] == 0 + assert "Invalid or misattributed" in result.task_results[0]["error"] + + +class TestBenchmarkAggregationIntegrity: + @pytest.mark.asyncio + async def test_worker_crash_retains_task_identity_and_denominator(self, tmp_path, monkeypatch): + from types import SimpleNamespace + import cua_bench.runners as runners + + for name in ("first", "second"): + task_dir = tmp_path / name + task_dir.mkdir() + (task_dir / "main.py").write_text("# discovery placeholder\n") + + monkeypatch.setattr( + runners, "make", + lambda path, split="train": SimpleNamespace(tasks_config_fn=lambda: [object()]), + ) + + async def fake_run(path, task_index=0, **kwargs): + if path.name == "first": + raise RuntimeError("worker exploded") + return runners.TaskResult(str(path), task_index, True, 1.0, 1) + + monkeypatch.setattr(runners, "run_single_task", fake_run) + result = await runners.run_benchmark(tmp_path, max_parallel=2) + + assert result.total_tasks == 2 + assert result.success_count == 1 + assert result.failed_count == 1 + assert result.avg_reward == 0.5 + assert result.task_results[0]["task_path"] == str(tmp_path / "first") + assert result.task_results[0]["variant_id"] == 0 + assert "worker exploded" in result.task_results[0]["error"] + assert result.task_results[1]["task_path"] == str(tmp_path / "second") + + +class TestBenchmarkScoreIntegrity: + """Invalid evaluator output must never increase benchmark success.""" + + @pytest.mark.asyncio + @pytest.mark.parametrize("expression", ["float('nan')", "float('inf')", "-0.25", "1.25", "'not-a-score'"]) + async def test_reject_invalid_evaluator_result(self, tmp_path, expression): + task_dir = tmp_path / "invalid-reward" + task_dir.mkdir() + (task_dir / "main.py").write_text( + "import cua_bench as cb\n" + "@cb.tasks_config(split='train')\n" + "def get_tasks():\n" + " return [cb.Task(description='score test')]\n" + "@cb.evaluate_task(split='train')\n" + "async def evaluate(task, session):\n" + f" return {expression}\n" + ) + result = await run_single_task(task_dir) + assert result.success is False + assert result.reward == 0.0 + assert result.error is not None + + @pytest.mark.asyncio + async def test_missing_evaluator_never_claims_success(self, tmp_path): + task_dir = tmp_path / "missing-evaluator" + task_dir.mkdir() + (task_dir / "main.py").write_text( + "import cua_bench as cb\n" + "@cb.tasks_config(split='train')\n" + "def get_tasks():\n" + " return [cb.Task(description='no evaluator')]\n" + ) + result = await run_single_task(task_dir) + assert result.success is False + assert result.error is not None + + +class TestBenchmarkDatasetDiscoveryIntegrity: + """Invalid task definitions must not be silently scored as one variant.""" + + @pytest.mark.asyncio + async def test_broken_task_configuration_aborts_benchmark(self, tmp_path): + task_dir = tmp_path / "broken-task" + task_dir.mkdir() + (task_dir / "main.py").write_text( + "import cua_bench as cb\n" + "@cb.tasks_config(split='train')\n" + "def get_tasks():\n" + " raise RuntimeError('broken fixture')\n" + ) + with pytest.raises(ValueError, match="Failed to load task variants"): + await run_benchmark(tmp_path) + + @pytest.mark.asyncio + async def test_zero_variants_aborts_instead_of_zero_denominator(self, tmp_path): + task_dir = tmp_path / "empty-task" + task_dir.mkdir() + (task_dir / "main.py").write_text( + "import cua_bench as cb\n" + "@cb.tasks_config(split='train')\n" + "def get_tasks():\n" + " return []\n" + ) + with pytest.raises(ValueError, match="has no variants"): + await run_benchmark(tmp_path) + + +class TestBenchmarkParameterValidation: + """Parameter failures must surface before scheduling asynchronous workers.""" + + @pytest.mark.asyncio + @pytest.mark.parametrize("value", [0, -1, True, 1.5, "2"]) + async def test_reject_invalid_parallelism(self, tmp_path, value): + with pytest.raises(ValueError, match="max_parallel"): + await run_benchmark(tmp_path, max_parallel=value) + + @pytest.mark.asyncio + @pytest.mark.parametrize("value", [0, -1, False, 2.5]) + async def test_reject_invalid_step_limit(self, tmp_path, value): + with pytest.raises(ValueError, match="max_steps"): + await run_benchmark(tmp_path, max_steps=value) + + @pytest.mark.asyncio + @pytest.mark.parametrize("value", [0, -1, True, "3"]) + async def test_reject_invalid_variant_limit(self, tmp_path, value): + with pytest.raises(ValueError, match="max_variants"): + await run_benchmark(tmp_path, max_variants=value) + + class TestRunBenchmark: """Tests for run_benchmark function.""" @@ -403,3 +591,247 @@ async def setup(task, session): assert task_cfg.description == "Interactive task" finally: await env.close() + + +class TestFailedTaskStepEvidence: + @pytest.mark.asyncio + async def test_agent_failure_preserves_completed_steps(self, tmp_path, monkeypatch): + from types import SimpleNamespace + import cua_bench.runners as runners + + observed = {"steps": 0, "closed": False} + + class BrokenEnvironment: + solve_task_fn = None + evaluate_task_fn = object() + + async def reset(self, task_id=0): + return b"initial", SimpleNamespace() + + async def step(self, action): + observed["steps"] += 1 + if observed["steps"] == 3: + raise RuntimeError("injected action failure") + return b"next" + + async def close(self): + observed["closed"] = True + + monkeypatch.setattr(runners, "make", lambda path, split="train": BrokenEnvironment()) + result = await runners.run_single_task( + tmp_path, agent_fn=lambda screenshot, task: object(), max_steps=10 + ) + assert result.success is False + assert result.steps == 2 + assert result.reward == 0.0 + assert "injected action failure" in result.error + assert observed["closed"] is True + + @pytest.mark.asyncio + async def test_reset_failure_reports_zero_steps(self, tmp_path, monkeypatch): + import cua_bench.runners as runners + + class BrokenReset: + async def reset(self, task_id=0): + raise RuntimeError("reset failed") + + async def close(self): + return None + + monkeypatch.setattr(runners, "make", lambda path, split="train": BrokenReset()) + result = await runners.run_single_task(tmp_path, oracle=True) + assert result.steps == 0 + assert "reset failed" in result.error + + +class TestActionTraceDigest: + @pytest.mark.asyncio + async def test_trace_digest_is_stable_and_redacts_payloads(self, tmp_path, monkeypatch): + import hashlib + import json + from types import SimpleNamespace + import cua_bench.runners as runners + + class Env: + evaluate_task_fn = object() + solve_task_fn = None + async def reset(self, task_id=0): + return b"private screenshot", SimpleNamespace() + async def step(self, action): + return b"private screenshot after action" + async def evaluate(self): + return 1.0 + async def close(self): + pass + + monkeypatch.setattr(runners, "make", lambda path, split="train": Env()) + one = await runners.run_single_task(tmp_path, agent_fn=lambda *_: DoneAction()) + two = await runners.run_single_task(tmp_path, agent_fn=lambda *_: DoneAction()) + expected = hashlib.sha256(json.dumps( + [{"step": 1, "action_type": "DoneAction"}], + sort_keys=True, separators=(",", ":") + ).encode()).hexdigest() + assert one.action_trace_digest == two.action_trace_digest == expected + assert one.success is True + + @pytest.mark.asyncio + async def test_failed_step_cannot_be_recorded_as_completed_action(self, tmp_path, monkeypatch): + from types import SimpleNamespace + import cua_bench.runners as runners + + class Env: + async def reset(self, task_id=0): + return b"screen", SimpleNamespace() + async def step(self, action): + raise RuntimeError("injected step failure") + async def close(self): + pass + + monkeypatch.setattr(runners, "make", lambda path, split="train": Env()) + result = await runners.run_single_task(tmp_path, agent_fn=lambda *_: DoneAction()) + assert result.action_trace_digest is None + assert result.steps == 0 + assert result.success is False + + +class TestActionTraceAggregateAttestation: + @pytest.mark.asyncio + @pytest.mark.parametrize("digest", ["broken", "g" * 64, "A" * 64, 123, "0" * 64]) + async def test_aggregator_rejects_bad_or_impossible_worker_digest(self, tmp_path, monkeypatch, digest): + import cua_bench.runners as runners + from types import SimpleNamespace + + task = tmp_path / "case" + task.mkdir() + (task / "main.py").write_text("# discovery stub") + + monkeypatch.setattr(runners, "make", lambda *args, **kwargs: SimpleNamespace( + tasks_config_fn=lambda: [object()] + )) + + async def forged(path, task_index=0, **kwargs): + return TaskResult( + task_path=str(path), variant_id=task_index, + success=True, reward=1.0, steps=0 if digest == "0" * 64 else 1, + action_trace_digest=digest, + ) + + monkeypatch.setattr(runners, "run_single_task", forged) + report = await runners.run_benchmark(tmp_path, agent_fn=lambda *_: DoneAction()) + assert report.success_count == 0 + assert report.avg_reward == 0.0 + assert "Invalid or misattributed" in report.task_results[0]["error"] + + +class TestIndependentlyRecomputableActionTrace: + @pytest.mark.asyncio + async def test_aggregator_accepts_only_matching_redacted_events(self, tmp_path, monkeypatch): + import hashlib + import json + from types import SimpleNamespace + import cua_bench.runners as runners + + task = tmp_path / "case" + task.mkdir() + (task / "main.py").write_text("# discovery stub") + monkeypatch.setattr(runners, "make", lambda *a, **k: SimpleNamespace( + tasks_config_fn=lambda: [object()] + )) + events = [{"step": 1, "action_type": "DoneAction"}] + digest = hashlib.sha256(json.dumps( + events, sort_keys=True, separators=(",", ":") + ).encode()).hexdigest() + + async def worker(path, task_index=0, **kwargs): + return TaskResult( + task_path=str(path), variant_id=task_index, success=True, + reward=1.0, steps=1, action_trace_digest=digest, + action_trace_events=[dict(event) for event in events], + ) + + monkeypatch.setattr(runners, "run_single_task", worker) + good = await runners.run_benchmark(tmp_path, agent_fn=lambda *_: DoneAction()) + assert good.success_count == 1 + assert good.task_results[0]["action_trace_digest"] == digest + assert good.task_results[0]["action_trace_events"] == events + + events[0]["action_type"] = "ClickAction" + tampered = await runners.run_benchmark(tmp_path, agent_fn=lambda *_: DoneAction()) + assert tampered.success_count == 0 + assert "Invalid or misattributed" in tampered.task_results[0]["error"] + + + @pytest.mark.asyncio + async def test_digest_without_events_cannot_claim_success(self, tmp_path, monkeypatch): + from types import SimpleNamespace + import cua_bench.runners as runners + + task = tmp_path / "case" + task.mkdir() + (task / "main.py").write_text("# discovery stub") + monkeypatch.setattr(runners, "make", lambda *a, **k: SimpleNamespace( + tasks_config_fn=lambda: [object()] + )) + + async def worker(path, task_index=0, **kwargs): + return TaskResult( + task_path=str(path), variant_id=task_index, + success=True, reward=1.0, steps=1, + action_trace_digest="a" * 64, + ) + + monkeypatch.setattr(runners, "run_single_task", worker) + report = await runners.run_benchmark(tmp_path, agent_fn=lambda *_: DoneAction()) + assert report.success_count == 0 + assert report.avg_reward == 0.0 + + +class TestPinnedDatasetRuntimeIntegration: + @pytest.mark.asyncio + async def test_modified_dataset_aborts_before_task_import_or_worker(self, tmp_path, monkeypatch): + import json + from cua_bench.dataset_manifest import scan_dataset + import cua_bench.runners as runners + + task = tmp_path / "case" + task.mkdir() + (task / "main.py").write_text("fixture = 1\n") + manifest_path = tmp_path.parent / (tmp_path.name + "-manifest.json") + manifest_path.write_text(json.dumps(scan_dataset(tmp_path))) + (task / "main.py").write_text("fixture = 2\n") + + def no_import(*args, **kwargs): + raise AssertionError("task discovery must not execute when manifest differs") + + monkeypatch.setattr(runners, "make", no_import) + with pytest.raises(ValueError, match="dataset differs from manifest"): + await runners.run_benchmark(tmp_path, dataset_manifest=manifest_path, oracle=True) + + @pytest.mark.asyncio + async def test_matching_manifest_is_bound_to_benchmark_result(self, tmp_path, monkeypatch): + import hashlib + import json + from types import SimpleNamespace + from cua_bench.dataset_manifest import scan_dataset + import cua_bench.runners as runners + + task = tmp_path / "case" + task.mkdir() + (task / "main.py").write_text("fixture = 1\n") + manifest = scan_dataset(tmp_path) + pinned = tmp_path.parent / (tmp_path.name + "-pinned.json") + pinned.write_text(json.dumps(manifest)) + monkeypatch.setattr(runners, "make", lambda *args, **kwargs: SimpleNamespace( + tasks_config_fn=lambda: [object()] + )) + + async def worker(path, task_index=0, **kwargs): + return TaskResult(task_path=str(path), variant_id=task_index, + success=True, reward=1.0, steps=1) + monkeypatch.setattr(runners, "run_single_task", worker) + outcome = await runners.run_benchmark(tmp_path, dataset_manifest=pinned, oracle=True) + expected = hashlib.sha256(json.dumps( + manifest, sort_keys=True, separators=(",", ":") + ).encode()).hexdigest() + assert outcome.success_count == 1 + assert outcome.dataset_manifest_sha256 == expected diff --git a/libs/cua-bench/cua_bench/tests/test_task_cleanup_result.py b/libs/cua-bench/cua_bench/tests/test_task_cleanup_result.py new file mode 100644 index 0000000000..ea593806fe --- /dev/null +++ b/libs/cua-bench/cua_bench/tests/test_task_cleanup_result.py @@ -0,0 +1,118 @@ +"""Regression for preserving failed task results through cleanup errors.""" +import asyncio +import json +from pathlib import Path + +import pytest + +from cua_bench.runner import task_runner as module + + +@pytest.mark.asyncio +async def test_cleanup_exception_preserves_timeout_result_and_receipt(tmp_path, monkeypatch): + runner = module.TaskRunner() + events = [] + + async def create_network(*args, **kwargs): + return None + + async def start_agent(**kwargs): + return None + + async def timeout(*args, **kwargs): + raise asyncio.TimeoutError() + + async def logs(*args, **kwargs): + return "logs" + + async def fail_cleanup(task_id): + events.append("cleanup") + raise RuntimeError("injected image removal failure") + + async def receipt(task_id, output_dir): + events.append("receipt") + (Path(output_dir) / "cleanup-result.json").write_text( + json.dumps({"verified": True}), encoding="utf-8" + ) + + monkeypatch.setattr(module, "create_network", create_network) + monkeypatch.setattr(module, "wait_for_container", timeout) + monkeypatch.setattr(module, "get_container_logs", logs) + monkeypatch.setattr(runner, "_start_agent_container", start_agent) + monkeypatch.setattr(runner, "_cleanup_task", fail_cleanup) + monkeypatch.setattr(runner, "_write_cleanup_receipt", receipt) + + result = await runner.run_task( + env_path=tmp_path, task_index=0, env_type="linux-docker", + provider_type="simulated", cleanup_before=False, output_dir=str(tmp_path), + ) + assert result.success is False + assert "Task timed out" in result.error + assert "Task cleanup failed: RuntimeError" in result.error + assert events == ["cleanup", "receipt"] + + +@pytest.mark.asyncio +async def test_receipt_write_failure_keeps_original_timeout_reason(tmp_path, monkeypatch): + runner = module.TaskRunner() + + async def noop(*args, **kwargs): + return None + + async def timeout(*args, **kwargs): + raise asyncio.TimeoutError() + + async def logs(*args, **kwargs): + return "logs" + + async def broken_receipt(*args, **kwargs): + raise OSError("injected disk full") + + monkeypatch.setattr(module, "create_network", noop) + monkeypatch.setattr(module, "wait_for_container", timeout) + monkeypatch.setattr(module, "get_container_logs", logs) + monkeypatch.setattr(runner, "_start_agent_container", noop) + monkeypatch.setattr(runner, "_cleanup_task", noop) + monkeypatch.setattr(runner, "_write_cleanup_receipt", broken_receipt) + + result = await runner.run_task( + env_path=tmp_path, task_index=0, env_type="linux-docker", + provider_type="simulated", cleanup_before=False, output_dir=str(tmp_path), + ) + assert result.success is False + assert "Task timed out" in result.error + assert "Unable to verify Docker cleanup: OSError" in result.error + + +@pytest.mark.asyncio +async def test_partial_overlay_failure_triggers_registered_cleanup(tmp_path, monkeypatch): + runner = module.TaskRunner() + seen = [] + monkeypatch.setattr(module, "generate_task_id", lambda: "overlay-test") + monkeypatch.setitem(module.ENV_CONFIGS, "overlay-test-env", {"use_overlays": True}) + monkeypatch.setattr(module, "get_overlays_path", lambda: tmp_path) + + def partial_overlay(task_id): + (tmp_path / task_id).mkdir() + raise OSError("injected overlay creation failure") + + async def cleanup(task_id): + tracked = runner._running_tasks.pop(task_id) + seen.append(tracked["overlay_path"]) + tracked["overlay_path"].rmdir() + + async def receipt(task_id, output_dir): + seen.append("receipt") + + monkeypatch.setattr(runner, "_create_task_overlay", partial_overlay) + monkeypatch.setattr(runner, "_cleanup_task", cleanup) + monkeypatch.setattr(runner, "_write_cleanup_receipt", receipt) + + result = await runner.run_task( + env_path=tmp_path, task_index=0, env_type="overlay-test-env", + cleanup_before=False, output_dir=str(tmp_path), + ) + assert result.success is False + assert "injected overlay creation failure" in result.error + assert seen == [tmp_path / "overlay-test", "receipt"] + assert not (tmp_path / "overlay-test").exists() diff --git a/libs/cua-bench/cua_bench/tests/test_verify_execution.py b/libs/cua-bench/cua_bench/tests/test_verify_execution.py new file mode 100644 index 0000000000..523d11611c --- /dev/null +++ b/libs/cua-bench/cua_bench/tests/test_verify_execution.py @@ -0,0 +1,173 @@ +"""Offline verifier contract tests. No Docker or model credentials required.""" +import json + +from cua_bench.verify_execution import verify_execution + + +def write_json(path, obj): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(obj) + "\n", encoding="utf-8") + + +def make_receipts(root, execution_id="run-1"): + write_json(root / "oracle-result.json", { + "schema_version": "cua-bench-oracle-result/v1", + "execution_id": execution_id, + "task_index": 0, + "evaluated": True, "success": True, + "rewards": [1.0], "success_threshold": 1.0, + }) + write_json(root / "cleanup-result.json", { + "schema_version": "cua-bench-cleanup/v1", + "task_id": execution_id, + "verified": True, + "resources": { + "agent_container": {"name": f"cua-agent-{execution_id}", "absent": True}, + "env_container": {"name": f"cua-env-{execution_id}", "absent": True}, + "network": {"name": f"cua-task-{execution_id}", "absent": True}, + "overlay": {"name": f"/tmp/overlays/{execution_id}", "absent": True}, + }, + }) + logs = root / "task_0_agent_logs" / "executed-actions.jsonl" + write_json(logs, { + "schema_version": "cua-bench-executed-action/v1", + "execution_id": execution_id, "sequence": 1, + "execution_status": "returned", "action": {"type": "click"}, + "before": {"sha256": "a" * 64, "size_bytes": 1}, + "after": {"sha256": "b" * 64, "size_bytes": 1}, + }) + + +def test_complete_receipts_pass_internal_consistency(tmp_path): + make_receipts(tmp_path) + result = verify_execution(tmp_path, 0, "run-1") + assert result["valid"] is True + assert result["verified_action_count"] == 1 + + +def test_wrong_execution_id_fails_closed(tmp_path): + make_receipts(tmp_path) + result = verify_execution(tmp_path, 0, "run-2") + assert result["valid"] is False + assert not result["checks"]["oracle"] + + +def test_missing_action_evidence_fails_closed(tmp_path): + make_receipts(tmp_path) + (tmp_path / "task_0_agent_logs" / "executed-actions.jsonl").unlink() + assert verify_execution(tmp_path, 0, "run-1")["valid"] is False + + +def test_forged_oracle_success_is_detected(tmp_path): + make_receipts(tmp_path) + p = tmp_path / "oracle-result.json" + value = json.loads(p.read_text()) + value["rewards"] = [0.0] + write_json(p, value) + assert verify_execution(tmp_path, 0, "run-1")["checks"]["oracle"] is False + + +def test_cleanup_residual_is_rejected(tmp_path): + make_receipts(tmp_path) + p = tmp_path / "cleanup-result.json" + value = json.loads(p.read_text()) + value["resources"]["agent_container"]["absent"] = False + write_json(p, value) + assert verify_execution(tmp_path, 0, "run-1")["checks"]["cleanup"] is False + + +def test_oracle_only_mode_allows_no_agent_stream(tmp_path): + make_receipts(tmp_path) + (tmp_path / "task_0_agent_logs" / "executed-actions.jsonl").unlink() + result = verify_execution(tmp_path, 0, "run-1", require_actions=False) + assert result["valid"] is True + assert result["checks"]["actions"] is None + + +def test_finite_negative_reward_uses_explicit_threshold(tmp_path): + make_receipts(tmp_path) + p = tmp_path / "oracle-result.json" + value = json.loads(p.read_text()) + value["rewards"] = [-0.25] + value["success_threshold"] = 0.0 + value["success"] = False + write_json(p, value) + result = verify_execution(tmp_path, 0, "run-1") + assert result["checks"]["oracle"] is True + assert result["evidence_consistent"] is True + assert result["task_success"] is False + assert result["run_accepted"] is False + + +def test_failed_action_cannot_prove_successful_execution(tmp_path): + make_receipts(tmp_path) + p = tmp_path / "task_0_agent_logs" / "executed-actions.jsonl" + value = json.loads(p.read_text()) + value["execution_status"] = "failed" + write_json(p, value) + result = verify_execution(tmp_path, 0, "run-1") + assert result["valid"] is False + assert result["checks"]["actions"] is True + assert result["evidence_consistent"] is True + assert result["failed_action_count"] == 1 + assert result["run_accepted"] is False + + +def test_missing_screenshot_observation_fails_closed(tmp_path): + make_receipts(tmp_path) + p = tmp_path / "task_0_agent_logs" / "executed-actions.jsonl" + value = json.loads(p.read_text()) + value["before"] = {"error_type": "TimeoutError"} + write_json(p, value) + result = verify_execution(tmp_path, 0, "run-1") + assert result["valid"] is False + assert result["checks"]["actions"] is False + + +def test_failed_task_can_have_consistent_evidence(tmp_path): + make_receipts(tmp_path) + p = tmp_path / "oracle-result.json" + value = json.loads(p.read_text()) + value["rewards"] = [0.0] + value["success"] = False + write_json(p, value) + result = verify_execution(tmp_path, 0, "run-1") + assert result["checks"]["oracle"] is True + assert result["evidence_consistent"] is True + assert result["task_success"] is False + assert result["run_accepted"] is False + assert result["valid"] is False + + +def test_finite_reward_above_one_is_consistent_with_solver_contract(tmp_path): + make_receipts(tmp_path) + p = tmp_path / "oracle-result.json" + value = json.loads(p.read_text()) + value["rewards"] = [2.0] + write_json(p, value) + result = verify_execution(tmp_path, 0, "run-1") + assert result["checks"]["oracle"] is True + assert result["task_success"] is True + assert result["run_accepted"] is True + + +def test_missing_cleanup_resource_fails_closed(tmp_path): + make_receipts(tmp_path) + path = tmp_path / "cleanup-result.json" + receipt = json.loads(path.read_text()) + del receipt["resources"]["overlay"] + write_json(path, receipt) + report = verify_execution(tmp_path, 0, "run-1") + assert report["checks"]["cleanup"] is False + assert report["run_accepted"] is False + + +def test_wrong_cleanup_resource_identity_is_rejected(tmp_path): + make_receipts(tmp_path) + path = tmp_path / "cleanup-result.json" + receipt = json.loads(path.read_text()) + receipt["resources"]["network"]["name"] = "cua-task-some-other-run" + write_json(path, receipt) + result = verify_execution(tmp_path, 0, "run-1") + assert result["checks"]["cleanup"] is False + assert result["run_accepted"] is False diff --git a/libs/cua-bench/cua_bench/verify_execution.py b/libs/cua-bench/cua_bench/verify_execution.py new file mode 100644 index 0000000000..3f3c4170b0 --- /dev/null +++ b/libs/cua-bench/cua_bench/verify_execution.py @@ -0,0 +1,124 @@ +"""Offline, read-only verifier for cua-bench execution artifacts. + +Checks internal consistency, not the truth of screenshots or the remote oracle. +Usage: python -m cua_bench.verify_execution OUTPUT_DIR --task-index 0 --execution-id ID +""" +import argparse +import json +from .evidence_contract import action_event_valid, oracle_receipt_valid +from pathlib import Path + + +def _json(path): + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError(f"{path.name}: expected object") + return value + + +def verify_execution(root: Path, task_index: int, execution_id: str, *, require_actions=True): + """Return a machine-readable fail-closed independent verification report.""" + errors = [] + checks = {} + task_success = False + failed_action_count = 0 + root = Path(root) + oracle_path = root / "oracle-result.json" + cleanup_path = root / "cleanup-result.json" + action_path = root / f"task_{task_index}_agent_logs" / "executed-actions.jsonl" + + try: + oracle = _json(oracle_path) + rewards = oracle.get("rewards") + threshold = oracle.get("success_threshold") + valid = oracle_receipt_valid(oracle, task_index, execution_id) + checks["oracle"] = bool(valid) + task_success = bool(valid and oracle["success"]) + if not checks["oracle"]: + errors.append("oracle_invalid") + except (OSError, ValueError, TypeError, KeyError) as exc: + checks["oracle"] = False + errors.append(f"oracle_unreadable:{type(exc).__name__}") + + try: + cleanup = _json(cleanup_path) + resources = cleanup.get("resources") + valid = ( + cleanup.get("schema_version") == "cua-bench-cleanup/v1" + and cleanup.get("task_id") == execution_id + and isinstance(resources, dict) + and {"agent_container", "env_container", "network", "overlay"} <= resources.keys() + and all(isinstance(v, dict) for v in resources.values()) + and resources["agent_container"].get("name") == f"cua-agent-{execution_id}" + and resources["env_container"].get("name") == f"cua-env-{execution_id}" + and resources["network"].get("name") == f"cua-task-{execution_id}" + and isinstance(resources["overlay"].get("name"), str) + and Path(resources["overlay"]["name"]).name == execution_id + and all(isinstance(v, dict) and v.get("absent") is True for v in resources.values()) + and cleanup.get("verified") is True + ) + checks["cleanup"] = valid + if not valid: + errors.append("cleanup_unverified") + except (OSError, ValueError, TypeError) as exc: + checks["cleanup"] = False + errors.append(f"cleanup_unreadable:{type(exc).__name__}") + + action_count = 0 + if require_actions: + try: + with action_path.open(encoding="utf-8") as stream: + for line in stream: + record = json.loads(line) + action_count += 1 + if not action_event_valid(record, action_count, execution_id): + raise ValueError("Invalid action event") + failed_action_count += record["execution_status"] == "failed" + checks["actions"] = action_count > 0 + if not checks["actions"]: + errors.append("actions_empty") + except (OSError, ValueError, TypeError) as exc: + checks["actions"] = False + errors.append(f"actions_unreadable_or_invalid:{type(exc).__name__}") + else: + checks["actions"] = None + + evidence_consistent = not errors + run_accepted = evidence_consistent and task_success and failed_action_count == 0 + return { + "schema_version": "cua-bench-verification/v1", + "execution_id": execution_id, + "task_index": task_index, + "valid": run_accepted, + "evidence_consistent": evidence_consistent, + "task_success": task_success, + "run_accepted": run_accepted, + "failed_action_count": failed_action_count, + "checks": checks, + "verified_action_count": action_count, + "errors": errors, + "limitations": [ + "Verifies internal artifact consistency only, not screenshot authenticity", + "Does not independently re-execute or trustlessly validate the environment oracle", + "Cannot establish that the recording intercepted all device actions", + ], + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("output_dir", type=Path) + parser.add_argument("--task-index", required=True, type=int) + parser.add_argument("--execution-id", required=True) + parser.add_argument("--no-agent", action="store_true", help="Oracle-only task; no agent action stream") + args = parser.parse_args() + result = verify_execution( + args.output_dir, args.task_index, args.execution_id, + require_actions=not args.no_agent, + ) + print(json.dumps(result, indent=2, sort_keys=True)) + raise SystemExit(0 if result["valid"] else 1) + + +if __name__ == "__main__": + main() diff --git a/libs/cua-bench/datasets/DATASET_MANIFEST.md b/libs/cua-bench/datasets/DATASET_MANIFEST.md new file mode 100644 index 0000000000..b0341ed815 --- /dev/null +++ b/libs/cua-bench/datasets/DATASET_MANIFEST.md @@ -0,0 +1,15 @@ +# Dataset Manifest v1 + +A new **read-only dataset integrity feature** for CUA Bench. It inventories each task's source and static assets and detects drift before a benchmark is compared with previous runs. + +```sh +cd libs/cua-bench +python3 scripts/dataset_manifest.py datasets/cua-bench-basic --output /tmp/cua-basic-manifest.json +python3 scripts/dataset_manifest.py datasets/cua-bench-basic --verify /tmp/cua-basic-manifest.json +``` + +The manifest format is `cua-dataset-manifest/v1`: deterministic task IDs (directory names), sorted relative file paths, raw-byte SHA-256 and sizes. Verification fails if task files are added, removed or edited, or if task directories are added or removed. Generated cache directories are excluded, but task-local hidden files (for example `.env` and `.config`) are included because they may affect execution. Avoid placing actual secrets inside benchmark task directories; manifests expose file names and their hashes. Symlinked task files are rejected. The scanner never imports `main.py` or runs task code. + +**Security limits:** SHA-256 validates file integrity against the provided manifest; the manifest itself must be authenticated by a trusted commit, source repository or signed release. This is not an Oracle execution receipt and cannot prove that a GUI action was performed. Pin the task code, data assets and environment separately for comparable results. + +**Acceptance:** negative tests cover drift, new tasks, symlinks and non-execution. CI executes a manifest round-trip against `cua-bench-basic`. Do not mark the feature verified until the relevant CI run passes. diff --git a/libs/cua-bench/schemas/README.md b/libs/cua-bench/schemas/README.md new file mode 100644 index 0000000000..3d990f6802 --- /dev/null +++ b/libs/cua-bench/schemas/README.md @@ -0,0 +1,38 @@ +# CUA Bench CI Receipt v1 — external consumer guide + +This is a **test-result integrity envelope**. It is not an independent oracle, a signed software supply-chain attestation, or proof that an agent performed desktop actions. + +## Implemented integration +- `pytest-json-report` (MIT) produces `core-report.json` and `full-report.json` from real pytest executions. +- `jsonschema` validates the published Draft 2020-12 contract at `schemas/ci-receipt-v1.schema.json`. +- `scripts/ci_receipt.py` binds the reports to their SHA-256 checksums, passing test counts and Git commit, and refuses altered or failed reports. +- GitHub Actions publishes these three files as `cua-bench-pytest-evidence` when available. + +## Generate in CI + +```sh +cd libs/cua-bench +uv run --extra dev --with 'jsonschema>=4.20,<5' python scripts/ci_receipt.py --commit "$GITHUB_SHA" +``` + +## Independently verify downloaded artifacts + +Place the downloaded `core-report.json`, `full-report.json`, and `ci-receipt-v1.json` in the same directory, then run: + +```sh +cd libs/cua-bench +uv run --extra dev --with 'jsonschema>=4.20,<5' python scripts/ci_receipt.py \ + --verify /path/to/artifacts/ci-receipt-v1.json \ + --evidence-dir /path/to/artifacts +``` + +The verifier checks the contract, two distinct report kinds, safe local basenames, exact report checksums, passing outcomes and mutually consistent counts. It does not download external URLs or execute downloaded code. + +## Limits and outstanding User Stories +1. Commit and SHA-256 metadata are unauthenticated; consumers must establish CI origin independently (e.g. official GitHub run URL, permissions and repository owner). +2. Reported tests may use mocks. This is not a full Computer-Use Execution Witness and contains neither per-action state observations nor independent oracle replays. +3. No real VM crash/cleanup inventory is included. Real desktop VM fault injection remains a separate P0 gate. +4. Compatibility: versioned schema `cua-bench-ci-receipt/v1`; breaking changes require a new schema version. +5. Evidence retention is currently 30 days. Export elsewhere before expiry if it is used as a long-lived credential. + +Success means an **authentic test run can be inspected with the raw reports and the same verifier**, not that the underlying benchmark's real-world task success is guaranteed. diff --git a/libs/cua-bench/schemas/ci-receipt-v1.schema.json b/libs/cua-bench/schemas/ci-receipt-v1.schema.json new file mode 100644 index 0000000000..78ea99c512 --- /dev/null +++ b/libs/cua-bench/schemas/ci-receipt-v1.schema.json @@ -0,0 +1,61 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/hippoley/cua/blob/fix/cua-bench-validate-parallelism-20261009/libs/cua-bench/schemas/ci-receipt-v1.schema.json", + "title": "CUA Bench CI test receipt v1", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "commit", + "reports" + ], + "properties": { + "schema_version": { + "const": "cua-bench-ci-receipt/v1" + }, + "commit": { + "type": "string", + "pattern": "^[a-f0-9]{40}$" + }, + "reports": { + "type": "array", + "minItems": 2, + "maxItems": 2, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "path", + "sha256", + "passed", + "distinct_tests" + ], + "properties": { + "kind": { + "enum": [ + "core", + "full" + ] + }, + "path": { + "type": "string", + "minLength": 1 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "passed": { + "type": "integer", + "minimum": 1 + }, + "distinct_tests": { + "type": "integer", + "minimum": 1 + } + } + } + } + } +} diff --git a/libs/cua-bench/scripts/ci_receipt.py b/libs/cua-bench/scripts/ci_receipt.py new file mode 100644 index 0000000000..7c0f307ca8 --- /dev/null +++ b/libs/cua-bench/scripts/ci_receipt.py @@ -0,0 +1,86 @@ +"""Build a versioned, hash-linked CI test receipt using the jsonschema project. + +This witnesses archived pytest report bytes, not actual computer-use actions. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path + +from jsonschema import Draft202012Validator + +from verify_pytest_receipt import verify_report + +SCHEMA = Path(__file__).resolve().parents[1] / "schemas" / "ci-receipt-v1.schema.json" + + +def build_receipt(core: Path, full: Path, commit: str): + reports = [] + for kind, path in (("core", core), ("full", full)): + raw = path.read_bytes() + result = verify_report(json.loads(raw)) + reports.append({ + "kind": kind, + "path": path.name, + "sha256": hashlib.sha256(raw).hexdigest(), + "passed": result["passed"], + "distinct_tests": result["distinct_tests"], + }) + receipt = {"schema_version": "cua-bench-ci-receipt/v1", "commit": commit, "reports": reports} + validator = Draft202012Validator(json.loads(SCHEMA.read_text())) + validator.validate(receipt) + if set(x["kind"] for x in reports) != {"core", "full"}: + raise ValueError("receipt must cover both core and full runs") + return receipt + + +def verify_receipt(receipt, folder: Path): + schema = json.loads(SCHEMA.read_text()) + Draft202012Validator(schema).validate(receipt) + if {r["kind"] for r in receipt["reports"]} != {"core", "full"}: + raise ValueError("receipt must contain exactly one core and full result") + if len({r["path"] for r in receipt["reports"]}) != len(receipt["reports"]): + raise ValueError("core and full reports must have distinct file paths") + root = folder.resolve(strict=True) + for item in receipt["reports"]: + name = item["path"] + # No paths outside evidence folder or URI-based remote fetches. + if Path(name).name != name: + raise ValueError("only local report basenames are allowed") + target = (root / name).resolve(strict=True) + if target.parent != root: + raise ValueError("report escapes evidence directory via symlink") + raw = target.read_bytes() + if hashlib.sha256(raw).hexdigest() != item["sha256"]: + raise ValueError(f"report checksum mismatch: {name}") + checked = verify_report(json.loads(raw)) + if checked["passed"] != item["passed"] or checked["distinct_tests"] != item["distinct_tests"]: + raise ValueError(f"report summary mismatch: {name}") + return True + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--verify", type=Path, help="verify an existing receipt without regenerating it") + parser.add_argument("--evidence-dir", type=Path, default=Path("."), help="folder containing the archived reports") + parser.add_argument("--core", type=Path, default=Path("core-report.json")) + parser.add_argument("--full", type=Path, default=Path("full-report.json")) + parser.add_argument("--commit", default=os.environ.get("GITHUB_SHA")) + parser.add_argument("--output", type=Path, default=Path("ci-receipt-v1.json")) + args = parser.parse_args() + if args.verify is not None: + receipt = json.loads(args.verify.read_text()) + verify_receipt(receipt, args.evidence_dir) + print(f"CUA CI receipt independently verified: {args.verify}") + return + if not args.commit: + parser.error("a real 40-character git commit is required") + receipt = build_receipt(args.core, args.full, args.commit) + verify_receipt(receipt, args.core.parent) + args.output.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n") + print(f"CUA CI receipt checked: {args.output}") + + +if __name__ == "__main__": + main() diff --git a/libs/cua-bench/scripts/dataset_manifest.py b/libs/cua-bench/scripts/dataset_manifest.py new file mode 100644 index 0000000000..41b149c496 --- /dev/null +++ b/libs/cua-bench/scripts/dataset_manifest.py @@ -0,0 +1,76 @@ +"""Reproducible, read-only dataset manifest for CUA Bench task directories. + +Does not import or execute untrusted task main.py files. +""" +import argparse +import hashlib +import json +from pathlib import Path + +VERSION = "cua-dataset-manifest/v1" +IGNORED = {"__pycache__", ".git", ".venv", "node_modules", ".pytest_cache"} + + +def scan_dataset(root: Path): + root = root.resolve(strict=True) + if not root.is_dir(): + raise ValueError("dataset must be a directory") + if (root / "main.py").is_file(): + tasks = [root] + else: + entries = sorted(root.iterdir()) + for entry in entries: + if entry.is_symlink() and entry.is_dir() and (entry / "main.py").is_file(): + raise ValueError(f"symlinked dataset task directory: {entry.name}") + tasks = [p for p in entries if p.is_dir() and (p / "main.py").is_file()] + if not tasks: + raise ValueError("no task directories containing main.py") + output = [] + for task in tasks: + files = [] + for path in sorted(task.rglob("*")): + relative = path.relative_to(root) + if any(part in IGNORED for part in relative.parts): + continue + if path.is_symlink(): + raise ValueError(f"symlink in dataset task: {relative}") + if not path.is_file(): + continue + data = path.read_bytes() + files.append({"path": relative.as_posix(), "sha256": hashlib.sha256(data).hexdigest(), "size": len(data)}) + if not files: + raise ValueError(f"empty task: {task}") + output.append({"task": task.relative_to(root).as_posix(), "files": files}) + return {"schema_version": VERSION, "tasks": output} + + +def verify_dataset(root: Path, manifest: dict): + if not isinstance(manifest, dict) or manifest.get("schema_version") != VERSION: + raise ValueError("unsupported dataset manifest") + expected = manifest.get("tasks") + if not isinstance(expected, list) or not expected: + raise ValueError("manifest has no tasks") + actual = scan_dataset(root) + if actual != manifest: + raise ValueError("dataset differs from manifest (added, removed, or modified task files)") + return len(actual["tasks"]) + + +def main(): + parser = argparse.ArgumentParser(description="Create or verify CUA dataset task manifest") + parser.add_argument("dataset", type=Path) + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--output", type=Path) + group.add_argument("--verify", type=Path) + args = parser.parse_args() + if args.verify: + count = verify_dataset(args.dataset, json.loads(args.verify.read_text())) + print(f"Verified {count} tasks without executing task source") + else: + manifest = scan_dataset(args.dataset) + args.output.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") + print(f"Recorded {len(manifest['tasks'])} tasks: {args.output}") + + +if __name__ == "__main__": + main() diff --git a/libs/cua-bench/scripts/verify_pytest_receipt.py b/libs/cua-bench/scripts/verify_pytest_receipt.py new file mode 100644 index 0000000000..001aedf761 --- /dev/null +++ b/libs/cua-bench/scripts/verify_pytest_receipt.py @@ -0,0 +1,48 @@ +"""Validate pytest-json-report output without trusting a green CI label alone. + +This is a structural receipt check, not independent task-oracle attestation. +""" +import argparse +import json +from pathlib import Path + + +def verify_report(doc): + if not isinstance(doc, dict): + raise ValueError("report must be an object") + tests = doc.get("tests") + summary = doc.get("summary") + if not isinstance(tests, list) or not tests: + raise ValueError("no executed test records") + if not isinstance(summary, dict): + raise ValueError("missing summary") + if type(doc.get("exitcode")) is not int or doc["exitcode"] != 0: + raise ValueError("pytest did not exit successfully") + nodeids = [x.get("nodeid") for x in tests if isinstance(x, dict)] + if len(nodeids) != len(tests) or any(not isinstance(x, str) or not x for x in nodeids): + raise ValueError("invalid test identity") + if len(nodeids) != len(set(nodeids)): + raise ValueError("duplicate test identities") + if any(x.get("outcome") != "passed" for x in tests): + raise ValueError("not all test outcomes passed") + if any(summary.get(k, 0) for k in ("failed", "error", "skipped", "xfailed", "xpassed", "deselected")): + raise ValueError("report contains non-passing or unexecuted tests") + if summary.get("passed") != len(tests) or summary.get("total") != len(tests): + raise ValueError("reported totals disagree with test records") + if summary.get("collected", len(tests)) != len(tests): + raise ValueError("collection count disagrees with test records") + if any(x.get("outcome") == "failed" for x in doc.get("collectors", []) if isinstance(x, dict)): + raise ValueError("collection failure") + return {"passed": len(tests), "distinct_tests": len(nodeids), "report_valid": True} + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("reports", nargs="+", type=Path) + args = parser.parse_args() + for path in args.reports: + print(json.dumps({"file": str(path), **verify_report(json.loads(path.read_text()))})) + + +if __name__ == "__main__": + main()