From 3a501836856468f0e26d5f155fbeb856e64cb63f Mon Sep 17 00:00:00 2001 From: "cua-release-bot[bot]" <254316371+cua-release-bot[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 01:51:33 +0000 Subject: [PATCH 1/3] chore(main): release bench 0.3.1 --- .release-please-manifest.json | 2 +- libs/cua-bench/CHANGELOG.md | 14 ++++++++++++++ libs/cua-bench/pyproject.toml | 2 +- libs/cua-bench/uv.lock | 2 +- 4 files changed, 17 insertions(+), 3 deletions(-) create mode 100644 libs/cua-bench/CHANGELOG.md diff --git a/.release-please-manifest.json b/.release-please-manifest.json index da638c7d2f..bf412d97ac 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -8,7 +8,7 @@ "apps/cua-spaces-macos": "0.10.0", "libs/python/agent": "0.9.0", "libs/python/core": "0.3.2", - "libs/cua-bench": "0.3.0", + "libs/cua-bench": "0.3.1", "libs/python/bench-ui": "0.7.1", "libs/python/som": "0.1.4", "libs/python/cua-sandbox-apps": "0.1.2", diff --git a/libs/cua-bench/CHANGELOG.md b/libs/cua-bench/CHANGELOG.md new file mode 100644 index 0000000000..8343915950 --- /dev/null +++ b/libs/cua-bench/CHANGELOG.md @@ -0,0 +1,14 @@ +# Changelog + +## [0.3.1](https://github.com/trycua/cua/compare/bench-v0.3.0...bench-v0.3.1) (2026-10-19) + + +### Bug Fixes + +* **cua-bench:** fail fast on unsupported Windows Arena hosts ([#5356](https://github.com/trycua/cua/issues/5356)) ([2236b53](https://github.com/trycua/cua/commit/2236b5372a372067239a4b5f44efc595e0f8aca1)) +* **cua-bench:** pin bench-osworld to verified-20261003-fe6d89d ([#4580](https://github.com/trycua/cua/issues/4580)) ([94e7e9f](https://github.com/trycua/cua/commit/94e7e9fbc1b1bf467ff93f272dccc527f5d61dc6)), closes [#4519](https://github.com/trycua/cua/issues/4519) +* **cua-bench:** raise when the sandbox does not answer in file_exists/directory_exists ([#5194](https://github.com/trycua/cua/issues/5194)) ([1ff5710](https://github.com/trycua/cua/commit/1ff57101e705d5964ca344e26207366a94467dee)) +* **cua-bench:** reject missing, nonfinite and nonnumeric evaluator rewards ([#5163](https://github.com/trycua/cua/issues/5163)) ([341d1e1](https://github.com/trycua/cua/commit/341d1e17237815340991376828992115a64ee15c)) +* lock the cua SDK checkout at 0.3.1 ([#4593](https://github.com/trycua/cua/issues/4593)) ([fcb386f](https://github.com/trycua/cua/commit/fcb386f4f958d1357af31ccb0056376f06d9db37)) +* **python:** cua-sandbox, cua-bench and the playground no longer depend on Fleet; remove cua-train (CUA-1834) ([#5445](https://github.com/trycua/cua/issues/5445)) ([7733289](https://github.com/trycua/cua/commit/773328941ead1b83a616e8b6c5a5ac4851af7114)) +* stop Cua Cloud (on="cloud") with a clear "Cua Cloud has ended" error (CUA-1854) ([#5415](https://github.com/trycua/cua/issues/5415)) ([5d78ec8](https://github.com/trycua/cua/commit/5d78ec8b7705d942a3d260b276bcce76614a634c)) diff --git a/libs/cua-bench/pyproject.toml b/libs/cua-bench/pyproject.toml index e628ad9b98..db2c82aada 100644 --- a/libs/cua-bench/pyproject.toml +++ b/libs/cua-bench/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "cua-bench" -version = "0.3.0" +version = "0.3.1" description = "Run computer-use benchmark datasets on local and cloud sandboxes" readme = "README.md" license = "MIT" diff --git a/libs/cua-bench/uv.lock b/libs/cua-bench/uv.lock index f749fdeac3..c571fb63cc 100644 --- a/libs/cua-bench/uv.lock +++ b/libs/cua-bench/uv.lock @@ -594,7 +594,7 @@ provides-extras = ["computer", "openai", "anthropic", "qwen", "uitars", "uitars- [[package]] name = "cua-bench" -version = "0.3.0" +version = "0.3.1" source = { editable = "." } dependencies = [ { name = "aiohttp" }, From b0c6972dfc7bd77f82dfba9ca6bf0bc0b0ea40c5 Mon Sep 17 00:00:00 2001 From: Francesco Bonacci Date: Sun, 18 Oct 2026 22:20:14 -0500 Subject: [PATCH 2/3] chore(release): relock cua-bench-rl for bench 0.3.1 Co-Authored-By: Claude Opus 5.5 --- libs/cua-bench-rl/uv.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/cua-bench-rl/uv.lock b/libs/cua-bench-rl/uv.lock index 22676fda15..26ce66dde3 100644 --- a/libs/cua-bench-rl/uv.lock +++ b/libs/cua-bench-rl/uv.lock @@ -522,7 +522,7 @@ provides-extras = ["computer", "openai", "anthropic", "qwen", "uitars", "uitars- [[package]] name = "cua-bench" -version = "0.3.0" +version = "0.3.1" source = { editable = "../cua-bench" } dependencies = [ { name = "aiohttp" }, From 78891f928461ccf8b82195275d24d43307dfeba6 Mon Sep 17 00:00:00 2001 From: Francesco Bonacci Date: Sun, 18 Oct 2026 22:54:09 -0500 Subject: [PATCH 3/3] chore(release): regenerate the cua-bench reference for 0.3.1 and let legacy baselines trail the manifest Co-Authored-By: Claude Opus 5.5 --- .github/scripts/tests/test_release_notes.py | 15 +++++++++++++-- .../docs/cua-bench/reference/cli/agent.mdx | 2 +- .../docs/cua-bench/reference/cli/dataset.mdx | 2 +- docs/content/docs/cua-bench/reference/cli/env.mdx | 2 +- .../docs/cua-bench/reference/cli/images.mdx | 2 +- .../docs/cua-bench/reference/cli/index.mdx | 2 +- docs/content/docs/cua-bench/reference/cli/run.mdx | 2 +- .../content/docs/cua-bench/reference/cli/task.mdx | 2 +- .../docs/cua-bench/reference/cli/trace.mdx | 2 +- docs/content/docs/cua-bench/reference/index.mdx | 2 +- docs/content/docs/cua-bench/reference/results.mdx | 2 +- .../docs/cua-bench/reference/task-definition.mdx | 2 +- scripts/docs-generators/cli-specs/cb.json | 2 +- 13 files changed, 25 insertions(+), 14 deletions(-) diff --git a/.github/scripts/tests/test_release_notes.py b/.github/scripts/tests/test_release_notes.py index 01e8fc4924..7bebece3c8 100644 --- a/.github/scripts/tests/test_release_notes.py +++ b/.github/scripts/tests/test_release_notes.py @@ -198,8 +198,19 @@ def test_version_check_catches_a_package_file_that_drifted(tmp_path): def test_legacy_baselines_are_exactly_the_enrolled_packages_last_legacy_tags(): - expected = {f"{component}-v{MANIFEST[path]}" for path, component in ENROLLED.items()} - assert release_notes.LEGACY_BASELINE_TAGS == expected + # One baseline per enrolled package, named for its last legacy tag. Until the + # package's first Release Please release that is the manifest version; the + # release PR then moves the manifest ahead of it (the baseline must stay for + # the notes of that release), so the baseline may be older, never newer. + by_component = {} + for tag in release_notes.LEGACY_BASELINE_TAGS: + component, version = tag.rsplit("-v", 1) + assert component not in by_component, f"two legacy baselines for {component}" + by_component[component] = tuple(int(part) for part in version.split(".")) + assert set(by_component) == set(ENROLLED.values()) + for path, component in ENROLLED.items(): + released = tuple(int(part) for part in MANIFEST[path].split(".")) + assert by_component[component] <= released, f"{component} baseline is newer than its manifest version" def test_only_a_legacy_baseline_relaxes_the_changelog_citation_check(monkeypatch, tmp_path): diff --git a/docs/content/docs/cua-bench/reference/cli/agent.mdx b/docs/content/docs/cua-bench/reference/cli/agent.mdx index 2d27ff6f55..8bca0485f8 100644 --- a/docs/content/docs/cua-bench/reference/cli/agent.mdx +++ b/docs/content/docs/cua-bench/reference/cli/agent.mdx @@ -7,7 +7,7 @@ description: "Scaffold custom agents." AUTO-GENERATED FILE - DO NOT EDIT DIRECTLY Generated by: pnpm --dir docs docs:generate:cua-bench Source: scripts/docs-generators/extract_cb_docs.py (cua_bench) - Version: 0.3.0 + Version: 0.3.1 */} | Command | Description | diff --git a/docs/content/docs/cua-bench/reference/cli/dataset.mdx b/docs/content/docs/cua-bench/reference/cli/dataset.mdx index 8632ed09bc..7d27783608 100644 --- a/docs/content/docs/cua-bench/reference/cli/dataset.mdx +++ b/docs/content/docs/cua-bench/reference/cli/dataset.mdx @@ -7,7 +7,7 @@ description: "List registry datasets and build training data from runs." AUTO-GENERATED FILE - DO NOT EDIT DIRECTLY Generated by: pnpm --dir docs docs:generate:cua-bench Source: scripts/docs-generators/extract_cb_docs.py (cua_bench) - Version: 0.3.0 + Version: 0.3.1 */} | Command | Description | diff --git a/docs/content/docs/cua-bench/reference/cli/env.mdx b/docs/content/docs/cua-bench/reference/cli/env.mdx index a200b8f5e3..f5981436dd 100644 --- a/docs/content/docs/cua-bench/reference/cli/env.mdx +++ b/docs/content/docs/cua-bench/reference/cli/env.mdx @@ -7,7 +7,7 @@ description: "Cloud pools and local sandboxes, sign-in, status and cleanup." AUTO-GENERATED FILE - DO NOT EDIT DIRECTLY Generated by: pnpm --dir docs docs:generate:cua-bench Source: scripts/docs-generators/extract_cb_docs.py (cua_bench) - Version: 0.3.0 + Version: 0.3.1 */} | Command | Description | diff --git a/docs/content/docs/cua-bench/reference/cli/images.mdx b/docs/content/docs/cua-bench/reference/cli/images.mdx index e4fc03132f..fe009b35d6 100644 --- a/docs/content/docs/cua-bench/reference/cli/images.mdx +++ b/docs/content/docs/cua-bench/reference/cli/images.mdx @@ -7,7 +7,7 @@ description: "The canonical images and platform configurations." AUTO-GENERATED FILE - DO NOT EDIT DIRECTLY Generated by: pnpm --dir docs docs:generate:cua-bench Source: scripts/docs-generators/extract_cb_docs.py (cua_bench) - Version: 0.3.0 + Version: 0.3.1 */} | Command | Description | diff --git a/docs/content/docs/cua-bench/reference/cli/index.mdx b/docs/content/docs/cua-bench/reference/cli/index.mdx index a99b129929..33bbc4707a 100644 --- a/docs/content/docs/cua-bench/reference/cli/index.mdx +++ b/docs/content/docs/cua-bench/reference/cli/index.mdx @@ -7,7 +7,7 @@ description: "cua-bench: run computer-use tasks and benchmark datasets on local AUTO-GENERATED FILE - DO NOT EDIT DIRECTLY Generated by: pnpm --dir docs docs:generate:cua-bench Source: scripts/docs-generators/extract_cb_docs.py (cua_bench) - Version: 0.3.0 + Version: 0.3.1 */} ```text output diff --git a/docs/content/docs/cua-bench/reference/cli/run.mdx b/docs/content/docs/cua-bench/reference/cli/run.mdx index 05159427eb..d662ceee3c 100644 --- a/docs/content/docs/cua-bench/reference/cli/run.mdx +++ b/docs/content/docs/cua-bench/reference/cli/run.mdx @@ -7,7 +7,7 @@ description: "Run a task or dataset with an agent or the oracle, run one interac AUTO-GENERATED FILE - DO NOT EDIT DIRECTLY Generated by: pnpm --dir docs docs:generate:cua-bench Source: scripts/docs-generators/extract_cb_docs.py (cua_bench) - Version: 0.3.0 + Version: 0.3.1 */} | Command | Description | diff --git a/docs/content/docs/cua-bench/reference/cli/task.mdx b/docs/content/docs/cua-bench/reference/cli/task.mdx index 6390550a02..dee69003b3 100644 --- a/docs/content/docs/cua-bench/reference/cli/task.mdx +++ b/docs/content/docs/cua-bench/reference/cli/task.mdx @@ -7,7 +7,7 @@ description: "Inspect, list, scaffold and generate tasks." AUTO-GENERATED FILE - DO NOT EDIT DIRECTLY Generated by: pnpm --dir docs docs:generate:cua-bench Source: scripts/docs-generators/extract_cb_docs.py (cua_bench) - Version: 0.3.0 + Version: 0.3.1 */} | Command | Description | diff --git a/docs/content/docs/cua-bench/reference/cli/trace.mdx b/docs/content/docs/cua-bench/reference/cli/trace.mdx index 30e3c024e1..b68f15acae 100644 --- a/docs/content/docs/cua-bench/reference/cli/trace.mdx +++ b/docs/content/docs/cua-bench/reference/cli/trace.mdx @@ -7,7 +7,7 @@ description: "View traces and agent trajectories." AUTO-GENERATED FILE - DO NOT EDIT DIRECTLY Generated by: pnpm --dir docs docs:generate:cua-bench Source: scripts/docs-generators/extract_cb_docs.py (cua_bench) - Version: 0.3.0 + Version: 0.3.1 */} | Command | Description | diff --git a/docs/content/docs/cua-bench/reference/index.mdx b/docs/content/docs/cua-bench/reference/index.mdx index be75e0f95f..b8a1ba6348 100644 --- a/docs/content/docs/cua-bench/reference/index.mdx +++ b/docs/content/docs/cua-bench/reference/index.mdx @@ -7,7 +7,7 @@ description: "Every cb command, the task module contract and the result files, g AUTO-GENERATED FILE - DO NOT EDIT DIRECTLY Generated by: pnpm --dir docs docs:generate:cua-bench Source: scripts/docs-generators/extract_cb_docs.py (cua_bench) - Version: 0.3.0 + Version: 0.3.1 */} ## Commands diff --git a/docs/content/docs/cua-bench/reference/results.mdx b/docs/content/docs/cua-bench/reference/results.mdx index 15eefe8fb4..0085d5c12f 100644 --- a/docs/content/docs/cua-bench/reference/results.mdx +++ b/docs/content/docs/cua-bench/reference/results.mdx @@ -7,7 +7,7 @@ description: "The result.json and summary.json a run writes: every key, its type AUTO-GENERATED FILE - DO NOT EDIT DIRECTLY Generated by: pnpm --dir docs docs:generate:cua-bench Source: scripts/docs-generators/extract_cb_docs.py (cua_bench) - Version: 0.3.0 + Version: 0.3.1 */} A run writes one directory per variant and a `summary.json` next to them: diff --git a/docs/content/docs/cua-bench/reference/task-definition.mdx b/docs/content/docs/cua-bench/reference/task-definition.mdx index 1883b2c4e0..9cf52665e6 100644 --- a/docs/content/docs/cua-bench/reference/task-definition.mdx +++ b/docs/content/docs/cua-bench/reference/task-definition.mdx @@ -7,7 +7,7 @@ description: "The task module contract: cb.Task, setup_config, the lifecycle dec AUTO-GENERATED FILE - DO NOT EDIT DIRECTLY Generated by: pnpm --dir docs docs:generate:cua-bench Source: scripts/docs-generators/extract_cb_docs.py (cua_bench) - Version: 0.3.0 + Version: 0.3.1 */} A task is a directory with a `main.py`. `cb` imports it, calls the `@cb.tasks_config` function to list the variants, then for each variant starts a sandbox from its `computer` and runs setup, the agent (or the `@cb.solve_task` oracle) and evaluation. Each lifecycle function receives the variant's `cb.Task` and the session on its sandbox. diff --git a/scripts/docs-generators/cli-specs/cb.json b/scripts/docs-generators/cli-specs/cb.json index fad4844554..59d6aac4f2 100644 --- a/scripts/docs-generators/cli-specs/cb.json +++ b/scripts/docs-generators/cli-specs/cb.json @@ -2771,5 +2771,5 @@ "global_options": [], "name": "cb", "usage": "cb [OPTIONS]", - "version": "0.3.0" + "version": "0.3.1" }