draft(cua-bench): fail-closed benchmark scoring and reproducible evidence receipts

.github/release-attribution-config.json

@@ -0,0 +1,49 @@
+{
+  "bots": [
+    "dependabot[bot]",
+    "github-actions[bot]",
+    "claude",
+    "copilot",
+    "release-please[bot]",
+    "renovate[bot]",
+    "trycua-release[bot]"
+  ],
+  "ignoredCoauthorEmails": [
+    "claude@anthropic.com",
+    "coding-sub-agent@trycua.com",
+    "codex@openai.com",
+    "cua@cua.localdomain",
+    "hermes@hermes.home.arpa",
+    "noreply@anthropic.com"
+  ],
+  "coauthorOverrides": {
+    "hippoley <noreply@github.com>": "hippoley"
+  },
+  "identityOverrides": {
+    "antoine.bonnet@hcompany.ai": "abonneth",
+    "anvit.mail@icloud.com": "AnvitDevadiga",
+    "ddupont@mit.edu": "ddupont808",
+    "dillon@trycua.com": "ddupont808",
+    "egnichtel@crosscode.com": "ngnichtel",
+    "f@trycua.com": "f-trycua",
+    "fbonacci@fbonacci-xfce-vm.b40fumexvs1ujlod2ppcyfh5qa.bx.internal.cloudapp.net": "f-trycua",
+    "i@jyunko.cn": "HsiangNianian",
+    "jf-mac-mini@jf-mac-mini-4.local": "0xjohnnydev",
+    "klymenko@adobe.com": "pavelklymenko",
+    "m.fuechtenkoetter@posteo.de": "ai-ag2026",
+    "manfred@ubuntu26.04-vm": "ai-ag2026",
+    "marko.civonafets@gmail.com": "civonafets",
+    "me@madhavajay.com": "madhavajay",
+    "robert@trycua.com": "enchanted-koala",
+    "rwendt1337@gmail.com": "r33drichards",
+    "rsyuzyov@gmail.com": "rsyuzyov",
+    "sarinajin.li@gmail.com": "sarinali",
+    "steipete@gmail.com": "steipete",
+    "tshtark@intelligo.ai": "tshtark",
+    "umtcgnd@gmail.com": "c8dhjp4tyv-bit",
+    "zane.chee.2023@scis.smu.edu.sg": "injaneity",
+    "zongxin_yang@hms.harvard.edu": "z-x-yang"
+  },
+  "internalHandles": ["ddupont808", "enchanted-koala", "f-trycua"],
+  "optOutHandles": []
+}

.github/workflows/ci-py-bench.yml

@@ -1,6 +1,14 @@
 name: "CI: cua-bench"
 
 on:
+  push:
+    branches:
+      - fix/cua-bench-validate-parallelism-20261009
+    paths:
+      - "libs/cua-bench/**"
+      - ".github/workflows/ci-py-bench.yml"
+      - ".github/workflows/py-reusable-build.yml"
+  workflow_dispatch:
   pull_request:
     paths:
       - "libs/cua-bench/**"
@@ -13,3 +21,54 @@ jobs:
     with:
       package_name: "bench"
       package_dir: "libs/cua-bench"
+
+  regression:
+    runs-on: ubuntu-latest
+    timeout-minutes: 35
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up uv and Python 3.12
+        uses: astral-sh/setup-uv@v6
+        with:
+          python-version: "3.12"
+      - name: Install Playwright Chromium for simulated desktop E2E
+        working-directory: libs/cua-bench
+        run: uv run --extra dev python -m playwright install --with-deps chromium
+      - name: Verify dataset manifest round trip on basic dataset
+        working-directory: libs/cua-bench
+        run: |
+          python3 scripts/dataset_manifest.py datasets/cua-bench-basic --output /tmp/cua-basic-manifest.json
+          python3 scripts/dataset_manifest.py datasets/cua-bench-basic --verify /tmp/cua-basic-manifest.json
+      - name: Exercise installed manifest CLI
+        working-directory: libs/cua-bench
+        run: |
+          uv run --extra dev python -m cua_bench.cli.main run manifest datasets/cua-bench-basic --output /tmp/cua-cli-pinned.json
+          uv run --extra dev python -c "import json; from pathlib import Path; from cua_bench.dataset_manifest import verify_dataset; verify_dataset(Path('datasets/cua-bench-basic'), json.loads(Path('/tmp/cua-cli-pinned.json').read_text()))"
+      - name: Run CUA benchmark core and oracle regression suite
+        working-directory: libs/cua-bench
+        run: |
+          xvfb-run -a uv run --extra dev --with 'pytest-json-report>=1.5,<2' --with 'jsonschema>=4.20,<5' --with 'pytest-timeout>=2.3,<3' --with 'hypothesis>=6.100,<7' pytest --timeout=120 --timeout-method=signal --json-report --json-report-file=core-report.json --json-report-omit=streams \
+            cua_bench/tests/test_run_benchmark.py \
+            cua_bench/tests/test_gym_interface.py \
+            cua_bench/tests/test_actions.py \
+            cua_bench/tests/test_reward_hypothesis.py -q
+      - name: Run full benchmark suite including RL workers
+        working-directory: libs/cua-bench
+        run: xvfb-run -a uv run --extra dev --extra rl --extra server --with 'httpx>=0.27,<1' --with 'pytest-json-report>=1.5,<2' --with 'jsonschema>=4.20,<5' --with 'pytest-timeout>=2.3,<3' --with 'hypothesis>=6.100,<7' pytest --timeout=120 --timeout-method=signal --json-report --json-report-file=full-report.json --json-report-omit=streams cua_bench/tests -q
+      - name: Independently verify test reports
+        working-directory: libs/cua-bench
+        run: python3 scripts/verify_pytest_receipt.py core-report.json full-report.json
+      - name: Build versioned checksum-linked receipt
+        working-directory: libs/cua-bench
+        run: uv run --extra dev --with 'jsonschema>=4.20,<5' python scripts/ci_receipt.py --commit "$GITHUB_SHA"
+      - name: Preserve CUA benchmark machine-readable evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: cua-bench-pytest-evidence
+          path: |
+            libs/cua-bench/core-report.json
+            libs/cua-bench/full-report.json
+            libs/cua-bench/ci-receipt-v1.json
+          if-no-files-found: warn
+          retention-days: 30