bench: UpgradeBench v0 pilot harness (UPT-24)

bench/check.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# The final type-check, lint and tests of one run, from a clean worktree of its head: a fresh
+# install with lifecycle scripts off, then the repository's own commands from selection.json.
+#
+#   BENCH_OUT=<dir> bench/check.sh <run>
+set -euo pipefail
+run=$1
+here=$(cd "$(dirname "$0")" && pwd)
+dir=${BENCH_OUT:?}/runs/$run
+id=$(jq -r .id "$dir/meta.json") head=$(jq -r .head "$dir/meta.json")
+export npm_config_ignore_scripts=true YARN_ENABLE_SCRIPTS=0 CI=1
+rm -rf "$dir/check"
+git -C "$dir/repo" worktree prune
+git -C "$dir/repo" worktree add -q --detach "$dir/check" "$head"
+cd "$dir/check"
+result='{}'
+for step in install typecheck lint test; do
+  cmd=$(jq -r --arg id "$id" --arg s "$step" '.repos[] | select(.id == $id) | .checks[$s] // ""' "$here/selection.json")
+  [ -z "$cmd" ] && continue
+  code=0
+  bash -c "$cmd" >"$dir/check-$step.log" 2>&1 || code=$?
+  result=$(jq --arg s "$step" --argjson c "$code" '.[$s] = $c' <<<"$result")
+  # Nothing else means anything without dependencies.
+  [ "$step" = install ] && [ "$code" != 0 ] && break
+done
+echo "$result" >"$dir/checks.json"
+echo "$run $result"

bench/claude-sandbox.json

@@ -0,0 +1,27 @@
+{
+  "sandbox": {
+    "enabled": true,
+    "failIfUnavailable": true,
+    "allowUnsandboxedCommands": false,
+    "autoAllowBashIfSandboxed": true,
+    "filesystem": {
+      "allowWrite": ["~/.npm", "~/Library/pnpm", "~/Library/Caches", "~/.cache"]
+    },
+    "network": {
+      "allowedDomains": [
+        "registry.npmjs.org",
+        "registry.yarnpkg.com",
+        "repo.yarnpkg.com",
+        "github.com",
+        "codeload.github.com",
+        "objects.githubusercontent.com"
+      ],
+      "strictAllowlist": true,
+      "allowLocalBinding": true
+    }
+  },
+  "permissions": {
+    "deny": ["Bash(dangerouslyDisableSandbox:true)", "WebFetch", "WebSearch"],
+    "allow": ["Bash"]
+  }
+}