From bf88ead20b74315b97c301a250106d801f56808c Mon Sep 17 00:00:00 2001
From: Lucas Braum <lucasbraum@gmail.com>
Date: Thu, 15 Oct 2026 21:59:57 -0700
Subject: [PATCH 1/7] bench: UpgradeBench v0 pilot selection, held out from
 every ground truth (UPT-24)

Three public repositories with a human's merged upgrade in a verified
pack's range (zod, stripe, react), recorded before either arm runs.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Lucas Braum <lucasbraum@gmail.com>
---
 bench/excluded.json  | 443 +++++++++++++++++++++++++++++++++++++++++++
 bench/selection.json |  83 ++++++++
 2 files changed, 526 insertions(+)
 create mode 100644 bench/excluded.json
 create mode 100644 bench/selection.json

diff --git a/bench/excluded.json b/bench/excluded.json
new file mode 100644
index 0000000..607a52d
--- /dev/null
+++ b/bench/excluded.json
@@ -0,0 +1,443 @@
+[
+  {
+    "pack": "react-router",
+    "repo": "Arize-ai/phoenix",
+    "commit": "4d6c1dfb01e9406cf769efab14f0350ea2d0a0f8",
+    "from": "6.25.1",
+    "to": "7.4.0"
+  },
+  {
+    "pack": "stripe",
+    "repo": "AtlasDevHQ/atlas",
+    "commit": "cfaf3c3c500995a4dc1e918ef89de20e535af43f",
+    "from": "21.0.1",
+    "to": "22.2.0"
+  },
+  {
+    "pack": "@sentry/browser",
+    "repo": "Automancer-Ltd/automancer-site",
+    "commit": "ea78c738e8de80efbb796b506fbc81bb4f649f70",
+    "from": "10.75.2",
+    "to": "11.4.0"
+  },
+  {
+    "pack": "typescript",
+    "repo": "Borewit/music-metadata",
+    "commit": "783509b27bac24e244dd75e66c9b4a0828c35a96",
+    "from": "5.9.3",
+    "to": "6.0.3"
+  },
+  {
+    "pack": "zod",
+    "repo": "CANShift/canshift-core",
+    "commit": "3ad0a35edd902e94cb3014f1331cd38528532e3f",
+    "from": "3.25.76",
+    "to": "4.4.3"
+  },
+  {
+    "pack": "typescript",
+    "repo": "CroffZ/github-release-notifier",
+    "commit": "d9353f2471c554f0ce339496f0058a27599f4965",
+    "from": "5.9.3",
+    "to": "6.0.3"
+  },
+  {
+    "pack": "next",
+    "repo": "HiroshiIgarash/koremina",
+    "commit": "e9a1595083f746c8cb481e1ca7156aca6db6d4d0",
+    "from": "15.5.10",
+    "to": "16.1.6"
+  },
+  {
+    "pack": "typescript",
+    "repo": "JosephMaynard/dependency-radar",
+    "commit": "1920eb3217b701e43afc1651db3f6197028e6f0e",
+    "from": "5.9.3",
+    "to": "6.0.3"
+  },
+  {
+    "pack": "typescript",
+    "repo": "SignalK/signalk-derived-data",
+    "commit": "02f635e8232a433462593ed1192ff22e2e9b6253",
+    "from": "5.9.3",
+    "to": "6.0.3"
+  },
+  {
+    "pack": "@sentry/node",
+    "repo": "TryGhost/ActivityPub",
+    "commit": "f8a8233219f927b39a41418c7cce0f2fa4867b67",
+    "from": "10.75.3",
+    "to": "11.0.0"
+  },
+  {
+    "pack": "redis",
+    "repo": "UNN-Devotek/FCM-Fallout-Chat-Mod",
+    "commit": "280190489fac1ac8b75bf5fda8fd54265f339d7e",
+    "from": "4.7.1",
+    "to": "6.0.0"
+  },
+  {
+    "pack": "express",
+    "repo": "amiantos/lurker",
+    "commit": "f7a0f3dac77570ec3fd93a487ccf0aaf2f6ca5db",
+    "from": "4.22.2",
+    "to": "5.2.1"
+  },
+  {
+    "pack": "zod",
+    "repo": "band-ai/band-sdk-typescript",
+    "commit": "5d9a66a1dde4989fbc0608200f291f0ca8194052",
+    "from": "3.25.76",
+    "to": "4.6.5"
+  },
+  {
+    "pack": "express",
+    "repo": "bcgov/tenant-management-system",
+    "commit": "6fd01755043a3db9ed0577ffa6b1b4a184ae670f",
+    "from": "4.22.2",
+    "to": "5.2.1"
+  },
+  {
+    "pack": "next",
+    "repo": "benshabbat/GamesForMyKids",
+    "commit": "793cb46f19f2bc279dbf741a74cf7083a5025310",
+    "from": "15.5.15",
+    "to": "16.2.6"
+  },
+  {
+    "pack": "@sentry/nextjs",
+    "repo": "cohstats/coh3-stats",
+    "commit": "5b9c60bd6575ed4a4bcd8d2bb71529766675a7e6",
+    "from": "10.75.3",
+    "to": "11.1.0"
+  },
+  {
+    "pack": "openai",
+    "repo": "csnyder256/ux-struggle-detector",
+    "commit": "b73084896208f7d2794b84bf23de3157eca5fc09",
+    "from": "4.104.0",
+    "to": "7.23.0"
+  },
+  {
+    "pack": "lucide-react",
+    "repo": "cubicsteak/keep",
+    "commit": "9380d10ea22b66e4559779d7cc3afb4f209cf198",
+    "from": "0.483.0",
+    "to": "1.47.0"
+  },
+  {
+    "pack": "redis",
+    "repo": "deanmoses/tacocat-gallery-sam",
+    "commit": "34e11760f6e767abef6afd53f973dc155f6f8c4b",
+    "from": "4.6.12",
+    "to": "5.10.0"
+  },
+  {
+    "pack": "redis",
+    "repo": "deanmoses/tacocat-gallery-sam",
+    "commit": "30724e21efd6db123a7385e73482cac237b1e9ba",
+    "from": "5.12.1",
+    "to": "6.2.1"
+  },
+  {
+    "pack": "openai",
+    "repo": "digitalchokro/askchokro",
+    "commit": "047fc4485538a3bff4224a9a604d83143c2cd5d2",
+    "from": "4.104.0",
+    "to": "7.4.0"
+  },
+  {
+    "pack": "joi",
+    "repo": "edtech4good/edtech-lms-api",
+    "commit": "ce2281005fcf0c41a883779c778f3123a48bbeec",
+    "from": "17.6.0",
+    "to": "18.2.9"
+  },
+  {
+    "pack": "react",
+    "repo": "excalidraw/excalidraw",
+    "commit": "9ee0b8ffcbd3664a47748a93262860321a203821",
+    "from": "18.2.0",
+    "to": "19.0.0"
+  },
+  {
+    "pack": "ioredis",
+    "repo": "ferni-ai/voiceai",
+    "commit": "c59ed0982f96e23d2b2f5728711da5b55e07483a",
+    "from": "5.8.2",
+    "to": "6.0.0"
+  },
+  {
+    "pack": "@sentry/node",
+    "repo": "frog-pond/ccc-server",
+    "commit": "75fdc083bbf719c5c577b4c3068528776bea6fa3",
+    "from": "10.75.3",
+    "to": "11.2.0"
+  },
+  {
+    "pack": "react-router",
+    "repo": "getlago/lago-front",
+    "commit": "84aa05d535edc326a4d104923e4e7293d1be6646",
+    "from": "6.30.6",
+    "to": "7.18.3"
+  },
+  {
+    "pack": "openai",
+    "repo": "grantsingleton/batch-ai",
+    "commit": "05c7e9f132b3e6ec6504de3d82371889fdc2d368",
+    "from": "4.83.0",
+    "to": "7.4.0"
+  },
+  {
+    "pack": "zod",
+    "repo": "howardyang2009/PATH",
+    "commit": "bb171417d911f373c8df28ab7a1e0698b62e0906",
+    "from": "3.25.76",
+    "to": "4.6.5"
+  },
+  {
+    "pack": "ioredis",
+    "repo": "kazumadev619-dev/FishingConditionsApp",
+    "commit": "8ba3f0d24bc709169b79df11daa87307df32e77c",
+    "from": "5.11.1",
+    "to": "6.0.0"
+  },
+  {
+    "pack": "typescript",
+    "repo": "m-lyon/recipe",
+    "commit": "f6e44b530c4dab5afc9ca6f3505774d5b44fcd33",
+    "from": "5.6.2",
+    "to": "6.0.3"
+  },
+  {
+    "pack": "@sentry/nextjs",
+    "repo": "maximunited/yomu",
+    "commit": "228fc3e4f7abf1850e4d03d32535e28b5173306c",
+    "from": "10.75.1",
+    "to": "11.1.0"
+  },
+  {
+    "pack": "openai",
+    "repo": "microsoft/prompty",
+    "commit": "e878b6b7789ff621265ca086b20d4e2d70914f81",
+    "from": "4.104.0",
+    "to": "7.10.0"
+  },
+  {
+    "pack": "ai",
+    "repo": "miurla/morphic",
+    "commit": "989a05697d973030f2fbb4b805326bb24e68fd0d",
+    "from": "6.0.180",
+    "to": "7.0.66"
+  },
+  {
+    "pack": "express",
+    "repo": "netlify/cli",
+    "commit": "63b6f1cbca316fa6c338d59e5b5053a11de25c6c",
+    "from": "4.22.1",
+    "to": "5.2.1"
+  },
+  {
+    "pack": "stripe",
+    "repo": "nextjs/saas-starter",
+    "commit": "3e5fd2bf626c4ee73c06dcc9e1aa503b8904a37c",
+    "from": "17.7.0",
+    "to": "18.0.0"
+  },
+  {
+    "pack": "openai",
+    "repo": "openai/openai-guardrails-js",
+    "commit": "6c13ba08ab021215fd6f8c900ae15ae29627f2d2",
+    "from": "4.104.0",
+    "to": "7.9.0"
+  },
+  {
+    "pack": "express",
+    "repo": "openhab/openhab-cloud",
+    "commit": "5a1866efe1c038e287d2d59091951e1912db12f8",
+    "from": "4.22.1",
+    "to": "5.2.1"
+  },
+  {
+    "pack": "@sentry/react",
+    "repo": "owine/youth-activity-scheduler",
+    "commit": "18b467f5c4f2fc066e14dc5edfe10874016ba3d3",
+    "from": "10.75.3",
+    "to": "11.0.0"
+  },
+  {
+    "pack": "typescript",
+    "repo": "robrechtme/recipes",
+    "commit": "0853747427b71f1d5de275e72aead59979fc1a03",
+    "from": "5.9.3",
+    "to": "6.0.3"
+  },
+  {
+    "pack": "ioredis",
+    "repo": "safe-global/safe-client-gateway",
+    "commit": "14bd05b98bf41df509de0bfee8bc662cb9d95d1e",
+    "from": "5.11.1",
+    "to": "6.0.0"
+  },
+  {
+    "pack": "next",
+    "repo": "sambecker/exif-photo-blog",
+    "commit": "3f1a36354d203e5a28f7c2cd55eceda95a13e3bd",
+    "from": "15.5.5",
+    "to": "16.0.1"
+  },
+  {
+    "pack": "@sentry/browser",
+    "repo": "sbomify/sbomify",
+    "commit": "4c2be20126779d91d629b9478c6a0aff0239abfe",
+    "from": "10.75.2",
+    "to": "11.0.0"
+  },
+  {
+    "pack": "joi",
+    "repo": "senacor/azure-function-middleware",
+    "commit": "5c2da676e7327681dde70db6229edeebfa76b0f4",
+    "from": "17.13.1",
+    "to": "18.0.1"
+  },
+  {
+    "pack": "typescript",
+    "repo": "shogo82148/typescript-action",
+    "commit": "0f03405c7267fafaebee5d57e773c1bb56e28b44",
+    "from": "5.9.3",
+    "to": "6.0.3"
+  },
+  {
+    "pack": "ioredis",
+    "repo": "simondietschi/dokunc",
+    "commit": "83681ed0ccc527fa1dd5c802f554f26a5eac575c",
+    "from": "5.11.1",
+    "to": "6.0.0"
+  },
+  {
+    "pack": "lucide-react",
+    "repo": "spizeck/SFPCA",
+    "commit": "7f5b9dcb8985260e7b9ab345be971b392bd129a5",
+    "from": "0.563.0",
+    "to": "1.47.0"
+  },
+  {
+    "pack": "stripe",
+    "repo": "starl3xx/wallet-to-social",
+    "commit": "b7f60f1167b4665746e455696f6aa34caa75ad0c",
+    "from": "20.2.0",
+    "to": "22.6.2"
+  },
+  {
+    "pack": "next",
+    "repo": "stoczec/wortkarte",
+    "commit": "e3bd3679ae6ea60ce2a78aefebec0caa48c4febd",
+    "from": "15.5.27",
+    "to": "16.3.8"
+  },
+  {
+    "pack": "@sentry/react",
+    "repo": "successible/cleanslate",
+    "commit": "5d45401e6562e6c028028774c9097ecb4737dc6e",
+    "from": "10.74.0",
+    "to": "11.0.0"
+  },
+  {
+    "pack": "redis",
+    "repo": "swoofer/mcp-coordinator",
+    "commit": "8b337f0a081d0d0e0c3724890b9cc847b6c5dd6f",
+    "from": "5.12.1",
+    "to": "6.2.0"
+  },
+  {
+    "pack": "ioredis",
+    "repo": "syzygyhack/open-foundry",
+    "commit": "73910018b5e8119cf4db99a81ff145726451aa3b",
+    "from": "5.10.1",
+    "to": "6.0.0"
+  },
+  {
+    "pack": "redis",
+    "repo": "tearingItUp786/blog",
+    "commit": "bb88f3afc40df07e29da461c6aed6c34709bc6bf",
+    "from": "4.7.0",
+    "to": "6.0.0"
+  },
+  {
+    "pack": "react",
+    "repo": "tldraw/tldraw",
+    "commit": "5957867a9c5c205a0b0c3665be443138b8014a79",
+    "from": "18.3.1",
+    "to": "19.2.1"
+  },
+  {
+    "pack": "lucide-react",
+    "repo": "tradeblocks-org/tradeblocks",
+    "commit": "918c0dadccacec7e0533ea1a208f252562ac711f",
+    "from": "0.577.0",
+    "to": "1.52.0"
+  },
+  {
+    "pack": "zod",
+    "repo": "ugurkocde/AwesomeIntune",
+    "commit": "dcf38603fd55cea72737e17274d5de03cece341e",
+    "from": "3.25.76",
+    "to": "4.6.5"
+  },
+  {
+    "pack": "stripe",
+    "repo": "unkeyed/unkey",
+    "commit": "37044fc57fe08c917e92b323933baf322d6df1c6",
+    "from": "14.23.0",
+    "to": "22.2.0"
+  },
+  {
+    "pack": "ai",
+    "repo": "vercel/chatbot",
+    "commit": "77d362e596e2c624638cad5a35199c0c8785c9fd",
+    "from": "6.0.116",
+    "to": "7.0.9"
+  },
+  {
+    "pack": "vitest",
+    "repo": "vitorvasc/opentelemetry-ecosystem-explorer",
+    "commit": "46d5b37d00bbf27f394e33cee64a0e206c31315f",
+    "from": "4.1.11",
+    "to": "5.0.0"
+  },
+  {
+    "pack": "vitest",
+    "repo": "votingworks/vxsuite",
+    "commit": "5cb3c7d9c1abc891e7b12de57c0278790dd6160a",
+    "from": "4.1.11",
+    "to": "5.0.2"
+  },
+  {
+    "pack": "zod",
+    "repo": "vzdc-artcc/website",
+    "commit": "48a93f65a0976e342d4c15145b51de8eef030214",
+    "from": "3.25.76",
+    "to": "4.6.5"
+  },
+  {
+    "pack": "openai",
+    "repo": "wardby/wardby",
+    "commit": "d46ec64dc0be781aa43967d9b6d26be9386aa67d",
+    "from": "4.104.0",
+    "to": "7.30.0"
+  },
+  {
+    "pack": "@sentry/nextjs",
+    "repo": "xdoubleu/tools.xdoubleu.com",
+    "commit": "d3b780d42c8937af42d37072f7b0948bced39d47",
+    "from": "10.75.3",
+    "to": "11.0.0"
+  },
+  {
+    "pack": "openai",
+    "repo": "zeikar/charivo",
+    "commit": "b9a476d9acb59f473e08f0e97ce6e91dc2201d99",
+    "from": "4.104.0",
+    "to": "7.10.0"
+  }
+]
diff --git a/bench/selection.json b/bench/selection.json
new file mode 100644
index 0000000..9f5aef4
--- /dev/null
+++ b/bench/selection.json
@@ -0,0 +1,83 @@
+{
+  "$comment": "UpgradeBench v0 pilot (UPT-24): three held-out public repositories, each with a human's merged upgrade in the range of a verified pack. None is in any pack's ground-truth.json (bench/excluded.json). Committed before either arm ran. checks are the repository's own commands, run by bench/check.sh with lifecycle scripts off.",
+  "selected": "2026-10-15",
+  "repos": [
+    {
+      "id": "zod-calmcp",
+      "repo": "consetto/calmcp",
+      "license": "MIT",
+      "pack": "zod",
+      "package": "zod",
+      "from": "3.25.76",
+      "to": "4.6.5",
+      "parent": "741de4eba98eb96b4bed817d8974e9e8c6eec867",
+      "upgrade": "bdaef39c34705b2dfc1065a59ec683288a2e99d2",
+      "note": "npm, single package. The upgrade (merged in PR #22) changes 3 source files, a test and a script: z.string().uuid() to z.guid(), .strict() tool schemas, a _def walker rewritten for zod 4. Mostly behavior, little a compiler sees. The commit carries a Co-Authored-By: Claude trailer.",
+      "ci": ".github/workflows/ci.yml: build (lint, typecheck, test, build, test:e2e), btp-descriptors, docker; tool-surface.yml",
+      "checks": {
+        "install": "npm ci --ignore-scripts",
+        "typecheck": "npm run typecheck",
+        "lint": "npm run lint",
+        "test": "npm test"
+      }
+    },
+    {
+      "id": "stripe-dms-saas",
+      "repo": "AntelopeJS/dms-saas",
+      "license": "Apache-2.0",
+      "pack": "stripe",
+      "package": "stripe",
+      "from": "18.5.0",
+      "to": "22.6.2",
+      "parent": "6390f434622db5d6e546854b435fb1017c3cc665",
+      "upgrade": "f992f335089cfb17ca2b86d8d9d6a908ddbaee2a",
+      "note": "pnpm 10.2.0 workspace of 2 packages plus a nested frontend-vue project with its own lockfile. The upgrade (PR #70) changes 14 source files: subscriptions.cancel options, schedule phase iterations to duration, allowed_payment_method_types, billing mode, apiVersion basil to dahlia. dms-saas type-checks only after the interface package is built.",
+      "ci": ".github/workflows/ci.yml: static (build, lint, typecheck, knip, check:exports), unit (vitest), frontend",
+      "checks": {
+        "install": "corepack pnpm install --frozen-lockfile --ignore-scripts && corepack pnpm --dir packages/dms-saas/frontend-vue install --frozen-lockfile --ignore-scripts",
+        "typecheck": "corepack pnpm --filter @antelopejs/interface-dms-saas build && corepack pnpm -r typecheck",
+        "lint": "corepack pnpm -r lint",
+        "test": "corepack pnpm --filter @antelopejs/dms-saas test:unit"
+      }
+    },
+    {
+      "id": "react-enki",
+      "repo": "entur/enki",
+      "license": "EUPL-1.2",
+      "pack": "react",
+      "package": "react",
+      "from": "18.3.1",
+      "to": "19.2.0",
+      "parent": "08cf2a70267185a74c362c15bab3b3826e4be3f2",
+      "upgrade": "ed37dd231b49628b7611760dfcde750e9c5a8e02",
+      "note": "npm, single package (Vite). The upgrade is the merge of PR #1760 (scored against its first parent): react, react-dom and their types to 19, react-leaflet 4 to 5, MutableRefObject to RefObject, useRef arguments, useReducer generics, 11 source files.",
+      "ci": ".github/workflows/pull-request-build-external.yml: fork pull requests, npm ci, prettier check, vitest, build (tsc and vite)",
+      "checks": {
+        "install": "npm ci --ignore-scripts",
+        "typecheck": "./node_modules/.bin/tsc --noEmit",
+        "lint": "npm run check",
+        "test": "./node_modules/.bin/vitest run"
+      }
+    }
+  ],
+  "backups": [
+    {
+      "repo": "sukhmal/servicenow-mcp",
+      "package": "zod",
+      "parent": "9c71205e26752ecb5a14cf95b26fcbe475af1839",
+      "upgrade": "0406fba4baf28c9eba8d0690d70eab1b5e38f8e7"
+    },
+    {
+      "repo": "likecoin/likecoin-api-public",
+      "package": "stripe",
+      "parent": "4526998344fb4905034c9248bbed4b80e0fde242",
+      "upgrade": "12480cda61716226155dec06a98a9cd67737741b"
+    },
+    {
+      "repo": "Tietokilta/web",
+      "package": "next",
+      "parent": "f6f51b539594504d2d91adff3b5a14e5dc075b9f",
+      "upgrade": "d96361254033243cb8165bc3277f0be15649aba7"
+    }
+  ]
+}

From de4f867cfec59ee1117e912ea012bff206e7055c Mon Sep 17 00:00:00 2001
From: Lucas Braum <lucasbraum@gmail.com>
Date: Thu, 15 Oct 2026 22:01:21 -0700
Subject: [PATCH 2/7] bench: run, check, judge and blind-score scripts for the
 UpgradeBench pilot (UPT-24)

run.sh starts both arms from the same fresh clone of the parent, with
lifecycle scripts off and pushes disabled; the Claude Code arm runs Bash
only inside Claude Code's sandbox. check.sh runs the repository's own
checks, judge.sh opens a pull request on an uptide-bench fork, and
score.mjs scores every run the same way without reading its arm.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Lucas Braum <lucasbraum@gmail.com>
---
 bench/check.sh            |  27 +++++++
 bench/claude-sandbox.json |  25 +++++++
 bench/judge.sh            |  41 +++++++++++
 bench/run.sh              |  66 +++++++++++++++++
 bench/score.mjs           | 150 ++++++++++++++++++++++++++++++++++++++
 bench/score.test.mjs      |  41 +++++++++++
 6 files changed, 350 insertions(+)
 create mode 100755 bench/check.sh
 create mode 100644 bench/claude-sandbox.json
 create mode 100755 bench/judge.sh
 create mode 100755 bench/run.sh
 create mode 100644 bench/score.mjs
 create mode 100644 bench/score.test.mjs

diff --git a/bench/check.sh b/bench/check.sh
new file mode 100755
index 0000000..134556d
--- /dev/null
+++ b/bench/check.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# The final type-check, lint and tests of one run, from a clean worktree of its head: a fresh
+# install with lifecycle scripts off, then the repository's own commands from selection.json.
+#
+#   BENCH_OUT=<dir> bench/check.sh <run>
+set -euo pipefail
+run=$1
+here=$(cd "$(dirname "$0")" && pwd)
+dir=${BENCH_OUT:?}/runs/$run
+id=$(jq -r .id "$dir/meta.json") head=$(jq -r .head "$dir/meta.json")
+export npm_config_ignore_scripts=true YARN_ENABLE_SCRIPTS=0 CI=1
+rm -rf "$dir/check"
+git -C "$dir/repo" worktree prune
+git -C "$dir/repo" worktree add -q --detach "$dir/check" "$head"
+cd "$dir/check"
+result='{}'
+for step in install typecheck lint test; do
+  cmd=$(jq -r --arg id "$id" --arg s "$step" '.repos[] | select(.id == $id) | .checks[$s] // ""' "$here/selection.json")
+  [ -z "$cmd" ] && continue
+  code=0
+  bash -c "$cmd" >"$dir/check-$step.log" 2>&1 || code=$?
+  result=$(jq --arg s "$step" --argjson c "$code" '.[$s] = $c' <<<"$result")
+  # Nothing else means anything without dependencies.
+  [ "$step" = install ] && [ "$code" != 0 ] && break
+done
+echo "$result" >"$dir/checks.json"
+echo "$run $result"
diff --git a/bench/claude-sandbox.json b/bench/claude-sandbox.json
new file mode 100644
index 0000000..ef0f97c
--- /dev/null
+++ b/bench/claude-sandbox.json
@@ -0,0 +1,25 @@
+{
+  "sandbox": {
+    "enabled": true,
+    "failIfUnavailable": true,
+    "allowUnsandboxedCommands": false,
+    "autoAllowBashIfSandboxed": true,
+    "filesystem": {
+      "allowWrite": ["~/.npm", "~/Library/pnpm", "~/Library/Caches", "~/.cache"]
+    },
+    "network": {
+      "allowedDomains": [
+        "registry.npmjs.org",
+        "registry.yarnpkg.com",
+        "repo.yarnpkg.com",
+        "github.com",
+        "codeload.github.com",
+        "objects.githubusercontent.com"
+      ],
+      "strictAllowlist": true
+    }
+  },
+  "permissions": {
+    "deny": ["Bash(dangerouslyDisableSandbox:true)", "WebFetch", "WebSearch"]
+  }
+}
diff --git a/bench/judge.sh b/bench/judge.sh
new file mode 100755
index 0000000..2820827
--- /dev/null
+++ b/bench/judge.sh
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# Let the repository's own CI judge one run: a fork in the uptide-bench organization whose
+# default branch is reset to the parent commit (so `pull_request` branch filters match), the
+# run's head pushed as bench/<run>, and a pull request between the two, on the fork only.
+# Pushes go to the fork's explicit URL; the original repository is never written to.
+#
+#   BENCH_OUT=<dir> bench/judge.sh <run>            fork, push, open the pull request
+#   BENCH_OUT=<dir> bench/judge.sh <run> --collect  wait for its checks once, store them
+set -euo pipefail
+run=$1
+here=$(cd "$(dirname "$0")" && pwd)
+dir=${BENCH_OUT:?}/runs/$run
+org=uptide-bench
+id=$(jq -r .id "$dir/meta.json") head=$(jq -r .head "$dir/meta.json")
+sel() { jq -er --arg id "$id" ".repos[] | select(.id == \$id) | $1" "$here/selection.json"; }
+repo=$(sel .repo) parent=$(sel .parent)
+fork=$org/${repo%%/*}--${repo##*/}
+
+if [ "${2:-}" = --collect ]; then
+  pr=$(jq -r .number "$dir/pr.json")
+  gh pr checks "$pr" --repo "$fork" --watch --interval 60 >/dev/null 2>&1 || :
+  gh pr checks "$pr" --repo "$fork" --json name,workflow,state,bucket >"$dir/ci.json" || :
+  echo "$run $(jq -c 'group_by(.bucket) | map({(.[0].bucket): length}) | add' "$dir/ci.json")"
+  exit 0
+fi
+
+if ! gh repo view "$fork" >/dev/null 2>&1; then
+  gh repo fork "$repo" --org "$org" --fork-name "${fork#*/}" --clone=false --default-branch-only
+  sleep 5
+fi
+# Forks start with Actions off.
+gh api -X PUT "repos/$fork/actions/permissions" -F enabled=true -f allowed_actions=all >/dev/null
+base=$(gh api "repos/$fork" --jq .default_branch)
+url=https://github.com/$fork.git
+git -C "$dir/repo" push -q --force "$url" "$parent:refs/heads/$base"
+git -C "$dir/repo" push -q --force "$url" "$head:refs/heads/bench/$run"
+head -c 60000 "$dir/summary.md" >"$dir/pr-body.md"
+[ -s "$dir/pr-body.md" ] || echo "(no summary)" >"$dir/pr-body.md"
+gh api "repos/$fork/pulls" -f title="bench $run" -f head="bench/$run" -f base="$base" \
+  -F body=@"$dir/pr-body.md" --jq '{number, url: .html_url}' >"$dir/pr.json"
+echo "$run $(cat "$dir/pr.json")"
diff --git a/bench/run.sh b/bench/run.sh
new file mode 100755
index 0000000..b5ed183
--- /dev/null
+++ b/bench/run.sh
@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+# One run of one arm on one repository of bench/selection.json.
+#
+#   BENCH_OUT=<dir outside this repo> bench/run.sh <claude|uptide|human> <id>
+#
+# Both arms start from the same fresh clone of the parent commit, with installs' lifecycle
+# scripts off and pushes to the original repository disabled. `human` checks out the human's
+# upgrade commit instead: the reference every score is measured against.
+# The run gets an opaque id; $BENCH_OUT/key.tsv is the only place its arm is written, so
+# check.sh, judge.sh and score.mjs never see it.
+set -euo pipefail
+arm=$1 id=$2
+here=$(cd "$(dirname "$0")" && pwd)
+out=${BENCH_OUT:?set BENCH_OUT to a directory outside this repository}
+sel() { jq -er --arg id "$id" ".repos[] | select(.id == \$id) | $1" "$here/selection.json"; }
+repo=$(sel .repo) parent=$(sel .parent) upgrade=$(sel .upgrade) pkg=$(sel .package)
+from=$(sel .from) to=$(sel .to)
+# Uptide's default executor model and effort (packages/core/src/llm/pricing.ts DEFAULT_MODELS,
+# capabilities.ts defaultEffort), used by both arms.
+model=claude-sonnet-5-5 effort=medium
+
+run=$(uuidgen | tr '[:upper:]' '[:lower:]' | cut -c1-8)
+dir=$out/runs/$run
+mkdir -p "$dir"
+printf '%s\t%s\t%s\n' "$run" "$id" "$arm" >>"$out/key.tsv"
+git clone -q "https://github.com/$repo.git" "$dir/repo"
+cd "$dir/repo"
+git checkout -q -b bench-base "$parent"
+git remote set-url --push origin no-push
+export npm_config_ignore_scripts=true YARN_ENABLE_SCRIPTS=0
+
+start=$(date +%s) status=0 cost=0
+case $arm in
+claude)
+  prompt="Upgrade $pkg from $from to $to. Make the code compile and work with the new version, keep behavior correct, and make sure type-check, lint and tests pass. Commit and summarize what to review."
+  # --bare: none of this machine's hooks, plugins, memory or CLAUDE.md; --add-dir loads the
+  # repository's own CLAUDE.md as a plain `claude -p` in it would. Bash runs only inside
+  # Claude Code's sandbox (claude-sandbox.json); anything that would prompt is denied.
+  claude -p --bare --add-dir "$PWD" --model "$model" --effort "$effort" \
+    --settings "$here/claude-sandbox.json" --permission-mode acceptEdits --permission-prompts none \
+    --no-session-persistence --output-format json --max-budget-usd "${CLAUDE_CAP_USD:-6}" \
+    "$prompt" >"$dir/agent.json" 2>"$dir/agent.log" || status=$?
+  cost=$(jq '.total_cost_usd // 0' "$dir/agent.json" 2>/dev/null || echo 0)
+  jq -r '.result // ""' "$dir/agent.json" >"$dir/summary.md" 2>/dev/null || : >"$dir/summary.md"
+  ;;
+uptide)
+  UPTIDE_MODEL=$model node "$here/../packages/cli/dist/index.js" fix "$pkg" --target "$to" \
+    --json >"$dir/agent.json" 2>"$dir/agent.log" || status=$?
+  branch=$(git for-each-ref --format='%(refname:short)' "refs/heads/uptide/$pkg-*" | head -1)
+  [ -n "$branch" ] && git checkout -q "$branch"
+  [ -f .uptide/report.json ] && cost=$(jq '.llm.costUsd // 0' .uptide/report.json)
+  cp .uptide/pr-body.md "$dir/summary.md" 2>/dev/null || : >"$dir/summary.md"
+  ;;
+human)
+  git checkout -q "$upgrade"
+  git log -1 --format=%B >"$dir/summary.md"
+  ;;
+*) echo "unknown arm $arm" >&2 && exit 2 ;;
+esac
+wall=$(($(date +%s) - start))
+dirty=$(git status --porcelain --untracked-files=no -- . ':!.uptide' | wc -l | tr -d ' ')
+jq -n --arg id "$id" --arg head "$(git rev-parse HEAD)" --argjson status "$status" \
+  --argjson cost "$cost" --argjson wall "$wall" --argjson dirty "$dirty" \
+  '{id: $id, head: $head, exit: $status, costUsd: $cost, wallSeconds: $wall, uncommittedFiles: $dirty}' \
+  >"$dir/meta.json"
+echo "$run $(cat "$dir/meta.json")"
diff --git a/bench/score.mjs b/bench/score.mjs
new file mode 100644
index 0000000..f490914
--- /dev/null
+++ b/bench/score.mjs
@@ -0,0 +1,150 @@
+// Scores every run in $BENCH_OUT/runs the same way, blind to arm: it never reads key.tsv.
+// The reference of a repository is the run whose head is the human's upgrade commit.
+//
+//   BENCH_OUT=<dir> node bench/score.mjs             writes $BENCH_OUT/scores.json
+//   BENCH_OUT=<dir> node bench/score.mjs --unblind   then prints the table with arms joined in
+//
+// Measures, on code files only (lockfiles, package.json and docs left out):
+// - coverage: human hunks (git diff -U0 parent..upgrade) whose old lines the run also changed;
+//   a pure insertion counts as the two lines around it. Missed hunks are the candidates for
+//   escaped regressions, read by hand.
+// - unrelated: run hunks in files the human left alone that never name the package.
+// - regressions: local checks and CI checks that fail on the run and pass on the reference.
+// - unverified: whether the run's summary or PR body says what it did not verify.
+import { execFileSync } from 'node:child_process';
+import { existsSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+const CODE = /\.(c|m)?(t|j)sx?$/;
+const UNVERIFIED =
+  /not (been )?(verified|tested|checked|run)|did(n't| not) (run|verify|test|check)|could(n't| not) (run|verify|test)|unable to|untested|unverified|manual(ly)? (review|check|test)|needs? (manual|human)|not covered|wasn't able/i;
+
+const read = (file, fallback) =>
+  existsSync(file) ? JSON.parse(readFileSync(file, 'utf8')) : fallback;
+const git = (repo, ...args) =>
+  execFileSync('git', ['-C', repo, ...args], { encoding: 'utf8', maxBuffer: 1 << 28 });
+
+/** Hunks of `git diff -U0 a b` on code files: file, old start, old count. */
+export function hunks(diff) {
+  const found = [];
+  let file;
+  for (const line of diff.split('\n')) {
+    const header = /^diff --git a\/(.+) b\//.exec(line);
+    if (header) file = CODE.test(header[1]) ? header[1] : undefined;
+    const hunk = /^@@ -(\d+)(?:,(\d+))? /.exec(line);
+    if (hunk && file)
+      found.push({
+        file,
+        start: Number(hunk[1]),
+        count: hunk[2] === undefined ? 1 : Number(hunk[2]),
+      });
+  }
+  return found;
+}
+/** Old-side lines a hunk touches; an insertion after line n touches n and n + 1. */
+export const lines = (h) =>
+  h.count === 0 ? [h.start, h.start + 1] : Array.from({ length: h.count }, (_, i) => h.start + i);
+
+export function compare(human, agent) {
+  const touched = new Set(agent.flatMap((h) => lines(h).map((l) => `${h.file}:${l}`)));
+  const missed = human.filter((h) => !lines(h).some((l) => touched.has(`${h.file}:${l}`)));
+  return { covered: human.length - missed.length, missed };
+}
+
+function score(out, selection, run) {
+  const dir = join(out, 'runs', run);
+  const meta = read(join(dir, 'meta.json'));
+  const entry = selection.repos.find((r) => r.id === meta.id);
+  const repo = join(dir, 'repo');
+  const human = hunks(git(repo, 'diff', '-U0', '--no-renames', entry.parent, entry.upgrade));
+  const agent = hunks(git(repo, 'diff', '-U0', '--no-renames', entry.parent, meta.head));
+  const { covered, missed } = compare(human, agent);
+  const humanFiles = new Set(human.map((h) => h.file));
+  const names = new RegExp(`['"]${entry.package.replace(/[/.]/g, '\\$&')}(/[^'"]*)?['"]`);
+  const unrelated = agent.filter((h) => {
+    if (humanFiles.has(h.file)) return false;
+    try {
+      return !names.test(git(repo, 'show', `${entry.parent}:${h.file}`));
+    } catch {
+      return false; // a file the run created
+    }
+  });
+  const summaryFile = join(dir, 'summary.md');
+  const summary = existsSync(summaryFile) ? readFileSync(summaryFile, 'utf8') : '';
+  return {
+    run,
+    id: meta.id,
+    head: meta.head,
+    reference: meta.head === entry.upgrade,
+    exit: meta.exit,
+    costUsd: meta.costUsd,
+    wallSeconds: meta.wallSeconds,
+    uncommittedFiles: meta.uncommittedFiles,
+    humanHunks: human.length,
+    covered,
+    missed: missed.map((h) => `${h.file}:${h.start}`),
+    agentHunks: agent.length,
+    agentFiles: new Set(agent.map((h) => h.file)).size,
+    unrelated: unrelated.map((h) => `${h.file}:${h.start}`),
+    unverified: UNVERIFIED.exec(summary)?.[0] ?? null,
+    checks: read(join(dir, 'checks.json'), {}),
+    ci: read(join(dir, 'ci.json'), []).map((c) => ({
+      name: `${c.workflow}/${c.name}`,
+      bucket: c.bucket,
+    })),
+  };
+}
+
+export function regressions(row, reference) {
+  if (!reference || row.reference) return [];
+  const local = Object.entries(row.checks)
+    .filter(([step, code]) => code !== 0 && reference.checks[step] === 0)
+    .map(([step]) => `local ${step}`);
+  const passed = new Set(reference.ci.filter((c) => c.bucket === 'pass').map((c) => c.name));
+  const ci = row.ci
+    .filter((c) => c.bucket === 'fail' && passed.has(c.name))
+    .map((c) => `ci ${c.name}`);
+  return [...local, ...ci];
+}
+
+function table(rows, arms) {
+  const ci = (r) => {
+    const n = (b) => r.ci.filter((c) => c.bucket === b).length;
+    return r.ci.length ? `${n('pass')} pass, ${n('fail')} fail` : 'none';
+  };
+  const checks = (r) =>
+    Object.entries(r.checks)
+      .map(([s, c]) => `${s} ${c === 0 ? 'ok' : 'FAIL'}`)
+      .join(', ') || 'not run';
+  const lines = [
+    '| repo | arm | exit | coverage | missed | unrelated | regressions vs human | states unverified | local checks | fork CI | cost | wall |',
+    '|---|---|---|---|---|---|---|---|---|---|---|---|',
+  ];
+  const sorted = [...rows].sort(
+    (a, b) => a.id.localeCompare(b.id) || arms[a.run].localeCompare(arms[b.run]),
+  );
+  for (const r of sorted)
+    lines.push(
+      `| ${r.id} | ${arms[r.run]} | ${r.exit} | ${r.covered}/${r.humanHunks} | ${r.missed.length} | ${r.unrelated.length} | ${r.regressions.join(', ') || 'none'} | ${r.unverified ? 'yes' : 'no'} | ${checks(r)} | ${ci(r)} | $${Number(r.costUsd).toFixed(2)} | ${Math.round(r.wallSeconds / 60)} min |`,
+    );
+  return lines.join('\n');
+}
+
+if (process.argv[1] === new URL(import.meta.url).pathname) {
+  const out = process.env.BENCH_OUT;
+  if (!out) throw new Error('set BENCH_OUT');
+  const selection = JSON.parse(readFileSync(new URL('./selection.json', import.meta.url), 'utf8'));
+  const rows = readdirSync(join(out, 'runs')).map((run) => score(out, selection, run));
+  for (const row of rows)
+    row.regressions = regressions(
+      row,
+      rows.find((r) => r.reference && r.id === row.id),
+    );
+  writeFileSync(join(out, 'scores.json'), `${JSON.stringify(rows, null, 2)}\n`);
+  if (process.argv.includes('--unblind')) {
+    const key = readFileSync(join(out, 'key.tsv'), 'utf8').trim().split('\n');
+    console.log(
+      table(rows, Object.fromEntries(key.map((l) => [l.split('\t')[0], l.split('\t')[2]]))),
+    );
+  }
+}
diff --git a/bench/score.test.mjs b/bench/score.test.mjs
new file mode 100644
index 0000000..ef0e8cc
--- /dev/null
+++ b/bench/score.test.mjs
@@ -0,0 +1,41 @@
+import assert from 'node:assert/strict';
+import { test } from 'node:test';
+import { compare, hunks, regressions } from './score.mjs';
+
+const diff = (...parts) => parts.join('\n');
+const human = hunks(
+  diff(
+    'diff --git a/src/a.ts b/src/a.ts',
+    '@@ -3 +3 @@',
+    '@@ -10,0 +11,2 @@',
+    'diff --git a/package-lock.json b/package-lock.json',
+    '@@ -1,4 +1,4 @@',
+    'diff --git a/src/b.tsx b/src/b.tsx',
+    '@@ -7,2 +7,0 @@',
+  ),
+);
+
+test('code hunks only, with old start and count', () => {
+  assert.deepEqual(human, [
+    { file: 'src/a.ts', start: 3, count: 1 },
+    { file: 'src/a.ts', start: 10, count: 0 },
+    { file: 'src/b.tsx', start: 7, count: 2 },
+  ]);
+});
+
+test('a hunk is covered when the run touches one of its old lines', () => {
+  const agent = hunks(diff('diff --git a/src/a.ts b/src/a.ts', '@@ -2,2 +2,2 @@', '@@ -11 +11 @@'));
+  const { covered, missed } = compare(human, agent);
+  assert.equal(covered, 2); // line 3, and the insertion after 10 by touching 11
+  assert.deepEqual(missed, [{ file: 'src/b.tsx', start: 7, count: 2 }]);
+});
+
+test('regressions are failures the reference does not have', () => {
+  const reference = {
+    reference: true,
+    checks: { test: 0, lint: 1 },
+    ci: [{ name: 'ci/build', bucket: 'pass' }],
+  };
+  const row = { checks: { test: 1, lint: 1 }, ci: [{ name: 'ci/build', bucket: 'fail' }] };
+  assert.deepEqual(regressions(row, reference), ['local test', 'ci ci/build']);
+});

From e38f672b080dc7fc5d2b08714ee894d003d1dcb8 Mon Sep 17 00:00:00 2001
From: Lucas Braum <lucasbraum@gmail.com>
Date: Thu, 15 Oct 2026 22:04:54 -0700
Subject: [PATCH 3/7] bench: let the sandboxed Claude Code arm commit, price
 both arms alike (UPT-24)

Probes showed headless Claude Code denying git commit and compound
commands that need approval; every Bash call already runs inside the OS
sandbox, so all Bash is allowed. Claude Code reports an unknown model's
cost at older rates, so its tokens are priced with Uptide's table.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Lucas Braum <lucasbraum@gmail.com>
---
 bench/claude-sandbox.json | 16 ++++++++++++++--
 bench/run.sh              | 10 +++++++++-
 2 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/bench/claude-sandbox.json b/bench/claude-sandbox.json
index ef0f97c..9ca295e 100644
--- a/bench/claude-sandbox.json
+++ b/bench/claude-sandbox.json
@@ -5,7 +5,12 @@
     "allowUnsandboxedCommands": false,
     "autoAllowBashIfSandboxed": true,
     "filesystem": {
-      "allowWrite": ["~/.npm", "~/Library/pnpm", "~/Library/Caches", "~/.cache"]
+      "allowWrite": [
+        "~/.npm",
+        "~/Library/pnpm",
+        "~/Library/Caches",
+        "~/.cache"
+      ]
     },
     "network": {
       "allowedDomains": [
@@ -20,6 +25,13 @@
     }
   },
   "permissions": {
-    "deny": ["Bash(dangerouslyDisableSandbox:true)", "WebFetch", "WebSearch"]
+    "deny": [
+      "Bash(dangerouslyDisableSandbox:true)",
+      "WebFetch",
+      "WebSearch"
+    ],
+    "allow": [
+      "Bash"
+    ]
   }
 }
diff --git a/bench/run.sh b/bench/run.sh
index b5ed183..c7c3872 100755
--- a/bench/run.sh
+++ b/bench/run.sh
@@ -27,6 +27,7 @@ git clone -q "https://github.com/$repo.git" "$dir/repo"
 cd "$dir/repo"
 git checkout -q -b bench-base "$parent"
 git remote set-url --push origin no-push
+git config user.name "UpgradeBench" && git config user.email bench@uptide.invalid
 export npm_config_ignore_scripts=true YARN_ENABLE_SCRIPTS=0
 
 start=$(date +%s) status=0 cost=0
@@ -40,7 +41,14 @@ claude)
     --settings "$here/claude-sandbox.json" --permission-mode acceptEdits --permission-prompts none \
     --no-session-persistence --output-format json --max-budget-usd "${CLAUDE_CAP_USD:-6}" \
     "$prompt" >"$dir/agent.json" 2>"$dir/agent.log" || status=$?
-  cost=$(jq '.total_cost_usd // 0' "$dir/agent.json" 2>/dev/null || echo 0)
+  # Claude Code does not know this model's price and reports it at an older Sonnet's; price its
+  # tokens with Uptide's table (pricing.ts, per million: input, output, cache read, 5m write),
+  # the one Uptide's own cost uses.
+  cost=$(jq '[.modelUsage // {} | to_entries[]
+    | (if (.key | test("haiku")) then [1, 5, 0.1, 1.25] else [2, 10, 0.2, 2.5] end) as $p
+    | (.value.inputTokens * $p[0] + .value.outputTokens * $p[1]
+      + .value.cacheReadInputTokens * $p[2] + .value.cacheCreationInputTokens * $p[3]) / 1e6]
+    | add // 0' "$dir/agent.json" 2>/dev/null || echo 0)
   jq -r '.result // ""' "$dir/agent.json" >"$dir/summary.md" 2>/dev/null || : >"$dir/summary.md"
   ;;
 uptide)

From f0cf19a9513e31850a38c7200f3d81e61548870f Mon Sep 17 00:00:00 2001
From: Lucas Braum <lucasbraum@gmail.com>
Date: Thu, 15 Oct 2026 22:05:11 -0700
Subject: [PATCH 4/7] bench: format the sandbox settings (UPT-24)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Lucas Braum <lucasbraum@gmail.com>
---
 bench/claude-sandbox.json | 17 +++--------------
 1 file changed, 3 insertions(+), 14 deletions(-)

diff --git a/bench/claude-sandbox.json b/bench/claude-sandbox.json
index 9ca295e..b30dd8f 100644
--- a/bench/claude-sandbox.json
+++ b/bench/claude-sandbox.json
@@ -5,12 +5,7 @@
     "allowUnsandboxedCommands": false,
     "autoAllowBashIfSandboxed": true,
     "filesystem": {
-      "allowWrite": [
-        "~/.npm",
-        "~/Library/pnpm",
-        "~/Library/Caches",
-        "~/.cache"
-      ]
+      "allowWrite": ["~/.npm", "~/Library/pnpm", "~/Library/Caches", "~/.cache"]
     },
     "network": {
       "allowedDomains": [
@@ -25,13 +20,7 @@
     }
   },
   "permissions": {
-    "deny": [
-      "Bash(dangerouslyDisableSandbox:true)",
-      "WebFetch",
-      "WebSearch"
-    ],
-    "allow": [
-      "Bash"
-    ]
+    "deny": ["Bash(dangerouslyDisableSandbox:true)", "WebFetch", "WebSearch"],
+    "allow": ["Bash"]
   }
 }

From 0c2a341d3c80a45472cb5767b2c949e213db3041 Mon Sep 17 00:00:00 2001
From: Lucas Braum <lucasbraum@gmail.com>
Date: Thu, 15 Oct 2026 22:07:38 -0700
Subject: [PATCH 5/7] bench: install the parent before either arm, allow local
 ports in the sandbox (UPT-24)

The first run showed Uptide refusing an uninstalled checkout (a
developer's is installed) and the sandbox failing supertest's local
listen, so Claude Code could not run tests Uptide runs.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Lucas Braum <lucasbraum@gmail.com>
---
 bench/claude-sandbox.json | 3 ++-
 bench/run.sh              | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/bench/claude-sandbox.json b/bench/claude-sandbox.json
index b30dd8f..9914615 100644
--- a/bench/claude-sandbox.json
+++ b/bench/claude-sandbox.json
@@ -16,7 +16,8 @@
         "codeload.github.com",
         "objects.githubusercontent.com"
       ],
-      "strictAllowlist": true
+      "strictAllowlist": true,
+      "allowLocalBinding": true
     }
   },
   "permissions": {
diff --git a/bench/run.sh b/bench/run.sh
index c7c3872..063824e 100755
--- a/bench/run.sh
+++ b/bench/run.sh
@@ -29,6 +29,8 @@ git checkout -q -b bench-base "$parent"
 git remote set-url --push origin no-push
 git config user.name "UpgradeBench" && git config user.email bench@uptide.invalid
 export npm_config_ignore_scripts=true YARN_ENABLE_SCRIPTS=0
+# A developer's checkout is installed; Uptide refuses one that is not. Untimed, both arms.
+bash -c "$(sel .checks.install)" >"$dir/setup.log" 2>&1
 
 start=$(date +%s) status=0 cost=0
 case $arm in

From 48aa022c260595b7217bfcf14670dc73968edfc3 Mon Sep 17 00:00:00 2001
From: Lucas Braum <lucasbraum@gmail.com>
Date: Thu, 15 Oct 2026 22:19:42 -0700
Subject: [PATCH 6/7] bench: read Uptide's cost and PR body from its JSON, skip
 PRs for runs without commits (UPT-24)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Lucas Braum <lucasbraum@gmail.com>
---
 bench/judge.sh | 6 ++++++
 bench/run.sh   | 7 ++++---
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/bench/judge.sh b/bench/judge.sh
index 2820827..6cf8f06 100755
--- a/bench/judge.sh
+++ b/bench/judge.sh
@@ -18,12 +18,18 @@ fork=$org/${repo%%/*}--${repo##*/}
 
 if [ "${2:-}" = --collect ]; then
   pr=$(jq -r .number "$dir/pr.json")
+  [ "$pr" = null ] && exit 0
   gh pr checks "$pr" --repo "$fork" --watch --interval 60 >/dev/null 2>&1 || :
   gh pr checks "$pr" --repo "$fork" --json name,workflow,state,bucket >"$dir/ci.json" || :
   echo "$run $(jq -c 'group_by(.bucket) | map({(.[0].bucket): length}) | add' "$dir/ci.json")"
   exit 0
 fi
 
+if [ "$head" = "$parent" ]; then
+  echo '{"number": null, "skipped": "no commits"}' >"$dir/pr.json"
+  echo "$run no commits, no pull request"
+  exit 0
+fi
 if ! gh repo view "$fork" >/dev/null 2>&1; then
   gh repo fork "$repo" --org "$org" --fork-name "${fork#*/}" --clone=false --default-branch-only
   sleep 5
diff --git a/bench/run.sh b/bench/run.sh
index 063824e..23fb116 100755
--- a/bench/run.sh
+++ b/bench/run.sh
@@ -58,8 +58,9 @@ uptide)
     --json >"$dir/agent.json" 2>"$dir/agent.log" || status=$?
   branch=$(git for-each-ref --format='%(refname:short)' "refs/heads/uptide/$pkg-*" | head -1)
   [ -n "$branch" ] && git checkout -q "$branch"
-  [ -f .uptide/report.json ] && cost=$(jq '.llm.costUsd // 0' .uptide/report.json)
-  cp .uptide/pr-body.md "$dir/summary.md" 2>/dev/null || : >"$dir/summary.md"
+  cost=$(jq '.llm.costUsd // 0' "$dir/agent.json" 2>/dev/null || echo 0)
+  cp "$(jq -r '.prBody // empty' "$dir/agent.json" 2>/dev/null)" "$dir/summary.md" 2>/dev/null ||
+    tail -5 "$dir/agent.log" >"$dir/summary.md"
   ;;
 human)
   git checkout -q "$upgrade"
@@ -67,7 +68,7 @@ human)
   ;;
 *) echo "unknown arm $arm" >&2 && exit 2 ;;
 esac
-wall=$(($(date +%s) - start))
+wall=$(($(date +%s) - start)) cost=${cost:-0}
 dirty=$(git status --porcelain --untracked-files=no -- . ':!.uptide' | wc -l | tr -d ' ')
 jq -n --arg id "$id" --arg head "$(git rev-parse HEAD)" --argjson status "$status" \
   --argjson cost "$cost" --argjson wall "$wall" --argjson dirty "$dirty" \

From 339e306bba1e7001ad1412dfe45ab513a1b28cd3 Mon Sep 17 00:00:00 2001
From: Lucas Braum <lucasbraum@gmail.com>
Date: Sun, 18 Oct 2026 17:14:44 -0700
Subject: [PATCH 7/7] bench: score silent changes caught and silent false
 positives (UPT-114)

selection.json gains calmcp's five `silent` sites, the inputSchema lines where the human's
upgrade added .strict() so tools/list keeps advertising additionalProperties: false (ADR 0017).
score.mjs counts a site as caught when the run's diff touches its line and the file at the
run's head has the keep edit, or when the PR body names the record; a site a PR body's
"Behavior change (<record>)" line names that the selection lacks is a silent false positive.
Both arms are scored by the same two columns.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Lucas Braum <lucasbraum@gmail.com>
---
 bench/score.mjs      | 44 +++++++++++++++++++++++++++++++++++++++++---
 bench/score.test.mjs | 34 +++++++++++++++++++++++++++++++++-
 bench/selection.json | 32 ++++++++++++++++++++++++++++++++
 3 files changed, 106 insertions(+), 4 deletions(-)

diff --git a/bench/score.mjs b/bench/score.mjs
index f490914..3c5ad20 100644
--- a/bench/score.mjs
+++ b/bench/score.mjs
@@ -11,6 +11,10 @@
 // - unrelated: run hunks in files the human left alone that never name the package.
 // - regressions: local checks and CI checks that fail on the run and pass on the reference.
 // - unverified: whether the run's summary or PR body says what it did not verify.
+// - silent changes (ADR 0017): the selection's `silent` sites, behavior the upgrade must keep that
+//   no check shows. Caught when the run's diff touches the site's line and the file at the run's
+//   head has the `keep` edit, or when the PR body names the record. A silent false positive is a
+//   site a PR body's "Behavior change (<record>)" line names that the selection does not.
 import { execFileSync } from 'node:child_process';
 import { existsSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
 import { join } from 'node:path';
@@ -51,6 +55,28 @@ export function compare(human, agent) {
   return { covered: human.length - missed.length, missed };
 }
 
+/** Silent sites caught by an edit or by a decision citing the record, and silent false positives. */
+export function silent(entry, agent, summary, headText) {
+  const truth = entry.silent ?? [];
+  const touched = new Set(agent.flatMap((h) => lines(h).map((l) => `${h.file}:${l}`)));
+  const edited = truth.filter(
+    (s) => touched.has(`${s.file}:${s.line}`) && new RegExp(s.keep).test(headText(s.file)),
+  );
+  const decided = truth.filter((s) => !edited.includes(s) && summary.includes(s.record));
+  const known = new Set(truth.map((s) => `${s.record} ${s.file}:${s.line}`));
+  const named = [...summary.matchAll(/\*\*Behavior change \(([^)\s]+)\):\*\*([^\n]*)/g)].flatMap(
+    ([, record, line]) =>
+      [...line.matchAll(/`([^`\s]+):(\d+)`/g)].map(([, file, n]) => `${record} ${file}:${n}`),
+  );
+  const at = (s) => `${s.file}:${s.line}`;
+  return {
+    silentSites: truth.length,
+    silentEdited: edited.map(at),
+    silentDecided: decided.map(at),
+    silentFalsePositives: [...new Set(named.filter((n) => !known.has(n)))],
+  };
+}
+
 function score(out, selection, run) {
   const dir = join(out, 'runs', run);
   const meta = read(join(dir, 'meta.json'));
@@ -71,6 +97,13 @@ function score(out, selection, run) {
   });
   const summaryFile = join(dir, 'summary.md');
   const summary = existsSync(summaryFile) ? readFileSync(summaryFile, 'utf8') : '';
+  const headText = (file) => {
+    try {
+      return git(repo, 'show', `${meta.head}:${file}`);
+    } catch {
+      return '';
+    }
+  };
   return {
     run,
     id: meta.id,
@@ -87,6 +120,7 @@ function score(out, selection, run) {
     agentFiles: new Set(agent.map((h) => h.file)).size,
     unrelated: unrelated.map((h) => `${h.file}:${h.start}`),
     unverified: UNVERIFIED.exec(summary)?.[0] ?? null,
+    ...silent(entry, agent, summary, headText),
     checks: read(join(dir, 'checks.json'), {}),
     ci: read(join(dir, 'ci.json'), []).map((c) => ({
       name: `${c.workflow}/${c.name}`,
@@ -117,15 +151,19 @@ function table(rows, arms) {
       .map(([s, c]) => `${s} ${c === 0 ? 'ok' : 'FAIL'}`)
       .join(', ') || 'not run';
   const lines = [
-    '| repo | arm | exit | coverage | missed | unrelated | regressions vs human | states unverified | local checks | fork CI | cost | wall |',
-    '|---|---|---|---|---|---|---|---|---|---|---|---|',
+    '| repo | arm | exit | coverage | missed | unrelated | regressions vs human | silent changes caught | silent FP | states unverified | local checks | fork CI | cost | wall |',
+    '|---|---|---|---|---|---|---|---|---|---|---|---|---|---|',
   ];
+  const caught = (r) =>
+    r.silentSites
+      ? `${r.silentEdited.length + r.silentDecided.length}/${r.silentSites} (edit ${r.silentEdited.length})`
+      : 'n/a';
   const sorted = [...rows].sort(
     (a, b) => a.id.localeCompare(b.id) || arms[a.run].localeCompare(arms[b.run]),
   );
   for (const r of sorted)
     lines.push(
-      `| ${r.id} | ${arms[r.run]} | ${r.exit} | ${r.covered}/${r.humanHunks} | ${r.missed.length} | ${r.unrelated.length} | ${r.regressions.join(', ') || 'none'} | ${r.unverified ? 'yes' : 'no'} | ${checks(r)} | ${ci(r)} | $${Number(r.costUsd).toFixed(2)} | ${Math.round(r.wallSeconds / 60)} min |`,
+      `| ${r.id} | ${arms[r.run]} | ${r.exit} | ${r.covered}/${r.humanHunks} | ${r.missed.length} | ${r.unrelated.length} | ${r.regressions.join(', ') || 'none'} | ${caught(r)} | ${r.silentFalsePositives.length} | ${r.unverified ? 'yes' : 'no'} | ${checks(r)} | ${ci(r)} | $${Number(r.costUsd).toFixed(2)} | ${Math.round(r.wallSeconds / 60)} min |`,
     );
   return lines.join('\n');
 }
diff --git a/bench/score.test.mjs b/bench/score.test.mjs
index ef0e8cc..cf1ec83 100644
--- a/bench/score.test.mjs
+++ b/bench/score.test.mjs
@@ -1,6 +1,6 @@
 import assert from 'node:assert/strict';
 import { test } from 'node:test';
-import { compare, hunks, regressions } from './score.mjs';
+import { compare, hunks, regressions, silent } from './score.mjs';
 
 const diff = (...parts) => parts.join('\n');
 const human = hunks(
@@ -39,3 +39,35 @@ test('regressions are failures the reference does not have', () => {
   const row = { checks: { test: 1, lint: 1 }, ci: [{ name: 'ci/build', bucket: 'fail' }] };
   assert.deepEqual(regressions(row, reference), ['local test', 'ci ci/build']);
 });
+
+test('silent changes: caught by the keep edit at the site or a decision naming the record; a named site the selection lacks is a false positive', () => {
+  const record = 'zod/mcp-input-schema-additional-properties';
+  const entry = {
+    silent: [69, 83, 102].map((line) => ({
+      file: 'src/tools/index.ts',
+      line,
+      record,
+      keep: '\\.strict\\(\\)|strictObject\\(',
+    })),
+  };
+  const agent = hunks(
+    diff('diff --git a/src/tools/index.ts b/src/tools/index.ts', '@@ -69 +69 @@'),
+  );
+  const head = () => 'function strictInput(shape) { return z.object(shape).strict(); }';
+  assert.deepEqual(silent(entry, agent, '', head), {
+    silentSites: 3,
+    silentEdited: ['src/tools/index.ts:69'],
+    silentDecided: [],
+    silentFalsePositives: [],
+  });
+  // The line touched without the edit is not caught.
+  assert.deepEqual(silent(entry, agent, '', () => 'z.object(shape)').silentEdited, []);
+  const body = `- **Behavior change (${record}):** … Not applied at \`src/tools/index.ts:83\`, \`src/other.ts:4\`: recommended`;
+  assert.deepEqual(silent(entry, agent, body, head), {
+    silentSites: 3,
+    silentEdited: ['src/tools/index.ts:69'],
+    silentDecided: ['src/tools/index.ts:83', 'src/tools/index.ts:102'],
+    silentFalsePositives: [`${record} src/other.ts:4`],
+  });
+  assert.equal(silent({}, agent, body, head).silentSites, 0);
+});
diff --git a/bench/selection.json b/bench/selection.json
index 9f5aef4..83a1e61 100644
--- a/bench/selection.json
+++ b/bench/selection.json
@@ -12,6 +12,38 @@
       "to": "4.6.5",
       "parent": "741de4eba98eb96b4bed817d8974e9e8c6eec867",
       "upgrade": "bdaef39c34705b2dfc1065a59ec683288a2e99d2",
+      "silent": [
+        {
+          "file": "src/tools/index.ts",
+          "line": 69,
+          "record": "zod/mcp-input-schema-additional-properties",
+          "keep": "\\.strict\\(\\)|strictObject\\("
+        },
+        {
+          "file": "src/tools/index.ts",
+          "line": 83,
+          "record": "zod/mcp-input-schema-additional-properties",
+          "keep": "\\.strict\\(\\)|strictObject\\("
+        },
+        {
+          "file": "src/tools/index.ts",
+          "line": 102,
+          "record": "zod/mcp-input-schema-additional-properties",
+          "keep": "\\.strict\\(\\)|strictObject\\("
+        },
+        {
+          "file": "src/tools/index.ts",
+          "line": 119,
+          "record": "zod/mcp-input-schema-additional-properties",
+          "keep": "\\.strict\\(\\)|strictObject\\("
+        },
+        {
+          "file": "src/tools/index.ts",
+          "line": 148,
+          "record": "zod/mcp-input-schema-additional-properties",
+          "keep": "\\.strict\\(\\)|strictObject\\("
+        }
+      ],
       "note": "npm, single package. The upgrade (merged in PR #22) changes 3 source files, a test and a script: z.string().uuid() to z.guid(), .strict() tool schemas, a _def walker rewritten for zod 4. Mostly behavior, little a compiler sees. The commit carries a Co-Authored-By: Claude trailer.",
       "ci": ".github/workflows/ci.yml: build (lint, typecheck, test, build, test:e2e), btp-descriptors, docker; tool-surface.yml",
       "checks": {