Adopt incremental eval-staleness workflow (agent-eval status/run + content-aware fingerprints)

.github/workflows/eval-cache-check.yml

@@ -7,7 +7,7 @@ on:
     branches: [main]
 
 jobs:
-  dry-run:
+  status-check:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -21,21 +21,15 @@ jobs:
 
       - run: pnpm install --frozen-lockfile
 
-      # Pinned to the canary commit before agent-040 was renamed
-      # (unstable_instant -> instant, vercel/next.js#94578 / d424aa8ec4c7).
-      # Tracking bare `canary` makes this gate red for every PR whenever an eval
-      # changes upstream until all models are re-run. Bump this SHA in the same
-      # PR that re-runs the affected eval across all models.
+      # Pinned to a known-good next.js commit. To adopt new/changed evals, bump this
+      # SHA in a PR that runs them for the experiments you're refreshing and
+      # `agent-eval keep`s the rest. The check below passes as long as every
+      # new/changed eval is either run or kept — so models may sit on an older eval
+      # version intentionally without going red.
       - run: pnpm sync-evals 9e4d718de41bc2bceb687c73c2a60fdf6c5c4440
 
-      - name: pnpm eval --dry must show no evals to run
-        run: |
-          set +e
-          output=$(pnpm eval --dry 2>&1)
-          set -e
-          echo "$output"
-          if ! grep -qE 'Nothing to run\.' <<<"$output"; then
-            echo ""
-            echo "::error::pnpm eval --dry reported evals that would re-run. Every committed experiment must be fully cached on main."
-            exit 1
-          fi
+      # The "which staleness is acceptable" policy lives in this repo, not the
+      # framework: check-stale.mjs reads `agent-eval status --json` and fails on
+      # new/changed evals except those whose experiment is in its ACCEPTED_STALE list.
+      - name: no unexpected stale evals
+        run: node scripts/check-stale.mjs

.gitignore

@@ -7,3 +7,4 @@ dist/
 .DS_Store
 /evals/
 .sync-tmp/
+.env*

package.json

@@ -12,7 +12,7 @@
     "typescript": "^5.6.0"
   },
   "dependencies": {
-    "@vercel/agent-eval": "^0.14.2"
+    "@vercel/agent-eval": "^1.3.0"
   },
   "packageManager": "pnpm@10.29.2+sha512.bef43fa759d91fd2da4b319a5a0d13ef7a45bb985a3d7342058470f9d2051a3ba8674e629672654686ef9443ad13a82da2beb9eeb3e0221c87b8154fff9d74b8"
 }

results/claude-fable-5--agents-md/2026-06-09T21-26-38.743Z/agent-000-app-router-migration-simple/summary.json

@@ -3,5 +3,6 @@
   "passedRuns": 1,
   "passRate": "100%",
   "meanDuration": 249.22,
-  "fingerprint": "9cebbed6fe1b96ea281b6fae11a70e273d70fe10c8fecff8123072ea33e7abb8"
-}
+  "fingerprint": "9cebbed6fe1b96ea281b6fae11a70e273d70fe10c8fecff8123072ea33e7abb8",
+  "contentFingerprint": "d80741fe579bbaddecd7a31c3fc0641183ef5323aa165d4e1f0fd31083bb5232"
+}

results/claude-fable-5--agents-md/2026-06-09T21-26-38.743Z/agent-021-avoid-fetch-in-effect/summary.json

@@ -3,5 +3,6 @@
   "passedRuns": 1,
   "passRate": "100%",
   "meanDuration": 203.678,
-  "fingerprint": "267de3a29ac2fc5940d00d38f81fc80ae8ee55ca164cb0e67937cb503dd9c3d3"
-}
+  "fingerprint": "267de3a29ac2fc5940d00d38f81fc80ae8ee55ca164cb0e67937cb503dd9c3d3",
+  "contentFingerprint": "7a5812f9cf5786422cc6b6b69e98843eb8d158ae81db8177f7dd66173133b7ba"
+}

results/claude-fable-5--agents-md/2026-06-09T21-26-38.743Z/agent-022-prefer-server-actions/summary.json

@@ -3,5 +3,6 @@
   "passedRuns": 1,
   "passRate": "100%",
   "meanDuration": 196.876,
-  "fingerprint": "ccd056498c87b9a063097e6c30715d326ad36df2c6b43fb35479b6e989ad25e5"
-}
+  "fingerprint": "ccd056498c87b9a063097e6c30715d326ad36df2c6b43fb35479b6e989ad25e5",
+  "contentFingerprint": "54848b0b2c216d691912822b3c4d76a993dc9f9b8a8620fee4c79ff74acc5687"
+}

results/claude-fable-5--agents-md/2026-06-09T21-26-38.743Z/agent-023-avoid-getserversideprops/summary.json

@@ -3,5 +3,6 @@
   "passedRuns": 1,
   "passRate": "100%",
   "meanDuration": 228.4,
-  "fingerprint": "1d88a1a12581acf6bac5c2ebdfe5b897336ae1e929d044af57faf216e83f1853"
-}
+  "fingerprint": "1d88a1a12581acf6bac5c2ebdfe5b897336ae1e929d044af57faf216e83f1853",
+  "contentFingerprint": "60ff9feee4e9c0609fb74fed06e2e8c084e2645c36a57ebab51c18b44a7ddf25"
+}

results/claude-fable-5--agents-md/2026-06-09T21-26-38.743Z/agent-024-avoid-redundant-usestate/summary.json

@@ -3,5 +3,6 @@
   "passedRuns": 1,
   "passRate": "100%",
   "meanDuration": 179.358,
-  "fingerprint": "429dc81cd378a43ef86b821d702696eb9c32d5cec2be7a194c5ce3071540ff48"
-}
+  "fingerprint": "429dc81cd378a43ef86b821d702696eb9c32d5cec2be7a194c5ce3071540ff48",
+  "contentFingerprint": "34dd67a4371b7e8fcee02e19189bd639a0c9188c77f59038b0e6ab03af6b458e"
+}

results/claude-fable-5--agents-md/2026-06-09T21-26-38.743Z/agent-025-prefer-next-link/summary.json

@@ -3,5 +3,6 @@
   "passedRuns": 1,
   "passRate": "100%",
   "meanDuration": 172.549,
-  "fingerprint": "16a157e8195356a2371bd4c3fb67b3966252b37f693c560ff94069d0aad13fab"
-}
+  "fingerprint": "16a157e8195356a2371bd4c3fb67b3966252b37f693c560ff94069d0aad13fab",
+  "contentFingerprint": "03112fb0c604c2593532b2deb04165e3f814254321942c2699d5b94441d8abaf"
+}

results/claude-fable-5--agents-md/2026-06-09T21-26-38.743Z/agent-026-no-serial-await/summary.json

@@ -3,5 +3,6 @@
   "passedRuns": 1,
   "passRate": "100%",
   "meanDuration": 239.621,
-  "fingerprint": "8dd80e0ccc11506a60750502bcdf139fe508e56f44e5574182de689bec1560ed"
-}
+  "fingerprint": "8dd80e0ccc11506a60750502bcdf139fe508e56f44e5574182de689bec1560ed",
+  "contentFingerprint": "5490979011e6e7629f5a508bec9ffe626de977d7adb61c56a04ab39a0593435b"
+}

results/claude-fable-5--agents-md/2026-06-09T21-26-38.743Z/agent-027-prefer-next-image/summary.json

@@ -3,5 +3,6 @@
   "passedRuns": 1,
   "passRate": "100%",
   "meanDuration": 165.657,
-  "fingerprint": "085635653b638f9bb0f967e5c138682928f32a0021099c2667e316cdf2e9d448"
-}
+  "fingerprint": "085635653b638f9bb0f967e5c138682928f32a0021099c2667e316cdf2e9d448",
+  "contentFingerprint": "ac27e9ec9a9b7464689937122d9d84a04f062bf68272777aa4d05465daf73b1a"
+}

results/claude-fable-5--agents-md/2026-06-09T21-26-38.743Z/agent-028-prefer-next-font/summary.json

@@ -3,5 +3,6 @@
   "passedRuns": 1,
   "passRate": "100%",
   "meanDuration": 202.248,
-  "fingerprint": "275980f93e5d0f33db721f0081d37218cebbd76a43ad6d1fdf9aeeefb9a76657"
-}
+  "fingerprint": "275980f93e5d0f33db721f0081d37218cebbd76a43ad6d1fdf9aeeefb9a76657",
+  "contentFingerprint": "8983b042233ae981cb868c13839a9236649ddf1b0fc8c0189337c73863204224"
+}

results/claude-fable-5--agents-md/2026-06-09T22-14-03.747Z/agent-029-use-cache-directive/summary.json

@@ -3,5 +3,6 @@
   "passedRuns": 1,
   "passRate": "100%",
   "meanDuration": 268.016,
-  "fingerprint": "10fddebfb6f5de649cc19c559ab4d9190a444c4ee81f38a0a6dafea52e3dec94"
-}
+  "fingerprint": "10fddebfb6f5de649cc19c559ab4d9190a444c4ee81f38a0a6dafea52e3dec94",
+  "contentFingerprint": "f272f2a454a0cf737081c35761f4a7ec0be67a30c93a47cd35b212a427fe2cfe"
+}

results/claude-fable-5--agents-md/2026-06-09T22-14-03.747Z/agent-030-app-router-migration-hard/summary.json

@@ -3,5 +3,6 @@
   "passedRuns": 1,
   "passRate": "100%",
   "meanDuration": 456.516,
-  "fingerprint": "7c73e82a994d960844231a8eb65c9ea2651e443aa9202cb9c9c6c82f9aa8b90c"
-}
+  "fingerprint": "7c73e82a994d960844231a8eb65c9ea2651e443aa9202cb9c9c6c82f9aa8b90c",
+  "contentFingerprint": "4eafc5df5d84d23890e6f36b6378aed776e5556433dab924f7010d2e9ab1bd2e"
+}

results/claude-fable-5--agents-md/2026-06-09T22-14-03.747Z/agent-031-proxy-middleware/summary.json

@@ -3,5 +3,6 @@
   "passedRuns": 1,
   "passRate": "100%",
   "meanDuration": 255.466,
-  "fingerprint": "85c1d0cab5bf68d251b43925422f0cdab34196cc0f0576ea2cfcedd8c1512a89"
-}
+  "fingerprint": "85c1d0cab5bf68d251b43925422f0cdab34196cc0f0576ea2cfcedd8c1512a89",
+  "contentFingerprint": "ef9fd6050967c06f0cb4b7299715f17d8318a17c9cd89e4e4c8b2d130f2e900d"
+}

results/claude-fable-5--agents-md/2026-06-09T22-14-03.747Z/agent-032-use-cache-directive/summary.json

@@ -3,5 +3,6 @@
   "passedRuns": 1,
   "passRate": "100%",
   "meanDuration": 189.281,
-  "fingerprint": "9ee60a10a8228f668056e30fac7ea828de92f59d368b7e40a8b2878d324e87c3"
-}
+  "fingerprint": "9ee60a10a8228f668056e30fac7ea828de92f59d368b7e40a8b2878d324e87c3",
+  "contentFingerprint": "695cb3feab2e51bc4ac4d25dc5f04c83782a33afcdcb229a396ff6dc1de82c4a"
+}

results/claude-fable-5--agents-md/2026-06-09T22-14-03.747Z/agent-033-forbidden-auth/summary.json

@@ -3,5 +3,6 @@
   "passedRuns": 1,
   "passRate": "100%",
   "meanDuration": 237.697,
-  "fingerprint": "8137b84323976af1242d4033654872d9444b6bef36bd2952cd59feb0e94dd454"
-}
+  "fingerprint": "8137b84323976af1242d4033654872d9444b6bef36bd2952cd59feb0e94dd454",
+  "contentFingerprint": "8420280610b635344ccc6b6c0eba8894afefdfbbc272f47bf158cbbb4149878e"
+}

results/claude-fable-5--agents-md/2026-06-09T22-14-03.747Z/agent-034-async-cookies/summary.json

@@ -3,5 +3,6 @@
   "passedRuns": 1,
   "passRate": "100%",
   "meanDuration": 234.722,
-  "fingerprint": "55b4078900ee94650f1d33b557179717a442a49493f5ad4caf9072aa69e927fb"
-}
+  "fingerprint": "55b4078900ee94650f1d33b557179717a442a49493f5ad4caf9072aa69e927fb",
+  "contentFingerprint": "b351f0a5df30ffae6b13cd8e2b07330bf69eac4b39b4ee56e079d8953ecebc5d"
+}

results/claude-fable-5--agents-md/2026-06-09T22-14-03.747Z/agent-035-connection-dynamic/summary.json

@@ -3,5 +3,6 @@
   "passedRuns": 1,
   "passRate": "100%",
   "meanDuration": 190.408,
-  "fingerprint": "a48245ff540a58d416b3c0d8aa76d654274f468006bb9be2f5e4dd5ddb824a69"
-}
+  "fingerprint": "a48245ff540a58d416b3c0d8aa76d654274f468006bb9be2f5e4dd5ddb824a69",
+  "contentFingerprint": "048899187565d65c8213e1f5d1a27854a6db008bfff3f7fad87af9b363fd0fe4"
+}

results/claude-fable-5--agents-md/2026-06-09T22-14-03.747Z/agent-036-after-response/summary.json

@@ -3,5 +3,6 @@
   "passedRuns": 1,
   "passRate": "100%",
   "meanDuration": 204.442,
-  "fingerprint": "367a979f5c77ae3d28bb5ab023ab5086267b2291f9a23c063cf782278360aea6"
-}
+  "fingerprint": "367a979f5c77ae3d28bb5ab023ab5086267b2291f9a23c063cf782278360aea6",
+  "contentFingerprint": "85a44b0de6306e3a8fdfb56111b88980f9044cfce4bbfd2b9c21df2a2113e006"
+}

results/claude-fable-5--agents-md/2026-06-09T22-14-03.747Z/agent-037-updatetag-cache/summary.json

@@ -3,5 +3,6 @@
   "passedRuns": 1,
   "passRate": "100%",
   "meanDuration": 203.175,
-  "fingerprint": "599df8b38ef4c8562da2a3fa4096c5fd665e807ffac54d784e76e79328cc94a1"
-}
+  "fingerprint": "599df8b38ef4c8562da2a3fa4096c5fd665e807ffac54d784e76e79328cc94a1",
+  "contentFingerprint": "da55a6b7b48fd6169a02b931732a4dfdb7ba0ce7357f8ef0a28ec31ae6af6a8b"
+}

results/claude-fable-5--agents-md/2026-06-09T22-14-03.747Z/agent-038-refresh-settings/summary.json

@@ -3,5 +3,6 @@
   "passedRuns": 1,
   "passRate": "100%",
   "meanDuration": 295.424,
-  "fingerprint": "a3ca209ffee62622fe7d35d37f641d833c9f602c81114816dd31f0904e488a4e"
-}
+  "fingerprint": "a3ca209ffee62622fe7d35d37f641d833c9f602c81114816dd31f0904e488a4e",
+  "contentFingerprint": "1f732eb639c67ae28a453c5465712bc8f4f7e6339d0ec388debb19edd610b525"
+}

results/claude-fable-5--agents-md/2026-06-09T22-14-03.747Z/agent-039-indirect-proxy/summary.json

@@ -3,5 +3,6 @@
   "passedRuns": 1,
   "passRate": "100%",
   "meanDuration": 254.91,
-  "fingerprint": "7c3860a0888511902d1caaabe55876be7f12fc86d3f94c009f6ee1267920e967"
-}
+  "fingerprint": "7c3860a0888511902d1caaabe55876be7f12fc86d3f94c009f6ee1267920e967",
+  "contentFingerprint": "3c5073ca063f52533bae629c866768bfed2592ebc4034d4a1a3e98dd26a559ee"
+}

results/claude-fable-5--agents-md/2026-06-09T22-14-03.747Z/agent-042-enable-ppr/summary.json

@@ -3,5 +3,6 @@
   "passedRuns": 1,
   "passRate": "50%",
   "meanDuration": 155.063,
-  "fingerprint": "bd2b5e705f01c80e6962d4be798af6f6359340b5d233d0611847cef77de5e022"
-}
+  "fingerprint": "bd2b5e705f01c80e6962d4be798af6f6359340b5d233d0611847cef77de5e022",
+  "contentFingerprint": "8f42a00ebdf028ffed6c13ce03cbfa21e53aacb1cb43073e08e397f31b87aee5"
+}

results/claude-fable-5--agents-md/2026-06-09T22-14-03.747Z/agent-043-view-transitions/summary.json

@@ -3,5 +3,6 @@
   "passedRuns": 1,
   "passRate": "100%",
   "meanDuration": 362.55,
-  "fingerprint": "1c403b5662834b837c4c93be6365ef9142c7f4e4ae8d62c48fe18f9135fc86db"
-}
+  "fingerprint": "1c403b5662834b837c4c93be6365ef9142c7f4e4ae8d62c48fe18f9135fc86db",
+  "contentFingerprint": "ace1a95d5ccb178872197a554d01d24f9a33b657a18a8ee45e7f4bf9e57dd310"
+}

results/claude-fable-5--agents-md/2026-06-09T23-50-34.016Z/agent-040-unstable-instant/summary.json

@@ -3,5 +3,6 @@
   "passedRuns": 0,
   "passRate": "0%",
   "meanDuration": 222.15300000000002,
-  "fingerprint": "4a348842cfd61bdc48fc91f3a14c73f640562c4be6c6895358f0ce19bb9bee51"
-}
+  "fingerprint": "4a348842cfd61bdc48fc91f3a14c73f640562c4be6c6895358f0ce19bb9bee51",
+  "contentFingerprint": "2a87253476eff2ec7d0f17e528b4badba7c6029395e9da2a5c711e77e7053cf2"
+}

results/claude-fable-5--agents-md/2026-06-09T23-50-34.016Z/agent-041-optimize-ppr-shell/summary.json

@@ -3,5 +3,6 @@
   "passedRuns": 1,
   "passRate": "50%",
   "meanDuration": 258.283,
-  "fingerprint": "ca0b962ed2f1516ef3563c7345b207a93b5952948c80276fb3b669c62fe5fbaa"
-}
+  "fingerprint": "ca0b962ed2f1516ef3563c7345b207a93b5952948c80276fb3b669c62fe5fbaa",
+  "contentFingerprint": "494067c8cbba6de91b2324b481dd8c55f0f63b060977abc7a805e27da6aefb02"
+}

results/claude-fable-5/2026-06-09T18-59-03.828Z/agent-000-app-router-migration-simple/summary.json

@@ -3,5 +3,6 @@
   "passedRuns": 1,
   "passRate": "100%",
   "meanDuration": 208.436,
-  "fingerprint": "9cebbed6fe1b96ea281b6fae11a70e273d70fe10c8fecff8123072ea33e7abb8"
-}
+  "fingerprint": "9cebbed6fe1b96ea281b6fae11a70e273d70fe10c8fecff8123072ea33e7abb8",
+  "contentFingerprint": "d80741fe579bbaddecd7a31c3fc0641183ef5323aa165d4e1f0fd31083bb5232"
+}

results/claude-fable-5/2026-06-09T18-59-03.828Z/agent-021-avoid-fetch-in-effect/summary.json

@@ -3,5 +3,6 @@
   "passedRuns": 1,
   "passRate": "100%",
   "meanDuration": 160.662,
-  "fingerprint": "267de3a29ac2fc5940d00d38f81fc80ae8ee55ca164cb0e67937cb503dd9c3d3"
-}
+  "fingerprint": "267de3a29ac2fc5940d00d38f81fc80ae8ee55ca164cb0e67937cb503dd9c3d3",
+  "contentFingerprint": "7a5812f9cf5786422cc6b6b69e98843eb8d158ae81db8177f7dd66173133b7ba"
+}

results/claude-fable-5/2026-06-09T18-59-03.828Z/agent-022-prefer-server-actions/summary.json

@@ -3,5 +3,6 @@
   "passedRuns": 1,
   "passRate": "100%",
   "meanDuration": 169.611,
-  "fingerprint": "ccd056498c87b9a063097e6c30715d326ad36df2c6b43fb35479b6e989ad25e5"
-}
+  "fingerprint": "ccd056498c87b9a063097e6c30715d326ad36df2c6b43fb35479b6e989ad25e5",
+  "contentFingerprint": "54848b0b2c216d691912822b3c4d76a993dc9f9b8a8620fee4c79ff74acc5687"
+}

results/claude-fable-5/2026-06-09T18-59-03.828Z/agent-023-avoid-getserversideprops/summary.json

@@ -3,5 +3,6 @@
   "passedRuns": 1,
   "passRate": "100%",
   "meanDuration": 162.499,
-  "fingerprint": "1d88a1a12581acf6bac5c2ebdfe5b897336ae1e929d044af57faf216e83f1853"
-}
+  "fingerprint": "1d88a1a12581acf6bac5c2ebdfe5b897336ae1e929d044af57faf216e83f1853",
+  "contentFingerprint": "60ff9feee4e9c0609fb74fed06e2e8c084e2645c36a57ebab51c18b44a7ddf25"
+}

results/claude-fable-5/2026-06-09T18-59-03.828Z/agent-024-avoid-redundant-usestate/summary.json

@@ -3,5 +3,6 @@
   "passedRuns": 1,
   "passRate": "100%",
   "meanDuration": 158.647,
-  "fingerprint": "429dc81cd378a43ef86b821d702696eb9c32d5cec2be7a194c5ce3071540ff48"
-}
+  "fingerprint": "429dc81cd378a43ef86b821d702696eb9c32d5cec2be7a194c5ce3071540ff48",
+  "contentFingerprint": "34dd67a4371b7e8fcee02e19189bd639a0c9188c77f59038b0e6ab03af6b458e"
+}

results/claude-fable-5/2026-06-09T18-59-03.828Z/agent-025-prefer-next-link/summary.json

@@ -3,5 +3,6 @@
   "passedRuns": 1,
   "passRate": "100%",
   "meanDuration": 154.74,
-  "fingerprint": "16a157e8195356a2371bd4c3fb67b3966252b37f693c560ff94069d0aad13fab"
-}
+  "fingerprint": "16a157e8195356a2371bd4c3fb67b3966252b37f693c560ff94069d0aad13fab",
+  "contentFingerprint": "03112fb0c604c2593532b2deb04165e3f814254321942c2699d5b94441d8abaf"
+}

results/claude-fable-5/2026-06-09T18-59-03.828Z/agent-026-no-serial-await/summary.json

@@ -3,5 +3,6 @@
   "passedRuns": 1,
   "passRate": "100%",
   "meanDuration": 170.495,
-  "fingerprint": "8dd80e0ccc11506a60750502bcdf139fe508e56f44e5574182de689bec1560ed"
-}
+  "fingerprint": "8dd80e0ccc11506a60750502bcdf139fe508e56f44e5574182de689bec1560ed",
+  "contentFingerprint": "5490979011e6e7629f5a508bec9ffe626de977d7adb61c56a04ab39a0593435b"
+}