From 130d5248f91658956a8d3f49bd497b7b91f2d3d3 Mon Sep 17 00:00:00 2001
From: jerilynzheng <zheng.jerilyn@gmail.com>
Date: Tue, 14 Apr 2026 12:36:00 -0700
Subject: [PATCH] Document how to run evals for unreleased models

Adds README section explaining how to build a patched OpenCode binary,
upload it, and reference it via agentOptions.binaryUrl in experiment
configs. Also documents extraProviders for custom provider configurations.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 README.md | 91 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 91 insertions(+)

diff --git a/README.md b/README.md
index e29820583..b7d1fb7d4 100644
--- a/README.md
+++ b/README.md
@@ -69,6 +69,97 @@ evals/agent-031-proxy-middleware/
 2. Add the display name to `MODEL_NAMES` in `scripts/export-results.ts`
 3. Run `npm run eval` — it will automatically run all evals for the new model
 
+### Unreleased models (OpenCode)
+
+If the model isn't yet available in OpenCode (i.e., not listed on [models.dev](https://models.dev)), you need a patched OpenCode binary with the model baked in. OpenCode resolves models from models.dev at **build time** via a macro, so runtime config alone won't work.
+
+**1. Create a patched models.dev JSON**
+
+Fetch the current models.dev API and add your model to **both** the provider's entry and the `vercel` provider entry (since we route through Vercel AI Gateway):
+
+```bash
+curl -s https://models.dev/api.json > /tmp/models-dev-patched.json
+```
+
+Then edit the JSON to add your model. For example, to add `moonshotai/kimi-k2.6-preview`:
+- Add a `kimi-k2.6-preview` entry under `moonshotai.models` (copy from `kimi-k2.5`)
+- Add a `moonshotai/kimi-k2.6-preview` entry under `vercel.models` (copy from `moonshotai/kimi-k2.5`)
+
+Both entries are required — the `moonshotai` entry defines the model, and the `vercel` entry makes it available through AI Gateway routing.
+
+**2. Check temperature/transform requirements**
+
+Some models require specific temperature settings. Check `packages/opencode/src/provider/transform.ts` in the opencode repo — if your model needs `temperature: 1` (common for reasoning models), add it to the temperature function:
+
+```typescript
+// packages/opencode/src/provider/transform.ts
+if (id.includes("kimi-k2")) {
+  if (id.includes("thinking") || id.includes("k2.6")) return 1.0
+  return 0.6
+}
+```
+
+If unsure, test first — a temperature error like `"invalid temperature: only 1 is allowed"` will tell you.
+
+**3. Build the binary**
+
+The build script enforces a specific Bun version (check `packageManager` in root `package.json`):
+
+```bash
+cd opencode/packages/opencode
+MODELS_DEV_API_JSON=/tmp/models-dev-patched.json bun run script/build.ts
+```
+
+This builds all platforms. The Vercel sandbox needs `dist/opencode-linux-x64/bin/opencode`.
+
+**4. Upload to Vercel Blob Storage**
+
+```bash
+BLOB_READ_WRITE_TOKEN="..." vercel blob put dist/opencode-linux-x64/bin/opencode \
+  --pathname opencode-linux-x64-<model-name> --access public
+```
+
+The token is in the `next-evals` Vercel project env vars. Pull with `vercel env pull`.
+
+**5. Create experiment config**
+
+```typescript
+import type { ExperimentConfig } from '@vercel/agent-eval';
+
+const config: ExperimentConfig = {
+  agent: 'vercel-ai-gateway/opencode',
+  model: 'vercel/moonshotai/kimi-k2.6-preview',
+  agentOptions: {
+    binaryUrl: 'https://ymdea60kblwwhidh.public.blob.vercel-storage.com/opencode-linux-x64-<model-name>',
+  },
+  scripts: ['build'],
+  runs: 4,
+  earlyExit: true,
+  timeout: 720,
+  sandbox: 'vercel',
+  setup: async (sandbox) => {
+    await sandbox.runCommand('npm', ['install', 'next@canary']);
+  },
+};
+
+export default config;
+```
+
+**6. Smoke test, then full run**
+
+```bash
+npm run eval <config-name> -- --smoke   # verify 1 eval works
+npm run eval <config-name>              # full run
+```
+
+**Troubleshooting**
+
+- **403 error at 0.2s**: `VERCEL_TOKEN` has expired SAML session. Re-authenticate with `vercel login --scope ai-gateway-early-access-models`, then create a new token at https://vercel.com/account/tokens with the correct team scope.
+- **"terminated" with no output**: OOM in sandbox. The `agent-eval` harness automatically requests 2 vCPUs (4GB RAM) when `binaryUrl` is set. If still OOM, the model's output may be too large.
+- **"invalid temperature"**: Add a temperature override in `transform.ts` and rebuild the binary.
+- **Zod validation error / "expected object, received undefined"**: Model is missing from the `vercel` provider in the patched models.dev JSON. Make sure it's added under both the model's native provider AND the `vercel` provider.
+- **"ProviderModelNotFoundError"**: The model isn't in the binary's baked-in models list. Verify your patched JSON includes the model and that `MODELS_DEV_API_JSON` was set during the build.
+
 ## Publishing to nextjs.org/evals
 
 After running evals: