From 0a1007b186bb80d47d771d954415be39159e7dd4 Mon Sep 17 00:00:00 2001 From: Seth Date: Mon, 31 Aug 2026 19:53:29 -0400 Subject: [PATCH 01/13] feat: refresh models from a hosted catalog --- .github/workflows/refresh-model-catalog.yml | 70 +++++ packages/ai/.changes/hosted-model-catalog.md | 1 + packages/ai/package.json | 2 + packages/ai/scripts/export-model-catalog.ts | 13 + packages/ai/scripts/generate-models.ts | 12 + packages/ai/scripts/model-catalog-format.ts | 105 +++++++ packages/ai/scripts/validate-model-catalog.ts | 14 + packages/ai/test/model-catalog-format.test.ts | 47 ++++ .../.changes/hosted-model-catalog.md | 1 + packages/coding-agent/docs/providers.md | 4 +- .../coding-agent/src/core/model-registry.ts | 69 +++-- .../src/core/remote-model-catalog.ts | 262 ++++++++++++++++++ .../test/remote-model-catalog.test.ts | 158 +++++++++++ 13 files changed, 734 insertions(+), 24 deletions(-) create mode 100644 .github/workflows/refresh-model-catalog.yml create mode 100644 packages/ai/.changes/hosted-model-catalog.md create mode 100644 packages/ai/scripts/export-model-catalog.ts create mode 100644 packages/ai/scripts/model-catalog-format.ts create mode 100644 packages/ai/scripts/validate-model-catalog.ts create mode 100644 packages/ai/test/model-catalog-format.test.ts create mode 100644 packages/coding-agent/.changes/hosted-model-catalog.md create mode 100644 packages/coding-agent/src/core/remote-model-catalog.ts create mode 100644 packages/coding-agent/test/remote-model-catalog.test.ts diff --git a/.github/workflows/refresh-model-catalog.yml b/.github/workflows/refresh-model-catalog.yml new file mode 100644 index 0000000000..ce8163b9d7 --- /dev/null +++ b/.github/workflows/refresh-model-catalog.yml @@ -0,0 +1,70 @@ +name: Refresh hosted model catalog + +on: + schedule: + # Daily, off the top of the hour to avoid the cron rush. + - cron: "17 5 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: refresh-hosted-model-catalog + cancel-in-progress: false + +env: + CATALOG_URL: ${{ vars.R2_PUBLIC_BASE_URL }}/model-catalog.json + +jobs: + publish: + name: Generate, validate, and publish + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout default branch + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 22 + cache: npm + + - name: Install dependencies + run: npm ci --ignore-scripts + + - name: Load validation baseline + working-directory: packages/ai + run: | + if ! curl --fail --silent --show-error --location "$CATALOG_URL" --output /tmp/model-catalog-baseline.json; then + npm run export-model-catalog -- /tmp/model-catalog-baseline.json + fi + + - name: Generate aggregate catalog + working-directory: packages/ai + env: + PRIME_AGENT_MODEL_CATALOG_OUTPUT: /tmp/model-catalog.json + run: npm run generate-models + + - name: Validate catalog + working-directory: packages/ai + run: npm run validate-model-catalog -- /tmp/model-catalog.json /tmp/model-catalog-baseline.json + + - name: Publish catalog to R2 + env: + AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + AWS_DEFAULT_REGION: auto + AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} + R2_BUCKET: ${{ secrets.R2_BUCKET }} + R2_ENDPOINT_URL: ${{ secrets.R2_ENDPOINT_URL }} + run: | + test -n "$R2_BUCKET" + test -n "$R2_ENDPOINT_URL" + aws s3 cp /tmp/model-catalog.json "s3://${R2_BUCKET}/model-catalog.json" \ + --endpoint-url "$R2_ENDPOINT_URL" \ + --content-type application/json \ + --cache-control 'public, max-age=3600, must-revalidate' diff --git a/packages/ai/.changes/hosted-model-catalog.md b/packages/ai/.changes/hosted-model-catalog.md new file mode 100644 index 0000000000..bc80fc69ea --- /dev/null +++ b/packages/ai/.changes/hosted-model-catalog.md @@ -0,0 +1 @@ +- Added a validated aggregate model catalog for daily publication from live provider catalogs. diff --git a/packages/ai/package.json b/packages/ai/package.json index 547984c386..c4d1d45197 100644 --- a/packages/ai/package.json +++ b/packages/ai/package.json @@ -65,6 +65,8 @@ "scripts": { "clean": "shx rm -rf dist", "generate-models": "npx tsx scripts/generate-models.ts", + "export-model-catalog": "npx tsx scripts/export-model-catalog.ts", + "validate-model-catalog": "npx tsx scripts/validate-model-catalog.ts", "build": "npm run generate-models && tsgo -p tsconfig.build.json", "dev": "tsgo -p tsconfig.build.json --watch --preserveWatchOutput", "dev:tsc": "tsgo -p tsconfig.build.json --watch --preserveWatchOutput", diff --git a/packages/ai/scripts/export-model-catalog.ts b/packages/ai/scripts/export-model-catalog.ts new file mode 100644 index 0000000000..a0db608692 --- /dev/null +++ b/packages/ai/scripts/export-model-catalog.ts @@ -0,0 +1,13 @@ +#!/usr/bin/env tsx + +import { writeFileSync } from "node:fs"; +import { resolve } from "node:path"; +import type { Api, Model } from "../src/types.js"; +import { MODELS } from "../src/models.generated.js"; +import { createModelCatalog } from "./model-catalog-format.js"; + +const outputPath = process.argv[2]; +if (!outputPath) throw new Error("Usage: export-model-catalog.ts "); +const models = Object.values(MODELS).flatMap((providerModels) => Object.values(providerModels)) as Model[]; +writeFileSync(resolve(outputPath), `${JSON.stringify(createModelCatalog(models), null, 2)}\n`); +console.log(`Exported ${models.length} models to ${resolve(outputPath)}`); diff --git a/packages/ai/scripts/generate-models.ts b/packages/ai/scripts/generate-models.ts index 1ba0fc83c4..01dbc24994 100644 --- a/packages/ai/scripts/generate-models.ts +++ b/packages/ai/scripts/generate-models.ts @@ -6,6 +6,7 @@ import { dirname, join } from "path"; import { fileURLToPath } from "url"; import { getAnthropicCacheCosts } from "../src/cache-pricing.js"; import { getOpenRouterReasoningCapabilities } from "../src/openrouter-reasoning.js"; +import { createModelCatalog } from "./model-catalog-format.js"; import { CLOUDFLARE_AI_GATEWAY_ANTHROPIC_BASE_URL, CLOUDFLARE_AI_GATEWAY_COMPAT_BASE_URL, @@ -2433,6 +2434,17 @@ export const MODELS = { writeFileSync(join(packageRoot, "src/models.generated.ts"), output); console.log("Generated src/models.generated.ts"); + const catalogOutputPath = process.env.PRIME_AGENT_MODEL_CATALOG_OUTPUT?.trim(); + if (catalogOutputPath) { + const uniqueModels = sortedProviderIds.flatMap((providerId) => + Object.keys(providers[providerId]) + .sort() + .map((modelId) => providers[providerId][modelId]), + ); + writeFileSync(catalogOutputPath, `${JSON.stringify(createModelCatalog(uniqueModels), null, 2)}\n`); + console.log(`Generated ${catalogOutputPath}`); + } + // Print statistics const totalModels = allModels.length; const reasoningModels = allModels.filter(m => m.reasoning).length; diff --git a/packages/ai/scripts/model-catalog-format.ts b/packages/ai/scripts/model-catalog-format.ts new file mode 100644 index 0000000000..69cb4c6e0d --- /dev/null +++ b/packages/ai/scripts/model-catalog-format.ts @@ -0,0 +1,105 @@ +import type { Api, Model } from "../src/types.js"; + +export const MODEL_CATALOG_SCHEMA_VERSION = 1 as const; +export const MAX_MODEL_CATALOG_MODELS = 20_000; + +export interface ModelCatalogV1 { + schemaVersion: typeof MODEL_CATALOG_SCHEMA_VERSION; + generatedAt: string; + models: Model[]; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function isFiniteNonNegative(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value) && value >= 0; +} + +function isCatalogModel(value: unknown): value is Model { + if (!isRecord(value) || !isRecord(value.cost)) return false; + return ( + typeof value.id === "string" && + value.id.length > 0 && + typeof value.name === "string" && + value.name.length > 0 && + typeof value.api === "string" && + value.api.length > 0 && + typeof value.provider === "string" && + value.provider.length > 0 && + typeof value.baseUrl === "string" && + typeof value.reasoning === "boolean" && + Array.isArray(value.input) && + value.input.length > 0 && + value.input.every((item) => item === "text" || item === "image") && + isFiniteNonNegative(value.cost.input) && + isFiniteNonNegative(value.cost.output) && + isFiniteNonNegative(value.cost.cacheRead) && + isFiniteNonNegative(value.cost.cacheWrite) && + typeof value.contextWindow === "number" && + Number.isSafeInteger(value.contextWindow) && + value.contextWindow > 0 && + typeof value.maxTokens === "number" && + Number.isSafeInteger(value.maxTokens) && + value.maxTokens > 0 + ); +} + +function cloneModel(model: Model): Model { + return structuredClone(model); +} + +export function createModelCatalog(models: readonly Model[], generatedAt = new Date()): ModelCatalogV1 { + const sorted = [...models] + .map(cloneModel) + .sort((left, right) => left.provider.localeCompare(right.provider) || left.id.localeCompare(right.id)); + return { + schemaVersion: MODEL_CATALOG_SCHEMA_VERSION, + generatedAt: generatedAt.toISOString(), + models: sorted, + }; +} + +export function parseModelCatalog(value: unknown): ModelCatalogV1 { + if (!isRecord(value) || value.schemaVersion !== MODEL_CATALOG_SCHEMA_VERSION) { + throw new Error("Model catalog has an unsupported schema version"); + } + if (typeof value.generatedAt !== "string" || !Number.isFinite(Date.parse(value.generatedAt))) { + throw new Error("Model catalog has an invalid generatedAt timestamp"); + } + if (!Array.isArray(value.models) || value.models.length === 0 || value.models.length > MAX_MODEL_CATALOG_MODELS) { + throw new Error("Model catalog has an invalid model count"); + } + + const seen = new Set(); + for (const model of value.models) { + if (!isCatalogModel(model)) throw new Error("Model catalog contains an invalid model"); + const key = `${model.provider}/${model.id}`; + if (seen.has(key)) throw new Error(`Model catalog contains duplicate model ${key}`); + seen.add(key); + } + return value as unknown as ModelCatalogV1; +} + +export function assertProviderCounts( + candidate: ModelCatalogV1, + baseline: ModelCatalogV1, + minimumRatio = 0.5, +): void { + if (!(minimumRatio > 0 && minimumRatio <= 1)) throw new Error("minimumRatio must be in (0, 1]"); + const count = (catalog: ModelCatalogV1): Map => { + const counts = new Map(); + for (const model of catalog.models) counts.set(model.provider, (counts.get(model.provider) ?? 0) + 1); + return counts; + }; + const baselineCounts = count(baseline); + const candidateCounts = count(candidate); + for (const [provider, previous] of baselineCounts) { + const current = candidateCounts.get(provider) ?? 0; + const minimum = Math.max(1, Math.floor(previous * minimumRatio)); + if (current < minimum) { + throw new Error(`Provider ${provider} dropped from ${previous} to ${current} models (minimum ${minimum})`); + } + } +} diff --git a/packages/ai/scripts/validate-model-catalog.ts b/packages/ai/scripts/validate-model-catalog.ts new file mode 100644 index 0000000000..3da5bb7ee2 --- /dev/null +++ b/packages/ai/scripts/validate-model-catalog.ts @@ -0,0 +1,14 @@ +#!/usr/bin/env tsx + +import { readFileSync } from "node:fs"; +import { assertProviderCounts, parseModelCatalog } from "./model-catalog-format.js"; + +const [candidatePath, baselinePath] = process.argv.slice(2); +if (!candidatePath || !baselinePath) { + throw new Error("Usage: validate-model-catalog.ts "); +} +const read = (path: string) => parseModelCatalog(JSON.parse(readFileSync(path, "utf8")) as unknown); +const candidate = read(candidatePath); +const baseline = read(baselinePath); +assertProviderCounts(candidate, baseline); +console.log(`Validated ${candidate.models.length} models against ${baseline.models.length} baseline models`); diff --git a/packages/ai/test/model-catalog-format.test.ts b/packages/ai/test/model-catalog-format.test.ts new file mode 100644 index 0000000000..0da28057fe --- /dev/null +++ b/packages/ai/test/model-catalog-format.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, test } from "vitest"; +import { assertProviderCounts, createModelCatalog, parseModelCatalog } from "../scripts/model-catalog-format.js"; +import type { Api, Model } from "../src/types.js"; + +function model(provider: string, id: string): Model { + return { + id, + name: id, + api: "openai-completions", + provider, + baseUrl: "https://example.test/v1", + reasoning: false, + input: ["text"], + cost: { input: 1, output: 2, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128_000, + maxTokens: 8_192, + }; +} + +describe("hosted model catalog format", () => { + test("creates deterministic provider/model ordering", () => { + const generatedAt = new Date("2026-08-31T00:00:00.000Z"); + const result = createModelCatalog([model("z", "b"), model("a", "z"), model("z", "a")], generatedAt); + expect(result.generatedAt).toBe(generatedAt.toISOString()); + expect(result.models.map((entry) => `${entry.provider}/${entry.id}`)).toEqual(["a/z", "z/a", "z/b"]); + }); + + test("rejects duplicate entries and invalid prices", () => { + const entry = model("provider", "model"); + expect(() => + parseModelCatalog({ schemaVersion: 1, generatedAt: new Date().toISOString(), models: [entry, entry] }), + ).toThrow("duplicate"); + expect(() => + parseModelCatalog({ + schemaVersion: 1, + generatedAt: new Date().toISOString(), + models: [{ ...entry, cost: { ...entry.cost, output: Number.NaN } }], + }), + ).toThrow("invalid model"); + }); + + test("fails closed on a provider-wide source outage", () => { + const baseline = createModelCatalog([model("large", "1"), model("large", "2"), model("small", "1")]); + const candidate = createModelCatalog([model("large", "1"), model("large", "2")]); + expect(() => assertProviderCounts(candidate, baseline)).toThrow("Provider small dropped"); + }); +}); diff --git a/packages/coding-agent/.changes/hosted-model-catalog.md b/packages/coding-agent/.changes/hosted-model-catalog.md new file mode 100644 index 0000000000..a51c1ba310 --- /dev/null +++ b/packages/coding-agent/.changes/hosted-model-catalog.md @@ -0,0 +1 @@ +- Added daily runtime refreshes of provider model metadata and pricing from the hosted Prime Agent catalog, with validated disk and bundled fallbacks. diff --git a/packages/coding-agent/docs/providers.md b/packages/coding-agent/docs/providers.md index 5c7ff364bb..875cca8ae6 100644 --- a/packages/coding-agent/docs/providers.md +++ b/packages/coding-agent/docs/providers.md @@ -1,6 +1,8 @@ # Providers -Prime Agent supports subscription-based providers via OAuth and API key providers via environment variables or the auth file. Its built-in model catalog is updated with each Prime Agent release. +Prime Agent supports subscription-based providers via OAuth and API key providers via environment variables or the auth file. It refreshes provider model names, capabilities, and pricing from Prime Intellect's hosted catalog once per day. A validated disk cache and the catalog bundled with each release keep model selection available when the endpoint is offline. The hosted catalog cannot change provider request URLs, APIs, headers, or compatibility settings unless that transport already exists in the bundled catalog. + +Set `PI_OFFLINE=1` to skip catalog network refreshes. Set `PRIME_AGENT_MODEL_CATALOG_URL` to use another catalog endpoint; `PRIME_AGENT_DOWNLOAD_BASE_URL` also changes the default catalog origin alongside the release origin. ## Table of Contents diff --git a/packages/coding-agent/src/core/model-registry.ts b/packages/coding-agent/src/core/model-registry.ts index 4ebf6c3b8d..052400c7be 100644 --- a/packages/coding-agent/src/core/model-registry.ts +++ b/packages/coding-agent/src/core/model-registry.ts @@ -36,6 +36,11 @@ import { isPrivatePrimeInferenceModel, } from "./prime-inference-models.js"; import { BUILT_IN_PROVIDER_DISPLAY_NAMES } from "./provider-display-names.js"; +import { + mergeRemoteModelCatalog, + readCachedRemoteModelCatalog, + refreshRemoteModelCatalog, +} from "./remote-model-catalog.js"; import { resolveConfigValueOrThrow, resolveConfigValueUncached, @@ -445,6 +450,7 @@ export class ModelRegistry { private explicitPrivatePrimeInferenceModelIds = new Set(); private openAICodexModelsCache: { authFingerprint: string; modelIds: Set; refreshedAt: number } | undefined; private backgroundPrivatePrimeAuthorization: { fingerprint: string; promise: Promise } | undefined; + private remoteCatalogModels: Model[] | undefined; private loadError: string | undefined = undefined; /** Re-register dynamic OAuth providers (e.g. user MCP servers) after refresh() resets the registry. */ @@ -479,6 +485,7 @@ export class ModelRegistry { this.authorizedPrivatePrimeInferenceModelIds.clear(); this.authorizedPrivatePrimeInferenceTeamId = undefined; this.explicitPrivatePrimeInferenceModelIds.clear(); + this.remoteCatalogModels = undefined; this.loadError = undefined; // Credentials may have been written by another process (e.g. the UI @@ -498,6 +505,10 @@ export class ModelRegistry { this.loadModels(); + this.reapplyRegisteredProviders(); + } + + private reapplyRegisteredProviders(): void { for (const [providerName, config] of this.registeredProviders.entries()) { this.applyProviderConfig(providerName, config); } @@ -510,6 +521,10 @@ export class ModelRegistry { return this.loadError; } + private remoteModelCatalogCachePath(): string | undefined { + return this.modelsJsonPath ? join(dirname(this.modelsJsonPath), "model-catalog-cache.json") : undefined; + } + private loadModels(): void { const { models: customModels, @@ -525,7 +540,13 @@ export class ModelRegistry { this.explicitPrivatePrimeInferenceModelIds = new Set( customModels.filter(isPrivatePrimeInferenceModel).map((model) => model.id), ); - const builtInModels = [...this.loadBuiltInModels(overrides, modelOverrides), ...getPrivatePrimeInferenceModels()]; + const cachePath = this.remoteModelCatalogCachePath(); + const remoteModels = + this.remoteCatalogModels ?? (cachePath ? readCachedRemoteModelCatalog(cachePath) : undefined); + const builtInModels = [ + ...this.loadBuiltInModels(overrides, modelOverrides, remoteModels), + ...getPrivatePrimeInferenceModels(), + ]; let combined = this.mergeCustomModels(builtInModels, customModels); for (const oauthProvider of this.authStorage.getOAuthProviders()) { @@ -542,30 +563,26 @@ export class ModelRegistry { private loadBuiltInModels( overrides: Map, modelOverrides: Map>, + remoteModels?: Model[], ): Model[] { - return getProviders().flatMap((provider) => { - const models = getModels(provider as KnownProvider) as Model[]; - const providerOverride = overrides.get(provider); - const perModelOverrides = modelOverrides.get(provider); - - return models.map((m) => { - let model = m; - - if (providerOverride) { - model = { - ...model, - baseUrl: providerOverride.baseUrl ?? model.baseUrl, - compat: mergeCompat(model.compat, providerOverride.compat), - }; - } - - const modelOverride = perModelOverrides?.get(m.id); - if (modelOverride) { - model = applyModelOverride(model, modelOverride); - } + const bundledModels = getProviders().flatMap((provider) => getModels(provider as KnownProvider) as Model[]); + const catalogModels = mergeRemoteModelCatalog(bundledModels, remoteModels); + return catalogModels.map((m) => { + const providerOverride = overrides.get(m.provider); + const perModelOverrides = modelOverrides.get(m.provider); + let model = m; + + if (providerOverride) { + model = { + ...model, + baseUrl: providerOverride.baseUrl ?? model.baseUrl, + compat: mergeCompat(model.compat, providerOverride.compat), + }; + } - return model; - }); + const modelOverride = perModelOverrides?.get(m.id); + if (modelOverride) model = applyModelOverride(model, modelOverride); + return model; }); } @@ -777,6 +794,12 @@ export class ModelRegistry { const previousPrivateModelIds = new Set(this.authorizedPrivatePrimeInferenceModelIds); const previousTeamId = this.authorizedPrivatePrimeInferenceTeamId; this.refresh(); + const cachePath = this.remoteModelCatalogCachePath(); + if (cachePath) { + this.remoteCatalogModels = await refreshRemoteModelCatalog(cachePath); + this.loadModels(); + this.reapplyRegisteredProviders(); + } await this.refreshPrivatePrimeInferenceAuthorization(previousPrivateModelIds, previousTeamId); return this.getAvailable(); } diff --git a/packages/coding-agent/src/core/remote-model-catalog.ts b/packages/coding-agent/src/core/remote-model-catalog.ts new file mode 100644 index 0000000000..d13fb3cb4d --- /dev/null +++ b/packages/coding-agent/src/core/remote-model-catalog.ts @@ -0,0 +1,262 @@ +import { Buffer } from "node:buffer"; +import { existsSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs"; +import type { Api, Model } from "@earendil-works/pi-ai"; + +const DEFAULT_PRIME_AGENT_DOWNLOAD_BASE_URL = "https://pub-728493de92a943e2a9b2d17b4719f318.r2.dev"; +const MODEL_CATALOG_PATH = "model-catalog.json"; +const MODEL_CATALOG_SCHEMA_VERSION = 1; +const MODEL_CATALOG_CACHE_TTL_MS = 24 * 60 * 60 * 1000; +const MODEL_CATALOG_FETCH_TIMEOUT_MS = 5_000; +const MAX_MODEL_CATALOG_BYTES = 8 * 1024 * 1024; +const MAX_MODEL_CATALOG_MODELS = 20_000; +const pendingRefreshes = new Map[] | undefined>>(); + +interface ModelCatalogV1 { + schemaVersion: 1; + generatedAt: string; + models: Model[]; +} + +interface CachedModelCatalog { + url: string; + fetchedAt: number; + catalog: ModelCatalogV1; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function isFiniteCost(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1_000_000; +} + +function isNonEmptyString(value: unknown, maxLength: number): value is string { + return typeof value === "string" && value.length > 0 && value.length <= maxLength; +} + +function isBoundedJsonObject(value: unknown, depth = 0): boolean { + if (!isRecord(value) || depth > 10 || Object.keys(value).length > 100) return false; + return Object.entries(value).every(([key, entry]) => { + if (key.length > 128) return false; + if (entry === null || typeof entry === "boolean") return true; + if (typeof entry === "string") return entry.length <= 4_096; + if (typeof entry === "number") return Number.isFinite(entry); + if (Array.isArray(entry)) + return entry.length <= 100 && entry.every((item) => isBoundedJsonObject({ item }, depth + 1)); + return isBoundedJsonObject(entry, depth + 1); + }); +} + +const THINKING_LEVELS = new Set(["off", "minimal", "low", "medium", "high", "xhigh", "max"]); + +function isThinkingLevelMap(value: unknown): boolean { + if (value === undefined) return true; + if (!isRecord(value) || Object.keys(value).some((key) => !THINKING_LEVELS.has(key))) return false; + return Object.values(value).every( + (entry) => entry === null || (typeof entry === "string" && entry.length > 0 && entry.length <= 128), + ); +} + +function isCatalogModel(value: unknown): value is Model { + if (!isRecord(value) || !isRecord(value.cost)) return false; + return ( + isNonEmptyString(value.id, 1_024) && + isNonEmptyString(value.name, 1_024) && + isNonEmptyString(value.api, 128) && + isNonEmptyString(value.provider, 128) && + typeof value.baseUrl === "string" && + value.baseUrl.length <= 2_048 && + typeof value.reasoning === "boolean" && + isThinkingLevelMap(value.thinkingLevelMap) && + Array.isArray(value.input) && + value.input.length > 0 && + value.input.length <= 2 && + value.input.every((item) => item === "text" || item === "image") && + isFiniteCost(value.cost.input) && + isFiniteCost(value.cost.output) && + isFiniteCost(value.cost.cacheRead) && + isFiniteCost(value.cost.cacheWrite) && + typeof value.contextWindow === "number" && + Number.isSafeInteger(value.contextWindow) && + value.contextWindow > 0 && + value.contextWindow <= 100_000_000 && + typeof value.maxTokens === "number" && + Number.isSafeInteger(value.maxTokens) && + value.maxTokens > 0 && + value.maxTokens <= 100_000_000 && + (value.featured === undefined || typeof value.featured === "boolean") && + (value.headers === undefined || isBoundedJsonObject(value.headers)) && + (value.compat === undefined || isBoundedJsonObject(value.compat)) + ); +} + +export function parseRemoteModelCatalog(value: unknown): ModelCatalogV1 { + if (!isRecord(value) || value.schemaVersion !== MODEL_CATALOG_SCHEMA_VERSION) { + throw new Error("Unsupported model catalog schema version"); + } + if (typeof value.generatedAt !== "string" || !Number.isFinite(Date.parse(value.generatedAt))) { + throw new Error("Invalid model catalog timestamp"); + } + if (!Array.isArray(value.models) || value.models.length === 0 || value.models.length > MAX_MODEL_CATALOG_MODELS) { + throw new Error("Invalid model catalog model count"); + } + const seen = new Set(); + for (const model of value.models) { + if (!isCatalogModel(model)) throw new Error("Invalid model catalog entry"); + const key = `${model.provider}/${model.id}`; + if (seen.has(key)) throw new Error(`Duplicate model catalog entry ${key}`); + seen.add(key); + } + return value as unknown as ModelCatalogV1; +} + +function canonicalize(value: unknown): unknown { + if (Array.isArray(value)) return value.map(canonicalize); + if (!isRecord(value)) return value; + return Object.fromEntries( + Object.keys(value) + .sort() + .map((key) => [key, canonicalize(value[key])]), + ); +} + +function transportSignature(model: Model): string { + return JSON.stringify( + canonicalize({ api: model.api, baseUrl: model.baseUrl, headers: model.headers, compat: model.compat }), + ); +} + +function cloneTransport(template: Model, remote: Model): Model { + return { + id: remote.id, + name: remote.name, + api: template.api, + provider: remote.provider, + baseUrl: template.baseUrl, + reasoning: remote.reasoning, + ...(remote.thinkingLevelMap ? { thinkingLevelMap: { ...remote.thinkingLevelMap } } : {}), + input: [...remote.input], + cost: { ...remote.cost }, + contextWindow: remote.contextWindow, + maxTokens: remote.maxTokens, + ...(remote.featured !== undefined ? { featured: remote.featured } : {}), + headers: template.headers ? { ...template.headers } : undefined, + compat: template.compat ? structuredClone(template.compat) : undefined, + }; +} + +export function mergeRemoteModelCatalog( + bundledModels: readonly Model[], + remoteModels: readonly Model[] | undefined, +): Model[] { + if (!remoteModels) return bundledModels.map((model) => structuredClone(model)); + const exact = new Map(bundledModels.map((model) => [`${model.provider}/${model.id}`, model])); + const transports = new Map>(); + for (const model of bundledModels) transports.set(`${model.provider}\0${transportSignature(model)}`, model); + + const merged = new Map(bundledModels.map((model) => [`${model.provider}/${model.id}`, structuredClone(model)])); + for (const remote of remoteModels) { + const key = `${remote.provider}/${remote.id}`; + const template = exact.get(key) ?? transports.get(`${remote.provider}\0${transportSignature(remote)}`); + if (template) merged.set(key, cloneTransport(template, remote)); + } + return [...merged.values()]; +} + +export function getRemoteModelCatalogUrl(): string { + const explicit = process.env.PRIME_AGENT_MODEL_CATALOG_URL?.trim(); + if (explicit) return explicit; + const base = (process.env.PRIME_AGENT_DOWNLOAD_BASE_URL?.trim() || DEFAULT_PRIME_AGENT_DOWNLOAD_BASE_URL).replace( + /\/+$/, + "", + ); + return `${base}/${MODEL_CATALOG_PATH}`; +} + +function readCache(cachePath: string, url: string): CachedModelCatalog | undefined { + if (!existsSync(cachePath)) return undefined; + try { + const value = JSON.parse(readFileSync(cachePath, "utf8")) as unknown; + if ( + !isRecord(value) || + value.url !== url || + typeof value.fetchedAt !== "number" || + !Number.isFinite(value.fetchedAt) + ) + return undefined; + return { url, fetchedAt: value.fetchedAt, catalog: parseRemoteModelCatalog(value.catalog) }; + } catch { + return undefined; + } +} + +function writeCache(cachePath: string, cache: CachedModelCatalog): void { + const temporaryPath = `${cachePath}.${process.pid}.${Date.now()}.tmp`; + try { + writeFileSync(temporaryPath, JSON.stringify(cache), { encoding: "utf8", mode: 0o600 }); + renameSync(temporaryPath, cachePath); + } catch { + // The bundled catalog remains available when the cache cannot be persisted. + } finally { + try { + if (existsSync(temporaryPath)) unlinkSync(temporaryPath); + } catch { + // Ignore cache cleanup failures. + } + } +} + +export function readCachedRemoteModelCatalog(cachePath: string): Model[] | undefined { + return readCache(cachePath, getRemoteModelCatalogUrl())?.catalog.models; +} + +function offlineModeEnabled(): boolean { + const value = process.env.PI_OFFLINE?.toLowerCase(); + return value === "1" || value === "true" || value === "yes"; +} + +async function fetchCatalog(url: string, fetchFn: typeof fetch): Promise { + const response = await fetchFn(url, { + headers: { accept: "application/json" }, + signal: AbortSignal.timeout(MODEL_CATALOG_FETCH_TIMEOUT_MS), + }); + if (!response.ok) throw new Error(`Model catalog request failed with status ${response.status}`); + const contentLength = Number(response.headers.get("content-length")); + if (Number.isFinite(contentLength) && contentLength > MAX_MODEL_CATALOG_BYTES) { + throw new Error("Model catalog response is too large"); + } + const text = await response.text(); + if (Buffer.byteLength(text, "utf8") > MAX_MODEL_CATALOG_BYTES) + throw new Error("Model catalog response is too large"); + return parseRemoteModelCatalog(JSON.parse(text) as unknown); +} + +export async function refreshRemoteModelCatalog( + cachePath: string, + options: { fetchFn?: typeof fetch; now?: number } = {}, +): Promise[] | undefined> { + const url = getRemoteModelCatalogUrl(); + const now = options.now ?? Date.now(); + const cached = readCache(cachePath, url); + if (cached && now - cached.fetchedAt < MODEL_CATALOG_CACHE_TTL_MS) return cached.catalog.models; + if (offlineModeEnabled()) return cached?.catalog.models; + + const key = `${cachePath}\0${url}`; + const existing = pendingRefreshes.get(key); + if (existing) return existing; + const promise = (async () => { + try { + const catalog = await fetchCatalog(url, options.fetchFn ?? fetch); + writeCache(cachePath, { url, fetchedAt: now, catalog }); + return catalog.models; + } catch { + return cached?.catalog.models; + } + })(); + pendingRefreshes.set(key, promise); + void promise.finally(() => { + if (pendingRefreshes.get(key) === promise) pendingRefreshes.delete(key); + }); + return promise; +} diff --git a/packages/coding-agent/test/remote-model-catalog.test.ts b/packages/coding-agent/test/remote-model-catalog.test.ts new file mode 100644 index 0000000000..06c70408dc --- /dev/null +++ b/packages/coding-agent/test/remote-model-catalog.test.ts @@ -0,0 +1,158 @@ +import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { type Api, getModels, type Model } from "@earendil-works/pi-ai"; +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; +import { AuthStorage } from "../src/core/auth-storage.js"; +import { ModelRegistry } from "../src/core/model-registry.js"; +import { + getRemoteModelCatalogUrl, + mergeRemoteModelCatalog, + parseRemoteModelCatalog, + refreshRemoteModelCatalog, +} from "../src/core/remote-model-catalog.js"; + +const generatedAt = "2026-08-31T00:00:00.000Z"; + +function catalog(models: Model[]) { + return { schemaVersion: 1, generatedAt, models }; +} + +function openAiModel(): Model { + return structuredClone(getModels("openai")[0] as Model); +} + +describe("remote model catalog", () => { + let tempDir: string; + let previousCatalogUrl: string | undefined; + let previousOffline: string | undefined; + + beforeEach(() => { + tempDir = join(tmpdir(), `prime-model-catalog-${Date.now()}-${Math.random().toString(36).slice(2)}`); + mkdirSync(tempDir, { recursive: true }); + previousCatalogUrl = process.env.PRIME_AGENT_MODEL_CATALOG_URL; + previousOffline = process.env.PI_OFFLINE; + process.env.PRIME_AGENT_MODEL_CATALOG_URL = `https://catalog.test/${Math.random()}.json`; + delete process.env.PI_OFFLINE; + }); + + afterEach(() => { + vi.restoreAllMocks(); + if (previousCatalogUrl === undefined) delete process.env.PRIME_AGENT_MODEL_CATALOG_URL; + else process.env.PRIME_AGENT_MODEL_CATALOG_URL = previousCatalogUrl; + if (previousOffline === undefined) delete process.env.PI_OFFLINE; + else process.env.PI_OFFLINE = previousOffline; + if (existsSync(tempDir)) rmSync(tempDir, { recursive: true, force: true }); + }); + + test("validates entries and rejects duplicate provider/model ids", () => { + const model = openAiModel(); + expect(parseRemoteModelCatalog(catalog([model])).models).toHaveLength(1); + expect(() => parseRemoteModelCatalog(catalog([model, structuredClone(model)]))).toThrow("Duplicate"); + expect(() => + parseRemoteModelCatalog(catalog([{ ...model, cost: { ...model.cost, input: Number.NaN } }])), + ).toThrow("Invalid model catalog entry"); + }); + + test("updates metadata while pinning bundled request transports", () => { + const bundled = openAiModel(); + const remote = { + ...structuredClone(bundled), + name: "Current provider name", + baseUrl: "https://attacker.test/v1", + api: "attacker-api", + headers: { Authorization: "exfiltrate" }, + cost: { ...bundled.cost, input: bundled.cost.input + 1 }, + }; + const [merged] = mergeRemoteModelCatalog([bundled], [remote]); + expect(merged).toMatchObject({ + name: "Current provider name", + baseUrl: bundled.baseUrl, + api: bundled.api, + cost: remote.cost, + }); + expect(merged.headers).toEqual(bundled.headers); + }); + + test("adds models only through a bundled provider transport", () => { + const bundled = openAiModel(); + const accepted = { ...structuredClone(bundled), id: "future-model", name: "Future Model" }; + const rejected = { ...structuredClone(accepted), id: "redirected-model", baseUrl: "https://other.test" }; + const merged = mergeRemoteModelCatalog([bundled], [accepted, rejected]); + expect(merged.map((model) => model.id)).toEqual([bundled.id, "future-model"]); + }); + + test("fetches once, validates, and reuses the fresh atomic cache", async () => { + const cachePath = join(tempDir, "cache.json"); + const model = openAiModel(); + const fetchFn = vi.fn(async () => new Response(JSON.stringify(catalog([model])), { status: 200 })); + const [first, concurrent] = await Promise.all([ + refreshRemoteModelCatalog(cachePath, { fetchFn, now: 1_000 }), + refreshRemoteModelCatalog(cachePath, { fetchFn, now: 1_000 }), + ]); + expect(fetchFn).toHaveBeenCalledOnce(); + expect(first?.[0].id).toBe(model.id); + expect(concurrent?.[0].id).toBe(model.id); + expect(JSON.parse(readFileSync(cachePath, "utf8"))).toMatchObject({ + url: getRemoteModelCatalogUrl(), + fetchedAt: 1_000, + }); + + await refreshRemoteModelCatalog(cachePath, { fetchFn, now: 2_000 }); + expect(fetchFn).toHaveBeenCalledOnce(); + }); + + test("uses a stale validated cache when offline or refresh fails", async () => { + const cachePath = join(tempDir, "cache.json"); + const model = openAiModel(); + writeFileSync( + cachePath, + JSON.stringify({ url: getRemoteModelCatalogUrl(), fetchedAt: 1, catalog: catalog([model]) }), + ); + const fetchFn = vi.fn(async () => new Response(null, { status: 503 })); + expect((await refreshRemoteModelCatalog(cachePath, { fetchFn, now: 100_000_000 }))?.[0].id).toBe(model.id); + expect(fetchFn).toHaveBeenCalledOnce(); + + process.env.PI_OFFLINE = "1"; + const offlineFetch = vi.fn(); + expect((await refreshRemoteModelCatalog(cachePath, { fetchFn: offlineFetch, now: 200_000_000 }))?.[0].id).toBe( + model.id, + ); + expect(offlineFetch).not.toHaveBeenCalled(); + }); + + test("keeps local model overrides and custom models above remote metadata", async () => { + const model = openAiModel(); + const modelsPath = join(tempDir, "models.json"); + writeFileSync( + modelsPath, + JSON.stringify({ + providers: { + openai: { + modelOverrides: { [model.id]: { name: "Local name", cost: { input: 99 } } }, + models: [{ ...model, id: "local-model", name: "Local model" }], + }, + }, + }), + ); + const remote = { ...structuredClone(model), name: "Remote name", cost: { ...model.cost, input: 42 } }; + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response(JSON.stringify(catalog([remote])), { status: 200 })), + ); + const registry = ModelRegistry.create( + AuthStorage.inMemory({ openai: { type: "api_key", key: "key" } }), + modelsPath, + ); + registry.registerProvider("extension-provider", { + api: "openai-completions", + apiKey: "extension-key", + baseUrl: "https://extension.test/v1", + models: [{ ...model, id: "extension-model", name: "Extension model" }], + }); + await registry.refreshAvailableModels(); + expect(registry.find("openai", model.id)).toMatchObject({ name: "Local name", cost: { input: 99 } }); + expect(registry.find("openai", "local-model")?.name).toBe("Local model"); + expect(registry.find("extension-provider", "extension-model")?.name).toBe("Extension model"); + }); +}); From e429b96245b05cf1661458864d0a19a0c01bd388 Mon Sep 17 00:00:00 2001 From: Seth Date: Mon, 31 Aug 2026 20:13:07 -0400 Subject: [PATCH 02/13] fix: make the hosted model list authoritative --- .github/workflows/refresh-model-catalog.yml | 13 ++-------- packages/ai/package.json | 1 - packages/ai/scripts/export-model-catalog.ts | 13 ---------- packages/ai/scripts/generate-models.ts | 24 +++++++++++++++++-- packages/ai/scripts/model-catalog-format.ts | 22 ----------------- packages/ai/scripts/validate-model-catalog.ts | 15 ++++-------- packages/ai/test/model-catalog-format.test.ts | 8 +------ .../.changes/hosted-model-catalog.md | 2 +- packages/coding-agent/docs/providers.md | 2 +- .../src/core/remote-model-catalog.ts | 6 ++--- .../test/remote-model-catalog.test.ts | 5 ++-- 11 files changed, 38 insertions(+), 73 deletions(-) delete mode 100644 packages/ai/scripts/export-model-catalog.ts diff --git a/.github/workflows/refresh-model-catalog.yml b/.github/workflows/refresh-model-catalog.yml index ce8163b9d7..29ba484f4f 100644 --- a/.github/workflows/refresh-model-catalog.yml +++ b/.github/workflows/refresh-model-catalog.yml @@ -13,9 +13,6 @@ concurrency: group: refresh-hosted-model-catalog cancel-in-progress: false -env: - CATALOG_URL: ${{ vars.R2_PUBLIC_BASE_URL }}/model-catalog.json - jobs: publish: name: Generate, validate, and publish @@ -37,22 +34,16 @@ jobs: - name: Install dependencies run: npm ci --ignore-scripts - - name: Load validation baseline - working-directory: packages/ai - run: | - if ! curl --fail --silent --show-error --location "$CATALOG_URL" --output /tmp/model-catalog-baseline.json; then - npm run export-model-catalog -- /tmp/model-catalog-baseline.json - fi - - name: Generate aggregate catalog working-directory: packages/ai env: PRIME_AGENT_MODEL_CATALOG_OUTPUT: /tmp/model-catalog.json + PRIME_AGENT_MODEL_CATALOG_STRICT: "1" run: npm run generate-models - name: Validate catalog working-directory: packages/ai - run: npm run validate-model-catalog -- /tmp/model-catalog.json /tmp/model-catalog-baseline.json + run: npm run validate-model-catalog -- /tmp/model-catalog.json - name: Publish catalog to R2 env: diff --git a/packages/ai/package.json b/packages/ai/package.json index c4d1d45197..b6d0ae3da5 100644 --- a/packages/ai/package.json +++ b/packages/ai/package.json @@ -65,7 +65,6 @@ "scripts": { "clean": "shx rm -rf dist", "generate-models": "npx tsx scripts/generate-models.ts", - "export-model-catalog": "npx tsx scripts/export-model-catalog.ts", "validate-model-catalog": "npx tsx scripts/validate-model-catalog.ts", "build": "npm run generate-models && tsgo -p tsconfig.build.json", "dev": "tsgo -p tsconfig.build.json --watch --preserveWatchOutput", diff --git a/packages/ai/scripts/export-model-catalog.ts b/packages/ai/scripts/export-model-catalog.ts deleted file mode 100644 index a0db608692..0000000000 --- a/packages/ai/scripts/export-model-catalog.ts +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env tsx - -import { writeFileSync } from "node:fs"; -import { resolve } from "node:path"; -import type { Api, Model } from "../src/types.js"; -import { MODELS } from "../src/models.generated.js"; -import { createModelCatalog } from "./model-catalog-format.js"; - -const outputPath = process.argv[2]; -if (!outputPath) throw new Error("Usage: export-model-catalog.ts "); -const models = Object.values(MODELS).flatMap((providerModels) => Object.values(providerModels)) as Model[]; -writeFileSync(resolve(outputPath), `${JSON.stringify(createModelCatalog(models), null, 2)}\n`); -console.log(`Exported ${models.length} models to ${resolve(outputPath)}`); diff --git a/packages/ai/scripts/generate-models.ts b/packages/ai/scripts/generate-models.ts index 01dbc24994..d10ff5d3f8 100644 --- a/packages/ai/scripts/generate-models.ts +++ b/packages/ai/scripts/generate-models.ts @@ -77,6 +77,7 @@ const KIMI_STATIC_HEADERS = { const AI_GATEWAY_MODELS_URL = "https://ai-gateway.vercel.sh/v1"; const AI_GATEWAY_BASE_URL = "https://ai-gateway.vercel.sh"; +const STRICT_MODEL_CATALOG_REFRESH = process.env.PRIME_AGENT_MODEL_CATALOG_STRICT === "1"; const ZAI_TOOL_STREAM_UNSUPPORTED_MODELS = new Set(["glm-4.5", "glm-4.5-air", "glm-4.5-flash", "glm-4.5v"]); const EAGER_TOOL_INPUT_STREAMING_UNSUPPORTED_ANTHROPIC_MODELS = new Set([ "github-copilot:claude-haiku-4.5", @@ -652,9 +653,12 @@ async function fetchPrimeInferenceModels(): Promise[ const response = await fetch(`${PRIME_INFERENCE_BASE_URL}/models`, { headers: getPrimeInferenceHeaders(apiKey, teamId), }); + if (!response.ok) throw new Error(`Prime Inference catalog request failed with status ${response.status}`); catalog = parsePrimeInferenceCatalog(await response.json()); + if (catalog.length === 0) throw new Error("Prime Inference catalog is empty or invalid"); } catch (error) { console.error("Failed to fetch Prime Inference models:", error); + if (STRICT_MODEL_CATALOG_REFRESH) throw error; } let openRouterIndex = new Map(); @@ -662,10 +666,12 @@ async function fetchPrimeInferenceModels(): Promise[ openRouterIndex = buildPrimeInferenceOpenRouterIndex(await fetchOpenRouterCatalog()); } catch (error) { console.error("Failed to fetch OpenRouter catalog for Prime Inference metadata:", error); + if (STRICT_MODEL_CATALOG_REFRESH) throw error; } if (openRouterIndex.size === 0) { // Without OpenRouter metadata every model would regress to the defaults; // keep the previous snapshot instead. + if (STRICT_MODEL_CATALOG_REFRESH) throw new Error("OpenRouter catalog has no Prime Inference metadata"); console.error("OpenRouter catalog unavailable; keeping snapshot Prime Inference models"); return getExistingPrimeInferenceModels(); } @@ -744,8 +750,10 @@ function fetchOpenRouterCatalog(): Promise { openRouterCatalogPromise ??= (async () => { console.log("Fetching models from OpenRouter API..."); const response = await fetch("https://openrouter.ai/api/v1/models"); + if (!response.ok) throw new Error(`OpenRouter catalog request failed with status ${response.status}`); const data = await response.json(); - return Array.isArray(data?.data) ? data.data : []; + if (!Array.isArray(data?.data) || data.data.length === 0) throw new Error("OpenRouter catalog is empty or invalid"); + return data.data; })(); return openRouterCatalogPromise; } @@ -810,6 +818,7 @@ async function fetchOpenRouterModels(): Promise[]> { return models; } catch (error) { console.error("Failed to fetch OpenRouter models:", error); + if (STRICT_MODEL_CATALOG_REFRESH) throw error; return []; } } @@ -818,7 +827,9 @@ async function fetchAiGatewayModels(): Promise[]> { try { console.log("Fetching models from Vercel AI Gateway API..."); const response = await fetch(`${AI_GATEWAY_MODELS_URL}/models`); + if (!response.ok) throw new Error(`Vercel AI Gateway catalog request failed with status ${response.status}`); const data = await response.json(); + if (!Array.isArray(data?.data)) throw new Error("Vercel AI Gateway catalog is invalid"); const models: Model[] = []; const toNumber = (value: string | number | undefined): number => { @@ -865,10 +876,12 @@ async function fetchAiGatewayModels(): Promise[]> { }); } + if (models.length === 0) throw new Error("Vercel AI Gateway catalog has no tool-capable models"); console.log(`Fetched ${models.length} tool-capable models from Vercel AI Gateway`); return models; } catch (error) { console.error("Failed to fetch Vercel AI Gateway models:", error); + if (STRICT_MODEL_CATALOG_REFRESH) throw error; return []; } } @@ -877,7 +890,9 @@ async function loadModelsDevData(): Promise[]> { try { console.log("Fetching models from models.dev API..."); const response = await fetch("https://models.dev/api.json"); + if (!response.ok) throw new Error(`models.dev catalog request failed with status ${response.status}`); const data = await response.json(); + if (!isRecord(data)) throw new Error("models.dev catalog is invalid"); const models: Model[] = []; @@ -1568,10 +1583,12 @@ async function loadModelsDevData(): Promise[]> { } } + if (models.length === 0) throw new Error("models.dev catalog has no tool-capable models"); console.log(`Loaded ${models.length} tool-capable models from models.dev`); return models; } catch (error) { console.error("Failed to load models.dev data:", error); + if (STRICT_MODEL_CATALOG_REFRESH) throw error; return []; } } @@ -2459,4 +2476,7 @@ export const MODELS = { } // Run the generator -generateModels().catch(console.error); +generateModels().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/packages/ai/scripts/model-catalog-format.ts b/packages/ai/scripts/model-catalog-format.ts index 69cb4c6e0d..761469baf1 100644 --- a/packages/ai/scripts/model-catalog-format.ts +++ b/packages/ai/scripts/model-catalog-format.ts @@ -81,25 +81,3 @@ export function parseModelCatalog(value: unknown): ModelCatalogV1 { } return value as unknown as ModelCatalogV1; } - -export function assertProviderCounts( - candidate: ModelCatalogV1, - baseline: ModelCatalogV1, - minimumRatio = 0.5, -): void { - if (!(minimumRatio > 0 && minimumRatio <= 1)) throw new Error("minimumRatio must be in (0, 1]"); - const count = (catalog: ModelCatalogV1): Map => { - const counts = new Map(); - for (const model of catalog.models) counts.set(model.provider, (counts.get(model.provider) ?? 0) + 1); - return counts; - }; - const baselineCounts = count(baseline); - const candidateCounts = count(candidate); - for (const [provider, previous] of baselineCounts) { - const current = candidateCounts.get(provider) ?? 0; - const minimum = Math.max(1, Math.floor(previous * minimumRatio)); - if (current < minimum) { - throw new Error(`Provider ${provider} dropped from ${previous} to ${current} models (minimum ${minimum})`); - } - } -} diff --git a/packages/ai/scripts/validate-model-catalog.ts b/packages/ai/scripts/validate-model-catalog.ts index 3da5bb7ee2..34b6a35a01 100644 --- a/packages/ai/scripts/validate-model-catalog.ts +++ b/packages/ai/scripts/validate-model-catalog.ts @@ -1,14 +1,9 @@ #!/usr/bin/env tsx import { readFileSync } from "node:fs"; -import { assertProviderCounts, parseModelCatalog } from "./model-catalog-format.js"; +import { parseModelCatalog } from "./model-catalog-format.js"; -const [candidatePath, baselinePath] = process.argv.slice(2); -if (!candidatePath || !baselinePath) { - throw new Error("Usage: validate-model-catalog.ts "); -} -const read = (path: string) => parseModelCatalog(JSON.parse(readFileSync(path, "utf8")) as unknown); -const candidate = read(candidatePath); -const baseline = read(baselinePath); -assertProviderCounts(candidate, baseline); -console.log(`Validated ${candidate.models.length} models against ${baseline.models.length} baseline models`); +const [candidatePath] = process.argv.slice(2); +if (!candidatePath) throw new Error("Usage: validate-model-catalog.ts "); +const candidate = parseModelCatalog(JSON.parse(readFileSync(candidatePath, "utf8")) as unknown); +console.log(`Validated ${candidate.models.length} models`); diff --git a/packages/ai/test/model-catalog-format.test.ts b/packages/ai/test/model-catalog-format.test.ts index 0da28057fe..7f925dd2a1 100644 --- a/packages/ai/test/model-catalog-format.test.ts +++ b/packages/ai/test/model-catalog-format.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "vitest"; -import { assertProviderCounts, createModelCatalog, parseModelCatalog } from "../scripts/model-catalog-format.js"; +import { createModelCatalog, parseModelCatalog } from "../scripts/model-catalog-format.js"; import type { Api, Model } from "../src/types.js"; function model(provider: string, id: string): Model { @@ -38,10 +38,4 @@ describe("hosted model catalog format", () => { }), ).toThrow("invalid model"); }); - - test("fails closed on a provider-wide source outage", () => { - const baseline = createModelCatalog([model("large", "1"), model("large", "2"), model("small", "1")]); - const candidate = createModelCatalog([model("large", "1"), model("large", "2")]); - expect(() => assertProviderCounts(candidate, baseline)).toThrow("Provider small dropped"); - }); }); diff --git a/packages/coding-agent/.changes/hosted-model-catalog.md b/packages/coding-agent/.changes/hosted-model-catalog.md index a51c1ba310..d2235f546f 100644 --- a/packages/coding-agent/.changes/hosted-model-catalog.md +++ b/packages/coding-agent/.changes/hosted-model-catalog.md @@ -1 +1 @@ -- Added daily runtime refreshes of provider model metadata and pricing from the hosted Prime Agent catalog, with validated disk and bundled fallbacks. +- Added daily authoritative refreshes of provider model availability, metadata, and pricing from the hosted Prime Agent catalog, with validated disk and bundled fallbacks. diff --git a/packages/coding-agent/docs/providers.md b/packages/coding-agent/docs/providers.md index 875cca8ae6..9eed3998da 100644 --- a/packages/coding-agent/docs/providers.md +++ b/packages/coding-agent/docs/providers.md @@ -1,6 +1,6 @@ # Providers -Prime Agent supports subscription-based providers via OAuth and API key providers via environment variables or the auth file. It refreshes provider model names, capabilities, and pricing from Prime Intellect's hosted catalog once per day. A validated disk cache and the catalog bundled with each release keep model selection available when the endpoint is offline. The hosted catalog cannot change provider request URLs, APIs, headers, or compatibility settings unless that transport already exists in the bundled catalog. +Prime Agent supports subscription-based providers via OAuth and API key providers via environment variables or the auth file. It treats Prime Intellect's hosted catalog as the authoritative public model list and refreshes model additions, removals, names, capabilities, and pricing once per day. A validated disk cache and the catalog bundled with each release keep model selection available when the endpoint is offline. The hosted catalog cannot change provider request URLs, APIs, headers, or compatibility settings unless that transport already exists in the bundled catalog. Set `PI_OFFLINE=1` to skip catalog network refreshes. Set `PRIME_AGENT_MODEL_CATALOG_URL` to use another catalog endpoint; `PRIME_AGENT_DOWNLOAD_BASE_URL` also changes the default catalog origin alongside the release origin. diff --git a/packages/coding-agent/src/core/remote-model-catalog.ts b/packages/coding-agent/src/core/remote-model-catalog.ts index d13fb3cb4d..4ac82f1aff 100644 --- a/packages/coding-agent/src/core/remote-model-catalog.ts +++ b/packages/coding-agent/src/core/remote-model-catalog.ts @@ -155,13 +155,13 @@ export function mergeRemoteModelCatalog( const transports = new Map>(); for (const model of bundledModels) transports.set(`${model.provider}\0${transportSignature(model)}`, model); - const merged = new Map(bundledModels.map((model) => [`${model.provider}/${model.id}`, structuredClone(model)])); + const merged: Model[] = []; for (const remote of remoteModels) { const key = `${remote.provider}/${remote.id}`; const template = exact.get(key) ?? transports.get(`${remote.provider}\0${transportSignature(remote)}`); - if (template) merged.set(key, cloneTransport(template, remote)); + if (template) merged.push(cloneTransport(template, remote)); } - return [...merged.values()]; + return merged; } export function getRemoteModelCatalogUrl(): string { diff --git a/packages/coding-agent/test/remote-model-catalog.test.ts b/packages/coding-agent/test/remote-model-catalog.test.ts index 06c70408dc..ab4554f582 100644 --- a/packages/coding-agent/test/remote-model-catalog.test.ts +++ b/packages/coding-agent/test/remote-model-catalog.test.ts @@ -74,12 +74,13 @@ describe("remote model catalog", () => { expect(merged.headers).toEqual(bundled.headers); }); - test("adds models only through a bundled provider transport", () => { + test("uses the hosted model list and only accepts additions through a bundled transport", () => { const bundled = openAiModel(); const accepted = { ...structuredClone(bundled), id: "future-model", name: "Future Model" }; const rejected = { ...structuredClone(accepted), id: "redirected-model", baseUrl: "https://other.test" }; const merged = mergeRemoteModelCatalog([bundled], [accepted, rejected]); - expect(merged.map((model) => model.id)).toEqual([bundled.id, "future-model"]); + expect(merged.map((model) => model.id)).toEqual(["future-model"]); + expect(mergeRemoteModelCatalog([bundled], undefined).map((model) => model.id)).toEqual([bundled.id]); }); test("fetches once, validates, and reuses the fresh atomic cache", async () => { From 871be56ba3f84b1c6316d155922be8c29d6086df Mon Sep 17 00:00:00 2001 From: Seth Date: Mon, 31 Aug 2026 20:14:31 -0400 Subject: [PATCH 03/13] fix: accept catalog metadata on known provider routes --- .../src/core/remote-model-catalog.ts | 22 +++---------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/packages/coding-agent/src/core/remote-model-catalog.ts b/packages/coding-agent/src/core/remote-model-catalog.ts index 4ac82f1aff..6a85734c03 100644 --- a/packages/coding-agent/src/core/remote-model-catalog.ts +++ b/packages/coding-agent/src/core/remote-model-catalog.ts @@ -111,22 +111,6 @@ export function parseRemoteModelCatalog(value: unknown): ModelCatalogV1 { return value as unknown as ModelCatalogV1; } -function canonicalize(value: unknown): unknown { - if (Array.isArray(value)) return value.map(canonicalize); - if (!isRecord(value)) return value; - return Object.fromEntries( - Object.keys(value) - .sort() - .map((key) => [key, canonicalize(value[key])]), - ); -} - -function transportSignature(model: Model): string { - return JSON.stringify( - canonicalize({ api: model.api, baseUrl: model.baseUrl, headers: model.headers, compat: model.compat }), - ); -} - function cloneTransport(template: Model, remote: Model): Model { return { id: remote.id, @@ -142,7 +126,7 @@ function cloneTransport(template: Model, remote: Model): Model { maxTokens: remote.maxTokens, ...(remote.featured !== undefined ? { featured: remote.featured } : {}), headers: template.headers ? { ...template.headers } : undefined, - compat: template.compat ? structuredClone(template.compat) : undefined, + compat: remote.compat ? structuredClone(remote.compat) : undefined, }; } @@ -153,12 +137,12 @@ export function mergeRemoteModelCatalog( if (!remoteModels) return bundledModels.map((model) => structuredClone(model)); const exact = new Map(bundledModels.map((model) => [`${model.provider}/${model.id}`, model])); const transports = new Map>(); - for (const model of bundledModels) transports.set(`${model.provider}\0${transportSignature(model)}`, model); + for (const model of bundledModels) transports.set(`${model.provider}\0${model.api}\0${model.baseUrl}`, model); const merged: Model[] = []; for (const remote of remoteModels) { const key = `${remote.provider}/${remote.id}`; - const template = exact.get(key) ?? transports.get(`${remote.provider}\0${transportSignature(remote)}`); + const template = exact.get(key) ?? transports.get(`${remote.provider}\0${remote.api}\0${remote.baseUrl}`); if (template) merged.push(cloneTransport(template, remote)); } return merged; From ee1538d4bcbd2b97ad0d1432bdaaa45c4b8ab5ac Mon Sep 17 00:00:00 2001 From: Seth Date: Mon, 31 Aug 2026 20:19:14 -0400 Subject: [PATCH 04/13] fix: harden hosted catalog validation --- packages/ai/scripts/model-catalog-format.ts | 85 +------------ packages/ai/src/index.ts | 1 + packages/ai/src/model-catalog.ts | 109 ++++++++++++++++ packages/ai/test/model-catalog-format.test.ts | 26 ++-- .../src/core/remote-model-catalog.ts | 118 ++++-------------- .../test/remote-model-catalog.test.ts | 20 +++ 6 files changed, 173 insertions(+), 186 deletions(-) create mode 100644 packages/ai/src/model-catalog.ts diff --git a/packages/ai/scripts/model-catalog-format.ts b/packages/ai/scripts/model-catalog-format.ts index 761469baf1..fdcf3f6b89 100644 --- a/packages/ai/scripts/model-catalog-format.ts +++ b/packages/ai/scripts/model-catalog-format.ts @@ -1,83 +1,2 @@ -import type { Api, Model } from "../src/types.js"; - -export const MODEL_CATALOG_SCHEMA_VERSION = 1 as const; -export const MAX_MODEL_CATALOG_MODELS = 20_000; - -export interface ModelCatalogV1 { - schemaVersion: typeof MODEL_CATALOG_SCHEMA_VERSION; - generatedAt: string; - models: Model[]; -} - -function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} - -function isFiniteNonNegative(value: unknown): value is number { - return typeof value === "number" && Number.isFinite(value) && value >= 0; -} - -function isCatalogModel(value: unknown): value is Model { - if (!isRecord(value) || !isRecord(value.cost)) return false; - return ( - typeof value.id === "string" && - value.id.length > 0 && - typeof value.name === "string" && - value.name.length > 0 && - typeof value.api === "string" && - value.api.length > 0 && - typeof value.provider === "string" && - value.provider.length > 0 && - typeof value.baseUrl === "string" && - typeof value.reasoning === "boolean" && - Array.isArray(value.input) && - value.input.length > 0 && - value.input.every((item) => item === "text" || item === "image") && - isFiniteNonNegative(value.cost.input) && - isFiniteNonNegative(value.cost.output) && - isFiniteNonNegative(value.cost.cacheRead) && - isFiniteNonNegative(value.cost.cacheWrite) && - typeof value.contextWindow === "number" && - Number.isSafeInteger(value.contextWindow) && - value.contextWindow > 0 && - typeof value.maxTokens === "number" && - Number.isSafeInteger(value.maxTokens) && - value.maxTokens > 0 - ); -} - -function cloneModel(model: Model): Model { - return structuredClone(model); -} - -export function createModelCatalog(models: readonly Model[], generatedAt = new Date()): ModelCatalogV1 { - const sorted = [...models] - .map(cloneModel) - .sort((left, right) => left.provider.localeCompare(right.provider) || left.id.localeCompare(right.id)); - return { - schemaVersion: MODEL_CATALOG_SCHEMA_VERSION, - generatedAt: generatedAt.toISOString(), - models: sorted, - }; -} - -export function parseModelCatalog(value: unknown): ModelCatalogV1 { - if (!isRecord(value) || value.schemaVersion !== MODEL_CATALOG_SCHEMA_VERSION) { - throw new Error("Model catalog has an unsupported schema version"); - } - if (typeof value.generatedAt !== "string" || !Number.isFinite(Date.parse(value.generatedAt))) { - throw new Error("Model catalog has an invalid generatedAt timestamp"); - } - if (!Array.isArray(value.models) || value.models.length === 0 || value.models.length > MAX_MODEL_CATALOG_MODELS) { - throw new Error("Model catalog has an invalid model count"); - } - - const seen = new Set(); - for (const model of value.models) { - if (!isCatalogModel(model)) throw new Error("Model catalog contains an invalid model"); - const key = `${model.provider}/${model.id}`; - if (seen.has(key)) throw new Error(`Model catalog contains duplicate model ${key}`); - seen.add(key); - } - return value as unknown as ModelCatalogV1; -} +export { createModelCatalog, parseModelCatalog } from "../src/model-catalog.js"; +export type { ModelCatalogV1 } from "../src/model-catalog.js"; diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts index 388dffd6e2..a9d3b3b6ae 100644 --- a/packages/ai/src/index.ts +++ b/packages/ai/src/index.ts @@ -4,6 +4,7 @@ export { Type } from "typebox"; export * from "./api-registry.js"; export * from "./env-api-keys.js"; export * from "./log.js"; +export * from "./model-catalog.js"; export * from "./models.js"; export type { BedrockOptions, BedrockThinkingDisplay } from "./providers/amazon-bedrock.js"; export type { AnthropicEffort, AnthropicOptions, AnthropicThinkingDisplay } from "./providers/anthropic.js"; diff --git a/packages/ai/src/model-catalog.ts b/packages/ai/src/model-catalog.ts new file mode 100644 index 0000000000..65734e163c --- /dev/null +++ b/packages/ai/src/model-catalog.ts @@ -0,0 +1,109 @@ +import type { Api, Model } from "./types.js"; + +const MODEL_CATALOG_SCHEMA_VERSION = 1 as const; +const MAX_MODEL_CATALOG_MODELS = 20_000; + +export interface ModelCatalogV1 { + schemaVersion: typeof MODEL_CATALOG_SCHEMA_VERSION; + generatedAt: string; + models: Model[]; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function isFiniteCost(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1_000_000; +} + +function isNonEmptyString(value: unknown, maxLength: number): value is string { + return typeof value === "string" && value.length > 0 && value.length <= maxLength; +} + +function isBoundedJsonObject(value: unknown, depth = 0): boolean { + if (!isRecord(value) || depth > 10 || Object.keys(value).length > 100) return false; + return Object.entries(value).every(([key, entry]) => { + if (key.length > 128) return false; + if (entry === null || typeof entry === "boolean") return true; + if (typeof entry === "string") return entry.length <= 4_096; + if (typeof entry === "number") return Number.isFinite(entry); + if (Array.isArray(entry)) + return entry.length <= 100 && entry.every((item) => isBoundedJsonObject({ item }, depth + 1)); + return isBoundedJsonObject(entry, depth + 1); + }); +} + +const THINKING_LEVELS = new Set(["off", "minimal", "low", "medium", "high", "xhigh", "max"]); + +function isThinkingLevelMap(value: unknown): boolean { + if (value === undefined) return true; + if (!isRecord(value) || Object.keys(value).some((key) => !THINKING_LEVELS.has(key))) return false; + return Object.values(value).every( + (entry) => entry === null || (typeof entry === "string" && entry.length > 0 && entry.length <= 128), + ); +} + +function isCatalogModel(value: unknown): value is Model { + if (!isRecord(value) || !isRecord(value.cost)) return false; + return ( + isNonEmptyString(value.id, 1_024) && + isNonEmptyString(value.name, 1_024) && + isNonEmptyString(value.api, 128) && + isNonEmptyString(value.provider, 128) && + typeof value.baseUrl === "string" && + value.baseUrl.length <= 2_048 && + typeof value.reasoning === "boolean" && + isThinkingLevelMap(value.thinkingLevelMap) && + Array.isArray(value.input) && + value.input.length > 0 && + value.input.length <= 2 && + value.input.every((item) => item === "text" || item === "image") && + isFiniteCost(value.cost.input) && + isFiniteCost(value.cost.output) && + isFiniteCost(value.cost.cacheRead) && + isFiniteCost(value.cost.cacheWrite) && + typeof value.contextWindow === "number" && + Number.isSafeInteger(value.contextWindow) && + value.contextWindow > 0 && + value.contextWindow <= 100_000_000 && + typeof value.maxTokens === "number" && + Number.isSafeInteger(value.maxTokens) && + value.maxTokens > 0 && + value.maxTokens <= 100_000_000 && + (value.featured === undefined || typeof value.featured === "boolean") && + (value.headers === undefined || isBoundedJsonObject(value.headers)) && + (value.compat === undefined || isBoundedJsonObject(value.compat)) + ); +} + +export function createModelCatalog(models: readonly Model[], generatedAt = new Date()): ModelCatalogV1 { + const sorted = [...models] + .map((model) => structuredClone(model)) + .sort((left, right) => left.provider.localeCompare(right.provider) || left.id.localeCompare(right.id)); + return { + schemaVersion: MODEL_CATALOG_SCHEMA_VERSION, + generatedAt: generatedAt.toISOString(), + models: sorted, + }; +} + +export function parseModelCatalog(value: unknown): ModelCatalogV1 { + if (!isRecord(value) || value.schemaVersion !== MODEL_CATALOG_SCHEMA_VERSION) { + throw new Error("Unsupported model catalog schema version"); + } + if (typeof value.generatedAt !== "string" || !Number.isFinite(Date.parse(value.generatedAt))) { + throw new Error("Invalid model catalog timestamp"); + } + if (!Array.isArray(value.models) || value.models.length === 0 || value.models.length > MAX_MODEL_CATALOG_MODELS) { + throw new Error("Invalid model catalog model count"); + } + const seen = new Set(); + for (const model of value.models) { + if (!isCatalogModel(model)) throw new Error("Invalid model catalog entry"); + const key = `${model.provider}/${model.id}`; + if (seen.has(key)) throw new Error(`Duplicate model catalog entry ${key}`); + seen.add(key); + } + return value as unknown as ModelCatalogV1; +} diff --git a/packages/ai/test/model-catalog-format.test.ts b/packages/ai/test/model-catalog-format.test.ts index 7f925dd2a1..a9864acb74 100644 --- a/packages/ai/test/model-catalog-format.test.ts +++ b/packages/ai/test/model-catalog-format.test.ts @@ -25,17 +25,23 @@ describe("hosted model catalog format", () => { expect(result.models.map((entry) => `${entry.provider}/${entry.id}`)).toEqual(["a/z", "z/a", "z/b"]); }); - test("rejects duplicate entries and invalid prices", () => { + test("applies the same strict schema used by clients", () => { const entry = model("provider", "model"); + const catalog = (modelEntry: Model) => ({ + schemaVersion: 1, + generatedAt: new Date().toISOString(), + models: [modelEntry], + }); + expect(() => parseModelCatalog({ ...catalog(entry), models: [entry, entry] })).toThrow(/duplicate/i); + expect(() => parseModelCatalog(catalog({ ...entry, cost: { ...entry.cost, output: Number.NaN } }))).toThrow( + /invalid model/i, + ); + expect(() => parseModelCatalog(catalog({ ...entry, cost: { ...entry.cost, input: 1_000_001 } }))).toThrow( + /invalid model/i, + ); + expect(() => parseModelCatalog(catalog({ ...entry, contextWindow: 100_000_001 }))).toThrow(/invalid model/i); expect(() => - parseModelCatalog({ schemaVersion: 1, generatedAt: new Date().toISOString(), models: [entry, entry] }), - ).toThrow("duplicate"); - expect(() => - parseModelCatalog({ - schemaVersion: 1, - generatedAt: new Date().toISOString(), - models: [{ ...entry, cost: { ...entry.cost, output: Number.NaN } }], - }), - ).toThrow("invalid model"); + parseModelCatalog(catalog({ ...entry, thinkingLevelMap: { unsupported: "value" } } as Model)), + ).toThrow(/invalid model/i); }); }); diff --git a/packages/coding-agent/src/core/remote-model-catalog.ts b/packages/coding-agent/src/core/remote-model-catalog.ts index 6a85734c03..c8aa9dfc09 100644 --- a/packages/coding-agent/src/core/remote-model-catalog.ts +++ b/packages/coding-agent/src/core/remote-model-catalog.ts @@ -1,22 +1,14 @@ import { Buffer } from "node:buffer"; import { existsSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs"; -import type { Api, Model } from "@earendil-works/pi-ai"; +import { type Api, type Model, type ModelCatalogV1, parseModelCatalog } from "@earendil-works/pi-ai"; const DEFAULT_PRIME_AGENT_DOWNLOAD_BASE_URL = "https://pub-728493de92a943e2a9b2d17b4719f318.r2.dev"; const MODEL_CATALOG_PATH = "model-catalog.json"; -const MODEL_CATALOG_SCHEMA_VERSION = 1; const MODEL_CATALOG_CACHE_TTL_MS = 24 * 60 * 60 * 1000; const MODEL_CATALOG_FETCH_TIMEOUT_MS = 5_000; const MAX_MODEL_CATALOG_BYTES = 8 * 1024 * 1024; -const MAX_MODEL_CATALOG_MODELS = 20_000; const pendingRefreshes = new Map[] | undefined>>(); -interface ModelCatalogV1 { - schemaVersion: 1; - generatedAt: string; - models: Model[]; -} - interface CachedModelCatalog { url: string; fetchedAt: number; @@ -27,88 +19,8 @@ function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } -function isFiniteCost(value: unknown): value is number { - return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1_000_000; -} - -function isNonEmptyString(value: unknown, maxLength: number): value is string { - return typeof value === "string" && value.length > 0 && value.length <= maxLength; -} - -function isBoundedJsonObject(value: unknown, depth = 0): boolean { - if (!isRecord(value) || depth > 10 || Object.keys(value).length > 100) return false; - return Object.entries(value).every(([key, entry]) => { - if (key.length > 128) return false; - if (entry === null || typeof entry === "boolean") return true; - if (typeof entry === "string") return entry.length <= 4_096; - if (typeof entry === "number") return Number.isFinite(entry); - if (Array.isArray(entry)) - return entry.length <= 100 && entry.every((item) => isBoundedJsonObject({ item }, depth + 1)); - return isBoundedJsonObject(entry, depth + 1); - }); -} - -const THINKING_LEVELS = new Set(["off", "minimal", "low", "medium", "high", "xhigh", "max"]); - -function isThinkingLevelMap(value: unknown): boolean { - if (value === undefined) return true; - if (!isRecord(value) || Object.keys(value).some((key) => !THINKING_LEVELS.has(key))) return false; - return Object.values(value).every( - (entry) => entry === null || (typeof entry === "string" && entry.length > 0 && entry.length <= 128), - ); -} - -function isCatalogModel(value: unknown): value is Model { - if (!isRecord(value) || !isRecord(value.cost)) return false; - return ( - isNonEmptyString(value.id, 1_024) && - isNonEmptyString(value.name, 1_024) && - isNonEmptyString(value.api, 128) && - isNonEmptyString(value.provider, 128) && - typeof value.baseUrl === "string" && - value.baseUrl.length <= 2_048 && - typeof value.reasoning === "boolean" && - isThinkingLevelMap(value.thinkingLevelMap) && - Array.isArray(value.input) && - value.input.length > 0 && - value.input.length <= 2 && - value.input.every((item) => item === "text" || item === "image") && - isFiniteCost(value.cost.input) && - isFiniteCost(value.cost.output) && - isFiniteCost(value.cost.cacheRead) && - isFiniteCost(value.cost.cacheWrite) && - typeof value.contextWindow === "number" && - Number.isSafeInteger(value.contextWindow) && - value.contextWindow > 0 && - value.contextWindow <= 100_000_000 && - typeof value.maxTokens === "number" && - Number.isSafeInteger(value.maxTokens) && - value.maxTokens > 0 && - value.maxTokens <= 100_000_000 && - (value.featured === undefined || typeof value.featured === "boolean") && - (value.headers === undefined || isBoundedJsonObject(value.headers)) && - (value.compat === undefined || isBoundedJsonObject(value.compat)) - ); -} - export function parseRemoteModelCatalog(value: unknown): ModelCatalogV1 { - if (!isRecord(value) || value.schemaVersion !== MODEL_CATALOG_SCHEMA_VERSION) { - throw new Error("Unsupported model catalog schema version"); - } - if (typeof value.generatedAt !== "string" || !Number.isFinite(Date.parse(value.generatedAt))) { - throw new Error("Invalid model catalog timestamp"); - } - if (!Array.isArray(value.models) || value.models.length === 0 || value.models.length > MAX_MODEL_CATALOG_MODELS) { - throw new Error("Invalid model catalog model count"); - } - const seen = new Set(); - for (const model of value.models) { - if (!isCatalogModel(model)) throw new Error("Invalid model catalog entry"); - const key = `${model.provider}/${model.id}`; - if (seen.has(key)) throw new Error(`Duplicate model catalog entry ${key}`); - seen.add(key); - } - return value as unknown as ModelCatalogV1; + return parseModelCatalog(value); } function cloneTransport(template: Model, remote: Model): Model { @@ -210,9 +122,29 @@ async function fetchCatalog(url: string, fetchFn: typeof fetch): Promise MAX_MODEL_CATALOG_BYTES) { throw new Error("Model catalog response is too large"); } - const text = await response.text(); - if (Buffer.byteLength(text, "utf8") > MAX_MODEL_CATALOG_BYTES) - throw new Error("Model catalog response is too large"); + if (!response.body) throw new Error("Model catalog response body is empty"); + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let bytesRead = 0; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + bytesRead += value.byteLength; + if (bytesRead > MAX_MODEL_CATALOG_BYTES) { + try { + await reader.cancel(); + } catch { + // Ignore cancellation errors and reject the oversized response. + } + throw new Error("Model catalog response is too large"); + } + chunks.push(value); + } + } finally { + reader.releaseLock(); + } + const text = Buffer.concat(chunks, bytesRead).toString("utf8"); return parseRemoteModelCatalog(JSON.parse(text) as unknown); } diff --git a/packages/coding-agent/test/remote-model-catalog.test.ts b/packages/coding-agent/test/remote-model-catalog.test.ts index ab4554f582..9e06e15c4b 100644 --- a/packages/coding-agent/test/remote-model-catalog.test.ts +++ b/packages/coding-agent/test/remote-model-catalog.test.ts @@ -62,6 +62,7 @@ describe("remote model catalog", () => { baseUrl: "https://attacker.test/v1", api: "attacker-api", headers: { Authorization: "exfiltrate" }, + compat: { supportsStore: true }, cost: { ...bundled.cost, input: bundled.cost.input + 1 }, }; const [merged] = mergeRemoteModelCatalog([bundled], [remote]); @@ -72,6 +73,7 @@ describe("remote model catalog", () => { cost: remote.cost, }); expect(merged.headers).toEqual(bundled.headers); + expect(merged.compat).toEqual(remote.compat); }); test("uses the hosted model list and only accepts additions through a bundled transport", () => { @@ -103,6 +105,24 @@ describe("remote model catalog", () => { expect(fetchFn).toHaveBeenCalledOnce(); }); + test("stops reading a chunked response at the byte limit", async () => { + const cachePath = join(tempDir, "cache.json"); + const cancel = vi.fn(); + let pulls = 0; + const body = new ReadableStream({ + pull(controller) { + pulls += 1; + controller.enqueue(new Uint8Array(1024 * 1024)); + }, + cancel, + }); + const fetchFn = vi.fn(async () => new Response(body, { status: 200 })); + expect(await refreshRemoteModelCatalog(cachePath, { fetchFn, now: 1_000 })).toBeUndefined(); + expect(cancel).toHaveBeenCalledOnce(); + expect(pulls).toBeLessThanOrEqual(10); + expect(existsSync(cachePath)).toBe(false); + }); + test("uses a stale validated cache when offline or refresh fails", async () => { const cachePath = join(tempDir, "cache.json"); const model = openAiModel(); From aeb795a7331018f49b50f0a3aa32ed32ad2abb28 Mon Sep 17 00:00:00 2001 From: Seth Date: Mon, 31 Aug 2026 20:20:48 -0400 Subject: [PATCH 05/13] refactor: share environment flag parsing --- .../coding-agent/src/core/model-registry.ts | 11 +++------- .../coding-agent/src/core/package-manager.ts | 21 +++++++------------ .../src/core/remote-model-catalog.ts | 8 ++----- packages/coding-agent/src/main.ts | 6 +----- packages/coding-agent/src/utils/env.ts | 4 ++++ .../coding-agent/src/utils/tools-manager.ts | 9 ++------ 6 files changed, 20 insertions(+), 39 deletions(-) create mode 100644 packages/coding-agent/src/utils/env.ts diff --git a/packages/coding-agent/src/core/model-registry.ts b/packages/coding-agent/src/core/model-registry.ts index 052400c7be..97e4072d25 100644 --- a/packages/coding-agent/src/core/model-registry.ts +++ b/packages/coding-agent/src/core/model-registry.ts @@ -28,6 +28,7 @@ import { type Static, type TProperties, Type } from "typebox"; import type { Validator } from "typebox/compile"; import type { TLocalizedValidationError } from "typebox/error"; import { getAgentDir } from "../config.js"; +import { isTruthyEnvFlag } from "../utils/env.js"; import type { AuthSourceToken, AuthStatus, AuthStorage } from "./auth-storage.js"; import { PRIME_INFERENCE_PROVIDER_ID } from "./prime-inference-auth.js"; import { @@ -429,12 +430,6 @@ function privatePrimeAuthorizationFingerprint(apiKey: string, teamId: string): s return createHash("sha256").update(apiKey).update("\0").update(teamId).digest("hex"); } -function isOfflineModeEnabled(): boolean { - const value = process.env.PI_OFFLINE; - if (!value) return false; - return value === "1" || value.toLowerCase() === "true" || value.toLowerCase() === "yes"; -} - /** * Model registry - loads and manages models, resolves API keys via AuthStorage. */ @@ -826,13 +821,13 @@ export class ModelRegistry { this.authorizedPrivatePrimeInferenceModelIds = new Set(cached.modelIds); this.authorizedPrivatePrimeInferenceTeamId = teamId; const cacheIsFresh = Date.now() - cached.refreshedAt < PRIVATE_PRIME_AUTHORIZATION_CACHE_TTL_MS; - if (cacheIsFresh || isOfflineModeEnabled()) { + if (cacheIsFresh || isTruthyEnvFlag(process.env.PI_OFFLINE)) { return; } this.startBackgroundPrivatePrimeAuthorizationRefresh(apiKey, teamHeaders, teamId, fingerprint); return; } - if (isOfflineModeEnabled()) { + if (isTruthyEnvFlag(process.env.PI_OFFLINE)) { this.authorizedPrivatePrimeInferenceModelIds.clear(); this.authorizedPrivatePrimeInferenceTeamId = undefined; return; diff --git a/packages/coding-agent/src/core/package-manager.ts b/packages/coding-agent/src/core/package-manager.ts index eaca1c746d..0fe3719587 100644 --- a/packages/coding-agent/src/core/package-manager.ts +++ b/packages/coding-agent/src/core/package-manager.ts @@ -29,6 +29,7 @@ import ignore from "ignore"; import { minimatch } from "minimatch"; import { CONFIG_DIR_NAME, getBundledSkillsDir } from "../config.js"; import { shouldUseWindowsShell } from "../utils/child-process.js"; +import { isTruthyEnvFlag } from "../utils/env.js"; import { type GitSource, parseGitUrl } from "../utils/git.js"; import { canonicalizePath, isLocalPath } from "../utils/paths.js"; import type { ResourceDiagnostic } from "./diagnostics.js"; @@ -39,12 +40,6 @@ const NETWORK_TIMEOUT_MS = 10000; const UPDATE_CHECK_CONCURRENCY = 4; const GIT_UPDATE_CONCURRENCY = 4; -function isOfflineModeEnabled(): boolean { - const value = process.env.PI_OFFLINE; - if (!value) return false; - return value === "1" || value.toLowerCase() === "true" || value.toLowerCase() === "yes"; -} - export interface PathMetadata { source: string; scope: SourceScope; @@ -1028,7 +1023,7 @@ export class DefaultPackageManager implements PackageManager { } private async updateConfiguredSources(sources: ConfiguredUpdateSource[]): Promise { - if (isOfflineModeEnabled() || sources.length === 0) { + if (isTruthyEnvFlag(process.env.PI_OFFLINE) || sources.length === 0) { return; } @@ -1126,7 +1121,7 @@ export class DefaultPackageManager implements PackageManager { } async checkForAvailableUpdates(): Promise { - if (isOfflineModeEnabled()) { + if (isTruthyEnvFlag(process.env.PI_OFFLINE)) { return []; } @@ -1208,7 +1203,7 @@ export class DefaultPackageManager implements PackageManager { } const installMissing = async (): Promise => { - if (isOfflineModeEnabled()) { + if (isTruthyEnvFlag(process.env.PI_OFFLINE)) { return false; } if (!onMissing) { @@ -1241,7 +1236,7 @@ export class DefaultPackageManager implements PackageManager { if (!existsSync(installedPath)) { const installed = await installMissing(); if (!installed) continue; - } else if (scope === "temporary" && !parsed.pinned && !isOfflineModeEnabled()) { + } else if (scope === "temporary" && !parsed.pinned && !isTruthyEnvFlag(process.env.PI_OFFLINE)) { await this.refreshTemporaryGitSource(parsed, sourceStr); } metadata.baseDir = installedPath; @@ -1407,7 +1402,7 @@ export class DefaultPackageManager implements PackageManager { } private async npmHasAvailableUpdate(source: NpmSource, installedPath: string): Promise { - if (isOfflineModeEnabled()) { + if (isTruthyEnvFlag(process.env.PI_OFFLINE)) { return false; } @@ -1449,7 +1444,7 @@ export class DefaultPackageManager implements PackageManager { } private async gitHasAvailableUpdate(installedPath: string): Promise { - if (isOfflineModeEnabled()) { + if (isTruthyEnvFlag(process.env.PI_OFFLINE)) { return false; } @@ -1761,7 +1756,7 @@ export class DefaultPackageManager implements PackageManager { } private async refreshTemporaryGitSource(source: GitSource, sourceStr: string): Promise { - if (isOfflineModeEnabled()) { + if (isTruthyEnvFlag(process.env.PI_OFFLINE)) { return; } try { diff --git a/packages/coding-agent/src/core/remote-model-catalog.ts b/packages/coding-agent/src/core/remote-model-catalog.ts index c8aa9dfc09..e3cf5d46b6 100644 --- a/packages/coding-agent/src/core/remote-model-catalog.ts +++ b/packages/coding-agent/src/core/remote-model-catalog.ts @@ -1,6 +1,7 @@ import { Buffer } from "node:buffer"; import { existsSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs"; import { type Api, type Model, type ModelCatalogV1, parseModelCatalog } from "@earendil-works/pi-ai"; +import { isTruthyEnvFlag } from "../utils/env.js"; const DEFAULT_PRIME_AGENT_DOWNLOAD_BASE_URL = "https://pub-728493de92a943e2a9b2d17b4719f318.r2.dev"; const MODEL_CATALOG_PATH = "model-catalog.json"; @@ -107,11 +108,6 @@ export function readCachedRemoteModelCatalog(cachePath: string): Model[] | return readCache(cachePath, getRemoteModelCatalogUrl())?.catalog.models; } -function offlineModeEnabled(): boolean { - const value = process.env.PI_OFFLINE?.toLowerCase(); - return value === "1" || value === "true" || value === "yes"; -} - async function fetchCatalog(url: string, fetchFn: typeof fetch): Promise { const response = await fetchFn(url, { headers: { accept: "application/json" }, @@ -156,7 +152,7 @@ export async function refreshRemoteModelCatalog( const now = options.now ?? Date.now(); const cached = readCache(cachePath, url); if (cached && now - cached.fetchedAt < MODEL_CATALOG_CACHE_TTL_MS) return cached.catalog.models; - if (offlineModeEnabled()) return cached?.catalog.models; + if (isTruthyEnvFlag(process.env.PI_OFFLINE)) return cached?.catalog.models; const key = `${cachePath}\0${url}`; const existing = pendingRefreshes.get(key); diff --git a/packages/coding-agent/src/main.ts b/packages/coding-agent/src/main.ts index 3c1a744f57..d2b4b52f9c 100644 --- a/packages/coding-agent/src/main.ts +++ b/packages/coding-agent/src/main.ts @@ -111,6 +111,7 @@ import { ExtensionSelectorComponent } from "./modes/interactive/components/exten import { shouldRunOnboarding } from "./modes/interactive/onboarding.js"; import { initTheme, preloadCodeHighlighter, stopThemeWatcher } from "./modes/interactive/theme/theme.js"; import { handleConfigCommand } from "./package-manager-cli.js"; +import { isTruthyEnvFlag } from "./utils/env.js"; import { isLocalPath } from "./utils/paths.js"; /** @@ -154,11 +155,6 @@ function reportDiagnostics(diagnostics: readonly AgentSessionRuntimeDiagnostic[] } } -function isTruthyEnvFlag(value: string | undefined): boolean { - if (!value) return false; - return value === "1" || value.toLowerCase() === "true" || value.toLowerCase() === "yes"; -} - export type ClientMode = AgentExecutionMode; /** Compatibility view of the CLI's internal daemon process entrypoint. */ export type AppMode = ClientMode | "daemon"; diff --git a/packages/coding-agent/src/utils/env.ts b/packages/coding-agent/src/utils/env.ts new file mode 100644 index 0000000000..b646498029 --- /dev/null +++ b/packages/coding-agent/src/utils/env.ts @@ -0,0 +1,4 @@ +export function isTruthyEnvFlag(value: string | undefined): boolean { + if (!value) return false; + return value === "1" || value.toLowerCase() === "true" || value.toLowerCase() === "yes"; +} diff --git a/packages/coding-agent/src/utils/tools-manager.ts b/packages/coding-agent/src/utils/tools-manager.ts index c3da7045ec..804149cffd 100644 --- a/packages/coding-agent/src/utils/tools-manager.ts +++ b/packages/coding-agent/src/utils/tools-manager.ts @@ -7,6 +7,7 @@ import { join } from "path"; import { Readable } from "stream"; import { pipeline } from "stream/promises"; import { APP_NAME, getBinDir } from "../config.js"; +import { isTruthyEnvFlag } from "./env.js"; const TOOLS_DIR = getBinDir(); const NETWORK_TIMEOUT_MS = 10_000; @@ -33,12 +34,6 @@ export interface ToolUnavailableResult { export type ToolEnsureResult = ToolAvailableResult | ToolUnavailableResult; -function isOfflineModeEnabled(): boolean { - const value = process.env.PI_OFFLINE; - if (!value) return false; - return value === "1" || value.toLowerCase() === "true" || value.toLowerCase() === "yes"; -} - interface ToolConfig { name: string; repo: string; // GitHub repo (e.g., "sharkdp/fd") @@ -330,7 +325,7 @@ export async function ensureToolWithStatus(tool: ManagedTool, silent: boolean = const platformName = platform(); const architecture = arch(); - if (isOfflineModeEnabled()) { + if (isTruthyEnvFlag(process.env.PI_OFFLINE)) { if (!silent) { console.log(chalk.yellow(`${config.name} not found. Offline mode enabled, skipping download.`)); } From b299f64031c3e949dc1daa365a09c4264450dfff Mon Sep 17 00:00:00 2001 From: Seth Date: Mon, 31 Aug 2026 20:30:29 -0400 Subject: [PATCH 06/13] fix: reject incompatible hosted catalogs --- packages/ai/src/model-catalog.ts | 180 +++++++++++++++++- packages/ai/test/model-catalog-format.test.ts | 30 +++ .../src/core/remote-model-catalog.ts | 10 +- .../test/remote-model-catalog.test.ts | 12 ++ 4 files changed, 226 insertions(+), 6 deletions(-) diff --git a/packages/ai/src/model-catalog.ts b/packages/ai/src/model-catalog.ts index 65734e163c..b356807613 100644 --- a/packages/ai/src/model-catalog.ts +++ b/packages/ai/src/model-catalog.ts @@ -34,6 +34,180 @@ function isBoundedJsonObject(value: unknown, depth = 0): boolean { }); } +function hasOnlyKeys(value: Record, allowed: ReadonlySet): boolean { + return Object.keys(value).every((key) => allowed.has(key)); +} + +function isStringArray(value: unknown): boolean { + return ( + Array.isArray(value) && + value.length <= 100 && + value.every((entry) => typeof entry === "string" && entry.length > 0 && entry.length <= 256) + ); +} + +function isStringRecord(value: unknown): boolean { + return ( + isRecord(value) && + Object.keys(value).length <= 100 && + Object.entries(value).every( + ([key, entry]) => key.length <= 128 && typeof entry === "string" && entry.length <= 4_096, + ) + ); +} + +const OPENROUTER_ROUTING_KEYS = new Set([ + "allow_fallbacks", + "require_parameters", + "data_collection", + "zdr", + "enforce_distillable_text", + "order", + "only", + "ignore", + "quantizations", + "sort", + "max_price", + "preferred_min_throughput", + "preferred_max_latency", +]); +const OPENROUTER_BOOLEAN_KEYS = ["allow_fallbacks", "require_parameters", "zdr", "enforce_distillable_text"] as const; +const OPENROUTER_ARRAY_KEYS = ["order", "only", "ignore", "quantizations"] as const; +const PERCENTILE_KEYS = new Set(["p50", "p75", "p90", "p99"]); +const OPENROUTER_SORT_KEYS = new Set(["by", "partition"]); +const OPENROUTER_PRICE_KEYS = new Set(["prompt", "completion", "image", "audio", "request"]); +const VERCEL_ROUTING_KEYS = new Set(["only", "order"]); +const OPENAI_RESPONSES_COMPAT_KEYS = new Set(["sendSessionIdHeader", "supportsLongCacheRetention"]); +const ANTHROPIC_MESSAGES_COMPAT_KEYS = new Set(["supportsEagerToolInputStreaming", "supportsLongCacheRetention"]); + +function isNonNegativeFiniteNumber(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value) && value >= 0; +} + +function isPercentileValue(value: unknown): boolean { + if (isNonNegativeFiniteNumber(value)) return true; + return ( + isRecord(value) && hasOnlyKeys(value, PERCENTILE_KEYS) && Object.values(value).every(isNonNegativeFiniteNumber) + ); +} + +function isOpenRouterRouting(value: unknown): boolean { + if (!isRecord(value) || !hasOnlyKeys(value, OPENROUTER_ROUTING_KEYS)) return false; + for (const key of OPENROUTER_BOOLEAN_KEYS) { + if (value[key] !== undefined && typeof value[key] !== "boolean") return false; + } + for (const key of OPENROUTER_ARRAY_KEYS) { + if (value[key] !== undefined && !isStringArray(value[key])) return false; + } + if (value.data_collection !== undefined && value.data_collection !== "allow" && value.data_collection !== "deny") + return false; + if (value.sort !== undefined) { + if (typeof value.sort !== "string") { + if (!isRecord(value.sort) || !hasOnlyKeys(value.sort, OPENROUTER_SORT_KEYS)) return false; + if (value.sort.by !== undefined && typeof value.sort.by !== "string") return false; + if ( + value.sort.partition !== undefined && + value.sort.partition !== null && + typeof value.sort.partition !== "string" + ) + return false; + } + } + if (value.max_price !== undefined) { + if (!isRecord(value.max_price) || !hasOnlyKeys(value.max_price, OPENROUTER_PRICE_KEYS)) return false; + if ( + !Object.values(value.max_price).every( + (entry) => isNonNegativeFiniteNumber(entry) || (typeof entry === "string" && entry.length <= 128), + ) + ) + return false; + } + return ( + (value.preferred_min_throughput === undefined || isPercentileValue(value.preferred_min_throughput)) && + (value.preferred_max_latency === undefined || isPercentileValue(value.preferred_max_latency)) + ); +} + +const OPENAI_COMPLETIONS_COMPAT_KEYS = new Set([ + "supportsStore", + "supportsDeveloperRole", + "supportsReasoningEffort", + "supportsUsageInStreaming", + "maxTokensField", + "requiresToolResultName", + "requiresAssistantAfterToolResult", + "requiresThinkingAsText", + "requiresReasoningContentOnAssistantMessages", + "thinkingFormat", + "openRouterRouting", + "vercelGatewayRouting", + "zaiToolStream", + "supportsStrictMode", + "cacheControlFormat", + "sendSessionAffinityHeaders", + "supportsLongCacheRetention", +]); +const OPENAI_COMPLETIONS_BOOLEAN_KEYS = [ + "supportsStore", + "supportsDeveloperRole", + "supportsReasoningEffort", + "supportsUsageInStreaming", + "requiresToolResultName", + "requiresAssistantAfterToolResult", + "requiresThinkingAsText", + "requiresReasoningContentOnAssistantMessages", + "zaiToolStream", + "supportsStrictMode", + "sendSessionAffinityHeaders", + "supportsLongCacheRetention", +] as const; +const THINKING_FORMATS = new Set(["openai", "openrouter", "deepseek", "zai", "qwen", "qwen-chat-template"]); + +function isOpenAiCompletionsCompat(value: Record): boolean { + if (!hasOnlyKeys(value, OPENAI_COMPLETIONS_COMPAT_KEYS)) return false; + for (const key of OPENAI_COMPLETIONS_BOOLEAN_KEYS) { + if (value[key] !== undefined && typeof value[key] !== "boolean") return false; + } + if ( + value.maxTokensField !== undefined && + value.maxTokensField !== "max_completion_tokens" && + value.maxTokensField !== "max_tokens" + ) + return false; + if (value.thinkingFormat !== undefined && !THINKING_FORMATS.has(value.thinkingFormat as string)) return false; + if (value.cacheControlFormat !== undefined && value.cacheControlFormat !== "anthropic") return false; + if (value.openRouterRouting !== undefined && !isOpenRouterRouting(value.openRouterRouting)) return false; + if (value.vercelGatewayRouting !== undefined) { + if ( + !isRecord(value.vercelGatewayRouting) || + !hasOnlyKeys(value.vercelGatewayRouting, VERCEL_ROUTING_KEYS) || + (value.vercelGatewayRouting.only !== undefined && !isStringArray(value.vercelGatewayRouting.only)) || + (value.vercelGatewayRouting.order !== undefined && !isStringArray(value.vercelGatewayRouting.order)) + ) + return false; + } + return true; +} + +function isCatalogCompat(api: string, value: unknown): boolean { + if (value === undefined) return true; + if (!isRecord(value) || !isBoundedJsonObject(value)) return false; + if (api === "openai-completions") return isOpenAiCompletionsCompat(value); + if (api === "openai-responses") { + return ( + hasOnlyKeys(value, OPENAI_RESPONSES_COMPAT_KEYS) && + Object.values(value).every((entry) => typeof entry === "boolean") + ); + } + if (api === "anthropic-messages") { + return ( + hasOnlyKeys(value, ANTHROPIC_MESSAGES_COMPAT_KEYS) && + Object.values(value).every((entry) => typeof entry === "boolean") + ); + } + return false; +} + const THINKING_LEVELS = new Set(["off", "minimal", "low", "medium", "high", "xhigh", "max"]); function isThinkingLevelMap(value: unknown): boolean { @@ -72,8 +246,8 @@ function isCatalogModel(value: unknown): value is Model { value.maxTokens > 0 && value.maxTokens <= 100_000_000 && (value.featured === undefined || typeof value.featured === "boolean") && - (value.headers === undefined || isBoundedJsonObject(value.headers)) && - (value.compat === undefined || isBoundedJsonObject(value.compat)) + (value.headers === undefined || isStringRecord(value.headers)) && + isCatalogCompat(value.api, value.compat) ); } @@ -101,7 +275,7 @@ export function parseModelCatalog(value: unknown): ModelCatalogV1 { const seen = new Set(); for (const model of value.models) { if (!isCatalogModel(model)) throw new Error("Invalid model catalog entry"); - const key = `${model.provider}/${model.id}`; + const key = JSON.stringify([model.provider, model.id]); if (seen.has(key)) throw new Error(`Duplicate model catalog entry ${key}`); seen.add(key); } diff --git a/packages/ai/test/model-catalog-format.test.ts b/packages/ai/test/model-catalog-format.test.ts index a9864acb74..9c3c708d55 100644 --- a/packages/ai/test/model-catalog-format.test.ts +++ b/packages/ai/test/model-catalog-format.test.ts @@ -25,6 +25,15 @@ describe("hosted model catalog format", () => { expect(result.models.map((entry) => `${entry.provider}/${entry.id}`)).toEqual(["a/z", "z/a", "z/b"]); }); + test("treats provider and model ids as an unambiguous pair", () => { + const parsed = parseModelCatalog({ + schemaVersion: 1, + generatedAt: new Date().toISOString(), + models: [model("a", "b/c"), model("a/b", "c")], + }); + expect(parsed.models).toHaveLength(2); + }); + test("applies the same strict schema used by clients", () => { const entry = model("provider", "model"); const catalog = (modelEntry: Model) => ({ @@ -43,5 +52,26 @@ describe("hosted model catalog format", () => { expect(() => parseModelCatalog(catalog({ ...entry, thinkingLevelMap: { unsupported: "value" } } as Model)), ).toThrow(/invalid model/i); + expect(() => + parseModelCatalog( + catalog({ + ...entry, + compat: { openRouterRouting: { only: ["anthropic"], max_price: { prompt: "1" } } }, + }), + ), + ).not.toThrow(); + expect(() => + parseModelCatalog(catalog({ ...entry, compat: { openRouterRouting: "invalid" } } as Model)), + ).toThrow(/invalid model/i); + expect(() => + parseModelCatalog( + catalog({ ...entry, api: "openai-responses", compat: { supportsStore: true } } as Model), + ), + ).toThrow(/invalid model/i); + expect(() => + parseModelCatalog( + catalog({ ...entry, headers: { authorization: { nested: true } } } as unknown as Model), + ), + ).toThrow(/invalid model/i); }); }); diff --git a/packages/coding-agent/src/core/remote-model-catalog.ts b/packages/coding-agent/src/core/remote-model-catalog.ts index e3cf5d46b6..0ba565c5d9 100644 --- a/packages/coding-agent/src/core/remote-model-catalog.ts +++ b/packages/coding-agent/src/core/remote-model-catalog.ts @@ -43,22 +43,26 @@ function cloneTransport(template: Model, remote: Model): Model { }; } +function modelKey(provider: string, id: string): string { + return JSON.stringify([provider, id]); +} + export function mergeRemoteModelCatalog( bundledModels: readonly Model[], remoteModels: readonly Model[] | undefined, ): Model[] { if (!remoteModels) return bundledModels.map((model) => structuredClone(model)); - const exact = new Map(bundledModels.map((model) => [`${model.provider}/${model.id}`, model])); + const exact = new Map(bundledModels.map((model) => [modelKey(model.provider, model.id), model])); const transports = new Map>(); for (const model of bundledModels) transports.set(`${model.provider}\0${model.api}\0${model.baseUrl}`, model); const merged: Model[] = []; for (const remote of remoteModels) { - const key = `${remote.provider}/${remote.id}`; + const key = modelKey(remote.provider, remote.id); const template = exact.get(key) ?? transports.get(`${remote.provider}\0${remote.api}\0${remote.baseUrl}`); if (template) merged.push(cloneTransport(template, remote)); } - return merged; + return merged.length > 0 ? merged : bundledModels.map((model) => structuredClone(model)); } export function getRemoteModelCatalogUrl(): string { diff --git a/packages/coding-agent/test/remote-model-catalog.test.ts b/packages/coding-agent/test/remote-model-catalog.test.ts index 9e06e15c4b..b1c3c1ae68 100644 --- a/packages/coding-agent/test/remote-model-catalog.test.ts +++ b/packages/coding-agent/test/remote-model-catalog.test.ts @@ -85,6 +85,18 @@ describe("remote model catalog", () => { expect(mergeRemoteModelCatalog([bundled], undefined).map((model) => model.id)).toEqual([bundled.id]); }); + test("falls back to bundled models when no hosted entry uses a known transport", () => { + const bundled = openAiModel(); + const incompatible = { + ...structuredClone(bundled), + id: "incompatible-model", + baseUrl: "https://unsupported.test/v1", + }; + const merged = mergeRemoteModelCatalog([bundled], [incompatible]); + expect(merged.map((model) => model.id)).toEqual([bundled.id]); + expect(merged[0]).not.toBe(bundled); + }); + test("fetches once, validates, and reuses the fresh atomic cache", async () => { const cachePath = join(tempDir, "cache.json"); const model = openAiModel(); From e4e286108f5d18511bed67ae0d996cb7f0f6a136 Mon Sep 17 00:00:00 2001 From: Seth Date: Mon, 31 Aug 2026 20:31:53 -0400 Subject: [PATCH 07/13] fix: fail closed on an empty OpenRouter slice --- packages/ai/scripts/generate-models.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/ai/scripts/generate-models.ts b/packages/ai/scripts/generate-models.ts index d10ff5d3f8..c27cc3b165 100644 --- a/packages/ai/scripts/generate-models.ts +++ b/packages/ai/scripts/generate-models.ts @@ -814,6 +814,7 @@ async function fetchOpenRouterModels(): Promise[]> { models.push(normalizedModel); } + if (models.length === 0) throw new Error("OpenRouter catalog has no tool-capable models"); console.log(`Fetched ${models.length} tool-capable models from OpenRouter`); return models; } catch (error) { From 91ea1c7cc20444cfc7d4db49167e7b9202b182f3 Mon Sep 17 00:00:00 2001 From: Seth Date: Mon, 31 Aug 2026 20:34:57 -0400 Subject: [PATCH 08/13] fix: require catalog transport matches --- .../coding-agent/src/core/remote-model-catalog.ts | 6 +++++- .../coding-agent/test/remote-model-catalog.test.ts | 14 ++++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/packages/coding-agent/src/core/remote-model-catalog.ts b/packages/coding-agent/src/core/remote-model-catalog.ts index 0ba565c5d9..6d1a020a35 100644 --- a/packages/coding-agent/src/core/remote-model-catalog.ts +++ b/packages/coding-agent/src/core/remote-model-catalog.ts @@ -59,7 +59,11 @@ export function mergeRemoteModelCatalog( const merged: Model[] = []; for (const remote of remoteModels) { const key = modelKey(remote.provider, remote.id); - const template = exact.get(key) ?? transports.get(`${remote.provider}\0${remote.api}\0${remote.baseUrl}`); + const exactTemplate = exact.get(key); + const template = + exactTemplate && exactTemplate.api === remote.api && exactTemplate.baseUrl === remote.baseUrl + ? exactTemplate + : transports.get(`${remote.provider}\0${remote.api}\0${remote.baseUrl}`); if (template) merged.push(cloneTransport(template, remote)); } return merged.length > 0 ? merged : bundledModels.map((model) => structuredClone(model)); diff --git a/packages/coding-agent/test/remote-model-catalog.test.ts b/packages/coding-agent/test/remote-model-catalog.test.ts index b1c3c1ae68..e756f1a8c1 100644 --- a/packages/coding-agent/test/remote-model-catalog.test.ts +++ b/packages/coding-agent/test/remote-model-catalog.test.ts @@ -54,15 +54,13 @@ describe("remote model catalog", () => { ).toThrow("Invalid model catalog entry"); }); - test("updates metadata while pinning bundled request transports", () => { + test("updates metadata while pinning headers and rejecting transport changes", () => { const bundled = openAiModel(); const remote = { ...structuredClone(bundled), name: "Current provider name", - baseUrl: "https://attacker.test/v1", - api: "attacker-api", headers: { Authorization: "exfiltrate" }, - compat: { supportsStore: true }, + compat: { supportsLongCacheRetention: false }, cost: { ...bundled.cost, input: bundled.cost.input + 1 }, }; const [merged] = mergeRemoteModelCatalog([bundled], [remote]); @@ -74,6 +72,14 @@ describe("remote model catalog", () => { }); expect(merged.headers).toEqual(bundled.headers); expect(merged.compat).toEqual(remote.compat); + + for (const redirected of [ + { ...remote, baseUrl: "https://attacker.test/v1" }, + { ...remote, api: "attacker-api", compat: undefined }, + ]) { + const [fallback] = mergeRemoteModelCatalog([bundled], [redirected]); + expect(fallback).toEqual(bundled); + } }); test("uses the hosted model list and only accepts additions through a bundled transport", () => { From 0634d93b00e74b5c877636b6329b9450fe6594ba Mon Sep 17 00:00:00 2001 From: Seth Date: Mon, 31 Aug 2026 20:54:25 -0400 Subject: [PATCH 09/13] refactor: share model catalog validation schemas --- packages/ai/scripts/generate-models.ts | 2 +- packages/ai/scripts/model-catalog-format.ts | 2 - packages/ai/scripts/validate-model-catalog.ts | 2 +- packages/ai/src/index.ts | 1 + packages/ai/src/model-catalog.ts | 299 ++++-------------- packages/ai/src/model-compat-schema.ts | 103 ++++++ packages/ai/test/model-catalog-format.test.ts | 53 ++-- .../coding-agent/src/core/model-registry.ts | 88 +----- .../src/core/remote-model-catalog.ts | 8 +- .../test/remote-model-catalog.test.ts | 10 - 10 files changed, 182 insertions(+), 386 deletions(-) delete mode 100644 packages/ai/scripts/model-catalog-format.ts create mode 100644 packages/ai/src/model-compat-schema.ts diff --git a/packages/ai/scripts/generate-models.ts b/packages/ai/scripts/generate-models.ts index c27cc3b165..98845a4b1a 100644 --- a/packages/ai/scripts/generate-models.ts +++ b/packages/ai/scripts/generate-models.ts @@ -6,7 +6,7 @@ import { dirname, join } from "path"; import { fileURLToPath } from "url"; import { getAnthropicCacheCosts } from "../src/cache-pricing.js"; import { getOpenRouterReasoningCapabilities } from "../src/openrouter-reasoning.js"; -import { createModelCatalog } from "./model-catalog-format.js"; +import { createModelCatalog } from "../src/model-catalog.js"; import { CLOUDFLARE_AI_GATEWAY_ANTHROPIC_BASE_URL, CLOUDFLARE_AI_GATEWAY_COMPAT_BASE_URL, diff --git a/packages/ai/scripts/model-catalog-format.ts b/packages/ai/scripts/model-catalog-format.ts deleted file mode 100644 index fdcf3f6b89..0000000000 --- a/packages/ai/scripts/model-catalog-format.ts +++ /dev/null @@ -1,2 +0,0 @@ -export { createModelCatalog, parseModelCatalog } from "../src/model-catalog.js"; -export type { ModelCatalogV1 } from "../src/model-catalog.js"; diff --git a/packages/ai/scripts/validate-model-catalog.ts b/packages/ai/scripts/validate-model-catalog.ts index 34b6a35a01..6797493377 100644 --- a/packages/ai/scripts/validate-model-catalog.ts +++ b/packages/ai/scripts/validate-model-catalog.ts @@ -1,7 +1,7 @@ #!/usr/bin/env tsx import { readFileSync } from "node:fs"; -import { parseModelCatalog } from "./model-catalog-format.js"; +import { parseModelCatalog } from "../src/model-catalog.js"; const [candidatePath] = process.argv.slice(2); if (!candidatePath) throw new Error("Usage: validate-model-catalog.ts "); diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts index a9d3b3b6ae..4698057816 100644 --- a/packages/ai/src/index.ts +++ b/packages/ai/src/index.ts @@ -5,6 +5,7 @@ export * from "./api-registry.js"; export * from "./env-api-keys.js"; export * from "./log.js"; export * from "./model-catalog.js"; +export { ProviderCompatSchema } from "./model-compat-schema.js"; export * from "./models.js"; export type { BedrockOptions, BedrockThinkingDisplay } from "./providers/amazon-bedrock.js"; export type { AnthropicEffort, AnthropicOptions, AnthropicThinkingDisplay } from "./providers/anthropic.js"; diff --git a/packages/ai/src/model-catalog.ts b/packages/ai/src/model-catalog.ts index b356807613..7ffdb2f74b 100644 --- a/packages/ai/src/model-catalog.ts +++ b/packages/ai/src/model-catalog.ts @@ -1,3 +1,6 @@ +import { type TProperties, Type } from "typebox"; +import { Value } from "typebox/value"; +import { isModelCompat } from "./model-compat-schema.js"; import type { Api, Model } from "./types.js"; const MODEL_CATALOG_SCHEMA_VERSION = 1 as const; @@ -9,256 +12,61 @@ export interface ModelCatalogV1 { models: Model[]; } +function strictObject(properties: T) { + return Type.Object(properties, { additionalProperties: false }); +} + +const ThinkingLevelValueSchema = Type.Union([Type.String({ minLength: 1, maxLength: 128 }), Type.Null()]); +const ThinkingLevelMapSchema = strictObject({ + off: Type.Optional(ThinkingLevelValueSchema), + minimal: Type.Optional(ThinkingLevelValueSchema), + low: Type.Optional(ThinkingLevelValueSchema), + medium: Type.Optional(ThinkingLevelValueSchema), + high: Type.Optional(ThinkingLevelValueSchema), + xhigh: Type.Optional(ThinkingLevelValueSchema), + max: Type.Optional(ThinkingLevelValueSchema), +}); +const CostSchema = Type.Number({ minimum: 0, maximum: 1_000_000 }); +const CatalogModelSchema = Type.Object({ + id: Type.String({ minLength: 1, maxLength: 1_024 }), + name: Type.String({ minLength: 1, maxLength: 1_024 }), + api: Type.String({ minLength: 1, maxLength: 128 }), + provider: Type.String({ minLength: 1, maxLength: 128 }), + baseUrl: Type.String({ maxLength: 2_048 }), + reasoning: Type.Boolean(), + thinkingLevelMap: Type.Optional(ThinkingLevelMapSchema), + input: Type.Array(Type.Union([Type.Literal("text"), Type.Literal("image")]), { minItems: 1, maxItems: 2 }), + cost: Type.Object({ + input: CostSchema, + output: CostSchema, + cacheRead: CostSchema, + cacheWrite: CostSchema, + }), + contextWindow: Type.Integer({ minimum: 1, maximum: 100_000_000 }), + maxTokens: Type.Integer({ minimum: 1, maximum: 100_000_000 }), + featured: Type.Optional(Type.Boolean()), + headers: Type.Optional( + Type.Record(Type.String({ maxLength: 128 }), Type.String({ maxLength: 4_096 }), { maxProperties: 100 }), + ), + compat: Type.Optional(Type.Unknown()), +}); +const ModelCatalogSchema = Type.Object({ + schemaVersion: Type.Literal(MODEL_CATALOG_SCHEMA_VERSION), + generatedAt: Type.String(), + models: Type.Array(CatalogModelSchema, { minItems: 1, maxItems: MAX_MODEL_CATALOG_MODELS }), +}); + function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } -function isFiniteCost(value: unknown): value is number { - return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1_000_000; -} - -function isNonEmptyString(value: unknown, maxLength: number): value is string { - return typeof value === "string" && value.length > 0 && value.length <= maxLength; -} - -function isBoundedJsonObject(value: unknown, depth = 0): boolean { - if (!isRecord(value) || depth > 10 || Object.keys(value).length > 100) return false; - return Object.entries(value).every(([key, entry]) => { - if (key.length > 128) return false; - if (entry === null || typeof entry === "boolean") return true; - if (typeof entry === "string") return entry.length <= 4_096; - if (typeof entry === "number") return Number.isFinite(entry); - if (Array.isArray(entry)) - return entry.length <= 100 && entry.every((item) => isBoundedJsonObject({ item }, depth + 1)); - return isBoundedJsonObject(entry, depth + 1); - }); -} - -function hasOnlyKeys(value: Record, allowed: ReadonlySet): boolean { - return Object.keys(value).every((key) => allowed.has(key)); -} - -function isStringArray(value: unknown): boolean { - return ( - Array.isArray(value) && - value.length <= 100 && - value.every((entry) => typeof entry === "string" && entry.length > 0 && entry.length <= 256) - ); -} - -function isStringRecord(value: unknown): boolean { - return ( - isRecord(value) && - Object.keys(value).length <= 100 && - Object.entries(value).every( - ([key, entry]) => key.length <= 128 && typeof entry === "string" && entry.length <= 4_096, - ) - ); -} - -const OPENROUTER_ROUTING_KEYS = new Set([ - "allow_fallbacks", - "require_parameters", - "data_collection", - "zdr", - "enforce_distillable_text", - "order", - "only", - "ignore", - "quantizations", - "sort", - "max_price", - "preferred_min_throughput", - "preferred_max_latency", -]); -const OPENROUTER_BOOLEAN_KEYS = ["allow_fallbacks", "require_parameters", "zdr", "enforce_distillable_text"] as const; -const OPENROUTER_ARRAY_KEYS = ["order", "only", "ignore", "quantizations"] as const; -const PERCENTILE_KEYS = new Set(["p50", "p75", "p90", "p99"]); -const OPENROUTER_SORT_KEYS = new Set(["by", "partition"]); -const OPENROUTER_PRICE_KEYS = new Set(["prompt", "completion", "image", "audio", "request"]); -const VERCEL_ROUTING_KEYS = new Set(["only", "order"]); -const OPENAI_RESPONSES_COMPAT_KEYS = new Set(["sendSessionIdHeader", "supportsLongCacheRetention"]); -const ANTHROPIC_MESSAGES_COMPAT_KEYS = new Set(["supportsEagerToolInputStreaming", "supportsLongCacheRetention"]); - -function isNonNegativeFiniteNumber(value: unknown): value is number { - return typeof value === "number" && Number.isFinite(value) && value >= 0; -} - -function isPercentileValue(value: unknown): boolean { - if (isNonNegativeFiniteNumber(value)) return true; - return ( - isRecord(value) && hasOnlyKeys(value, PERCENTILE_KEYS) && Object.values(value).every(isNonNegativeFiniteNumber) - ); -} - -function isOpenRouterRouting(value: unknown): boolean { - if (!isRecord(value) || !hasOnlyKeys(value, OPENROUTER_ROUTING_KEYS)) return false; - for (const key of OPENROUTER_BOOLEAN_KEYS) { - if (value[key] !== undefined && typeof value[key] !== "boolean") return false; - } - for (const key of OPENROUTER_ARRAY_KEYS) { - if (value[key] !== undefined && !isStringArray(value[key])) return false; - } - if (value.data_collection !== undefined && value.data_collection !== "allow" && value.data_collection !== "deny") - return false; - if (value.sort !== undefined) { - if (typeof value.sort !== "string") { - if (!isRecord(value.sort) || !hasOnlyKeys(value.sort, OPENROUTER_SORT_KEYS)) return false; - if (value.sort.by !== undefined && typeof value.sort.by !== "string") return false; - if ( - value.sort.partition !== undefined && - value.sort.partition !== null && - typeof value.sort.partition !== "string" - ) - return false; - } - } - if (value.max_price !== undefined) { - if (!isRecord(value.max_price) || !hasOnlyKeys(value.max_price, OPENROUTER_PRICE_KEYS)) return false; - if ( - !Object.values(value.max_price).every( - (entry) => isNonNegativeFiniteNumber(entry) || (typeof entry === "string" && entry.length <= 128), - ) - ) - return false; - } - return ( - (value.preferred_min_throughput === undefined || isPercentileValue(value.preferred_min_throughput)) && - (value.preferred_max_latency === undefined || isPercentileValue(value.preferred_max_latency)) - ); -} - -const OPENAI_COMPLETIONS_COMPAT_KEYS = new Set([ - "supportsStore", - "supportsDeveloperRole", - "supportsReasoningEffort", - "supportsUsageInStreaming", - "maxTokensField", - "requiresToolResultName", - "requiresAssistantAfterToolResult", - "requiresThinkingAsText", - "requiresReasoningContentOnAssistantMessages", - "thinkingFormat", - "openRouterRouting", - "vercelGatewayRouting", - "zaiToolStream", - "supportsStrictMode", - "cacheControlFormat", - "sendSessionAffinityHeaders", - "supportsLongCacheRetention", -]); -const OPENAI_COMPLETIONS_BOOLEAN_KEYS = [ - "supportsStore", - "supportsDeveloperRole", - "supportsReasoningEffort", - "supportsUsageInStreaming", - "requiresToolResultName", - "requiresAssistantAfterToolResult", - "requiresThinkingAsText", - "requiresReasoningContentOnAssistantMessages", - "zaiToolStream", - "supportsStrictMode", - "sendSessionAffinityHeaders", - "supportsLongCacheRetention", -] as const; -const THINKING_FORMATS = new Set(["openai", "openrouter", "deepseek", "zai", "qwen", "qwen-chat-template"]); - -function isOpenAiCompletionsCompat(value: Record): boolean { - if (!hasOnlyKeys(value, OPENAI_COMPLETIONS_COMPAT_KEYS)) return false; - for (const key of OPENAI_COMPLETIONS_BOOLEAN_KEYS) { - if (value[key] !== undefined && typeof value[key] !== "boolean") return false; - } - if ( - value.maxTokensField !== undefined && - value.maxTokensField !== "max_completion_tokens" && - value.maxTokensField !== "max_tokens" - ) - return false; - if (value.thinkingFormat !== undefined && !THINKING_FORMATS.has(value.thinkingFormat as string)) return false; - if (value.cacheControlFormat !== undefined && value.cacheControlFormat !== "anthropic") return false; - if (value.openRouterRouting !== undefined && !isOpenRouterRouting(value.openRouterRouting)) return false; - if (value.vercelGatewayRouting !== undefined) { - if ( - !isRecord(value.vercelGatewayRouting) || - !hasOnlyKeys(value.vercelGatewayRouting, VERCEL_ROUTING_KEYS) || - (value.vercelGatewayRouting.only !== undefined && !isStringArray(value.vercelGatewayRouting.only)) || - (value.vercelGatewayRouting.order !== undefined && !isStringArray(value.vercelGatewayRouting.order)) - ) - return false; - } - return true; -} - -function isCatalogCompat(api: string, value: unknown): boolean { - if (value === undefined) return true; - if (!isRecord(value) || !isBoundedJsonObject(value)) return false; - if (api === "openai-completions") return isOpenAiCompletionsCompat(value); - if (api === "openai-responses") { - return ( - hasOnlyKeys(value, OPENAI_RESPONSES_COMPAT_KEYS) && - Object.values(value).every((entry) => typeof entry === "boolean") - ); - } - if (api === "anthropic-messages") { - return ( - hasOnlyKeys(value, ANTHROPIC_MESSAGES_COMPAT_KEYS) && - Object.values(value).every((entry) => typeof entry === "boolean") - ); - } - return false; -} - -const THINKING_LEVELS = new Set(["off", "minimal", "low", "medium", "high", "xhigh", "max"]); - -function isThinkingLevelMap(value: unknown): boolean { - if (value === undefined) return true; - if (!isRecord(value) || Object.keys(value).some((key) => !THINKING_LEVELS.has(key))) return false; - return Object.values(value).every( - (entry) => entry === null || (typeof entry === "string" && entry.length > 0 && entry.length <= 128), - ); -} - -function isCatalogModel(value: unknown): value is Model { - if (!isRecord(value) || !isRecord(value.cost)) return false; - return ( - isNonEmptyString(value.id, 1_024) && - isNonEmptyString(value.name, 1_024) && - isNonEmptyString(value.api, 128) && - isNonEmptyString(value.provider, 128) && - typeof value.baseUrl === "string" && - value.baseUrl.length <= 2_048 && - typeof value.reasoning === "boolean" && - isThinkingLevelMap(value.thinkingLevelMap) && - Array.isArray(value.input) && - value.input.length > 0 && - value.input.length <= 2 && - value.input.every((item) => item === "text" || item === "image") && - isFiniteCost(value.cost.input) && - isFiniteCost(value.cost.output) && - isFiniteCost(value.cost.cacheRead) && - isFiniteCost(value.cost.cacheWrite) && - typeof value.contextWindow === "number" && - Number.isSafeInteger(value.contextWindow) && - value.contextWindow > 0 && - value.contextWindow <= 100_000_000 && - typeof value.maxTokens === "number" && - Number.isSafeInteger(value.maxTokens) && - value.maxTokens > 0 && - value.maxTokens <= 100_000_000 && - (value.featured === undefined || typeof value.featured === "boolean") && - (value.headers === undefined || isStringRecord(value.headers)) && - isCatalogCompat(value.api, value.compat) - ); -} - export function createModelCatalog(models: readonly Model[], generatedAt = new Date()): ModelCatalogV1 { - const sorted = [...models] - .map((model) => structuredClone(model)) - .sort((left, right) => left.provider.localeCompare(right.provider) || left.id.localeCompare(right.id)); return { schemaVersion: MODEL_CATALOG_SCHEMA_VERSION, generatedAt: generatedAt.toISOString(), - models: sorted, + models: models + .map((model) => structuredClone(model)) + .sort((left, right) => left.provider.localeCompare(right.provider) || left.id.localeCompare(right.id)), }; } @@ -272,9 +80,12 @@ export function parseModelCatalog(value: unknown): ModelCatalogV1 { if (!Array.isArray(value.models) || value.models.length === 0 || value.models.length > MAX_MODEL_CATALOG_MODELS) { throw new Error("Invalid model catalog model count"); } + if (!Value.Check(ModelCatalogSchema, value)) throw new Error("Invalid model catalog entry"); + + const models = value.models as Model[]; const seen = new Set(); - for (const model of value.models) { - if (!isCatalogModel(model)) throw new Error("Invalid model catalog entry"); + for (const model of models) { + if (!isModelCompat(model.api, model.compat)) throw new Error("Invalid model catalog entry"); const key = JSON.stringify([model.provider, model.id]); if (seen.has(key)) throw new Error(`Duplicate model catalog entry ${key}`); seen.add(key); diff --git a/packages/ai/src/model-compat-schema.ts b/packages/ai/src/model-compat-schema.ts new file mode 100644 index 0000000000..b4d49cfc36 --- /dev/null +++ b/packages/ai/src/model-compat-schema.ts @@ -0,0 +1,103 @@ +import { type TProperties, Type } from "typebox"; +import { Value } from "typebox/value"; + +function createCompatSchemas(strict: boolean) { + const object = (properties: T) => Type.Object(properties, { additionalProperties: !strict }); + const number = Type.Number(strict ? { minimum: 0 } : {}); + const string = (maxLength: number, minLength = 0) => Type.String(strict ? { minLength, maxLength } : {}); + const stringList = Type.Array(string(256, 1), strict ? { maxItems: 100 } : {}); + const percentileCutoffs = object({ + p50: Type.Optional(number), + p75: Type.Optional(number), + p90: Type.Optional(number), + p99: Type.Optional(number), + }); + const openRouterRouting = object({ + allow_fallbacks: Type.Optional(Type.Boolean()), + require_parameters: Type.Optional(Type.Boolean()), + data_collection: Type.Optional(Type.Union([Type.Literal("deny"), Type.Literal("allow")])), + zdr: Type.Optional(Type.Boolean()), + enforce_distillable_text: Type.Optional(Type.Boolean()), + order: Type.Optional(stringList), + only: Type.Optional(stringList), + ignore: Type.Optional(stringList), + quantizations: Type.Optional(stringList), + sort: Type.Optional( + Type.Union([ + string(256), + object({ + by: Type.Optional(string(256)), + partition: Type.Optional(Type.Union([string(256), Type.Null()])), + }), + ]), + ), + max_price: Type.Optional( + object({ + prompt: Type.Optional(Type.Union([number, string(128)])), + completion: Type.Optional(Type.Union([number, string(128)])), + image: Type.Optional(Type.Union([number, string(128)])), + audio: Type.Optional(Type.Union([number, string(128)])), + request: Type.Optional(Type.Union([number, string(128)])), + }), + ), + preferred_min_throughput: Type.Optional(Type.Union([number, percentileCutoffs])), + preferred_max_latency: Type.Optional(Type.Union([number, percentileCutoffs])), + }); + const vercelGatewayRouting = object({ + only: Type.Optional(stringList), + order: Type.Optional(stringList), + }); + const openAICompletions = object({ + supportsStore: Type.Optional(Type.Boolean()), + supportsDeveloperRole: Type.Optional(Type.Boolean()), + supportsReasoningEffort: Type.Optional(Type.Boolean()), + supportsUsageInStreaming: Type.Optional(Type.Boolean()), + maxTokensField: Type.Optional(Type.Union([Type.Literal("max_completion_tokens"), Type.Literal("max_tokens")])), + requiresToolResultName: Type.Optional(Type.Boolean()), + requiresAssistantAfterToolResult: Type.Optional(Type.Boolean()), + requiresThinkingAsText: Type.Optional(Type.Boolean()), + requiresReasoningContentOnAssistantMessages: Type.Optional(Type.Boolean()), + thinkingFormat: Type.Optional( + Type.Union([ + Type.Literal("openai"), + Type.Literal("openrouter"), + Type.Literal("deepseek"), + Type.Literal("zai"), + Type.Literal("qwen"), + Type.Literal("qwen-chat-template"), + ]), + ), + openRouterRouting: Type.Optional(openRouterRouting), + vercelGatewayRouting: Type.Optional(vercelGatewayRouting), + zaiToolStream: Type.Optional(Type.Boolean()), + supportsStrictMode: Type.Optional(Type.Boolean()), + cacheControlFormat: Type.Optional(Type.Literal("anthropic")), + sendSessionAffinityHeaders: Type.Optional(Type.Boolean()), + supportsLongCacheRetention: Type.Optional(Type.Boolean()), + }); + const openAIResponses = object({ + sendSessionIdHeader: Type.Optional(Type.Boolean()), + supportsLongCacheRetention: Type.Optional(Type.Boolean()), + }); + const anthropicMessages = object({ + supportsEagerToolInputStreaming: Type.Optional(Type.Boolean()), + supportsLongCacheRetention: Type.Optional(Type.Boolean()), + }); + return { + openAICompletions, + openAIResponses, + anthropicMessages, + provider: Type.Union([openAICompletions, openAIResponses, anthropicMessages]), + }; +} + +const strictSchemas = createCompatSchemas(true); +export const ProviderCompatSchema = createCompatSchemas(false).provider; + +export function isModelCompat(api: string, value: unknown): boolean { + if (value === undefined) return true; + if (api === "openai-completions") return Value.Check(strictSchemas.openAICompletions, value); + if (api === "openai-responses") return Value.Check(strictSchemas.openAIResponses, value); + if (api === "anthropic-messages") return Value.Check(strictSchemas.anthropicMessages, value); + return false; +} diff --git a/packages/ai/test/model-catalog-format.test.ts b/packages/ai/test/model-catalog-format.test.ts index 9c3c708d55..cffa556be7 100644 --- a/packages/ai/test/model-catalog-format.test.ts +++ b/packages/ai/test/model-catalog-format.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "vitest"; -import { createModelCatalog, parseModelCatalog } from "../scripts/model-catalog-format.js"; +import { createModelCatalog, parseModelCatalog } from "../src/model-catalog.js"; import type { Api, Model } from "../src/types.js"; function model(provider: string, id: string): Model { @@ -36,42 +36,25 @@ describe("hosted model catalog format", () => { test("applies the same strict schema used by clients", () => { const entry = model("provider", "model"); - const catalog = (modelEntry: Model) => ({ - schemaVersion: 1, - generatedAt: new Date().toISOString(), - models: [modelEntry], - }); - expect(() => parseModelCatalog({ ...catalog(entry), models: [entry, entry] })).toThrow(/duplicate/i); - expect(() => parseModelCatalog(catalog({ ...entry, cost: { ...entry.cost, output: Number.NaN } }))).toThrow( - /invalid model/i, - ); - expect(() => parseModelCatalog(catalog({ ...entry, cost: { ...entry.cost, input: 1_000_001 } }))).toThrow( - /invalid model/i, - ); - expect(() => parseModelCatalog(catalog({ ...entry, contextWindow: 100_000_001 }))).toThrow(/invalid model/i); - expect(() => - parseModelCatalog(catalog({ ...entry, thinkingLevelMap: { unsupported: "value" } } as Model)), - ).toThrow(/invalid model/i); + const parse = (modelEntry: Model) => + parseModelCatalog({ schemaVersion: 1, generatedAt: new Date().toISOString(), models: [modelEntry] }); + expect(() => - parseModelCatalog( - catalog({ - ...entry, - compat: { openRouterRouting: { only: ["anthropic"], max_price: { prompt: "1" } } }, - }), - ), + parse({ ...entry, compat: { openRouterRouting: { only: ["anthropic"], max_price: { prompt: "1" } } } }), ).not.toThrow(); expect(() => - parseModelCatalog(catalog({ ...entry, compat: { openRouterRouting: "invalid" } } as Model)), - ).toThrow(/invalid model/i); - expect(() => - parseModelCatalog( - catalog({ ...entry, api: "openai-responses", compat: { supportsStore: true } } as Model), - ), - ).toThrow(/invalid model/i); - expect(() => - parseModelCatalog( - catalog({ ...entry, headers: { authorization: { nested: true } } } as unknown as Model), - ), - ).toThrow(/invalid model/i); + parseModelCatalog({ schemaVersion: 1, generatedAt: new Date().toISOString(), models: [entry, entry] }), + ).toThrow(/duplicate/i); + + const invalidEntries = [ + { ...entry, cost: { ...entry.cost, output: Number.NaN } }, + { ...entry, cost: { ...entry.cost, input: 1_000_001 } }, + { ...entry, contextWindow: 100_000_001 }, + { ...entry, thinkingLevelMap: { unsupported: "value" } }, + { ...entry, compat: { openRouterRouting: "invalid" } }, + { ...entry, api: "openai-responses", compat: { supportsStore: true } }, + { ...entry, headers: { authorization: { nested: true } } }, + ] as unknown as Model[]; + for (const invalid of invalidEntries) expect(() => parse(invalid)).toThrow(/invalid model/i); }); }); diff --git a/packages/coding-agent/src/core/model-registry.ts b/packages/coding-agent/src/core/model-registry.ts index 97e4072d25..477df32223 100644 --- a/packages/coding-agent/src/core/model-registry.ts +++ b/packages/coding-agent/src/core/model-registry.ts @@ -16,6 +16,7 @@ import { type OAuthProviderInterface, type OpenAICompletionsCompat, type OpenAIResponsesCompat, + ProviderCompatSchema, registerApiProvider, resetApiProviders, type SimpleStreamOptions, @@ -48,50 +49,6 @@ import { resolveHeadersOrThrow, } from "./resolve-config-value.js"; -const PercentileCutoffsSchema = Type.Object({ - p50: Type.Optional(Type.Number()), - p75: Type.Optional(Type.Number()), - p90: Type.Optional(Type.Number()), - p99: Type.Optional(Type.Number()), -}); - -const OpenRouterRoutingSchema = Type.Object({ - allow_fallbacks: Type.Optional(Type.Boolean()), - require_parameters: Type.Optional(Type.Boolean()), - data_collection: Type.Optional(Type.Union([Type.Literal("deny"), Type.Literal("allow")])), - zdr: Type.Optional(Type.Boolean()), - enforce_distillable_text: Type.Optional(Type.Boolean()), - order: Type.Optional(Type.Array(Type.String())), - only: Type.Optional(Type.Array(Type.String())), - ignore: Type.Optional(Type.Array(Type.String())), - quantizations: Type.Optional(Type.Array(Type.String())), - sort: Type.Optional( - Type.Union([ - Type.String(), - Type.Object({ - by: Type.Optional(Type.String()), - partition: Type.Optional(Type.Union([Type.String(), Type.Null()])), - }), - ]), - ), - max_price: Type.Optional( - Type.Object({ - prompt: Type.Optional(Type.Union([Type.Number(), Type.String()])), - completion: Type.Optional(Type.Union([Type.Number(), Type.String()])), - image: Type.Optional(Type.Union([Type.Number(), Type.String()])), - audio: Type.Optional(Type.Union([Type.Number(), Type.String()])), - request: Type.Optional(Type.Union([Type.Number(), Type.String()])), - }), - ), - preferred_min_throughput: Type.Optional(Type.Union([Type.Number(), PercentileCutoffsSchema])), - preferred_max_latency: Type.Optional(Type.Union([Type.Number(), PercentileCutoffsSchema])), -}); - -const VercelGatewayRoutingSchema = Type.Object({ - only: Type.Optional(Type.Array(Type.String())), - order: Type.Optional(Type.Array(Type.String())), -}); - const ThinkingLevelMapValueSchema = Type.Union([Type.String(), Type.Null()]); const ThinkingLevelMapSchema = Type.Object({ off: Type.Optional(ThinkingLevelMapValueSchema), @@ -103,49 +60,6 @@ const ThinkingLevelMapSchema = Type.Object({ max: Type.Optional(ThinkingLevelMapValueSchema), }); -const OpenAICompletionsCompatSchema = Type.Object({ - supportsStore: Type.Optional(Type.Boolean()), - supportsDeveloperRole: Type.Optional(Type.Boolean()), - supportsReasoningEffort: Type.Optional(Type.Boolean()), - supportsUsageInStreaming: Type.Optional(Type.Boolean()), - maxTokensField: Type.Optional(Type.Union([Type.Literal("max_completion_tokens"), Type.Literal("max_tokens")])), - requiresToolResultName: Type.Optional(Type.Boolean()), - requiresAssistantAfterToolResult: Type.Optional(Type.Boolean()), - requiresThinkingAsText: Type.Optional(Type.Boolean()), - requiresReasoningContentOnAssistantMessages: Type.Optional(Type.Boolean()), - thinkingFormat: Type.Optional( - Type.Union([ - Type.Literal("openai"), - Type.Literal("openrouter"), - Type.Literal("deepseek"), - Type.Literal("zai"), - Type.Literal("qwen"), - Type.Literal("qwen-chat-template"), - ]), - ), - cacheControlFormat: Type.Optional(Type.Literal("anthropic")), - openRouterRouting: Type.Optional(OpenRouterRoutingSchema), - vercelGatewayRouting: Type.Optional(VercelGatewayRoutingSchema), - supportsStrictMode: Type.Optional(Type.Boolean()), - supportsLongCacheRetention: Type.Optional(Type.Boolean()), -}); - -const OpenAIResponsesCompatSchema = Type.Object({ - sendSessionIdHeader: Type.Optional(Type.Boolean()), - supportsLongCacheRetention: Type.Optional(Type.Boolean()), -}); - -const AnthropicMessagesCompatSchema = Type.Object({ - supportsEagerToolInputStreaming: Type.Optional(Type.Boolean()), - supportsLongCacheRetention: Type.Optional(Type.Boolean()), -}); - -const ProviderCompatSchema = Type.Union([ - OpenAICompletionsCompatSchema, - OpenAIResponsesCompatSchema, - AnthropicMessagesCompatSchema, -]); - // Most fields are optional with sensible defaults for local models (Ollama, LM Studio, etc.) const ModelDefinitionSchema = Type.Object({ id: Type.String({ minLength: 1 }), diff --git a/packages/coding-agent/src/core/remote-model-catalog.ts b/packages/coding-agent/src/core/remote-model-catalog.ts index 6d1a020a35..c2f9be74f6 100644 --- a/packages/coding-agent/src/core/remote-model-catalog.ts +++ b/packages/coding-agent/src/core/remote-model-catalog.ts @@ -20,10 +20,6 @@ function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } -export function parseRemoteModelCatalog(value: unknown): ModelCatalogV1 { - return parseModelCatalog(value); -} - function cloneTransport(template: Model, remote: Model): Model { return { id: remote.id, @@ -90,7 +86,7 @@ function readCache(cachePath: string, url: string): CachedModelCatalog | undefin !Number.isFinite(value.fetchedAt) ) return undefined; - return { url, fetchedAt: value.fetchedAt, catalog: parseRemoteModelCatalog(value.catalog) }; + return { url, fetchedAt: value.fetchedAt, catalog: parseModelCatalog(value.catalog) }; } catch { return undefined; } @@ -149,7 +145,7 @@ async function fetchCatalog(url: string, fetchFn: typeof fetch): Promise { if (existsSync(tempDir)) rmSync(tempDir, { recursive: true, force: true }); }); - test("validates entries and rejects duplicate provider/model ids", () => { - const model = openAiModel(); - expect(parseRemoteModelCatalog(catalog([model])).models).toHaveLength(1); - expect(() => parseRemoteModelCatalog(catalog([model, structuredClone(model)]))).toThrow("Duplicate"); - expect(() => - parseRemoteModelCatalog(catalog([{ ...model, cost: { ...model.cost, input: Number.NaN } }])), - ).toThrow("Invalid model catalog entry"); - }); - test("updates metadata while pinning headers and rejecting transport changes", () => { const bundled = openAiModel(); const remote = { From 73322c5ed27b590a07a69781e41d7f865a4f425c Mon Sep 17 00:00:00 2001 From: Seth Date: Mon, 31 Aug 2026 20:55:38 -0400 Subject: [PATCH 10/13] fix: preserve local compat validation behavior --- packages/ai/src/model-compat-schema.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ai/src/model-compat-schema.ts b/packages/ai/src/model-compat-schema.ts index b4d49cfc36..b8dc04909c 100644 --- a/packages/ai/src/model-compat-schema.ts +++ b/packages/ai/src/model-compat-schema.ts @@ -69,10 +69,10 @@ function createCompatSchemas(strict: boolean) { ), openRouterRouting: Type.Optional(openRouterRouting), vercelGatewayRouting: Type.Optional(vercelGatewayRouting), - zaiToolStream: Type.Optional(Type.Boolean()), + ...(strict ? { zaiToolStream: Type.Optional(Type.Boolean()) } : {}), supportsStrictMode: Type.Optional(Type.Boolean()), cacheControlFormat: Type.Optional(Type.Literal("anthropic")), - sendSessionAffinityHeaders: Type.Optional(Type.Boolean()), + ...(strict ? { sendSessionAffinityHeaders: Type.Optional(Type.Boolean()) } : {}), supportsLongCacheRetention: Type.Optional(Type.Boolean()), }); const openAIResponses = object({ From a1abe483f1139341e7e54ca7fcfd3f785793e583 Mon Sep 17 00:00:00 2001 From: Seth Date: Mon, 31 Aug 2026 21:26:34 -0400 Subject: [PATCH 11/13] fix: validate every local compat field --- packages/ai/src/model-compat-schema.ts | 7 ++++++- packages/ai/test/model-compat-schema.test.ts | 11 +++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 packages/ai/test/model-compat-schema.test.ts diff --git a/packages/ai/src/model-compat-schema.ts b/packages/ai/src/model-compat-schema.ts index b8dc04909c..8a923562e9 100644 --- a/packages/ai/src/model-compat-schema.ts +++ b/packages/ai/src/model-compat-schema.ts @@ -92,7 +92,12 @@ function createCompatSchemas(strict: boolean) { } const strictSchemas = createCompatSchemas(true); -export const ProviderCompatSchema = createCompatSchemas(false).provider; +const localSchemas = createCompatSchemas(false); +export const ProviderCompatSchema = Type.Intersect([ + localSchemas.openAICompletions, + localSchemas.openAIResponses, + localSchemas.anthropicMessages, +]) as unknown as typeof localSchemas.provider; export function isModelCompat(api: string, value: unknown): boolean { if (value === undefined) return true; diff --git a/packages/ai/test/model-compat-schema.test.ts b/packages/ai/test/model-compat-schema.test.ts new file mode 100644 index 0000000000..2892bb22aa --- /dev/null +++ b/packages/ai/test/model-compat-schema.test.ts @@ -0,0 +1,11 @@ +import { Value } from "typebox/value"; +import { describe, expect, test } from "vitest"; +import { ProviderCompatSchema } from "../src/model-compat-schema.js"; + +describe("provider compatibility schema", () => { + test("validates known fields while allowing provider extensions", () => { + expect(Value.Check(ProviderCompatSchema, { supportsDeveloperRole: false })).toBe(true); + expect(Value.Check(ProviderCompatSchema, { supportsDeveloperRole: "false" })).toBe(false); + expect(Value.Check(ProviderCompatSchema, { providerExtension: "value" })).toBe(true); + }); +}); From 95dfe25177f3e2b486befa5e7b50f64d3222e3bb Mon Sep 17 00:00:00 2001 From: Seth Date: Tue, 1 Sep 2026 13:44:39 -0400 Subject: [PATCH 12/13] fix: harden hosted catalog refreshes --- packages/ai/src/model-catalog.ts | 37 ++++++--- packages/ai/src/model-compat-schema.ts | 23 ++++-- packages/ai/test/model-catalog-format.test.ts | 18 +++- packages/ai/test/model-compat-schema.test.ts | 17 +++- packages/coding-agent/docs/providers.md | 2 +- .../coding-agent/src/core/model-registry.ts | 22 ++++- .../src/core/remote-model-catalog.ts | 60 ++++++++++---- .../src/utils/prime-agent-download.ts | 8 ++ .../coding-agent/src/utils/version-check.ts | 9 +- .../test/remote-model-catalog.test.ts | 82 ++++++++++++++++++- 10 files changed, 221 insertions(+), 57 deletions(-) create mode 100644 packages/coding-agent/src/utils/prime-agent-download.ts diff --git a/packages/ai/src/model-catalog.ts b/packages/ai/src/model-catalog.ts index 7ffdb2f74b..29137197c5 100644 --- a/packages/ai/src/model-catalog.ts +++ b/packages/ai/src/model-catalog.ts @@ -27,7 +27,7 @@ const ThinkingLevelMapSchema = strictObject({ max: Type.Optional(ThinkingLevelValueSchema), }); const CostSchema = Type.Number({ minimum: 0, maximum: 1_000_000 }); -const CatalogModelSchema = Type.Object({ +const CatalogModelSchema = strictObject({ id: Type.String({ minLength: 1, maxLength: 1_024 }), name: Type.String({ minLength: 1, maxLength: 1_024 }), api: Type.String({ minLength: 1, maxLength: 128 }), @@ -45,15 +45,12 @@ const CatalogModelSchema = Type.Object({ contextWindow: Type.Integer({ minimum: 1, maximum: 100_000_000 }), maxTokens: Type.Integer({ minimum: 1, maximum: 100_000_000 }), featured: Type.Optional(Type.Boolean()), - headers: Type.Optional( - Type.Record(Type.String({ maxLength: 128 }), Type.String({ maxLength: 4_096 }), { maxProperties: 100 }), - ), compat: Type.Optional(Type.Unknown()), }); -const ModelCatalogSchema = Type.Object({ +const ModelCatalogEnvelopeSchema = Type.Object({ schemaVersion: Type.Literal(MODEL_CATALOG_SCHEMA_VERSION), generatedAt: Type.String(), - models: Type.Array(CatalogModelSchema, { minItems: 1, maxItems: MAX_MODEL_CATALOG_MODELS }), + models: Type.Array(Type.Unknown(), { minItems: 1, maxItems: MAX_MODEL_CATALOG_MODELS }), }); function isRecord(value: unknown): value is Record { @@ -65,12 +62,16 @@ export function createModelCatalog(models: readonly Model[], generatedAt = schemaVersion: MODEL_CATALOG_SCHEMA_VERSION, generatedAt: generatedAt.toISOString(), models: models - .map((model) => structuredClone(model)) + .map((model) => { + const catalogModel = structuredClone(model); + delete catalogModel.headers; + return catalogModel; + }) .sort((left, right) => left.provider.localeCompare(right.provider) || left.id.localeCompare(right.id)), }; } -export function parseModelCatalog(value: unknown): ModelCatalogV1 { +export function parseModelCatalog(value: unknown, options: { skipInvalidModels?: boolean } = {}): ModelCatalogV1 { if (!isRecord(value) || value.schemaVersion !== MODEL_CATALOG_SCHEMA_VERSION) { throw new Error("Unsupported model catalog schema version"); } @@ -80,15 +81,25 @@ export function parseModelCatalog(value: unknown): ModelCatalogV1 { if (!Array.isArray(value.models) || value.models.length === 0 || value.models.length > MAX_MODEL_CATALOG_MODELS) { throw new Error("Invalid model catalog model count"); } - if (!Value.Check(ModelCatalogSchema, value)) throw new Error("Invalid model catalog entry"); + if (!Value.Check(ModelCatalogEnvelopeSchema, value)) throw new Error("Invalid model catalog entry"); - const models = value.models as Model[]; + const models: Model[] = []; const seen = new Set(); - for (const model of models) { - if (!isModelCompat(model.api, model.compat)) throw new Error("Invalid model catalog entry"); + for (const candidate of value.models) { + if (!Value.Check(CatalogModelSchema, candidate)) { + if (options.skipInvalidModels) continue; + throw new Error("Invalid model catalog entry"); + } + const model = candidate as Model; + if (!isModelCompat(model.api, model.compat)) { + if (options.skipInvalidModels) continue; + throw new Error("Invalid model catalog entry"); + } const key = JSON.stringify([model.provider, model.id]); if (seen.has(key)) throw new Error(`Duplicate model catalog entry ${key}`); seen.add(key); + models.push(model); } - return value as unknown as ModelCatalogV1; + if (models.length === 0) throw new Error("Model catalog has no compatible entries"); + return { schemaVersion: MODEL_CATALOG_SCHEMA_VERSION, generatedAt: value.generatedAt, models }; } diff --git a/packages/ai/src/model-compat-schema.ts b/packages/ai/src/model-compat-schema.ts index 8a923562e9..3f1e8ac809 100644 --- a/packages/ai/src/model-compat-schema.ts +++ b/packages/ai/src/model-compat-schema.ts @@ -1,5 +1,6 @@ -import { type TProperties, Type } from "typebox"; +import { type TProperties, type TSchema, Type } from "typebox"; import { Value } from "typebox/value"; +import type { KnownApi } from "./types.js"; function createCompatSchemas(strict: boolean) { const object = (properties: T) => Type.Object(properties, { additionalProperties: !strict }); @@ -99,10 +100,20 @@ export const ProviderCompatSchema = Type.Intersect([ localSchemas.anthropicMessages, ]) as unknown as typeof localSchemas.provider; +const apiCompatSchemas = { + "openai-completions": strictSchemas.openAICompletions, + "openai-responses": strictSchemas.openAIResponses, + "anthropic-messages": strictSchemas.anthropicMessages, + "mistral-conversations": null, + "azure-openai-responses": null, + "openai-codex-responses": null, + "bedrock-converse-stream": null, + "google-generative-ai": null, + "google-vertex": null, +} satisfies Record; + export function isModelCompat(api: string, value: unknown): boolean { - if (value === undefined) return true; - if (api === "openai-completions") return Value.Check(strictSchemas.openAICompletions, value); - if (api === "openai-responses") return Value.Check(strictSchemas.openAIResponses, value); - if (api === "anthropic-messages") return Value.Check(strictSchemas.anthropicMessages, value); - return false; + if (!Object.hasOwn(apiCompatSchemas, api)) return false; + const schema = apiCompatSchemas[api as KnownApi]; + return schema ? value === undefined || Value.Check(schema, value) : value === undefined; } diff --git a/packages/ai/test/model-catalog-format.test.ts b/packages/ai/test/model-catalog-format.test.ts index cffa556be7..d1f1e078d4 100644 --- a/packages/ai/test/model-catalog-format.test.ts +++ b/packages/ai/test/model-catalog-format.test.ts @@ -20,9 +20,11 @@ function model(provider: string, id: string): Model { describe("hosted model catalog format", () => { test("creates deterministic provider/model ordering", () => { const generatedAt = new Date("2026-08-31T00:00:00.000Z"); - const result = createModelCatalog([model("z", "b"), model("a", "z"), model("z", "a")], generatedAt); + const input = { ...model("z", "b"), headers: { Authorization: "bundled secret" } }; + const result = createModelCatalog([input, model("a", "z"), model("z", "a")], generatedAt); expect(result.generatedAt).toBe(generatedAt.toISOString()); expect(result.models.map((entry) => `${entry.provider}/${entry.id}`)).toEqual(["a/z", "z/a", "z/b"]); + expect(result.models.find((entry) => entry.id === "b")?.headers).toBeUndefined(); }); test("treats provider and model ids as an unambiguous pair", () => { @@ -34,7 +36,17 @@ describe("hosted model catalog format", () => { expect(parsed.models).toHaveLength(2); }); - test("applies the same strict schema used by clients", () => { + test("keeps compatible entries for older clients", () => { + const current = model("provider", "current"); + const future = { ...model("provider", "future"), api: "future-api" }; + const parsed = parseModelCatalog( + { schemaVersion: 1, generatedAt: new Date().toISOString(), models: [current, future] }, + { skipInvalidModels: true }, + ); + expect(parsed.models).toEqual([current]); + }); + + test("applies strict publication and entry validation", () => { const entry = model("provider", "model"); const parse = (modelEntry: Model) => parseModelCatalog({ schemaVersion: 1, generatedAt: new Date().toISOString(), models: [modelEntry] }); @@ -53,6 +65,8 @@ describe("hosted model catalog format", () => { { ...entry, thinkingLevelMap: { unsupported: "value" } }, { ...entry, compat: { openRouterRouting: "invalid" } }, { ...entry, api: "openai-responses", compat: { supportsStore: true } }, + { ...entry, api: "future-api", compat: undefined }, + { ...entry, headers: { authorization: "remote secret" } }, { ...entry, headers: { authorization: { nested: true } } }, ] as unknown as Model[]; for (const invalid of invalidEntries) expect(() => parse(invalid)).toThrow(/invalid model/i); diff --git a/packages/ai/test/model-compat-schema.test.ts b/packages/ai/test/model-compat-schema.test.ts index 2892bb22aa..96e04c8998 100644 --- a/packages/ai/test/model-compat-schema.test.ts +++ b/packages/ai/test/model-compat-schema.test.ts @@ -1,6 +1,6 @@ import { Value } from "typebox/value"; import { describe, expect, test } from "vitest"; -import { ProviderCompatSchema } from "../src/model-compat-schema.js"; +import { isModelCompat, ProviderCompatSchema } from "../src/model-compat-schema.js"; describe("provider compatibility schema", () => { test("validates known fields while allowing provider extensions", () => { @@ -8,4 +8,19 @@ describe("provider compatibility schema", () => { expect(Value.Check(ProviderCompatSchema, { supportsDeveloperRole: "false" })).toBe(false); expect(Value.Check(ProviderCompatSchema, { providerExtension: "value" })).toBe(true); }); + + test("defines compatibility handling for every bundled API", () => { + for (const api of [ + "mistral-conversations", + "azure-openai-responses", + "openai-codex-responses", + "bedrock-converse-stream", + "google-generative-ai", + "google-vertex", + ]) { + expect(isModelCompat(api, undefined)).toBe(true); + expect(isModelCompat(api, {})).toBe(false); + } + expect(isModelCompat("future-api", undefined)).toBe(false); + }); }); diff --git a/packages/coding-agent/docs/providers.md b/packages/coding-agent/docs/providers.md index 9eed3998da..b1e2dd2194 100644 --- a/packages/coding-agent/docs/providers.md +++ b/packages/coding-agent/docs/providers.md @@ -2,7 +2,7 @@ Prime Agent supports subscription-based providers via OAuth and API key providers via environment variables or the auth file. It treats Prime Intellect's hosted catalog as the authoritative public model list and refreshes model additions, removals, names, capabilities, and pricing once per day. A validated disk cache and the catalog bundled with each release keep model selection available when the endpoint is offline. The hosted catalog cannot change provider request URLs, APIs, headers, or compatibility settings unless that transport already exists in the bundled catalog. -Set `PI_OFFLINE=1` to skip catalog network refreshes. Set `PRIME_AGENT_MODEL_CATALOG_URL` to use another catalog endpoint; `PRIME_AGENT_DOWNLOAD_BASE_URL` also changes the default catalog origin alongside the release origin. +Set `PI_OFFLINE=1` to skip catalog network refreshes. Set `PRIME_AGENT_MODEL_CATALOG_URL` to use another HTTPS catalog endpoint; `PRIME_AGENT_DOWNLOAD_BASE_URL` also changes the default catalog origin alongside the release origin. ## Table of Contents diff --git a/packages/coding-agent/src/core/model-registry.ts b/packages/coding-agent/src/core/model-registry.ts index 477df32223..c5327c6d17 100644 --- a/packages/coding-agent/src/core/model-registry.ts +++ b/packages/coding-agent/src/core/model-registry.ts @@ -39,6 +39,7 @@ import { } from "./prime-inference-models.js"; import { BUILT_IN_PROVIDER_DISPLAY_NAMES } from "./provider-display-names.js"; import { + getMinimumRemoteModelCatalogSize, mergeRemoteModelCatalog, readCachedRemoteModelCatalog, refreshRemoteModelCatalog, @@ -434,6 +435,11 @@ export class ModelRegistry { return this.modelsJsonPath ? join(dirname(this.modelsJsonPath), "model-catalog-cache.json") : undefined; } + private minimumRemoteModelCatalogSize(): number { + const bundledModelCount = getProviders().reduce((count, provider) => count + getModels(provider).length, 0); + return getMinimumRemoteModelCatalogSize(bundledModelCount); + } + private loadModels(): void { const { models: customModels, @@ -451,7 +457,8 @@ export class ModelRegistry { ); const cachePath = this.remoteModelCatalogCachePath(); const remoteModels = - this.remoteCatalogModels ?? (cachePath ? readCachedRemoteModelCatalog(cachePath) : undefined); + this.remoteCatalogModels ?? + (cachePath ? readCachedRemoteModelCatalog(cachePath, this.minimumRemoteModelCatalogSize()) : undefined); const builtInModels = [ ...this.loadBuiltInModels(overrides, modelOverrides, remoteModels), ...getPrivatePrimeInferenceModels(), @@ -705,9 +712,16 @@ export class ModelRegistry { this.refresh(); const cachePath = this.remoteModelCatalogCachePath(); if (cachePath) { - this.remoteCatalogModels = await refreshRemoteModelCatalog(cachePath); - this.loadModels(); - this.reapplyRegisteredProviders(); + void refreshRemoteModelCatalog(cachePath, { + minimumModels: this.minimumRemoteModelCatalogSize(), + }) + .then((remoteModels) => { + if (!remoteModels || remoteModels === this.remoteCatalogModels) return; + this.remoteCatalogModels = remoteModels; + this.loadModels(); + this.reapplyRegisteredProviders(); + }) + .catch(() => {}); } await this.refreshPrivatePrimeInferenceAuthorization(previousPrivateModelIds, previousTeamId); return this.getAvailable(); diff --git a/packages/coding-agent/src/core/remote-model-catalog.ts b/packages/coding-agent/src/core/remote-model-catalog.ts index c2f9be74f6..6bcf84a637 100644 --- a/packages/coding-agent/src/core/remote-model-catalog.ts +++ b/packages/coding-agent/src/core/remote-model-catalog.ts @@ -2,12 +2,13 @@ import { Buffer } from "node:buffer"; import { existsSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs"; import { type Api, type Model, type ModelCatalogV1, parseModelCatalog } from "@earendil-works/pi-ai"; import { isTruthyEnvFlag } from "../utils/env.js"; +import { getPrimeAgentDownloadBaseUrl } from "../utils/prime-agent-download.js"; -const DEFAULT_PRIME_AGENT_DOWNLOAD_BASE_URL = "https://pub-728493de92a943e2a9b2d17b4719f318.r2.dev"; const MODEL_CATALOG_PATH = "model-catalog.json"; const MODEL_CATALOG_CACHE_TTL_MS = 24 * 60 * 60 * 1000; const MODEL_CATALOG_FETCH_TIMEOUT_MS = 5_000; const MAX_MODEL_CATALOG_BYTES = 8 * 1024 * 1024; +const MIN_REMOTE_MODEL_CATALOG_COVERAGE = 0.5; const pendingRefreshes = new Map[] | undefined>>(); interface CachedModelCatalog { @@ -43,6 +44,14 @@ function modelKey(provider: string, id: string): string { return JSON.stringify([provider, id]); } +function transportKey(model: Model): string { + return `${model.provider}\0${model.api}\0${model.baseUrl}`; +} + +export function getMinimumRemoteModelCatalogSize(bundledModelCount: number): number { + return Math.ceil(bundledModelCount * MIN_REMOTE_MODEL_CATALOG_COVERAGE); +} + export function mergeRemoteModelCatalog( bundledModels: readonly Model[], remoteModels: readonly Model[] | undefined, @@ -50,7 +59,11 @@ export function mergeRemoteModelCatalog( if (!remoteModels) return bundledModels.map((model) => structuredClone(model)); const exact = new Map(bundledModels.map((model) => [modelKey(model.provider, model.id), model])); const transports = new Map>(); - for (const model of bundledModels) transports.set(`${model.provider}\0${model.api}\0${model.baseUrl}`, model); + for (const model of bundledModels) { + const key = transportKey(model); + const current = transports.get(key); + if (!current || model.id < current.id) transports.set(key, model); + } const merged: Model[] = []; for (const remote of remoteModels) { @@ -59,23 +72,21 @@ export function mergeRemoteModelCatalog( const template = exactTemplate && exactTemplate.api === remote.api && exactTemplate.baseUrl === remote.baseUrl ? exactTemplate - : transports.get(`${remote.provider}\0${remote.api}\0${remote.baseUrl}`); + : transports.get(transportKey(remote)); if (template) merged.push(cloneTransport(template, remote)); } - return merged.length > 0 ? merged : bundledModels.map((model) => structuredClone(model)); + const minimumModels = getMinimumRemoteModelCatalogSize(bundledModels.length); + return merged.length >= minimumModels ? merged : bundledModels.map((model) => structuredClone(model)); } export function getRemoteModelCatalogUrl(): string { const explicit = process.env.PRIME_AGENT_MODEL_CATALOG_URL?.trim(); - if (explicit) return explicit; - const base = (process.env.PRIME_AGENT_DOWNLOAD_BASE_URL?.trim() || DEFAULT_PRIME_AGENT_DOWNLOAD_BASE_URL).replace( - /\/+$/, - "", - ); - return `${base}/${MODEL_CATALOG_PATH}`; + const url = explicit || `${getPrimeAgentDownloadBaseUrl()}/${MODEL_CATALOG_PATH}`; + if (new URL(url).protocol !== "https:") throw new Error("Model catalog URL must use HTTPS"); + return url; } -function readCache(cachePath: string, url: string): CachedModelCatalog | undefined { +function readCache(cachePath: string, url: string, minimumModels = 1): CachedModelCatalog | undefined { if (!existsSync(cachePath)) return undefined; try { const value = JSON.parse(readFileSync(cachePath, "utf8")) as unknown; @@ -86,7 +97,9 @@ function readCache(cachePath: string, url: string): CachedModelCatalog | undefin !Number.isFinite(value.fetchedAt) ) return undefined; - return { url, fetchedAt: value.fetchedAt, catalog: parseModelCatalog(value.catalog) }; + const catalog = parseModelCatalog(value.catalog, { skipInvalidModels: true }); + if (catalog.models.length < minimumModels) return undefined; + return { url, fetchedAt: value.fetchedAt, catalog }; } catch { return undefined; } @@ -108,8 +121,12 @@ function writeCache(cachePath: string, cache: CachedModelCatalog): void { } } -export function readCachedRemoteModelCatalog(cachePath: string): Model[] | undefined { - return readCache(cachePath, getRemoteModelCatalogUrl())?.catalog.models; +export function readCachedRemoteModelCatalog(cachePath: string, minimumModels = 1): Model[] | undefined { + try { + return readCache(cachePath, getRemoteModelCatalogUrl(), minimumModels)?.catalog.models; + } catch { + return undefined; + } } async function fetchCatalog(url: string, fetchFn: typeof fetch): Promise { @@ -145,16 +162,22 @@ async function fetchCatalog(url: string, fetchFn: typeof fetch): Promise[] | undefined> { - const url = getRemoteModelCatalogUrl(); + let url: string; + try { + url = getRemoteModelCatalogUrl(); + } catch { + return undefined; + } const now = options.now ?? Date.now(); - const cached = readCache(cachePath, url); + const minimumModels = options.minimumModels ?? 1; + const cached = readCache(cachePath, url, minimumModels); if (cached && now - cached.fetchedAt < MODEL_CATALOG_CACHE_TTL_MS) return cached.catalog.models; if (isTruthyEnvFlag(process.env.PI_OFFLINE)) return cached?.catalog.models; @@ -164,6 +187,7 @@ export async function refreshRemoteModelCatalog( const promise = (async () => { try { const catalog = await fetchCatalog(url, options.fetchFn ?? fetch); + if (catalog.models.length < minimumModels) throw new Error("Model catalog has too few compatible entries"); writeCache(cachePath, { url, fetchedAt: now, catalog }); return catalog.models; } catch { diff --git a/packages/coding-agent/src/utils/prime-agent-download.ts b/packages/coding-agent/src/utils/prime-agent-download.ts new file mode 100644 index 0000000000..b252e6fbf5 --- /dev/null +++ b/packages/coding-agent/src/utils/prime-agent-download.ts @@ -0,0 +1,8 @@ +export const DEFAULT_PRIME_AGENT_DOWNLOAD_BASE_URL = "https://pub-728493de92a943e2a9b2d17b4719f318.r2.dev"; + +export function getPrimeAgentDownloadBaseUrl(): string { + return (process.env.PRIME_AGENT_DOWNLOAD_BASE_URL?.trim() || DEFAULT_PRIME_AGENT_DOWNLOAD_BASE_URL).replace( + /\/+$/, + "", + ); +} diff --git a/packages/coding-agent/src/utils/version-check.ts b/packages/coding-agent/src/utils/version-check.ts index 79ec3f824e..7522e51cbe 100644 --- a/packages/coding-agent/src/utils/version-check.ts +++ b/packages/coding-agent/src/utils/version-check.ts @@ -1,6 +1,6 @@ import { getPiUserAgent } from "./pi-user-agent.js"; +import { getPrimeAgentDownloadBaseUrl } from "./prime-agent-download.js"; -const DEFAULT_PRIME_AGENT_DOWNLOAD_BASE_URL = "https://pub-728493de92a943e2a9b2d17b4719f318.r2.dev"; const STABLE_VERSION_MANIFEST_PATH = "latest.json"; const BETA_VERSION_MANIFEST_PATH = "beta.json"; const DEFAULT_VERSION_CHECK_TIMEOUT_MS = 10000; @@ -85,13 +85,6 @@ export function isNewerPackageVersion(candidateVersion: string, currentVersion: return candidateVersion.trim() !== currentVersion.trim(); } -function getPrimeAgentDownloadBaseUrl(): string { - return (process.env.PRIME_AGENT_DOWNLOAD_BASE_URL?.trim() || DEFAULT_PRIME_AGENT_DOWNLOAD_BASE_URL).replace( - /\/+$/, - "", - ); -} - function normalizeReleaseVersion(version: string): string { return version.trim().replace(/^v/, ""); } diff --git a/packages/coding-agent/test/remote-model-catalog.test.ts b/packages/coding-agent/test/remote-model-catalog.test.ts index db782e5d71..6351e68dab 100644 --- a/packages/coding-agent/test/remote-model-catalog.test.ts +++ b/packages/coding-agent/test/remote-model-catalog.test.ts @@ -1,7 +1,7 @@ import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { type Api, getModels, type Model } from "@earendil-works/pi-ai"; +import { type Api, getModels, getProviders, type Model } from "@earendil-works/pi-ai"; import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; import { AuthStorage } from "../src/core/auth-storage.js"; import { ModelRegistry } from "../src/core/model-registry.js"; @@ -21,6 +21,12 @@ function openAiModel(): Model { return structuredClone(getModels("openai")[0] as Model); } +function bundledModels(): Model[] { + return getProviders() + .flatMap((provider) => getModels(provider) as Model[]) + .map((model) => structuredClone(model)); +} + describe("remote model catalog", () => { let tempDir: string; let previousCatalogUrl: string | undefined; @@ -70,6 +76,11 @@ describe("remote model catalog", () => { const [fallback] = mergeRemoteModelCatalog([bundled], [redirected]); expect(fallback).toEqual(bundled); } + + const first = { ...structuredClone(bundled), id: "a", headers: { "X-Template": "first" } }; + const last = { ...structuredClone(bundled), id: "z", headers: { "X-Template": "last" } }; + const addition = { ...structuredClone(bundled), id: "future" }; + expect(mergeRemoteModelCatalog([last, first], [addition])[0].headers).toEqual(first.headers); }); test("uses the hosted model list and only accepts additions through a bundled transport", () => { @@ -93,6 +104,21 @@ describe("remote model catalog", () => { expect(merged[0]).not.toBe(bundled); }); + test("rejects a materially truncated hosted catalog", () => { + const bundled = openAiModel(); + const bundledModels = ["a", "b", "c"].map((id) => ({ ...structuredClone(bundled), id })); + const remote = [{ ...structuredClone(bundled), id: "a", name: "Remote A" }]; + expect(mergeRemoteModelCatalog(bundledModels, remote)).toEqual(bundledModels); + }); + + test("rejects insecure catalog URL overrides without fetching", async () => { + process.env.PRIME_AGENT_MODEL_CATALOG_URL = "http://catalog.test/model-catalog.json"; + expect(() => getRemoteModelCatalogUrl()).toThrow(/HTTPS/); + const fetchFn = vi.fn(); + expect(await refreshRemoteModelCatalog(join(tempDir, "cache.json"), { fetchFn })).toBeUndefined(); + expect(fetchFn).not.toHaveBeenCalled(); + }); + test("fetches once, validates, and reuses the fresh atomic cache", async () => { const cachePath = join(tempDir, "cache.json"); const model = openAiModel(); @@ -113,6 +139,21 @@ describe("remote model catalog", () => { expect(fetchFn).toHaveBeenCalledOnce(); }); + test("keeps a stale catalog when a refresh is materially truncated", async () => { + const cachePath = join(tempDir, "cache.json"); + const model = openAiModel(); + const staleModels = ["a", "b"].map((id) => ({ ...structuredClone(model), id })); + writeFileSync( + cachePath, + JSON.stringify({ url: getRemoteModelCatalogUrl(), fetchedAt: 1, catalog: catalog(staleModels) }), + ); + const truncated = [{ ...structuredClone(model), id: "a" }]; + const fetchFn = vi.fn(async () => new Response(JSON.stringify(catalog(truncated)), { status: 200 })); + const refreshed = await refreshRemoteModelCatalog(cachePath, { fetchFn, minimumModels: 2, now: 100_000_000 }); + expect(refreshed?.map((entry) => entry.id)).toEqual(["a", "b"]); + expect(JSON.parse(readFileSync(cachePath, "utf8")).fetchedAt).toBe(1); + }); + test("stops reading a chunked response at the byte limit", async () => { const cachePath = join(tempDir, "cache.json"); const cancel = vi.fn(); @@ -150,6 +191,24 @@ describe("remote model catalog", () => { expect(offlineFetch).not.toHaveBeenCalled(); }); + test("serves bundled models while a cold catalog refresh runs in the background", async () => { + const modelsPath = join(tempDir, "models.json"); + let respond: ((response: Response) => void) | undefined; + vi.stubGlobal( + "fetch", + vi.fn(() => new Promise((resolve) => (respond = resolve))), + ); + const registry = ModelRegistry.create( + AuthStorage.inMemory({ openai: { type: "api_key", key: "key" } }), + modelsPath, + ); + const available = await registry.refreshAvailableModels(); + expect(available.length).toBeGreaterThan(0); + expect(respond).toBeDefined(); + respond?.(new Response(JSON.stringify(catalog(bundledModels())), { status: 200 })); + await vi.waitFor(() => expect(existsSync(join(tempDir, "model-catalog-cache.json"))).toBe(true)); + }); + test("keeps local model overrides and custom models above remote metadata", async () => { const model = openAiModel(); const modelsPath = join(tempDir, "models.json"); @@ -164,10 +223,19 @@ describe("remote model catalog", () => { }, }), ); - const remote = { ...structuredClone(model), name: "Remote name", cost: { ...model.cost, input: 42 } }; + const remoteModels = bundledModels().map((entry) => + entry.provider === model.provider && entry.id === model.id + ? { + ...entry, + name: "Remote name", + cost: { ...entry.cost, input: 42 }, + contextWindow: entry.contextWindow + 1, + } + : entry, + ); vi.stubGlobal( "fetch", - vi.fn(async () => new Response(JSON.stringify(catalog([remote])), { status: 200 })), + vi.fn(async () => new Response(JSON.stringify(catalog(remoteModels)), { status: 200 })), ); const registry = ModelRegistry.create( AuthStorage.inMemory({ openai: { type: "api_key", key: "key" } }), @@ -180,7 +248,13 @@ describe("remote model catalog", () => { models: [{ ...model, id: "extension-model", name: "Extension model" }], }); await registry.refreshAvailableModels(); - expect(registry.find("openai", model.id)).toMatchObject({ name: "Local name", cost: { input: 99 } }); + await vi.waitFor(() => + expect(registry.find("openai", model.id)).toMatchObject({ + name: "Local name", + cost: { input: 99 }, + contextWindow: model.contextWindow + 1, + }), + ); expect(registry.find("openai", "local-model")?.name).toBe("Local model"); expect(registry.find("extension-provider", "extension-model")?.name).toBe("Extension model"); }); From 97d6e39a235f939e5d9ee8afbd94e3791484d264 Mon Sep 17 00:00:00 2001 From: Seth Date: Tue, 1 Sep 2026 13:53:54 -0400 Subject: [PATCH 13/13] fix: refresh future-dated model caches --- packages/coding-agent/src/core/remote-model-catalog.ts | 4 +++- packages/coding-agent/test/remote-model-catalog.test.ts | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/coding-agent/src/core/remote-model-catalog.ts b/packages/coding-agent/src/core/remote-model-catalog.ts index 6bcf84a637..b6c5c86694 100644 --- a/packages/coding-agent/src/core/remote-model-catalog.ts +++ b/packages/coding-agent/src/core/remote-model-catalog.ts @@ -178,7 +178,9 @@ export async function refreshRemoteModelCatalog( const now = options.now ?? Date.now(); const minimumModels = options.minimumModels ?? 1; const cached = readCache(cachePath, url, minimumModels); - if (cached && now - cached.fetchedAt < MODEL_CATALOG_CACHE_TTL_MS) return cached.catalog.models; + if (cached && now >= cached.fetchedAt && now - cached.fetchedAt < MODEL_CATALOG_CACHE_TTL_MS) { + return cached.catalog.models; + } if (isTruthyEnvFlag(process.env.PI_OFFLINE)) return cached?.catalog.models; const key = `${cachePath}\0${url}`; diff --git a/packages/coding-agent/test/remote-model-catalog.test.ts b/packages/coding-agent/test/remote-model-catalog.test.ts index 6351e68dab..bbaa48cf24 100644 --- a/packages/coding-agent/test/remote-model-catalog.test.ts +++ b/packages/coding-agent/test/remote-model-catalog.test.ts @@ -137,6 +137,9 @@ describe("remote model catalog", () => { await refreshRemoteModelCatalog(cachePath, { fetchFn, now: 2_000 }); expect(fetchFn).toHaveBeenCalledOnce(); + + await refreshRemoteModelCatalog(cachePath, { fetchFn, now: 500 }); + expect(fetchFn).toHaveBeenCalledTimes(2); }); test("keeps a stale catalog when a refresh is materially truncated", async () => {