diff --git a/platform/backend/src/models/llm-provider-api-key-model.test.ts b/platform/backend/src/models/llm-provider-api-key-model.test.ts index 9162227bfd0..d54687d40b8 100644 --- a/platform/backend/src/models/llm-provider-api-key-model.test.ts +++ b/platform/backend/src/models/llm-provider-api-key-model.test.ts @@ -600,11 +600,23 @@ describe("LlmProviderApiKeyModelLinkModel", () => { expected: "gemini-2.5-flash", }, { - // Current generation must outrank a legacy model so "best" is never a - // model the selector also badges "old". + // Pro always outranks Flash, even a newer-generation Flash: "best" + // means highest quality, so an older Pro wins over a newer Flash. provider: "gemini", catalog: ["gemini-2.5-pro", "gemini-3.5-flash"], - expected: "gemini-3.5-flash", + expected: "gemini-2.5-pro", + }, + { + // The real-world case: no 3.5 Pro exists yet, so the newest available + // Pro (a preview) must beat the 3.5/3.6 Flash models in the account. + provider: "gemini", + catalog: [ + "gemini-3.6-flash", + "gemini-3.5-flash", + "gemini-3.1-pro-preview", + "gemini-2.5-pro", + ], + expected: "gemini-3.1-pro-preview", }, { provider: "bedrock", diff --git a/platform/backend/src/models/llm-provider-api-key-model.ts b/platform/backend/src/models/llm-provider-api-key-model.ts index 71e56c655be..f8a8370fd19 100644 --- a/platform/backend/src/models/llm-provider-api-key-model.ts +++ b/platform/backend/src/models/llm-provider-api-key-model.ts @@ -1,6 +1,7 @@ import { type CompleteModelSelection, MODEL_MARKER_PATTERNS, + pickBestGeminiModelId, type SupportedProvider, } from "@archestra/shared"; import { and, asc, desc, eq, inArray, or, sql } from "drizzle-orm"; @@ -118,15 +119,8 @@ class LlmProviderApiKeyModelLinkModel { // Insert new links if (uniqueModels.length > 0) { - // Detect the best model using pattern matching - // Patterns are checked in order (first pattern = highest priority) - const patterns = MODEL_MARKER_PATTERNS[provider]; - const sorted = [...uniqueModels].sort((a, b) => - a.modelId.localeCompare(b.modelId), - ); - - // Find first matching model respecting pattern priority order - const bestModel = findFirstMatchByPatternPriority(sorted, patterns); + // Detect the best (highest-quality) model for this provider. + const bestModel = findBestModel(provider, uniqueModels); // Build values with markers const values = uniqueModels.map((model) => ({ @@ -655,16 +649,26 @@ function toDateOrNull(value: Date | string | null): Date | null { } /** - * Find the first model matching patterns, respecting pattern priority order. - * Patterns are checked in order (first pattern = highest priority). - * For each pattern, returns the first alphabetically sorted match. + * Pick the best (highest-quality) model for a provider. Gemini is ranked by a + * computed tier + version comparator (Google's catalog changes too often for a + * static id list); every other provider uses the ordered pattern list. */ -function findFirstMatchByPatternPriority( - sortedModels: Array<{ id: string; modelId: string }>, - patterns: string[], +function findBestModel( + provider: SupportedProvider, + models: Array<{ id: string; modelId: string }>, ): { id: string; modelId: string } | undefined { - for (const pattern of patterns) { - const match = sortedModels.find((m) => + if (provider === "gemini") { + const bestModelId = pickBestGeminiModelId(models.map((m) => m.modelId)); + return bestModelId === null + ? undefined + : models.find((m) => m.modelId === bestModelId); + } + + // Patterns are checked in order (first pattern = highest priority); for each + // pattern the first alphabetically sorted match wins. + const sorted = [...models].sort((a, b) => a.modelId.localeCompare(b.modelId)); + for (const pattern of MODEL_MARKER_PATTERNS[provider]) { + const match = sorted.find((m) => m.modelId.toLowerCase().includes(pattern.toLowerCase()), ); if (match) { diff --git a/platform/shared/gemini-models.test.ts b/platform/shared/gemini-models.test.ts index 6ddeb52b205..825dffa3f56 100644 --- a/platform/shared/gemini-models.test.ts +++ b/platform/shared/gemini-models.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test } from "vitest"; import { isLegacyGeminiModel, isUsableGeminiCatalogModel, + pickBestGeminiModelId, } from "./gemini-models"; describe("isUsableGeminiCatalogModel", () => { @@ -71,3 +72,88 @@ describe("isLegacyGeminiModel", () => { expect(isLegacyGeminiModel(modelId)).toBe(expected); }); }); + +describe("pickBestGeminiModelId", () => { + test("Pro always beats a newer-generation Flash", () => { + expect(pickBestGeminiModelId(["gemini-2.5-pro", "gemini-3.5-flash"])).toBe( + "gemini-2.5-pro", + ); + }); + + test("picks the newest Pro (the reported bug's real catalog)", () => { + // Direct Gemini API fetch on the dev key: a Flash used to be marked best + // because no `gemini-3.5-pro` exists. The newest Pro must win instead. + expect( + pickBestGeminiModelId([ + "gemini-2.5-pro", + "gemini-2.5-flash", + "gemini-3-pro-preview", + "gemini-3-flash-preview", + "gemini-3.1-pro-preview", + "gemini-3.1-pro-preview-customtools", + "gemini-3.5-flash", + "gemini-3.5-flash-lite", + "gemini-3.6-flash", + ]), + ).toBe("gemini-3.1-pro-preview"); + }); + + test("prefers GA over -preview at the same tier and version", () => { + expect( + pickBestGeminiModelId(["gemini-2.5-pro-preview-06-05", "gemini-2.5-pro"]), + ).toBe("gemini-2.5-pro"); + }); + + test("plain id wins over a longer variant at the same tier/version/preview", () => { + expect( + pickBestGeminiModelId([ + "gemini-3.1-pro-preview-customtools", + "gemini-3.1-pro-preview", + ]), + ).toBe("gemini-3.1-pro-preview"); + }); + + test("falls to Flash, then Flash-lite, when no Pro is present", () => { + expect( + pickBestGeminiModelId(["gemini-3.5-flash-lite", "gemini-3.5-flash"]), + ).toBe("gemini-3.5-flash"); + expect(pickBestGeminiModelId(["gemini-2.5-flash-lite"])).toBe( + "gemini-2.5-flash-lite", + ); + }); + + test("higher generation Flash beats a lower generation Flash", () => { + expect( + pickBestGeminiModelId(["gemini-2.5-flash", "gemini-3.6-flash"]), + ).toBe("gemini-3.6-flash"); + }); + + test("ignores non-chat models (embeddings, computer-use, robotics)", () => { + expect( + pickBestGeminiModelId([ + "gemini-embedding-001", + "gemini-2.5-computer-use-preview-10-2025", + "gemini-robotics-er-1.5-preview", + "gemini-2.5-pro", + ]), + ).toBe("gemini-2.5-pro"); + }); + + test("ignores unversioned -latest aliases and legacy generations", () => { + expect( + pickBestGeminiModelId([ + "gemini-pro-latest", + "gemini-flash-latest", + "gemini-1.5-pro", + "gemini-2.5-flash", + ]), + ).toBe("gemini-2.5-flash"); + }); + + test("returns null when there is no rankable Gemini chat model", () => { + expect(pickBestGeminiModelId([])).toBeNull(); + expect( + pickBestGeminiModelId(["gemini-embedding-001", "gemini-pro-latest"]), + ).toBeNull(); + }); +}); diff --git a/platform/shared/gemini-models.ts b/platform/shared/gemini-models.ts index a2143a38810..9e151a6e474 100644 --- a/platform/shared/gemini-models.ts +++ b/platform/shared/gemini-models.ts @@ -47,6 +47,23 @@ export function isUsableGeminiCatalogModel(modelId: string): boolean { ); } +/** + * Highest-quality Gemini chat model, computed from the id so new releases rank + * correctly with no hardcoded list to maintain. Best = highest tier (Pro > + * Flash > Flash-lite), then newest generation, then GA over -preview, then the + * shortest/lowest id. Null when the account has no rankable Gemini chat model. + */ +export function pickBestGeminiModelId(modelIds: string[]): string | null { + return ( + modelIds + .filter((id) => geminiChatScore(id) !== null) + .sort( + (a, b) => + geminiChatScore(b)! - geminiChatScore(a)! || a.localeCompare(b), + )[0] ?? null + ); +} + /** * True when a Gemini-family chat model is an older generation * (≤ {@link GEMINI_FAMILY_LEGACY_MAX_VERSION}) and should carry an "old" badge. @@ -94,3 +111,33 @@ function compareVersion(a: GeminiVersion, b: GeminiVersion): number { } return a[1] - b[1]; } + +/** Tier -> quality rank, listed match-first so "flash-lite" wins over "flash". */ +const GEMINI_TIER_RANKS: ReadonlyArray = [ + ["flash-lite", 1], + ["flash", 2], + ["pro", 3], +]; + +/** + * Sortable quality score for a Gemini chat model, or null when it is not + * rankable (embeddings, computer-use/robotics, unversioned `-latest` aliases, + * or legacy/non-text families filtered by {@link isUsableGeminiCatalogModel}). + * Digits, most significant first: tier, generation, GA-over-preview. + */ +function geminiChatScore(modelId: string): number | null { + const id = modelId.toLowerCase(); + const tierRank = GEMINI_TIER_RANKS.find(([tier]) => id.includes(tier))?.[1]; + const version = parseGeminiFamilyVersion(id); + if ( + tierRank === undefined || + version === null || + !isUsableGeminiCatalogModel(id) || + id.startsWith(GEMINI_EMBEDDING_PREFIX) + ) { + return null; + } + const [major, minor] = version; + const isGA = id.includes("preview") ? 0 : 1; + return ((tierRank * 100 + major) * 100 + minor) * 10 + isGA; +} diff --git a/platform/shared/model-constants.ts b/platform/shared/model-constants.ts index eb9b760044f..f12ef6ad6f7 100644 --- a/platform/shared/model-constants.ts +++ b/platform/shared/model-constants.ts @@ -347,14 +347,11 @@ export const MODEL_MARKER_PATTERNS: Record = { "gpt-4.1", "gpt-4o", ], - gemini: [ - "gemini-3.5-pro", - "gemini-3.5-flash", - "gemini-3.1-pro-preview", - "gemini-3-pro", - "gemini-2.5-pro", - "gemini-2.5-flash", - ], + // Gemini's "best" is computed from tier + version by pickBestGeminiModelId + // (shared/gemini-models.ts), not matched against a hardcoded id list — Google + // ships Pro/Flash/Flash-lite on different cadences with unpredictable preview + // names, so any static list goes stale and re-marks a Flash as best. + gemini: [], cerebras: ["zai-glm-4.7"], cohere: ["command-a-plus", "command-a", "command-r-plus", "command-r"], mistral: [