Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,23 @@ GH_TOKEN=ghp_...
# baked-in defaults.
# SWE_DEFAULT_MODEL=openrouter/minimax/minimax-m2.6

# Per-tier models. Each of the 17 agent roles belongs to one of three tiers:
# high = planning-heavy reasoning (pm, architect, tech_lead, replan)
# med = coding / review / QA (coder, qa, code_reviewer, sprint_planner,
# retry_advisor, issue_writer, issue_advisor, verifier, merger,
# integration_tester, ci_fixer)
# low = mechanical transformation (qa_synthesizer, git)
# Set any subset; a tier var applies to every role in its tier. Precedence:
# beats SWE_DEFAULT_MODEL / AI_MODEL / HARNESS_MODEL, loses to caller config
# (`models.default` and `models.<role>`). SWE_MODEL_HIGH also becomes the
# default for the standalone `plan` pipeline (its reasoners are high-tier).
# A model id may carry an optional "#variant" reasoning-effort suffix
# (e.g. openrouter/z-ai/glm-5.2#high), passed through to opencode (--variant)
# and codex (model_reasoning_effort).
# SWE_MODEL_HIGH=openrouter/z-ai/glm-5.2
# SWE_MODEL_MED=openrouter/deepseek/deepseek-v4-pro
# SWE_MODEL_LOW=openrouter/deepseek/deepseek-v4-flash

# Runtime/model selection is configured via API request config (V2):
# {
# "runtime": "claude_code" | "open_code" | "codex",
Expand Down
3 changes: 3 additions & 0 deletions go/internal/config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@ var providerEnvKeys = []string{
"SWE_DEFAULT_MODEL",
"AI_MODEL",
"HARNESS_MODEL",
"SWE_MODEL_LOW",
"SWE_MODEL_MED",
"SWE_MODEL_HIGH",
"SWE_CODEX_AUTH_MODE",
"OPENAI_API_KEY",
}
Expand Down
224 changes: 224 additions & 0 deletions go/internal/config/modeltiers_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,224 @@
package config

import "testing"

// Ports tests/test_model_tiers.py — the per-tier model env vars
// (SWE_MODEL_LOW / _MED / _HIGH).
//
// Validation contract:
// - No tier envs set → resolution is unchanged for every runtime.
// - A tier var applies to exactly the roles in its tier (see RoleToTier).
// - Tier vars beat the SWE_DEFAULT_MODEL → AI_MODEL → HARNESS_MODEL cascade,
// and lose to caller config (models["default"], models["<role>"]).
// - SWE_MODEL_HIGH also wins the DefaultPlanningModel cascade (the planning
// reasoners are high-tier roles).

// highTierFields ports _HIGH_FIELDS.
var highTierFields = map[string]bool{
"pm_model": true,
"architect_model": true,
"tech_lead_model": true,
"replan_model": true,
}

// openCodeBaseModel ports _OPEN_CODE_BASE.
const openCodeBaseModel = "openrouter/minimax/minimax-m2.5"

// TestModelTiers_NoTierEnvsUnchanged ports TestNoTierEnvsUnchanged: no tier
// envs set → resolution unchanged for all runtimes.
func TestModelTiers_NoTierEnvsUnchanged(t *testing.T) {
tests := []struct {
name string
runtime string
env map[string]string
want func(field string) string
}{
{"claude_code base defaults", "claude_code", nil, func(field string) string {
if field == "qa_synthesizer_model" {
return "haiku"
}
return "sonnet"
}},
{"open_code base defaults", "open_code", nil,
func(string) string { return openCodeBaseModel }},
{"codex base defaults", "codex", map[string]string{"SWE_CODEX_AUTH_MODE": "api_key"},
func(string) string { return "gpt-5.3-codex" }},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
clearProviderEnv(t)
for k, v := range tc.env {
t.Setenv(k, v)
}
got := mustResolve(t, tc.runtime, nil)
for _, field := range AllModelFields {
if want := tc.want(field); got[field] != want {
t.Errorf("field %s = %q, want %q", field, got[field], want)
}
}
})
}
}

// TestModelTiers_TierEnvApplication ports TestTierEnvApplication: a tier var
// applies to exactly the roles in its tier.
func TestModelTiers_TierEnvApplication(t *testing.T) {
t.Run("high only changes exactly the high-tier fields", func(t *testing.T) {
clearProviderEnv(t)
t.Setenv("SWE_MODEL_HIGH", "openrouter/z-ai/glm-5.2")
got := mustResolve(t, "open_code", nil)
for _, field := range AllModelFields {
want := openCodeBaseModel
if highTierFields[field] {
want = "openrouter/z-ai/glm-5.2"
}
if got[field] != want {
t.Errorf("field %s = %q, want %q", field, got[field], want)
}
}
})

t.Run("all three tiers resolve every field by role tier", func(t *testing.T) {
clearProviderEnv(t)
tierModels := map[string]string{
"high": "openrouter/z-ai/glm-5.2",
"med": "openrouter/deepseek/deepseek-v4-pro",
"low": "openrouter/deepseek/deepseek-v4-flash",
}
for tier, model := range tierModels {
t.Setenv(tierModelEnvVars[tier], model)
}
got := mustResolve(t, "open_code", nil)
for role, field := range RoleToModelField {
if want := tierModels[RoleToTier[role]]; got[field] != want {
t.Errorf("role %s (%s) = %q, want %q", role, field, got[field], want)
}
}
})

t.Run("empty tier value treated as unset", func(t *testing.T) {
clearProviderEnv(t)
t.Setenv("SWE_MODEL_HIGH", " ")
got := mustResolve(t, "open_code", nil)
for _, field := range AllModelFields {
if got[field] != openCodeBaseModel {
t.Errorf("field %s = %q, want %q", field, got[field], openCodeBaseModel)
}
}
})
}

// TestModelTiers_Precedence ports TestTierPrecedence: tier vars beat the
// default-model env cascade and lose to caller config.
func TestModelTiers_Precedence(t *testing.T) {
t.Run("models.default beats all tier vars", func(t *testing.T) {
clearProviderEnv(t)
t.Setenv("SWE_MODEL_HIGH", "tier-high")
t.Setenv("SWE_MODEL_MED", "tier-med")
t.Setenv("SWE_MODEL_LOW", "tier-low")
got := mustResolve(t, "open_code", map[string]string{"default": "caller-default"})
for _, field := range AllModelFields {
if got[field] != "caller-default" {
t.Errorf("field %s = %q, want caller-default", field, got[field])
}
}
})

t.Run("models.<role> beats everything for that role only", func(t *testing.T) {
clearProviderEnv(t)
t.Setenv("SWE_MODEL_MED", "tier-med")
got := mustResolve(t, "open_code", map[string]string{"coder": "caller-coder"})
if got["coder_model"] != "caller-coder" {
t.Errorf("coder_model = %q, want caller-coder", got["coder_model"])
}
// Other med-tier roles still pick up the tier env value.
if got["qa_model"] != "tier-med" {
t.Errorf("qa_model = %q, want tier-med", got["qa_model"])
}
})

t.Run("tier var beats default env cascade for its roles", func(t *testing.T) {
clearProviderEnv(t)
t.Setenv("SWE_DEFAULT_MODEL", "env-default")
t.Setenv("AI_MODEL", "env-ai-model")
t.Setenv("SWE_MODEL_HIGH", "tier-high")
got := mustResolve(t, "open_code", nil)
for _, field := range AllModelFields {
// Unset tiers still get the cascade winner (SWE_DEFAULT_MODEL).
want := "env-default"
if highTierFields[field] {
want = "tier-high"
}
if got[field] != want {
t.Errorf("field %s = %q, want %q", field, got[field], want)
}
}
})
}

// TestDefaultPlanningModel_HighTier ports TestDefaultPlanningModelHighTier:
// SWE_MODEL_HIGH wins the planning-model cascade; without it the prior
// behavior is unchanged.
func TestDefaultPlanningModel_HighTier(t *testing.T) {
tests := []struct {
name string
env map[string]string
want string
}{
{"high tier var wins over default-model env",
map[string]string{"SWE_DEFAULT_MODEL": "env-default", "SWE_MODEL_HIGH": "tier-high"}, "tier-high"},
{"unset high tier falls back to env cascade",
map[string]string{"SWE_DEFAULT_MODEL": "env-default"}, "env-default"},
{"whitespace high tier treated as unset",
map[string]string{"SWE_MODEL_HIGH": " "}, "sonnet"},
{"no env at all defaults to sonnet", nil, "sonnet"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
clearProviderEnv(t)
for k, v := range tc.env {
t.Setenv(k, v)
}
if got := DefaultPlanningModel(); got != tc.want {
t.Fatalf("DefaultPlanningModel() = %q, want %q", got, tc.want)
}
})
}
}

// TestModelTiers_MappingCompleteness ports TestTierMappingCompleteness: every
// role has a tier, and every tier is a known tier.
func TestModelTiers_MappingCompleteness(t *testing.T) {
t.Run("every role has a tier", func(t *testing.T) {
for role := range RoleToModelField {
if _, ok := RoleToTier[role]; !ok {
t.Errorf("role %q missing from RoleToTier", role)
}
}
for role := range RoleToTier {
if _, ok := RoleToModelField[role]; !ok {
t.Errorf("RoleToTier has unknown role %q", role)
}
}
})

t.Run("every tier value is known", func(t *testing.T) {
known := make(map[string]bool, len(modelTiers))
for _, tier := range modelTiers {
known[tier] = true
}
for role, tier := range RoleToTier {
if !known[tier] {
t.Errorf("role %q has unknown tier %q", role, tier)
}
}
if len(tierModelEnvVars) != len(modelTiers) {
t.Errorf("tierModelEnvVars has %d tiers, want %d", len(tierModelEnvVars), len(modelTiers))
}
for _, tier := range modelTiers {
if _, ok := tierModelEnvVars[tier]; !ok {
t.Errorf("tier %q missing from tierModelEnvVars", tier)
}
}
})
}
74 changes: 70 additions & 4 deletions go/internal/config/resolve.go
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,40 @@ var allowedModelKeys = func() map[string]struct{} {
return m
}()

// modelTiers ports MODEL_TIERS (ordered).
var modelTiers = []string{"low", "med", "high"}

// RoleToTier ports ROLE_TO_TIER: capability tier per role. "high" =
// planning-heavy reasoning, "med" = coding/review/QA work, "low" = mechanical
// transformation. Each tier can be pointed at a model via its env var (see
// tierModelEnvVars).
var RoleToTier = map[string]string{
"pm": "high",
"architect": "high",
"tech_lead": "high",
"replan": "high",
"sprint_planner": "med",
"coder": "med",
"qa": "med",
"code_reviewer": "med",
"retry_advisor": "med",
"issue_writer": "med",
"issue_advisor": "med",
"verifier": "med",
"merger": "med",
"integration_tester": "med",
"ci_fixer": "med",
"qa_synthesizer": "low",
"git": "low",
}

// tierModelEnvVars ports TIER_MODEL_ENV_VARS (tier → env var).
var tierModelEnvVars = map[string]string{
"low": "SWE_MODEL_LOW",
"med": "SWE_MODEL_MED",
"high": "SWE_MODEL_HIGH",
}

// ---------------------------------------------------------------------------
// Model default strings
// ---------------------------------------------------------------------------
Expand Down Expand Up @@ -207,9 +241,30 @@ func defaultModelFromEnv() string {
return ""
}

// DefaultPlanningModel ports _default_planning_model: env cascade, then the
// OpenRouter default when only an OpenRouter key is present, else "sonnet".
// tierModelsFromEnv ports _tier_models_from_env: tier → model id for each
// SWE_MODEL_<TIER> env var that is set. Lets the deployer point each role
// class at a different model without enumerating every role (see RoleToTier).
// Only tiers whose env var is non-empty (stripped) appear in the result; unset
// tiers fall through to the lower precedence layers in ResolveRuntimeModels.
func tierModelsFromEnv() map[string]string {
tiers := make(map[string]string, len(tierModelEnvVars))
for tier, v := range tierModelEnvVars {
if value := envStripped(v); value != "" {
tiers[tier] = value
}
}
return tiers
}

// DefaultPlanningModel ports _default_planning_model: SWE_MODEL_HIGH first
// (the planning reasoners are high-tier roles, see RoleToTier — the same
// relative precedence tier env vars have in ResolveRuntimeModels), then the
// env cascade, then the OpenRouter default when only an OpenRouter key is
// present, else "sonnet".
func DefaultPlanningModel() string {
if highModel := tierModelsFromEnv()["high"]; highModel != "" {
return highModel
}
if envModel := defaultModelFromEnv(); envModel != "" {
return envModel
}
Expand Down Expand Up @@ -263,8 +318,10 @@ func sortedAllowedModelKeys() []string {
}

// ResolveRuntimeModels ports resolve_runtime_models. Resolution order (lowest →
// highest precedence): runtime base defaults → env cascade → models["default"]
// → models["<role>"]. fieldNames nil defaults to AllModelFields.
// highest precedence): runtime base defaults → env cascade → tier env vars
// (SWE_MODEL_LOW / SWE_MODEL_MED / SWE_MODEL_HIGH, each applying to the roles
// in its tier, see RoleToTier) → models["default"] → models["<role>"].
// fieldNames nil defaults to AllModelFields.
func ResolveRuntimeModels(runtime string, models map[string]string, fieldNames []string) (map[string]string, error) {
if fieldNames == nil {
fieldNames = AllModelFields
Expand Down Expand Up @@ -306,6 +363,15 @@ func ResolveRuntimeModels(runtime string, models map[string]string, fieldNames [
}
}

if tierModels := tierModelsFromEnv(); len(tierModels) > 0 {
for _, field := range fieldNames {
tier := RoleToTier[modelFieldToRole[field]]
if model, ok := tierModels[tier]; ok {
resolved[field] = model
}
}
}

if defaultModel, ok := flatModels["default"]; ok && defaultModel != "" {
for _, field := range fieldNames {
resolved[field] = defaultModel
Expand Down
3 changes: 2 additions & 1 deletion go/internal/orch/plan_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -421,7 +421,8 @@ func TestPlanWritesArtifactsAtExactPaths(t *testing.T) {

func TestPlanOpenRouterOnlyDefaults(t *testing.T) {
for _, k := range []string{"ANTHROPIC_API_KEY", "SWE_DEFAULT_RUNTIME",
"SWE_DEFAULT_MODEL", "AI_MODEL", "HARNESS_MODEL"} {
"SWE_DEFAULT_MODEL", "AI_MODEL", "HARNESS_MODEL",
"SWE_MODEL_LOW", "SWE_MODEL_MED", "SWE_MODEL_HIGH"} {
t.Setenv(k, "")
}
t.Setenv("OPENROUTER_API_KEY", "sk-or-test")
Expand Down
Loading
Loading