diff --git a/.env.example b/.env.example index 1abc19c..1a5de27 100644 --- a/.env.example +++ b/.env.example @@ -124,6 +124,23 @@ GH_TOKEN=ghp_... # baked-in defaults. # SWE_DEFAULT_MODEL=openrouter/minimax/minimax-m2.6 +# Per-tier models. Each of the 17 agent roles belongs to one of three tiers: +# high = planning-heavy reasoning (pm, architect, tech_lead, replan) +# med = coding / review / QA (coder, qa, code_reviewer, sprint_planner, +# retry_advisor, issue_writer, issue_advisor, verifier, merger, +# integration_tester, ci_fixer) +# low = mechanical transformation (qa_synthesizer, git) +# Set any subset; a tier var applies to every role in its tier. Precedence: +# beats SWE_DEFAULT_MODEL / AI_MODEL / HARNESS_MODEL, loses to caller config +# (`models.default` and `models.`). SWE_MODEL_HIGH also becomes the +# default for the standalone `plan` pipeline (its reasoners are high-tier). +# A model id may carry an optional "#variant" reasoning-effort suffix +# (e.g. openrouter/z-ai/glm-5.2#high), passed through to opencode (--variant) +# and codex (model_reasoning_effort). +# SWE_MODEL_HIGH=openrouter/z-ai/glm-5.2 +# SWE_MODEL_MED=openrouter/deepseek/deepseek-v4-pro +# SWE_MODEL_LOW=openrouter/deepseek/deepseek-v4-flash + # Runtime/model selection is configured via API request config (V2): # { # "runtime": "claude_code" | "open_code" | "codex", diff --git a/go/internal/config/config_test.go b/go/internal/config/config_test.go index 2025865..82a47b9 100644 --- a/go/internal/config/config_test.go +++ b/go/internal/config/config_test.go @@ -14,6 +14,9 @@ var providerEnvKeys = []string{ "SWE_DEFAULT_MODEL", "AI_MODEL", "HARNESS_MODEL", + "SWE_MODEL_LOW", + "SWE_MODEL_MED", + "SWE_MODEL_HIGH", "SWE_CODEX_AUTH_MODE", "OPENAI_API_KEY", } diff --git a/go/internal/config/modeltiers_test.go b/go/internal/config/modeltiers_test.go new file mode 100644 index 0000000..24d76c4 --- /dev/null +++ b/go/internal/config/modeltiers_test.go @@ -0,0 +1,224 @@ +package config + +import "testing" + +// Ports tests/test_model_tiers.py — the per-tier model env vars +// (SWE_MODEL_LOW / _MED / _HIGH). +// +// Validation contract: +// - No tier envs set → resolution is unchanged for every runtime. +// - A tier var applies to exactly the roles in its tier (see RoleToTier). +// - Tier vars beat the SWE_DEFAULT_MODEL → AI_MODEL → HARNESS_MODEL cascade, +// and lose to caller config (models["default"], models[""]). +// - SWE_MODEL_HIGH also wins the DefaultPlanningModel cascade (the planning +// reasoners are high-tier roles). + +// highTierFields ports _HIGH_FIELDS. +var highTierFields = map[string]bool{ + "pm_model": true, + "architect_model": true, + "tech_lead_model": true, + "replan_model": true, +} + +// openCodeBaseModel ports _OPEN_CODE_BASE. +const openCodeBaseModel = "openrouter/minimax/minimax-m2.5" + +// TestModelTiers_NoTierEnvsUnchanged ports TestNoTierEnvsUnchanged: no tier +// envs set → resolution unchanged for all runtimes. +func TestModelTiers_NoTierEnvsUnchanged(t *testing.T) { + tests := []struct { + name string + runtime string + env map[string]string + want func(field string) string + }{ + {"claude_code base defaults", "claude_code", nil, func(field string) string { + if field == "qa_synthesizer_model" { + return "haiku" + } + return "sonnet" + }}, + {"open_code base defaults", "open_code", nil, + func(string) string { return openCodeBaseModel }}, + {"codex base defaults", "codex", map[string]string{"SWE_CODEX_AUTH_MODE": "api_key"}, + func(string) string { return "gpt-5.3-codex" }}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + clearProviderEnv(t) + for k, v := range tc.env { + t.Setenv(k, v) + } + got := mustResolve(t, tc.runtime, nil) + for _, field := range AllModelFields { + if want := tc.want(field); got[field] != want { + t.Errorf("field %s = %q, want %q", field, got[field], want) + } + } + }) + } +} + +// TestModelTiers_TierEnvApplication ports TestTierEnvApplication: a tier var +// applies to exactly the roles in its tier. +func TestModelTiers_TierEnvApplication(t *testing.T) { + t.Run("high only changes exactly the high-tier fields", func(t *testing.T) { + clearProviderEnv(t) + t.Setenv("SWE_MODEL_HIGH", "openrouter/z-ai/glm-5.2") + got := mustResolve(t, "open_code", nil) + for _, field := range AllModelFields { + want := openCodeBaseModel + if highTierFields[field] { + want = "openrouter/z-ai/glm-5.2" + } + if got[field] != want { + t.Errorf("field %s = %q, want %q", field, got[field], want) + } + } + }) + + t.Run("all three tiers resolve every field by role tier", func(t *testing.T) { + clearProviderEnv(t) + tierModels := map[string]string{ + "high": "openrouter/z-ai/glm-5.2", + "med": "openrouter/deepseek/deepseek-v4-pro", + "low": "openrouter/deepseek/deepseek-v4-flash", + } + for tier, model := range tierModels { + t.Setenv(tierModelEnvVars[tier], model) + } + got := mustResolve(t, "open_code", nil) + for role, field := range RoleToModelField { + if want := tierModels[RoleToTier[role]]; got[field] != want { + t.Errorf("role %s (%s) = %q, want %q", role, field, got[field], want) + } + } + }) + + t.Run("empty tier value treated as unset", func(t *testing.T) { + clearProviderEnv(t) + t.Setenv("SWE_MODEL_HIGH", " ") + got := mustResolve(t, "open_code", nil) + for _, field := range AllModelFields { + if got[field] != openCodeBaseModel { + t.Errorf("field %s = %q, want %q", field, got[field], openCodeBaseModel) + } + } + }) +} + +// TestModelTiers_Precedence ports TestTierPrecedence: tier vars beat the +// default-model env cascade and lose to caller config. +func TestModelTiers_Precedence(t *testing.T) { + t.Run("models.default beats all tier vars", func(t *testing.T) { + clearProviderEnv(t) + t.Setenv("SWE_MODEL_HIGH", "tier-high") + t.Setenv("SWE_MODEL_MED", "tier-med") + t.Setenv("SWE_MODEL_LOW", "tier-low") + got := mustResolve(t, "open_code", map[string]string{"default": "caller-default"}) + for _, field := range AllModelFields { + if got[field] != "caller-default" { + t.Errorf("field %s = %q, want caller-default", field, got[field]) + } + } + }) + + t.Run("models. beats everything for that role only", func(t *testing.T) { + clearProviderEnv(t) + t.Setenv("SWE_MODEL_MED", "tier-med") + got := mustResolve(t, "open_code", map[string]string{"coder": "caller-coder"}) + if got["coder_model"] != "caller-coder" { + t.Errorf("coder_model = %q, want caller-coder", got["coder_model"]) + } + // Other med-tier roles still pick up the tier env value. + if got["qa_model"] != "tier-med" { + t.Errorf("qa_model = %q, want tier-med", got["qa_model"]) + } + }) + + t.Run("tier var beats default env cascade for its roles", func(t *testing.T) { + clearProviderEnv(t) + t.Setenv("SWE_DEFAULT_MODEL", "env-default") + t.Setenv("AI_MODEL", "env-ai-model") + t.Setenv("SWE_MODEL_HIGH", "tier-high") + got := mustResolve(t, "open_code", nil) + for _, field := range AllModelFields { + // Unset tiers still get the cascade winner (SWE_DEFAULT_MODEL). + want := "env-default" + if highTierFields[field] { + want = "tier-high" + } + if got[field] != want { + t.Errorf("field %s = %q, want %q", field, got[field], want) + } + } + }) +} + +// TestDefaultPlanningModel_HighTier ports TestDefaultPlanningModelHighTier: +// SWE_MODEL_HIGH wins the planning-model cascade; without it the prior +// behavior is unchanged. +func TestDefaultPlanningModel_HighTier(t *testing.T) { + tests := []struct { + name string + env map[string]string + want string + }{ + {"high tier var wins over default-model env", + map[string]string{"SWE_DEFAULT_MODEL": "env-default", "SWE_MODEL_HIGH": "tier-high"}, "tier-high"}, + {"unset high tier falls back to env cascade", + map[string]string{"SWE_DEFAULT_MODEL": "env-default"}, "env-default"}, + {"whitespace high tier treated as unset", + map[string]string{"SWE_MODEL_HIGH": " "}, "sonnet"}, + {"no env at all defaults to sonnet", nil, "sonnet"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + clearProviderEnv(t) + for k, v := range tc.env { + t.Setenv(k, v) + } + if got := DefaultPlanningModel(); got != tc.want { + t.Fatalf("DefaultPlanningModel() = %q, want %q", got, tc.want) + } + }) + } +} + +// TestModelTiers_MappingCompleteness ports TestTierMappingCompleteness: every +// role has a tier, and every tier is a known tier. +func TestModelTiers_MappingCompleteness(t *testing.T) { + t.Run("every role has a tier", func(t *testing.T) { + for role := range RoleToModelField { + if _, ok := RoleToTier[role]; !ok { + t.Errorf("role %q missing from RoleToTier", role) + } + } + for role := range RoleToTier { + if _, ok := RoleToModelField[role]; !ok { + t.Errorf("RoleToTier has unknown role %q", role) + } + } + }) + + t.Run("every tier value is known", func(t *testing.T) { + known := make(map[string]bool, len(modelTiers)) + for _, tier := range modelTiers { + known[tier] = true + } + for role, tier := range RoleToTier { + if !known[tier] { + t.Errorf("role %q has unknown tier %q", role, tier) + } + } + if len(tierModelEnvVars) != len(modelTiers) { + t.Errorf("tierModelEnvVars has %d tiers, want %d", len(tierModelEnvVars), len(modelTiers)) + } + for _, tier := range modelTiers { + if _, ok := tierModelEnvVars[tier]; !ok { + t.Errorf("tier %q missing from tierModelEnvVars", tier) + } + } + }) +} diff --git a/go/internal/config/resolve.go b/go/internal/config/resolve.go index 5f13759..f8ee24d 100644 --- a/go/internal/config/resolve.go +++ b/go/internal/config/resolve.go @@ -88,6 +88,40 @@ var allowedModelKeys = func() map[string]struct{} { return m }() +// modelTiers ports MODEL_TIERS (ordered). +var modelTiers = []string{"low", "med", "high"} + +// RoleToTier ports ROLE_TO_TIER: capability tier per role. "high" = +// planning-heavy reasoning, "med" = coding/review/QA work, "low" = mechanical +// transformation. Each tier can be pointed at a model via its env var (see +// tierModelEnvVars). +var RoleToTier = map[string]string{ + "pm": "high", + "architect": "high", + "tech_lead": "high", + "replan": "high", + "sprint_planner": "med", + "coder": "med", + "qa": "med", + "code_reviewer": "med", + "retry_advisor": "med", + "issue_writer": "med", + "issue_advisor": "med", + "verifier": "med", + "merger": "med", + "integration_tester": "med", + "ci_fixer": "med", + "qa_synthesizer": "low", + "git": "low", +} + +// tierModelEnvVars ports TIER_MODEL_ENV_VARS (tier → env var). +var tierModelEnvVars = map[string]string{ + "low": "SWE_MODEL_LOW", + "med": "SWE_MODEL_MED", + "high": "SWE_MODEL_HIGH", +} + // --------------------------------------------------------------------------- // Model default strings // --------------------------------------------------------------------------- @@ -207,9 +241,30 @@ func defaultModelFromEnv() string { return "" } -// DefaultPlanningModel ports _default_planning_model: env cascade, then the -// OpenRouter default when only an OpenRouter key is present, else "sonnet". +// tierModelsFromEnv ports _tier_models_from_env: tier → model id for each +// SWE_MODEL_ env var that is set. Lets the deployer point each role +// class at a different model without enumerating every role (see RoleToTier). +// Only tiers whose env var is non-empty (stripped) appear in the result; unset +// tiers fall through to the lower precedence layers in ResolveRuntimeModels. +func tierModelsFromEnv() map[string]string { + tiers := make(map[string]string, len(tierModelEnvVars)) + for tier, v := range tierModelEnvVars { + if value := envStripped(v); value != "" { + tiers[tier] = value + } + } + return tiers +} + +// DefaultPlanningModel ports _default_planning_model: SWE_MODEL_HIGH first +// (the planning reasoners are high-tier roles, see RoleToTier — the same +// relative precedence tier env vars have in ResolveRuntimeModels), then the +// env cascade, then the OpenRouter default when only an OpenRouter key is +// present, else "sonnet". func DefaultPlanningModel() string { + if highModel := tierModelsFromEnv()["high"]; highModel != "" { + return highModel + } if envModel := defaultModelFromEnv(); envModel != "" { return envModel } @@ -263,8 +318,10 @@ func sortedAllowedModelKeys() []string { } // ResolveRuntimeModels ports resolve_runtime_models. Resolution order (lowest → -// highest precedence): runtime base defaults → env cascade → models["default"] -// → models[""]. fieldNames nil defaults to AllModelFields. +// highest precedence): runtime base defaults → env cascade → tier env vars +// (SWE_MODEL_LOW / SWE_MODEL_MED / SWE_MODEL_HIGH, each applying to the roles +// in its tier, see RoleToTier) → models["default"] → models[""]. +// fieldNames nil defaults to AllModelFields. func ResolveRuntimeModels(runtime string, models map[string]string, fieldNames []string) (map[string]string, error) { if fieldNames == nil { fieldNames = AllModelFields @@ -306,6 +363,15 @@ func ResolveRuntimeModels(runtime string, models map[string]string, fieldNames [ } } + if tierModels := tierModelsFromEnv(); len(tierModels) > 0 { + for _, field := range fieldNames { + tier := RoleToTier[modelFieldToRole[field]] + if model, ok := tierModels[tier]; ok { + resolved[field] = model + } + } + } + if defaultModel, ok := flatModels["default"]; ok && defaultModel != "" { for _, field := range fieldNames { resolved[field] = defaultModel diff --git a/go/internal/orch/plan_test.go b/go/internal/orch/plan_test.go index 8e83c6a..fe745b5 100644 --- a/go/internal/orch/plan_test.go +++ b/go/internal/orch/plan_test.go @@ -421,7 +421,8 @@ func TestPlanWritesArtifactsAtExactPaths(t *testing.T) { func TestPlanOpenRouterOnlyDefaults(t *testing.T) { for _, k := range []string{"ANTHROPIC_API_KEY", "SWE_DEFAULT_RUNTIME", - "SWE_DEFAULT_MODEL", "AI_MODEL", "HARNESS_MODEL"} { + "SWE_DEFAULT_MODEL", "AI_MODEL", "HARNESS_MODEL", + "SWE_MODEL_LOW", "SWE_MODEL_MED", "SWE_MODEL_HIGH"} { t.Setenv(k, "") } t.Setenv("OPENROUTER_API_KEY", "sk-or-test") diff --git a/swe_af/execution/schemas.py b/swe_af/execution/schemas.py index ba4c5c3..009c356 100644 --- a/swe_af/execution/schemas.py +++ b/swe_af/execution/schemas.py @@ -493,6 +493,37 @@ class QASynthesisResult(BaseModel): } _ALLOWED_MODEL_KEYS: set[str] = set(MODEL_ROLE_KEYS) | {"default"} +MODEL_TIERS: tuple[str, ...] = ("low", "med", "high") + +# Capability tier per role: "high" = planning-heavy reasoning, "med" = +# coding/review/QA work, "low" = mechanical transformation. Each tier can be +# pointed at a model via its env var (see TIER_MODEL_ENV_VARS). +ROLE_TO_TIER: dict[str, str] = { + "pm": "high", + "architect": "high", + "tech_lead": "high", + "replan": "high", + "sprint_planner": "med", + "coder": "med", + "qa": "med", + "code_reviewer": "med", + "retry_advisor": "med", + "issue_writer": "med", + "issue_advisor": "med", + "verifier": "med", + "merger": "med", + "integration_tester": "med", + "ci_fixer": "med", + "qa_synthesizer": "low", + "git": "low", +} + +TIER_MODEL_ENV_VARS: dict[str, str] = { + "low": "SWE_MODEL_LOW", + "med": "SWE_MODEL_MED", + "high": "SWE_MODEL_HIGH", +} + _LEGACY_GROUP_EQUIVALENTS: dict[str, str] = { "planning": "models.pm, models.architect, models.tech_lead, models.sprint_planner", "coding": "models.coder, models.qa, models.code_reviewer", @@ -631,6 +662,22 @@ def _default_model_from_env() -> str | None: return None +def _tier_models_from_env() -> dict[str, str]: + """Tier → model id for each ``SWE_MODEL_`` env var that is set. + + Lets the deployer point each role class at a different model without + enumerating every role (see ``ROLE_TO_TIER``). Only tiers whose env var is + non-empty appear in the result; unset tiers fall through to the lower + precedence layers in ``resolve_runtime_models``. + """ + tiers: dict[str, str] = {} + for tier, var in TIER_MODEL_ENV_VARS.items(): + value = os.getenv(var, "").strip() + if value: + tiers[tier] = value + return tiers + + def _default_planning_model() -> str: """Model for the planning reasoners (the ``plan`` pipeline) when the caller passes no model. @@ -638,12 +685,19 @@ def _default_planning_model() -> str: The planning reasoners take an explicit ``model`` argument rather than a runtime ``models={}`` config, so the ``resolve_runtime_models`` cascade doesn't apply to them. This mirrors that cascade for the planning path so an - OpenRouter-only deployment is zero-config. Precedence, first match wins: - - 1. deployer env (``SWE_DEFAULT_MODEL`` → ``AI_MODEL`` → ``HARNESS_MODEL``) - 2. the OpenRouter default when only an OpenRouter key is present - 3. the Claude ``sonnet`` alias (historical default) + OpenRouter-only deployment is zero-config. The planning reasoners are + high-tier roles (see ``ROLE_TO_TIER``), so ``SWE_MODEL_HIGH`` beats the + generic default-model env — the same relative precedence tier env vars have + in ``resolve_runtime_models``. Precedence, first match wins: + + 1. ``SWE_MODEL_HIGH`` (planning reasoners are high-tier) + 2. deployer env (``SWE_DEFAULT_MODEL`` → ``AI_MODEL`` → ``HARNESS_MODEL``) + 3. the OpenRouter default when only an OpenRouter key is present + 4. the Claude ``sonnet`` alias (historical default) """ + high_model = _tier_models_from_env().get("high") + if high_model: + return high_model env_model = _default_model_from_env() if env_model: return env_model @@ -724,8 +778,11 @@ def resolve_runtime_models( 1. runtime base defaults (``_RUNTIME_BASE_MODELS[runtime]``) 2. env-var cascade: ``SWE_DEFAULT_MODEL`` → ``AI_MODEL`` → ``HARNESS_MODEL`` (first non-empty wins, applies to all roles) - 3. caller's ``models["default"]`` - 4. caller's ``models[""]`` + 3. tier env vars: ``SWE_MODEL_LOW`` / ``SWE_MODEL_MED`` / + ``SWE_MODEL_HIGH``, each applying to the roles in its tier + (see ``ROLE_TO_TIER``) + 4. caller's ``models["default"]`` + 5. caller's ``models[""]`` """ if field_names is None: field_names = ALL_MODEL_FIELDS @@ -754,6 +811,13 @@ def resolve_runtime_models( for field in field_names: resolved[field] = env_default + tier_models = _tier_models_from_env() + if tier_models: + for field in field_names: + tier = ROLE_TO_TIER[_MODEL_FIELD_TO_ROLE[field]] + if tier in tier_models: + resolved[field] = tier_models[tier] + default_model = flat_models.get("default") if default_model: for field in field_names: diff --git a/tests/test_model_config.py b/tests/test_model_config.py index 3fcd92f..04495fc 100644 --- a/tests/test_model_config.py +++ b/tests/test_model_config.py @@ -27,6 +27,9 @@ "SWE_DEFAULT_MODEL", "AI_MODEL", "HARNESS_MODEL", + "SWE_MODEL_LOW", + "SWE_MODEL_MED", + "SWE_MODEL_HIGH", ) diff --git a/tests/test_model_tiers.py b/tests/test_model_tiers.py new file mode 100644 index 0000000..207e5c3 --- /dev/null +++ b/tests/test_model_tiers.py @@ -0,0 +1,189 @@ +"""Tests for the per-tier model env vars (SWE_MODEL_LOW / _MED / _HIGH). + +Validation contract: +- No tier envs set → resolution is unchanged for every runtime. +- A tier var applies to exactly the roles in its tier (see ROLE_TO_TIER). +- Tier vars beat the SWE_DEFAULT_MODEL → AI_MODEL → HARNESS_MODEL cascade, + and lose to caller config (models["default"], models[""]). +- SWE_MODEL_HIGH also wins the _default_planning_model cascade (the planning + reasoners are high-tier roles). +""" + +from __future__ import annotations + +import pytest + +from swe_af.execution.schemas import ( + ALL_MODEL_FIELDS, + MODEL_TIERS, + ROLE_TO_MODEL_FIELD, + ROLE_TO_TIER, + TIER_MODEL_ENV_VARS, + _default_planning_model, + resolve_runtime_models, +) + +_HIGH_FIELDS = {"pm_model", "architect_model", "tech_lead_model", "replan_model"} +_LOW_FIELDS = {"qa_synthesizer_model", "git_model"} + +_OPEN_CODE_BASE = "openrouter/minimax/minimax-m2.5" + +# Env vars that steer provider/runtime/model selection. Cleared before every +# test so assertions never depend on the developer's ambient shell. +_STEERING_ENV_KEYS = ( + "SWE_MODEL_LOW", + "SWE_MODEL_MED", + "SWE_MODEL_HIGH", + "SWE_DEFAULT_MODEL", + "AI_MODEL", + "HARNESS_MODEL", + "SWE_DEFAULT_RUNTIME", + "SWE_CODEX_AUTH_MODE", + "ANTHROPIC_API_KEY", + "OPENROUTER_API_KEY", + "OPENAI_API_KEY", +) + + +@pytest.fixture(autouse=True) +def _clean_env(monkeypatch: pytest.MonkeyPatch) -> None: + for key in _STEERING_ENV_KEYS: + monkeypatch.delenv(key, raising=False) + + +class TestNoTierEnvsUnchanged: + """Contract: no tier envs set → resolution unchanged for all runtimes.""" + + def test_claude_code_base_defaults(self) -> None: + resolved = resolve_runtime_models(runtime="claude_code", models=None) + for field in ALL_MODEL_FIELDS: + expected = "haiku" if field == "qa_synthesizer_model" else "sonnet" + assert resolved[field] == expected + + def test_open_code_base_defaults(self) -> None: + resolved = resolve_runtime_models(runtime="open_code", models=None) + for field in ALL_MODEL_FIELDS: + assert resolved[field] == _OPEN_CODE_BASE + + def test_codex_base_defaults(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("SWE_CODEX_AUTH_MODE", "api_key") + resolved = resolve_runtime_models(runtime="codex", models=None) + for field in ALL_MODEL_FIELDS: + assert resolved[field] == "gpt-5.3-codex" + + +class TestTierEnvApplication: + """Contract: a tier var applies to exactly the roles in its tier.""" + + def test_high_only_changes_exactly_the_high_tier_fields( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("SWE_MODEL_HIGH", "openrouter/z-ai/glm-5.2") + resolved = resolve_runtime_models(runtime="open_code", models=None) + for field in ALL_MODEL_FIELDS: + if field in _HIGH_FIELDS: + assert resolved[field] == "openrouter/z-ai/glm-5.2" + else: + assert resolved[field] == _OPEN_CODE_BASE + + def test_all_three_tiers_resolve_every_field_by_role_tier( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + tier_models = { + "high": "openrouter/z-ai/glm-5.2", + "med": "openrouter/deepseek/deepseek-v4-pro", + "low": "openrouter/deepseek/deepseek-v4-flash", + } + for tier, model in tier_models.items(): + monkeypatch.setenv(TIER_MODEL_ENV_VARS[tier], model) + resolved = resolve_runtime_models(runtime="open_code", models=None) + for role, field in ROLE_TO_MODEL_FIELD.items(): + assert resolved[field] == tier_models[ROLE_TO_TIER[role]] + + def test_empty_tier_value_treated_as_unset( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("SWE_MODEL_HIGH", " ") + resolved = resolve_runtime_models(runtime="open_code", models=None) + for field in ALL_MODEL_FIELDS: + assert resolved[field] == _OPEN_CODE_BASE + + +class TestTierPrecedence: + """Contract: tier vars beat the default-model env cascade and lose to + caller config.""" + + def test_models_default_beats_all_tier_vars( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("SWE_MODEL_HIGH", "tier-high") + monkeypatch.setenv("SWE_MODEL_MED", "tier-med") + monkeypatch.setenv("SWE_MODEL_LOW", "tier-low") + resolved = resolve_runtime_models( + runtime="open_code", models={"default": "caller-default"} + ) + for field in ALL_MODEL_FIELDS: + assert resolved[field] == "caller-default" + + def test_models_role_beats_everything_for_that_role_only( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("SWE_MODEL_MED", "tier-med") + resolved = resolve_runtime_models( + runtime="open_code", models={"coder": "caller-coder"} + ) + assert resolved["coder_model"] == "caller-coder" + # Other med-tier roles still pick up the tier env value. + assert resolved["qa_model"] == "tier-med" + + def test_tier_var_beats_default_env_cascade_for_its_roles( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("SWE_DEFAULT_MODEL", "env-default") + monkeypatch.setenv("AI_MODEL", "env-ai-model") + monkeypatch.setenv("SWE_MODEL_HIGH", "tier-high") + resolved = resolve_runtime_models(runtime="open_code", models=None) + for field in ALL_MODEL_FIELDS: + if field in _HIGH_FIELDS: + assert resolved[field] == "tier-high" + else: + # Unset tiers still get the cascade winner (SWE_DEFAULT_MODEL). + assert resolved[field] == "env-default" + + +class TestDefaultPlanningModelHighTier: + """Contract: SWE_MODEL_HIGH wins the planning-model cascade; without it + the prior behavior is unchanged.""" + + def test_high_tier_var_wins_over_default_model_env( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("SWE_DEFAULT_MODEL", "env-default") + monkeypatch.setenv("SWE_MODEL_HIGH", "tier-high") + assert _default_planning_model() == "tier-high" + + def test_unset_high_tier_falls_back_to_env_cascade( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("SWE_DEFAULT_MODEL", "env-default") + assert _default_planning_model() == "env-default" + + def test_whitespace_high_tier_treated_as_unset( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("SWE_MODEL_HIGH", " ") + assert _default_planning_model() == "sonnet" + + def test_no_env_at_all_defaults_to_sonnet(self) -> None: + assert _default_planning_model() == "sonnet" + + +class TestTierMappingCompleteness: + """Contract: every role has a tier, and every tier is a known tier.""" + + def test_every_role_has_a_tier(self) -> None: + assert set(ROLE_TO_TIER) == set(ROLE_TO_MODEL_FIELD) + + def test_every_tier_value_is_known(self) -> None: + assert set(ROLE_TO_TIER.values()) <= set(MODEL_TIERS) + assert set(TIER_MODEL_ENV_VARS) == set(MODEL_TIERS)