Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions config-schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -341,6 +341,11 @@
"additionalProperties": false
}
},
"defaultProfile": {
"type": "string",
"default": "",
"description": "Profile to activate on startup and after a configuration reload. Must name a key under profiles. An empty value starts with no profile active."
},
"selectors": {
"type": "object",
"default": {},
Expand Down
9 changes: 8 additions & 1 deletion config.example.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,8 @@ upstream:

# profiles: named model ID replacements switched at runtime through the UI or API
# - optional, default: empty dictionary
# - one profile or none is active; startup and configuration reload select none
# - one profile or none is active; startup and configuration reload select the
# defaultProfile below, or none when it is unset
# - pins are applied before aliases, filters, and routing
# - targets may be a local model, fully qualified peer model, alias, setParamsByID alias, or selector
# - an empty string or YAML null (~) disables the pin with a 404; it is not
Expand All @@ -188,6 +189,12 @@ profiles:
"llm-plan": "qwen-unlisted"
"image-gen": ~

# defaultProfile: profile to activate on startup and after a configuration reload
# - optional, default: empty string, which starts with no profile active
# - must name a key under profiles or the configuration fails to load
# - the runtime selection is not persisted; restarts return to this profile
defaultProfile: "coding"

# selectors: virtual model IDs resolved to concrete targets per request
# - optional, default: empty dictionary
# - profiles run first, so a profile pin may target a selector
Expand Down
10 changes: 9 additions & 1 deletion docs/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ llama-swap supports many more features to customize how you want to manage your
| `aliases` | serve a model with different names |
| `filters` | modify requests before sending to the upstream |
| `profiles` | switch model ID replacements at runtime |
| `defaultProfile` | activate a profile on startup |
| `...` | And many more tweaks |

## Full Configuration Example
Expand Down Expand Up @@ -254,7 +255,8 @@ apiKeys:

# profiles: named model ID replacements switched at runtime through the UI or API
# - optional, default: empty dictionary
# - one profile or none is active; startup and configuration reload select none
# - one profile or none is active; startup and configuration reload select the
# defaultProfile below, or none when it is unset
# - pins are applied before aliases, filters, and routing
# - targets may be a local model, fully qualified peer model, alias, setParamsByID alias, or selector
# - an empty string or YAML null (~) disables the pin with a 404; it is not
Expand All @@ -267,6 +269,12 @@ profiles:
"llm-plan": "qwen-unlisted"
"image-gen": ~

# defaultProfile: profile to activate on startup and after a configuration reload
# - optional, default: empty string, which starts with no profile active
# - must name a key under profiles or the configuration fails to load
# - the runtime selection is not persisted; restarts return to this profile
defaultProfile: "coding"

# selectors: virtual model IDs resolved to concrete targets per request
# - optional, default: empty dictionary
# - profiles run first, so a profile pin may target a selector
Expand Down
1 change: 1 addition & 0 deletions internal/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,7 @@ type Config struct {
UnloadTimeout int `yaml:"unloadTimeout"`
Models map[string]ModelConfig `yaml:"models"` /* key is model ID */
Profiles map[string]ProfileConfig `yaml:"profiles"`
DefaultProfile string `yaml:"defaultProfile"`
Selectors map[string]SelectorConfig `yaml:"selectors"`

// routing is the canonical source for swap/scheduling configuration.
Expand Down
5 changes: 5 additions & 0 deletions internal/config/load.go
Original file line number Diff line number Diff line change
Expand Up @@ -325,6 +325,11 @@ func validateProfiles(config Config) error {
}
}
}
if config.DefaultProfile != "" {
if _, found := config.Profiles[config.DefaultProfile]; !found {
return fmt.Errorf("defaultProfile references unknown profile %q", config.DefaultProfile)
}
}
return nil
}

Expand Down
13 changes: 13 additions & 0 deletions internal/config/profile_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,12 @@ profiles:
disabled-empty: ""
disabled-null: ~
local: peer-model
defaultProfile: coding
`))
require.NoError(t, err)

assert.Equal(t, "coding", cfg.DefaultProfile)

profile := cfg.Profiles["coding"]
assert.Equal(t, "Coding profile", profile.Description)
assert.Equal(t, "", profile.Pins["disabled-empty"])
Expand Down Expand Up @@ -107,6 +110,16 @@ func TestConfig_Profiles_Validation(t *testing.T) {
`,
wantErr: "references unknown model",
},
{
name: "unknown default profile",
profile: `profiles:
good:
pins:
public: model
defaultProfile: missing
`,
wantErr: "defaultProfile references unknown profile",
},
}

for _, tc := range tests {
Expand Down
29 changes: 15 additions & 14 deletions internal/server/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -183,20 +183,21 @@ func New(cfg config.Config, muxlog *logmon.Monitor, proxylog *logmon.Monitor, up

shutdownCtx, shutdownFn := context.WithCancel(context.Background())
s := &Server{
cfg: cfg,
muxlog: muxlog,
proxylog: proxylog,
upstreamlog: upstreamlog,
perf: perfMon,
inflight: newInflightTracker(),
metrics: newMetricsMonitor(proxylog, cfg.MetricsMaxInMemory, cfg.CaptureBuffer, st),
store: st,
build: build,
hardware: hardware,
local: local,
peer: peer,
shutdownCtx: shutdownCtx,
shutdownFn: shutdownFn,
cfg: cfg,
muxlog: muxlog,
proxylog: proxylog,
upstreamlog: upstreamlog,
perf: perfMon,
inflight: newInflightTracker(),
metrics: newMetricsMonitor(proxylog, cfg.MetricsMaxInMemory, cfg.CaptureBuffer, st),
store: st,
build: build,
hardware: hardware,
activeProfile: cfg.DefaultProfile,
local: local,
peer: peer,
shutdownCtx: shutdownCtx,
shutdownFn: shutdownFn,
}
s.routes()
s.startPreload()
Expand Down
24 changes: 24 additions & 0 deletions internal/server/server_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,30 @@ func TestServer_New_MatrixConfig(t *testing.T) {
}
}

func TestServer_New_DefaultProfile(t *testing.T) {
discard := logmon.NewWriter(io.Discard)
cfg := config.Config{HealthCheckTimeout: 15}
cfg.Profiles = map[string]config.ProfileConfig{
"coding": {Pins: map[string]string{"llm-code": "model"}},
}
cfg.DefaultProfile = "coding"
st, err := store.New("")
if err != nil {
t.Fatalf("store.New: %v", err)
}
defer st.Close()
s, err := New(cfg, discard, discard, discard, nil, st, BuildInfo{}, nil)
if err != nil {
t.Fatalf("New (default profile): %v", err)
}
if got := s.ActiveProfile(); got != "coding" {
t.Fatalf("ActiveProfile()=%q want %q", got, "coding")
}
if err := s.Shutdown(time.Second); err != nil {
t.Fatalf("Shutdown: %v", err)
}
}

func TestServer_RouteToLocalModel(t *testing.T) {
s := newTestServer(
newStubRouter([]string{"local-model"}, "local response"),
Expand Down