feat: configurable upstreams for local and self-hosted models

README.md

@@ -151,6 +151,35 @@ claude-review = "anthropic:claude-sonnet-5"
 The slugs above are illustrative — run `modelmux check-latest` to see which
 models actually exist on OpenRouter right now.
 
+## Local & self-hosted models
+
+`anthropic` and `openrouter` are built in, but you can point an alias at any
+**Anthropic-Messages-compatible** endpoint by declaring it under `[upstreams]`,
+then using it like any other model (`<name>:<slug>`).
+
+The obvious use is a **local model**. Recent Ollama (v0.14+) speaks the Anthropic
+Messages API natively, so no translation shim is needed — run your grunt-work
+subagents on a local Qwen while the orchestrator stays on Claude:
+
+```toml
+[upstreams]
+local = { base = "http://localhost:11434", auth = "none" }
+
+[models]
+orchestrator = "anthropic:passthrough" # brain stays on Claude
+flagship = "local:qwen3-coder:30b" # subagents run on your local Qwen
+cheap = "local:qwen3:8b"
+```
+
+`auth` is one of `passthrough` (forward Claude Code's own auth), `bearer:ENV_VAR`
+(send `Authorization: Bearer $ENV_VAR`), or `none`. modelmux never forwards your
+Claude auth to a `none`/`bearer` upstream, so your Anthropic token stays out of
+the local process.
+
+Runners that only speak the OpenAI format (LM Studio, llama.cpp, vLLM) need a
+[LiteLLM](https://github.com/BerriAI/litellm) proxy in front to expose an
+Anthropic endpoint; point the upstream `base` at that.
+
 ## Managing config from the CLI
 
 `modelmux` with no arguments runs the proxy; the subcommands manage `routes.toml`:
@@ -188,9 +217,9 @@ Everything is controlled by `routes.toml` and a few environment variables:
 
 ## Troubleshooting
 
-- **HTTP 400, `OPENROUTER_API_KEY is not set but a route needs OpenRouter`** — a
-  request routed to OpenRouter but the key is unset where modelmux runs. Set it
-  and restart the binary.
+- **HTTP 400, `OPENROUTER_API_KEY is not set but a route needs it`** — a request
+  routed to OpenRouter but the key is unset where modelmux runs. Set it and
+  restart the binary.
 - **Claude Code reports connection refused** — modelmux isn't running, or it's on
   a different port than `ANTHROPIC_BASE_URL`. Start it and check the listening
   line matches.

docs/development.md

@@ -150,7 +150,7 @@ If a request routes to OpenRouter and `OPENROUTER_API_KEY` is unset, the request
 fails HTTP 400 with the body:
 
 ```text
-OPENROUTER_API_KEY is not set but a route needs OpenRouter
+OPENROUTER_API_KEY is not set but a route needs it
 ```
 
 Override one alias for a single run without editing `routes.toml`:

routes.toml

@@ -39,3 +39,23 @@ reasoner = "openrouter:deepseek/deepseek-v4-pro"
 review = "openrouter:minimax/minimax-m3"
 cheap = "openrouter:deepseek/deepseek-v4-flash"
 claude-review = "anthropic:claude-sonnet-5"
+
+# --- upstreams (optional) ---------------------------------------------------
+# "anthropic" (passthrough) and "openrouter" (Bearer OPENROUTER_API_KEY) are
+# built in. Add your own Anthropic-Messages-compatible endpoints here, then use
+# them like any other alias: <name>:<slug>. auth = passthrough | passthrough:ENV
+# | bearer:ENV | none.
+#
+# LOCAL MODELS: recent Ollama (v0.14+) speaks the Anthropic Messages API
+# natively, so no translation shim is needed — point an alias at it and run your
+# grunt-work subagents on a local model while the orchestrator stays on Claude:
+#
+#   [upstreams]
+#   local = { base = "http://localhost:11434", auth = "none" }
+#
+#   # then in [models] above:
+#   flagship = "local:qwen3-coder:30b"    # subagents run on your local Qwen
+#
+# (Non-Ollama runners like LM Studio / llama.cpp / vLLM speak the OpenAI format;
+# put a LiteLLM proxy in front to expose an Anthropic endpoint, then point the
+# upstream base at that.)

src/config.ts

@@ -1,8 +1,8 @@
-import type { Config, ModelRef, RouteRule, Upstream, WorkType } from "./types.ts";
+import type { AuthMode, Config, ModelRef, RouteRule, UpstreamDef, WorkType } from "./types.ts";
 import { readFileSync, watch } from "node:fs";
 import process from "node:process";
+import { BUILTIN_UPSTREAMS } from "./upstreams.ts";
 
-const UPSTREAMS: Upstream[] = ["anthropic", "openrouter"];
 const WORK_TYPES = new Set<WorkType>(["background", "think", "longContext", "webSearch"]);
 
 // A route's `when` must name exactly one recognized condition. Reject anything
@@ -23,19 +23,57 @@ function validateWhen(when: RouteRule["when"] | undefined): void {
     throw new Error("route `when.anySubagent` must be true when present");
 }
 
+// Split "<upstream>:<slug>" into its parts (shape only). Whether the upstream
+// name actually exists is validated against the [upstreams] table in loadConfig.
 export function parseModelRef(spec: string): ModelRef {
   const idx = spec.indexOf(":");
   if (idx === -1)
     throw new Error(`bad model spec (need "<upstream>:<slug>"): ${spec}`);
-  const upstream = spec.slice(0, idx) as Upstream;
+  const upstream = spec.slice(0, idx);
   const slug = spec.slice(idx + 1);
-  if (!UPSTREAMS.includes(upstream))
-    throw new Error(`unknown upstream: ${upstream}`);
+  if (!upstream)
+    throw new Error(`empty upstream in: ${spec}`);
   if (!slug)
     throw new Error(`empty slug in: ${spec}`);
   return { upstream, slug };
 }
 
+interface RawUpstream {
+  base?: string;
+  auth?: string;
+  stripBeta?: boolean;
+}
+
+// Parse an auth spec: "passthrough" | "passthrough:ENV" | "bearer:ENV" | "none".
+export function parseAuth(spec: string): AuthMode {
+  if (spec === "none")
+    return { kind: "none" };
+  if (spec === "passthrough")
+    return { kind: "passthrough" };
+  if (spec.startsWith("passthrough:"))
+    return { kind: "passthrough", envKey: spec.slice("passthrough:".length) };
+  if (spec.startsWith("bearer:")) {
+    const envKey = spec.slice("bearer:".length);
+    if (!envKey)
+      throw new Error("bearer auth needs an env var, e.g. auth = \"bearer:MY_API_KEY\"");
+    return { kind: "bearer", envKey };
+  }
+  throw new Error(`unknown auth "${spec}" (use passthrough | passthrough:ENV | bearer:ENV | none)`);
+}
+
+// Merge any user-declared [upstreams] over the built-ins (anthropic, openrouter).
+function buildUpstreams(raw: Record<string, RawUpstream> | undefined): Record<string, UpstreamDef> {
+  const out: Record<string, UpstreamDef> = { ...BUILTIN_UPSTREAMS };
+  for (const [name, u] of Object.entries(raw ?? {})) {
+    if (!u || typeof u !== "object" || typeof u.base !== "string" || !u.base)
+      throw new Error(`upstream "${name}" needs a base URL, e.g. base = "http://localhost:11434"`);
+    const auth = parseAuth(u.auth ?? "none");
+    const stripBeta = u.stripBeta ?? (auth.kind !== "passthrough");
+    out[name] = { base: u.base, auth, stripBeta };
+  }
+  return out;
+}
+
 export function resolveMenu(
   config: Config,
   env: Record<string, string | undefined>,
@@ -60,6 +98,7 @@ interface RawConfig {
   default: string;
   routes: Config["routes"];
   longContextThreshold?: number;
+  upstreams?: Record<string, RawUpstream>;
 }
 
 export function loadConfig(
@@ -78,6 +117,7 @@ export function loadConfig(
     default: raw.default,
     routes: raw.routes ?? [],
     longContextThreshold: raw.longContextThreshold ?? 200000,
+    upstreams: buildUpstreams(raw.upstreams),
   };
   if (!config.default)
     throw new Error("routes.toml is missing a `default` alias");
@@ -90,7 +130,13 @@ export function loadConfig(
     if (!models[r.use])
       throw new Error(`route uses unknown alias "${r.use}" (not in models)`);
   }
-  return resolveMenu(config, env);
+  const resolved = resolveMenu(config, env);
+  // Every model's upstream must be built in or declared in [upstreams].
+  for (const [alias, ref] of Object.entries(resolved.models)) {
+    if (!resolved.upstreams?.[ref.upstream])
+      throw new Error(`model "${alias}" uses unknown upstream "${ref.upstream}" — built-ins are anthropic and openrouter; add others under [upstreams]`);
+  }
+  return resolved;
 }
 
 function readText(path: string): string {

src/server.ts

@@ -44,7 +44,7 @@ export function buildServer(opts: ServerOpts): Bun.Server<never> {
       let headers: Headers;
       try {
         decision = route(signals, config);
-        headers = rewriteHeaders(decision, req.headers, opts.env);
+        headers = rewriteHeaders(decision, req.headers, opts.env, config.upstreams);
       }
       catch (e) {
         logError(opts.logPath, signals, e as Error);
@@ -58,7 +58,7 @@ export function buildServer(opts: ServerOpts): Bun.Server<never> {
       const base = opts.baseOverride?.[decision.upstream];
       const target = base
         ? base + url.pathname + url.search
-        : forwardUrl(decision.upstream, url.pathname, url.search);
+        : forwardUrl(decision.upstream, url.pathname, url.search, config.upstreams);
 
       let upstream: Response;
       try {

src/types.ts

@@ -1,4 +1,18 @@
-export type Upstream = "anthropic" | "openrouter";
+// An upstream name. "anthropic" and "openrouter" are built in; more can be
+// defined in the [upstreams] table of routes.toml (e.g. a local model server).
+export type Upstream = string;
+
+// How modelmux authenticates the outbound leg to an upstream.
+export type AuthMode
+  = | { kind: "passthrough"; envKey?: string } // forward Claude Code's own inbound auth; if envKey is set and present, send it as x-api-key instead
+    | { kind: "bearer"; envKey: string } // Authorization: Bearer <env[envKey]>
+    | { kind: "none" }; // send no auth (e.g. a local model server)
+
+export interface UpstreamDef {
+  base: string; // base URL, e.g. https://api.anthropic.com or http://localhost:11434
+  auth: AuthMode;
+  stripBeta: boolean; // drop anthropic-beta — non-Anthropic endpoints don't understand Claude Code's betas
+}
 
 export interface ModelRef {
   upstream: Upstream;
@@ -17,6 +31,7 @@ export interface Config {
   default: string; // alias used when no route matches (the orchestrator)
   routes: RouteRule[];
   longContextThreshold: number;
+  upstreams?: Record<string, UpstreamDef>; // built-ins merged with any [upstreams] overrides; falls back to built-ins when absent
 }
 
 export interface Signals {

src/upstreams.ts

@@ -1,25 +1,49 @@
-import type { Decision, Upstream } from "./types.ts";
+import type { AuthMode, Decision, Upstream, UpstreamDef } from "./types.ts";
 
-const BASE: Record<Upstream, string> = {
-  anthropic: "https://api.anthropic.com",
-  openrouter: "https://openrouter.ai/api",
+// Built-in upstreams. Users can add more (or override these) via the [upstreams]
+// table in routes.toml; these are the defaults when a name isn't configured.
+export const BUILTIN_UPSTREAMS: Record<string, UpstreamDef> = {
+  anthropic: {
+    base: "https://api.anthropic.com",
+    auth: { kind: "passthrough", envKey: "ANTHROPIC_API_KEY" },
+    stripBeta: false,
+  },
+  openrouter: {
+    base: "https://openrouter.ai/api",
+    auth: { kind: "bearer", envKey: "OPENROUTER_API_KEY" },
+    stripBeta: true, // OpenRouter's Anthropic endpoint may reject Claude Code's betas
+  },
 };
 
 const HOP_BY_HOP = new Set(["host", "content-length", "connection", "accept-encoding"]);
 
 export class MissingKeyError extends Error {}
 
-export function forwardUrl(upstream: Upstream, inboundPath: string, inboundSearch: string): string {
-  return BASE[upstream] + inboundPath + inboundSearch;
+export function resolveUpstream(name: Upstream, upstreams?: Record<string, UpstreamDef>): UpstreamDef {
+  const def = upstreams?.[name] ?? BUILTIN_UPSTREAMS[name];
+  if (!def)
+    throw new Error(`unknown upstream "${name}" (define it in an [upstreams] table in routes.toml)`);
+  return def;
+}
+
+export function forwardUrl(
+  upstream: Upstream,
+  inboundPath: string,
+  inboundSearch: string,
+  upstreams?: Record<string, UpstreamDef>,
+): string {
+  return resolveUpstream(upstream, upstreams).base + inboundPath + inboundSearch;
 }
 
 export function rewriteHeaders(
   decision: Decision,
   inbound: Headers,
   env: Record<string, string | undefined>,
+  upstreams?: Record<string, UpstreamDef>,
 ): Headers {
+  const def = resolveUpstream(decision.upstream, upstreams);
   const out = new Headers();
-  // Copy inbound headers except hop-by-hop and auth (auth handled per-leg).
+  // Copy inbound headers except hop-by-hop and auth (auth is set per-upstream below).
   for (const [k, v] of inbound) {
     const key = k.toLowerCase();
     if (HOP_BY_HOP.has(key))
@@ -28,29 +52,40 @@ export function rewriteHeaders(
       continue;
     out.set(k, v);
   }
+  applyAuth(out, def.auth, inbound, env);
+  if (def.stripBeta)
+    out.delete("anthropic-beta");
+  return out;
+}
 
-  if (decision.upstream === "openrouter") {
-    const key = env.OPENROUTER_API_KEY;
+function applyAuth(
+  out: Headers,
+  auth: AuthMode,
+  inbound: Headers,
+  env: Record<string, string | undefined>,
+): void {
+  if (auth.kind === "bearer") {
+    const key = env[auth.envKey];
     if (!key)
-      throw new MissingKeyError("OPENROUTER_API_KEY is not set but a route needs OpenRouter");
+      throw new MissingKeyError(`${auth.envKey} is not set but a route needs it`);
     out.set("authorization", `Bearer ${key}`);
-    out.delete("anthropic-beta"); // OpenRouter's Anthropic endpoint may reject CC betas
-    return out;
-  }
-
-  // anthropic leg: prefer an explicit env key, else pass Claude Code's own auth through.
-  if (env.ANTHROPIC_API_KEY) {
-    out.set("x-api-key", env.ANTHROPIC_API_KEY);
+    return;
   }
-  else {
+  if (auth.kind === "passthrough") {
+    // Prefer an explicit env key; otherwise pass Claude Code's own auth through.
+    const envKey = auth.envKey ? env[auth.envKey] : undefined;
+    if (envKey) {
+      out.set("x-api-key", envKey);
+      return;
+    }
     const inboundAuth = inbound.get("authorization");
     const inboundKey = inbound.get("x-api-key");
     if (inboundAuth)
       out.set("authorization", inboundAuth);
     if (inboundKey)
       out.set("x-api-key", inboundKey);
   }
-  return out;
+  // kind === "none": send no auth (a local model server that doesn't want one).
 }
 
 export function rewriteBody(decision: Decision, body: any): any {

test/cli.test.ts

@@ -33,8 +33,8 @@ test("setModel throws on an unknown alias", () => {
   expect(() => setModel(ROUTES, "ghost", "openrouter:x/y")).toThrow(/not found/);
 });
 
-test("setModel rejects an invalid spec", () => {
-  expect(() => setModel(`[models]\nx = "a:b"\n`, "x", "bogus:y")).toThrow();
+test("setModel rejects a malformed spec", () => {
+  expect(() => setModel(`[models]\nx = "a:b"\n`, "x", "no-colon-spec")).toThrow();
 });
 
 test("listModels renders each alias", () => {

test/config.test.ts

@@ -1,6 +1,6 @@
 import { mkdirSync, rmSync, writeFileSync } from "node:fs";
 import { expect, test } from "bun:test";
-import { loadConfig, parseModelRef, resolveMenu, watchConfig } from "../src/config.ts";
+import { loadConfig, parseAuth, parseModelRef, resolveMenu, watchConfig } from "../src/config.ts";
 
 // Build a minimal routes.toml body for temp-file tests.
 function toml(models: Record<string, string>, def: string, extra = ""): string {
@@ -19,8 +19,37 @@ test("parseModelRef splits upstream and slug", () => {
   });
 });
 
-test("parseModelRef rejects unknown upstream", () => {
-  expect(() => parseModelRef("bogus:x")).toThrow();
+test("parseModelRef validates shape, not the upstream name", () => {
+  expect(() => parseModelRef("no-colon-here")).toThrow(/upstream.*slug/);
+  expect(() => parseModelRef("openrouter:")).toThrow(/empty slug/);
+  expect(() => parseModelRef(":slug")).toThrow(/empty upstream/);
+  // an unknown upstream name is fine at parse time — it's checked against [upstreams] at load
+  expect(parseModelRef("local:qwen3-coder:30b")).toEqual({ upstream: "local", slug: "qwen3-coder:30b" });
+});
+
+test("parseAuth understands the auth shorthands", () => {
+  expect(parseAuth("none")).toEqual({ kind: "none" });
+  expect(parseAuth("passthrough")).toEqual({ kind: "passthrough" });
+  expect(parseAuth("passthrough:ANTHROPIC_API_KEY")).toEqual({ kind: "passthrough", envKey: "ANTHROPIC_API_KEY" });
+  expect(parseAuth("bearer:OPENROUTER_API_KEY")).toEqual({ kind: "bearer", envKey: "OPENROUTER_API_KEY" });
+  expect(() => parseAuth("weird")).toThrow();
+});
+
+test("loadConfig parses [upstreams] and accepts a model that targets one", () => {
+  const tmp = "test/.tmp-upstreams.toml";
+  writeFileSync(tmp, `default = "a"\n\n[models]\na = "local:qwen3-coder:30b"\n\n[upstreams]\nlocal = { base = "http://localhost:11434", auth = "none" }\n`);
+  const cfg = loadConfig(tmp, {});
+  expect(cfg.models.a).toEqual({ upstream: "local", slug: "qwen3-coder:30b" });
+  expect(cfg.upstreams?.local).toEqual({ base: "http://localhost:11434", auth: { kind: "none" }, stripBeta: true });
+  expect(cfg.upstreams?.anthropic).toBeDefined(); // built-ins still present
+  rmSync(tmp, { force: true });
+});
+
+test("loadConfig rejects a model whose upstream is neither built in nor declared", () => {
+  const tmp = "test/.tmp-bad-upstream.toml";
+  writeFileSync(tmp, toml({ a: "local:qwen" }, "a")); // no [upstreams] table -> "local" is undefined
+  expect(() => loadConfig(tmp, {})).toThrow(/unknown upstream/);
+  rmSync(tmp, { force: true });
 });
 
 test("resolveMenu applies MUX_MODEL_<ALIAS> overrides", () => {