diff --git a/README.md b/README.md index dc4afa9..b934c53 100644 --- a/README.md +++ b/README.md @@ -151,6 +151,35 @@ claude-review = "anthropic:claude-sonnet-5" The slugs above are illustrative — run `modelmux check-latest` to see which models actually exist on OpenRouter right now. +## Local & self-hosted models + +`anthropic` and `openrouter` are built in, but you can point an alias at any +**Anthropic-Messages-compatible** endpoint by declaring it under `[upstreams]`, +then using it like any other model (`:`). + +The obvious use is a **local model**. Recent Ollama (v0.14+) speaks the Anthropic +Messages API natively, so no translation shim is needed — run your grunt-work +subagents on a local Qwen while the orchestrator stays on Claude: + +```toml +[upstreams] +local = { base = "http://localhost:11434", auth = "none" } + +[models] +orchestrator = "anthropic:passthrough" # brain stays on Claude +flagship = "local:qwen3-coder:30b" # subagents run on your local Qwen +cheap = "local:qwen3:8b" +``` + +`auth` is one of `passthrough` (forward Claude Code's own auth), `bearer:ENV_VAR` +(send `Authorization: Bearer $ENV_VAR`), or `none`. modelmux never forwards your +Claude auth to a `none`/`bearer` upstream, so your Anthropic token stays out of +the local process. + +Runners that only speak the OpenAI format (LM Studio, llama.cpp, vLLM) need a +[LiteLLM](https://github.com/BerriAI/litellm) proxy in front to expose an +Anthropic endpoint; point the upstream `base` at that. + ## Managing config from the CLI `modelmux` with no arguments runs the proxy; the subcommands manage `routes.toml`: @@ -188,9 +217,9 @@ Everything is controlled by `routes.toml` and a few environment variables: ## Troubleshooting -- **HTTP 400, `OPENROUTER_API_KEY is not set but a route needs OpenRouter`** — a - request routed to OpenRouter but the key is unset where modelmux runs. Set it - and restart the binary. +- **HTTP 400, `OPENROUTER_API_KEY is not set but a route needs it`** — a request + routed to OpenRouter but the key is unset where modelmux runs. Set it and + restart the binary. - **Claude Code reports connection refused** — modelmux isn't running, or it's on a different port than `ANTHROPIC_BASE_URL`. Start it and check the listening line matches. diff --git a/docs/development.md b/docs/development.md index 6381a41..5615854 100644 --- a/docs/development.md +++ b/docs/development.md @@ -150,7 +150,7 @@ If a request routes to OpenRouter and `OPENROUTER_API_KEY` is unset, the request fails HTTP 400 with the body: ```text -OPENROUTER_API_KEY is not set but a route needs OpenRouter +OPENROUTER_API_KEY is not set but a route needs it ``` Override one alias for a single run without editing `routes.toml`: diff --git a/routes.toml b/routes.toml index ce612c2..5df1381 100644 --- a/routes.toml +++ b/routes.toml @@ -39,3 +39,23 @@ reasoner = "openrouter:deepseek/deepseek-v4-pro" review = "openrouter:minimax/minimax-m3" cheap = "openrouter:deepseek/deepseek-v4-flash" claude-review = "anthropic:claude-sonnet-5" + +# --- upstreams (optional) --------------------------------------------------- +# "anthropic" (passthrough) and "openrouter" (Bearer OPENROUTER_API_KEY) are +# built in. Add your own Anthropic-Messages-compatible endpoints here, then use +# them like any other alias: :. auth = passthrough | passthrough:ENV +# | bearer:ENV | none. +# +# LOCAL MODELS: recent Ollama (v0.14+) speaks the Anthropic Messages API +# natively, so no translation shim is needed — point an alias at it and run your +# grunt-work subagents on a local model while the orchestrator stays on Claude: +# +# [upstreams] +# local = { base = "http://localhost:11434", auth = "none" } +# +# # then in [models] above: +# flagship = "local:qwen3-coder:30b" # subagents run on your local Qwen +# +# (Non-Ollama runners like LM Studio / llama.cpp / vLLM speak the OpenAI format; +# put a LiteLLM proxy in front to expose an Anthropic endpoint, then point the +# upstream base at that.) diff --git a/src/config.ts b/src/config.ts index 46a0339..26264d2 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,8 +1,8 @@ -import type { Config, ModelRef, RouteRule, Upstream, WorkType } from "./types.ts"; +import type { AuthMode, Config, ModelRef, RouteRule, UpstreamDef, WorkType } from "./types.ts"; import { readFileSync, watch } from "node:fs"; import process from "node:process"; +import { BUILTIN_UPSTREAMS } from "./upstreams.ts"; -const UPSTREAMS: Upstream[] = ["anthropic", "openrouter"]; const WORK_TYPES = new Set(["background", "think", "longContext", "webSearch"]); // A route's `when` must name exactly one recognized condition. Reject anything @@ -23,19 +23,57 @@ function validateWhen(when: RouteRule["when"] | undefined): void { throw new Error("route `when.anySubagent` must be true when present"); } +// Split ":" into its parts (shape only). Whether the upstream +// name actually exists is validated against the [upstreams] table in loadConfig. export function parseModelRef(spec: string): ModelRef { const idx = spec.indexOf(":"); if (idx === -1) throw new Error(`bad model spec (need ":"): ${spec}`); - const upstream = spec.slice(0, idx) as Upstream; + const upstream = spec.slice(0, idx); const slug = spec.slice(idx + 1); - if (!UPSTREAMS.includes(upstream)) - throw new Error(`unknown upstream: ${upstream}`); + if (!upstream) + throw new Error(`empty upstream in: ${spec}`); if (!slug) throw new Error(`empty slug in: ${spec}`); return { upstream, slug }; } +interface RawUpstream { + base?: string; + auth?: string; + stripBeta?: boolean; +} + +// Parse an auth spec: "passthrough" | "passthrough:ENV" | "bearer:ENV" | "none". +export function parseAuth(spec: string): AuthMode { + if (spec === "none") + return { kind: "none" }; + if (spec === "passthrough") + return { kind: "passthrough" }; + if (spec.startsWith("passthrough:")) + return { kind: "passthrough", envKey: spec.slice("passthrough:".length) }; + if (spec.startsWith("bearer:")) { + const envKey = spec.slice("bearer:".length); + if (!envKey) + throw new Error("bearer auth needs an env var, e.g. auth = \"bearer:MY_API_KEY\""); + return { kind: "bearer", envKey }; + } + throw new Error(`unknown auth "${spec}" (use passthrough | passthrough:ENV | bearer:ENV | none)`); +} + +// Merge any user-declared [upstreams] over the built-ins (anthropic, openrouter). +function buildUpstreams(raw: Record | undefined): Record { + const out: Record = { ...BUILTIN_UPSTREAMS }; + for (const [name, u] of Object.entries(raw ?? {})) { + if (!u || typeof u !== "object" || typeof u.base !== "string" || !u.base) + throw new Error(`upstream "${name}" needs a base URL, e.g. base = "http://localhost:11434"`); + const auth = parseAuth(u.auth ?? "none"); + const stripBeta = u.stripBeta ?? (auth.kind !== "passthrough"); + out[name] = { base: u.base, auth, stripBeta }; + } + return out; +} + export function resolveMenu( config: Config, env: Record, @@ -60,6 +98,7 @@ interface RawConfig { default: string; routes: Config["routes"]; longContextThreshold?: number; + upstreams?: Record; } export function loadConfig( @@ -78,6 +117,7 @@ export function loadConfig( default: raw.default, routes: raw.routes ?? [], longContextThreshold: raw.longContextThreshold ?? 200000, + upstreams: buildUpstreams(raw.upstreams), }; if (!config.default) throw new Error("routes.toml is missing a `default` alias"); @@ -90,7 +130,13 @@ export function loadConfig( if (!models[r.use]) throw new Error(`route uses unknown alias "${r.use}" (not in models)`); } - return resolveMenu(config, env); + const resolved = resolveMenu(config, env); + // Every model's upstream must be built in or declared in [upstreams]. + for (const [alias, ref] of Object.entries(resolved.models)) { + if (!resolved.upstreams?.[ref.upstream]) + throw new Error(`model "${alias}" uses unknown upstream "${ref.upstream}" — built-ins are anthropic and openrouter; add others under [upstreams]`); + } + return resolved; } function readText(path: string): string { diff --git a/src/server.ts b/src/server.ts index 578d15b..c73b543 100644 --- a/src/server.ts +++ b/src/server.ts @@ -44,7 +44,7 @@ export function buildServer(opts: ServerOpts): Bun.Server { let headers: Headers; try { decision = route(signals, config); - headers = rewriteHeaders(decision, req.headers, opts.env); + headers = rewriteHeaders(decision, req.headers, opts.env, config.upstreams); } catch (e) { logError(opts.logPath, signals, e as Error); @@ -58,7 +58,7 @@ export function buildServer(opts: ServerOpts): Bun.Server { const base = opts.baseOverride?.[decision.upstream]; const target = base ? base + url.pathname + url.search - : forwardUrl(decision.upstream, url.pathname, url.search); + : forwardUrl(decision.upstream, url.pathname, url.search, config.upstreams); let upstream: Response; try { diff --git a/src/types.ts b/src/types.ts index d2caab0..ded98d7 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,4 +1,18 @@ -export type Upstream = "anthropic" | "openrouter"; +// An upstream name. "anthropic" and "openrouter" are built in; more can be +// defined in the [upstreams] table of routes.toml (e.g. a local model server). +export type Upstream = string; + +// How modelmux authenticates the outbound leg to an upstream. +export type AuthMode + = | { kind: "passthrough"; envKey?: string } // forward Claude Code's own inbound auth; if envKey is set and present, send it as x-api-key instead + | { kind: "bearer"; envKey: string } // Authorization: Bearer + | { kind: "none" }; // send no auth (e.g. a local model server) + +export interface UpstreamDef { + base: string; // base URL, e.g. https://api.anthropic.com or http://localhost:11434 + auth: AuthMode; + stripBeta: boolean; // drop anthropic-beta — non-Anthropic endpoints don't understand Claude Code's betas +} export interface ModelRef { upstream: Upstream; @@ -17,6 +31,7 @@ export interface Config { default: string; // alias used when no route matches (the orchestrator) routes: RouteRule[]; longContextThreshold: number; + upstreams?: Record; // built-ins merged with any [upstreams] overrides; falls back to built-ins when absent } export interface Signals { diff --git a/src/upstreams.ts b/src/upstreams.ts index 6bf9fc8..5c31b7a 100644 --- a/src/upstreams.ts +++ b/src/upstreams.ts @@ -1,25 +1,49 @@ -import type { Decision, Upstream } from "./types.ts"; +import type { AuthMode, Decision, Upstream, UpstreamDef } from "./types.ts"; -const BASE: Record = { - anthropic: "https://api.anthropic.com", - openrouter: "https://openrouter.ai/api", +// Built-in upstreams. Users can add more (or override these) via the [upstreams] +// table in routes.toml; these are the defaults when a name isn't configured. +export const BUILTIN_UPSTREAMS: Record = { + anthropic: { + base: "https://api.anthropic.com", + auth: { kind: "passthrough", envKey: "ANTHROPIC_API_KEY" }, + stripBeta: false, + }, + openrouter: { + base: "https://openrouter.ai/api", + auth: { kind: "bearer", envKey: "OPENROUTER_API_KEY" }, + stripBeta: true, // OpenRouter's Anthropic endpoint may reject Claude Code's betas + }, }; const HOP_BY_HOP = new Set(["host", "content-length", "connection", "accept-encoding"]); export class MissingKeyError extends Error {} -export function forwardUrl(upstream: Upstream, inboundPath: string, inboundSearch: string): string { - return BASE[upstream] + inboundPath + inboundSearch; +export function resolveUpstream(name: Upstream, upstreams?: Record): UpstreamDef { + const def = upstreams?.[name] ?? BUILTIN_UPSTREAMS[name]; + if (!def) + throw new Error(`unknown upstream "${name}" (define it in an [upstreams] table in routes.toml)`); + return def; +} + +export function forwardUrl( + upstream: Upstream, + inboundPath: string, + inboundSearch: string, + upstreams?: Record, +): string { + return resolveUpstream(upstream, upstreams).base + inboundPath + inboundSearch; } export function rewriteHeaders( decision: Decision, inbound: Headers, env: Record, + upstreams?: Record, ): Headers { + const def = resolveUpstream(decision.upstream, upstreams); const out = new Headers(); - // Copy inbound headers except hop-by-hop and auth (auth handled per-leg). + // Copy inbound headers except hop-by-hop and auth (auth is set per-upstream below). for (const [k, v] of inbound) { const key = k.toLowerCase(); if (HOP_BY_HOP.has(key)) @@ -28,21 +52,32 @@ export function rewriteHeaders( continue; out.set(k, v); } + applyAuth(out, def.auth, inbound, env); + if (def.stripBeta) + out.delete("anthropic-beta"); + return out; +} - if (decision.upstream === "openrouter") { - const key = env.OPENROUTER_API_KEY; +function applyAuth( + out: Headers, + auth: AuthMode, + inbound: Headers, + env: Record, +): void { + if (auth.kind === "bearer") { + const key = env[auth.envKey]; if (!key) - throw new MissingKeyError("OPENROUTER_API_KEY is not set but a route needs OpenRouter"); + throw new MissingKeyError(`${auth.envKey} is not set but a route needs it`); out.set("authorization", `Bearer ${key}`); - out.delete("anthropic-beta"); // OpenRouter's Anthropic endpoint may reject CC betas - return out; - } - - // anthropic leg: prefer an explicit env key, else pass Claude Code's own auth through. - if (env.ANTHROPIC_API_KEY) { - out.set("x-api-key", env.ANTHROPIC_API_KEY); + return; } - else { + if (auth.kind === "passthrough") { + // Prefer an explicit env key; otherwise pass Claude Code's own auth through. + const envKey = auth.envKey ? env[auth.envKey] : undefined; + if (envKey) { + out.set("x-api-key", envKey); + return; + } const inboundAuth = inbound.get("authorization"); const inboundKey = inbound.get("x-api-key"); if (inboundAuth) @@ -50,7 +85,7 @@ export function rewriteHeaders( if (inboundKey) out.set("x-api-key", inboundKey); } - return out; + // kind === "none": send no auth (a local model server that doesn't want one). } export function rewriteBody(decision: Decision, body: any): any { diff --git a/test/cli.test.ts b/test/cli.test.ts index fca78de..ca1dd59 100644 --- a/test/cli.test.ts +++ b/test/cli.test.ts @@ -33,8 +33,8 @@ test("setModel throws on an unknown alias", () => { expect(() => setModel(ROUTES, "ghost", "openrouter:x/y")).toThrow(/not found/); }); -test("setModel rejects an invalid spec", () => { - expect(() => setModel(`[models]\nx = "a:b"\n`, "x", "bogus:y")).toThrow(); +test("setModel rejects a malformed spec", () => { + expect(() => setModel(`[models]\nx = "a:b"\n`, "x", "no-colon-spec")).toThrow(); }); test("listModels renders each alias", () => { diff --git a/test/config.test.ts b/test/config.test.ts index 81d4a3c..f5913ee 100644 --- a/test/config.test.ts +++ b/test/config.test.ts @@ -1,6 +1,6 @@ import { mkdirSync, rmSync, writeFileSync } from "node:fs"; import { expect, test } from "bun:test"; -import { loadConfig, parseModelRef, resolveMenu, watchConfig } from "../src/config.ts"; +import { loadConfig, parseAuth, parseModelRef, resolveMenu, watchConfig } from "../src/config.ts"; // Build a minimal routes.toml body for temp-file tests. function toml(models: Record, def: string, extra = ""): string { @@ -19,8 +19,37 @@ test("parseModelRef splits upstream and slug", () => { }); }); -test("parseModelRef rejects unknown upstream", () => { - expect(() => parseModelRef("bogus:x")).toThrow(); +test("parseModelRef validates shape, not the upstream name", () => { + expect(() => parseModelRef("no-colon-here")).toThrow(/upstream.*slug/); + expect(() => parseModelRef("openrouter:")).toThrow(/empty slug/); + expect(() => parseModelRef(":slug")).toThrow(/empty upstream/); + // an unknown upstream name is fine at parse time — it's checked against [upstreams] at load + expect(parseModelRef("local:qwen3-coder:30b")).toEqual({ upstream: "local", slug: "qwen3-coder:30b" }); +}); + +test("parseAuth understands the auth shorthands", () => { + expect(parseAuth("none")).toEqual({ kind: "none" }); + expect(parseAuth("passthrough")).toEqual({ kind: "passthrough" }); + expect(parseAuth("passthrough:ANTHROPIC_API_KEY")).toEqual({ kind: "passthrough", envKey: "ANTHROPIC_API_KEY" }); + expect(parseAuth("bearer:OPENROUTER_API_KEY")).toEqual({ kind: "bearer", envKey: "OPENROUTER_API_KEY" }); + expect(() => parseAuth("weird")).toThrow(); +}); + +test("loadConfig parses [upstreams] and accepts a model that targets one", () => { + const tmp = "test/.tmp-upstreams.toml"; + writeFileSync(tmp, `default = "a"\n\n[models]\na = "local:qwen3-coder:30b"\n\n[upstreams]\nlocal = { base = "http://localhost:11434", auth = "none" }\n`); + const cfg = loadConfig(tmp, {}); + expect(cfg.models.a).toEqual({ upstream: "local", slug: "qwen3-coder:30b" }); + expect(cfg.upstreams?.local).toEqual({ base: "http://localhost:11434", auth: { kind: "none" }, stripBeta: true }); + expect(cfg.upstreams?.anthropic).toBeDefined(); // built-ins still present + rmSync(tmp, { force: true }); +}); + +test("loadConfig rejects a model whose upstream is neither built in nor declared", () => { + const tmp = "test/.tmp-bad-upstream.toml"; + writeFileSync(tmp, toml({ a: "local:qwen" }, "a")); // no [upstreams] table -> "local" is undefined + expect(() => loadConfig(tmp, {})).toThrow(/unknown upstream/); + rmSync(tmp, { force: true }); }); test("resolveMenu applies MUX_MODEL_ overrides", () => { diff --git a/test/integration.test.ts b/test/integration.test.ts index d2121fb..9615727 100644 --- a/test/integration.test.ts +++ b/test/integration.test.ts @@ -107,6 +107,40 @@ test("missing OpenRouter key fails loud with 400 AND logs an error record", asyn noKey.stop(true); }); +test("a custom local upstream routes there and leaks no Claude auth", async () => { + let seenAuth: string | null = "unset"; + const localSrv = Bun.serve({ + port: 0, + fetch: (req) => { + seenAuth = req.headers.get("authorization"); + return sse("local"); + }, + }); + const cfg: Config = { + models: { + orchestrator: { upstream: "anthropic", slug: "passthrough" }, + flagship: { upstream: "local", slug: "qwen3-coder:30b" }, + }, + default: "orchestrator", + longContextThreshold: 200000, + routes: [{ when: { anySubagent: true }, use: "flagship" }], + upstreams: { local: { base: localSrv.url.origin, auth: { kind: "none" }, stripBeta: true } }, + }; + const p = buildServer({ config: cfg, env: {}, logPath: LOG, port: 0 }); + const res = await fetch(`${p.url.origin}/v1/messages`, { + method: "POST", + headers: { "content-type": "application/json", "x-claude-code-agent-id": "a", "authorization": "Bearer real-claude-oauth" }, + body: JSON.stringify({ model: "claude-x" }), + }); + expect(await res.text()).toContain("\"upstream\":\"local\""); + expect(seenAuth).toBeNull(); // Claude's own token did NOT reach the local server + const last = readDecisions(LOG).at(-1)!; + expect(last.upstream).toBe("local"); + expect(last.resolvedModel).toBe("qwen3-coder:30b"); + p.stop(true); + localSrv.stop(true); +}); + test("unreachable upstream fails loud: 502 + a logged error record", async () => { const dead = Bun.serve({ port: 0, fetch: () => new Response("x") }); const deadOrigin = dead.url.origin; diff --git a/test/upstreams.test.ts b/test/upstreams.test.ts index 32de0a4..6f0277b 100644 --- a/test/upstreams.test.ts +++ b/test/upstreams.test.ts @@ -1,6 +1,6 @@ import type { Decision } from "../src/types.ts"; import { expect, test } from "bun:test"; -import { forwardUrl, MissingKeyError, passthroughHeaders, rewriteBody, rewriteHeaders } from "../src/upstreams.ts"; +import { forwardUrl, MissingKeyError, passthroughHeaders, resolveUpstream, rewriteBody, rewriteHeaders } from "../src/upstreams.ts"; const toOR: Decision = { alias: "flagship", upstream: "openrouter", model: "z-ai/glm-5.2", matchedRule: "tag:flagship" }; const toAnthropic: Decision = { alias: "orchestrator", upstream: "anthropic", model: "passthrough", matchedRule: "default" }; @@ -81,3 +81,30 @@ test("passthroughHeaders defaults content-type when the upstream omits it", () = expect(out.get("content-type")).toBe("application/json"); expect(out.get("cache-control")).toBe("no-cache"); }); + +test("resolveUpstream falls back to built-ins and honors config overrides", () => { + expect(resolveUpstream("anthropic").base).toBe("https://api.anthropic.com"); + expect(resolveUpstream("openrouter").auth).toEqual({ kind: "bearer", envKey: "OPENROUTER_API_KEY" }); + const custom = { local: { base: "http://localhost:11434", auth: { kind: "none" as const }, stripBeta: true } }; + expect(resolveUpstream("local", custom).base).toBe("http://localhost:11434"); + expect(() => resolveUpstream("nope")).toThrow(/unknown upstream/); +}); + +test("a local (none-auth) upstream sends no auth and forwards to its configured base", () => { + const toLocal: Decision = { alias: "flagship", upstream: "local", model: "qwen3-coder:30b", matchedRule: "anySubagent" }; + const upstreams = { local: { base: "http://localhost:11434", auth: { kind: "none" as const }, stripBeta: true } }; + const inbound = new Headers({ "authorization": "Bearer oauth-tok", "x-api-key": "sk-ant", "anthropic-beta": "x", "content-type": "application/json" }); + const out = rewriteHeaders(toLocal, inbound, {}, upstreams); + expect(out.get("authorization")).toBeNull(); // no Claude auth leaked to the local server + expect(out.get("x-api-key")).toBeNull(); + expect(out.get("anthropic-beta")).toBeNull(); // stripped + expect(out.get("content-type")).toBe("application/json"); + expect(forwardUrl("local", "/v1/messages", "", upstreams)).toBe("http://localhost:11434/v1/messages"); +}); + +test("a config-defined bearer upstream injects its own env key", () => { + const toGw: Decision = { alias: "x", upstream: "gw", model: "m", matchedRule: "tag:x" }; + const upstreams = { gw: { base: "https://gw.example", auth: { kind: "bearer" as const, envKey: "GW_KEY" }, stripBeta: true } }; + expect(rewriteHeaders(toGw, new Headers(), { GW_KEY: "secret" }, upstreams).get("authorization")).toBe("Bearer secret"); + expect(() => rewriteHeaders(toGw, new Headers(), {}, upstreams)).toThrow(MissingKeyError); +});