From 5ed0cbc1f4a8f3e084982e41bf9a7dd0ce60aa1c Mon Sep 17 00:00:00 2001 From: Manvendra Date: Tue, 30 Jun 2026 22:35:21 +0530 Subject: [PATCH] feat: multi-key round-robin per provider (C5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lets a provider hold a pool of API keys and rotates across them per request, so free-tier users can spread load over several keys instead of hammering one key's limit. (Closes the round-robin gap from the v0.1.x audit.) What changed - Config: providers accept `apiKeyEnvs` (an array) alongside `apiKeyEnv`; `ResolvedProvider.apiKey` → `apiKeys: string[]` (unset/empty env vars skipped). - Adapters take an `ApiKeySource` (a fixed string OR a supplier); the registry builds a round-robin supplier over the resolved pool, so each outbound call picks the next key. Single-key and keyless providers behave exactly as before. - Docs/example: `apiKeyEnvs` on the Gemini example + `GEMINI_API_KEY_2` in `.env.example`; README note; SECURITY_REVIEW_LOG SR-008 (wider secret surface). Follow-up (not in this PR): on a 429, retry the *next key* before model fallback (needs executor/adapter awareness) — today rotation is per-request distribution. Tests - config resolves a pool from `apiKeyEnvs` (+ `apiKeyEnv`, skipping empties); the registry round-robins keys across successive requests (k1 → k2 → k1) and sends no auth header for a keyless pool. How to verify - `pnpm verify` → green (233 tests, coverage ≥90%) - `pnpm build` → green --- .env.example | 3 + CHANGELOG.md | 1 + README.md | 2 +- SECURITY_REVIEW_LOG.md | 2 + packages/gateway/src/config.test.ts | 26 ++++++++- packages/gateway/src/config.ts | 24 +++++++- packages/gateway/src/providers/anthropic.ts | 10 ++-- .../src/providers/openai-compatible.ts | 10 ++-- .../gateway/src/providers/registry.test.ts | 58 ++++++++++++++++++- packages/gateway/src/providers/registry.ts | 17 +++++- packages/gateway/src/providers/types.ts | 6 ++ sentinel.config.example.json | 2 +- 12 files changed, 140 insertions(+), 21 deletions(-) diff --git a/.env.example b/.env.example index e9d848b..46f3b56 100644 --- a/.env.example +++ b/.env.example @@ -4,6 +4,9 @@ OPENAI_API_KEY= ANTHROPIC_API_KEY= GEMINI_API_KEY= +# Optional extra key(s) for a provider — Sentinel round-robins across a provider's key pool +# (configured via "apiKeyEnvs" in sentinel.config.json) to spread load across free-tier limits. +GEMINI_API_KEY_2= GROQ_API_KEY= # ── Sentinel ───────────────────────────────────────────────────────── diff --git a/CHANGELOG.md b/CHANGELOG.md index 9944b03..0c725c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ All notable changes to Sentinel are documented here. The format follows - Per-request **cost tracking**: set a `pricing` map (USD per 1K tokens, per model) in `sentinel.config.json` and every trace records a `costUsd`; the dashboard shows total spend, spend saved by the cache, and cost over time. - **Native Anthropic provider**: set `"type": "anthropic"` on a provider and Sentinel speaks Anthropic's Messages API (`x-api-key`, request/response/stream translation) while clients keep using the OpenAI shape. +- **Multi-key round-robin**: a provider can list `apiKeyEnvs` (a key pool); Sentinel rotates across the keys per request to spread load and survive free-tier rate limits. ### Fixed diff --git a/README.md b/README.md index 663bd64..8d90cd0 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ Two git-ignored files: - **`.env`** — secrets only: provider API keys, and the `SENTINEL_API_KEYS` clients use to authenticate to Sentinel. See [`.env.example`](./.env.example). - **`sentinel.config.json`** — which providers exist and which model routes where. See [`sentinel.config.example.json`](./sentinel.config.example.json). -Each provider sets a `baseUrl` (or `baseUrlEnv` to read it from the environment) and an optional `apiKeyEnv` naming the env var that holds its key. The `models` map routes a model name to a provider; `defaultProvider` handles anything not listed. +Each provider sets a `baseUrl` (or `baseUrlEnv` to read it from the environment) and an optional `apiKeyEnv` naming the env var that holds its key — or `apiKeyEnvs` (an array) to give a provider a **pool of keys Sentinel round-robins across**, spreading load to survive free-tier limits. The `models` map routes a model name to a provider; `defaultProvider` handles anything not listed. ```json { diff --git a/SECURITY_REVIEW_LOG.md b/SECURITY_REVIEW_LOG.md index 8c5661c..ac2ea83 100644 --- a/SECURITY_REVIEW_LOG.md +++ b/SECURITY_REVIEW_LOG.md @@ -77,4 +77,6 @@ Each item: assume an attacker is actively trying it. Tick only when a test prove | SR-007 | 2026-06-30 | 1, 4 | PR3 native Anthropic adapter | A second outbound auth path (`x-api-key`) and a new response/stream shape: risk of leaking the Anthropic key to logs, following a malicious redirect, or trusting a malformed upstream body | high | mitigated | The Anthropic key is sent only in the outbound `x-api-key` header — never logged (`logRedaction` already redacts inbound `x-api-key`; Fastify omits request headers by default) and never returned to clients; it comes from `ANTHROPIC_API_KEY` via `apiKeyEnv`, never hard-coded. The adapter reuses `redirect: 'error'`, so a malicious provider can't 3xx the prompt to another host (tested), and the base URL still comes only from the config allow-list, never request input. The Anthropic response is parsed through a Zod schema before translation; an unexpected shape ⇒ `UpstreamError` (502), never a malformed pass-through (tested). | +| SR-008 | 2026-06-30 | 1 | PR4 multi-key round-robin | Holding several provider keys per provider widens the secret surface — a key could leak via logs/errors, or a blank slot send an empty key | low | mitigated | Keys come only from env (`apiKeyEnv`/`apiKeyEnvs`), never the repo or request input; each is sent only in the outbound auth header (`Authorization`/`x-api-key`), redacted in logs and never returned to clients. Rotation is an in-memory counter over the resolved pool — no provider key is persisted or traced (traces still store only the SHA-256 of the *Sentinel client* key). Unset/empty env vars are skipped, so a misconfigured slot can't send a blank key (tested). | + > Add a row per high-risk change. Status ∈ {open, mitigated, accepted}. Severity ∈ {low, med, high, critical}. diff --git a/packages/gateway/src/config.test.ts b/packages/gateway/src/config.test.ts index a2531fd..ff73de3 100644 --- a/packages/gateway/src/config.test.ts +++ b/packages/gateway/src/config.test.ts @@ -89,8 +89,8 @@ describe('loadConfig', () => { it('resolves providers, base URLs, and keys', () => { const cfg = loadConfig({ path: 'x', env, readFile: () => validConfig }); expect(cfg.providers.get('ollama')?.baseUrl).toBe('http://localhost:11434/v1'); - expect(cfg.providers.get('ollama')?.apiKey).toBeUndefined(); - expect(cfg.providers.get('openai')?.apiKey).toBe('sk-test'); + expect(cfg.providers.get('ollama')?.apiKeys).toEqual([]); + expect(cfg.providers.get('openai')?.apiKeys).toEqual(['sk-test']); expect(cfg.models.get('gpt-4o-mini')).toBe('openai'); expect(cfg.defaultProvider).toBe('ollama'); }); @@ -132,7 +132,27 @@ describe('loadConfig', () => { readFile: () => cfg, }); expect(resolved.providers.get('claude')?.type).toBe('anthropic'); - expect(resolved.providers.get('claude')?.apiKey).toBe('sk-ant'); + expect(resolved.providers.get('claude')?.apiKeys).toEqual(['sk-ant']); + }); + + it('collects a round-robin key pool from apiKeyEnvs (+ apiKeyEnv), skipping unset keys', () => { + const cfg = JSON.stringify({ + providers: { + groq: { + type: 'openai-compatible', + baseUrl: 'https://g', + apiKeyEnv: 'G1', + apiKeyEnvs: ['G2', 'G3'], + }, + }, + models: { m: 'groq' }, + }); + const resolved = loadConfig({ + path: 'x', + env: { G1: 'k1', G2: 'k2', G3: '' }, // G3 empty → skipped + readFile: () => cfg, + }); + expect(resolved.providers.get('groq')?.apiKeys).toEqual(['k1', 'k2']); }); it('rejects an unknown provider adapter type', () => { diff --git a/packages/gateway/src/config.ts b/packages/gateway/src/config.ts index b839e09..3c97f4c 100644 --- a/packages/gateway/src/config.ts +++ b/packages/gateway/src/config.ts @@ -113,6 +113,8 @@ const providerConfigSchema = z baseUrl: z.string().url().optional(), baseUrlEnv: z.string().min(1).optional(), apiKeyEnv: z.string().min(1).optional(), + /** Multiple key env vars to round-robin across (combined with `apiKeyEnv` if both are set). */ + apiKeyEnvs: z.array(z.string().min(1)).optional(), rpm: z.coerce.number().int().nonnegative().optional(), }) .refine((p) => (p.baseUrl === undefined) !== (p.baseUrlEnv === undefined), { @@ -168,7 +170,8 @@ export interface ResolvedProvider { /** Which adapter serves this provider: a generic OpenAI-compatible HTTP API or Anthropic's Messages API. */ type: 'openai-compatible' | 'anthropic'; baseUrl: string; - apiKey: string | undefined; + /** Resolved API keys for this provider (0 = keyless, >1 = round-robin pool). */ + apiKeys: string[]; /** Per-provider requests-per-minute limit for the throttle (omitted = unlimited). */ rpm?: number; } @@ -230,12 +233,12 @@ export function loadConfig(options: LoadConfigOptions): ResolvedConfig { const providers = new Map(); for (const [name, p] of Object.entries(parsed.data.providers)) { const baseUrl = p.baseUrl ?? readEnvOrThrow(options.env, p.baseUrlEnv, name); - const apiKey = p.apiKeyEnv === undefined ? undefined : options.env[p.apiKeyEnv]; + const apiKeys = resolveApiKeys(options.env, p.apiKeyEnv, p.apiKeyEnvs); providers.set(name, { name, type: p.type, baseUrl, - apiKey, + apiKeys, ...(p.rpm !== undefined ? { rpm: p.rpm } : {}), }); } @@ -250,6 +253,21 @@ export function loadConfig(options: LoadConfigOptions): ResolvedConfig { }; } +/** Resolves a provider's key pool from `apiKeyEnv` (single) + `apiKeyEnvs` (many); skips unset/empty. */ +function resolveApiKeys( + env: NodeJS.ProcessEnv, + single: string | undefined, + multiple: string[] | undefined, +): string[] { + const names = [...(single !== undefined ? [single] : []), ...(multiple ?? [])]; + const keys: string[] = []; + for (const varName of names) { + const value = env[varName]; + if (value !== undefined && value.length > 0) keys.push(value); + } + return keys; +} + function readEnvOrThrow(env: NodeJS.ProcessEnv, key: string | undefined, provider: string): string { if (key === undefined) { throw new ConfigError(`provider "${provider}" is missing a base URL`); diff --git a/packages/gateway/src/providers/anthropic.ts b/packages/gateway/src/providers/anthropic.ts index 70b0e57..1f7c56e 100644 --- a/packages/gateway/src/providers/anthropic.ts +++ b/packages/gateway/src/providers/anthropic.ts @@ -1,6 +1,6 @@ import { z } from 'zod'; import type { ChatCompletionRequest } from '../schemas.js'; -import type { FetchLike, Provider } from './types.js'; +import type { ApiKeySource, FetchLike, Provider } from './types.js'; import { UpstreamError } from '../errors.js'; /** @@ -18,7 +18,8 @@ const DEFAULT_MAX_TOKENS = 4096; export interface AnthropicOptions { name: string; baseUrl: string; - apiKey?: string | undefined; + /** A fixed key, or a supplier that returns the next key (round-robin across a pool). */ + apiKey?: ApiKeySource | undefined; /** Injectable fetch (defaults to global `fetch`); handy for tests. */ fetchImpl?: FetchLike; } @@ -50,8 +51,9 @@ export function createAnthropicProvider(options: AnthropicOptions): Provider { 'content-type': 'application/json', 'anthropic-version': ANTHROPIC_VERSION, }; - if (options.apiKey !== undefined && options.apiKey.length > 0) { - headers['x-api-key'] = options.apiKey; + const key = typeof options.apiKey === 'function' ? options.apiKey() : options.apiKey; + if (key !== undefined && key.length > 0) { + headers['x-api-key'] = key; } return headers; } diff --git a/packages/gateway/src/providers/openai-compatible.ts b/packages/gateway/src/providers/openai-compatible.ts index f038ec7..845be90 100644 --- a/packages/gateway/src/providers/openai-compatible.ts +++ b/packages/gateway/src/providers/openai-compatible.ts @@ -1,11 +1,12 @@ import type { ChatCompletionRequest } from '../schemas.js'; -import type { FetchLike, Provider } from './types.js'; +import type { ApiKeySource, FetchLike, Provider } from './types.js'; import { UpstreamError } from '../errors.js'; export interface OpenAICompatibleOptions { name: string; baseUrl: string; - apiKey?: string | undefined; + /** A fixed key, or a supplier that returns the next key (round-robin across a pool). */ + apiKey?: ApiKeySource | undefined; /** Injectable fetch (defaults to global `fetch`); handy for tests. */ fetchImpl?: FetchLike; } @@ -20,8 +21,9 @@ export function createOpenAICompatibleProvider(options: OpenAICompatibleOptions) function buildHeaders(): Record { const headers: Record = { 'content-type': 'application/json' }; - if (options.apiKey !== undefined && options.apiKey.length > 0) { - headers.authorization = `Bearer ${options.apiKey}`; + const key = typeof options.apiKey === 'function' ? options.apiKey() : options.apiKey; + if (key !== undefined && key.length > 0) { + headers.authorization = `Bearer ${key}`; } return headers; } diff --git a/packages/gateway/src/providers/registry.test.ts b/packages/gateway/src/providers/registry.test.ts index 6ead0b8..336f842 100644 --- a/packages/gateway/src/providers/registry.test.ts +++ b/packages/gateway/src/providers/registry.test.ts @@ -11,7 +11,7 @@ const config: ResolvedConfig = { name: 'ollama', type: 'openai-compatible', baseUrl: 'http://localhost:11434/v1', - apiKey: undefined, + apiKeys: [], }, ], [ @@ -20,7 +20,7 @@ const config: ResolvedConfig = { name: 'openai', type: 'openai-compatible', baseUrl: 'https://api.openai.com/v1', - apiKey: 'k', + apiKeys: ['k'], }, ], ]), @@ -67,7 +67,10 @@ describe('createRegistry', () => { it('builds the Anthropic adapter for an anthropic-typed provider', async () => { const anthropicConfig: ResolvedConfig = { providers: new Map([ - ['claude', { name: 'claude', type: 'anthropic', baseUrl: 'http://h/v1', apiKey: 'sk-ant' }], + [ + 'claude', + { name: 'claude', type: 'anthropic', baseUrl: 'http://h/v1', apiKeys: ['sk-ant'] }, + ], ]), models: new Map([['claude-3-5-sonnet', 'claude']]), defaultProvider: undefined, @@ -89,4 +92,53 @@ describe('createRegistry', () => { expect(call[0]).toContain('/messages'); expect(call[1].headers['x-api-key']).toBe('sk-ant'); }); + + it('round-robins across a provider key pool on successive requests', async () => { + const pooled: ResolvedConfig = { + providers: new Map([ + [ + 'groq', + { + name: 'groq', + type: 'openai-compatible', + baseUrl: 'http://h/v1', + apiKeys: ['k1', 'k2'], + }, + ], + ]), + models: new Map([['m', 'groq']]), + defaultProvider: undefined, + pricing: new Map(), + }; + const seen: (string | undefined)[] = []; + const fetchImpl = vi.fn(async (_url: string, init: { headers: Record }) => { + seen.push(init.headers.authorization); + return Promise.resolve(new Response('{}', { status: 200 })); + }); + const registry = createRegistry(pooled, { fetchImpl }); + for (let i = 0; i < 3; i++) { + await registry.resolve('m').chat({ model: 'm', messages: [{ role: 'user', content: 'hi' }] }); + } + expect(seen).toEqual(['Bearer k1', 'Bearer k2', 'Bearer k1']); + }); + + it('uses no auth header for a keyless provider pool', async () => { + const keyless: ResolvedConfig = { + providers: new Map([ + [ + 'ollama', + { name: 'ollama', type: 'openai-compatible', baseUrl: 'http://h/v1', apiKeys: [] }, + ], + ]), + models: new Map([['m', 'ollama']]), + defaultProvider: undefined, + pricing: new Map(), + }; + const fetchImpl = vi.fn(async (_url: string, _init: { headers: Record }) => + Promise.resolve(new Response('{}', { status: 200 })), + ); + const registry = createRegistry(keyless, { fetchImpl }); + await registry.resolve('m').chat({ model: 'm', messages: [{ role: 'user', content: 'hi' }] }); + expect(fetchImpl.mock.calls[0]![1].headers.authorization).toBeUndefined(); + }); }); diff --git a/packages/gateway/src/providers/registry.ts b/packages/gateway/src/providers/registry.ts index 36743bc..2153cd6 100644 --- a/packages/gateway/src/providers/registry.ts +++ b/packages/gateway/src/providers/registry.ts @@ -1,9 +1,21 @@ import type { ResolvedConfig } from '../config.js'; -import type { FetchLike, Provider } from './types.js'; +import type { ApiKeySource, FetchLike, Provider } from './types.js'; import { createOpenAICompatibleProvider } from './openai-compatible.js'; import { createAnthropicProvider } from './anthropic.js'; import { ModelNotFoundError } from '../errors.js'; +/** A key source for a provider: undefined (keyless), the single key, or a round-robin over the pool. */ +function keySupplier(keys: readonly string[]): ApiKeySource | undefined { + if (keys.length === 0) return undefined; + if (keys.length === 1) return keys[0]; + let index = 0; + return () => { + const key = keys[index % keys.length]; + index += 1; + return key; + }; +} + export interface ProviderRegistry { /** Resolves a model name to the provider that serves it. */ resolve(model: string): Provider; @@ -20,10 +32,11 @@ export function createRegistry( ): ProviderRegistry { const providers = new Map(); for (const resolved of config.providers.values()) { + const apiKey = keySupplier(resolved.apiKeys); const adapterOptions = { name: resolved.name, baseUrl: resolved.baseUrl, - apiKey: resolved.apiKey, + ...(apiKey !== undefined ? { apiKey } : {}), ...(options.fetchImpl ? { fetchImpl: options.fetchImpl } : {}), }; providers.set( diff --git a/packages/gateway/src/providers/types.ts b/packages/gateway/src/providers/types.ts index 09c40eb..f089b3d 100644 --- a/packages/gateway/src/providers/types.ts +++ b/packages/gateway/src/providers/types.ts @@ -12,6 +12,12 @@ export type FetchLike = ( }, ) => Promise; +/** + * A provider API key: either a fixed string, or a supplier that returns the next + * key to use (so the registry can round-robin across a multi-key pool per call). + */ +export type ApiKeySource = string | (() => string | undefined); + /** A provider Sentinel can forward chat-completion requests to. */ export interface Provider { readonly name: string; diff --git a/sentinel.config.example.json b/sentinel.config.example.json index 647f598..b17a844 100644 --- a/sentinel.config.example.json +++ b/sentinel.config.example.json @@ -13,7 +13,7 @@ "gemini": { "type": "openai-compatible", "baseUrl": "https://generativelanguage.googleapis.com/v1beta/openai", - "apiKeyEnv": "GEMINI_API_KEY" + "apiKeyEnvs": ["GEMINI_API_KEY", "GEMINI_API_KEY_2"] }, "groq": { "type": "openai-compatible",